diff --git a/.dockerignore b/.dockerignore index af957fe9a..cc4718f39 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,10 @@ *~ *.pyc +**/__pycache__/ +tests/.privacy-build/ +tests/.review-output/ +tests/.startup-review/ +tests/.aggregate-review/ en */auto */build diff --git a/.github/workflows/privacy-candidate.yml b/.github/workflows/privacy-candidate.yml new file mode 100644 index 000000000..5b3d1607c --- /dev/null +++ b/.github/workflows/privacy-candidate.yml @@ -0,0 +1,102 @@ +name: Privacy candidate checks + +on: + push: + paths: + - 'Docker/privacy-diagnostic/**' + - 'Docker/privacy/**' + - 'Docker/config/docassemble*.ini*' + - 'Docker/config/nginx-*.dist' + - 'Docker/nginx.conf' + - 'Docker/docassemble-supervisor.conf' + - 'Docker/docassemble-syslog-ng.conf' + - 'Docker/syslog-ng.conf' + - 'Docker/syslog-ng-docker.conf' + - 'Docker/docassemble.logrotate' + - 'Docker/restart-post-logrotate.sh' + - 'Docker/run-nginx.sh' + - 'Docker/run-uwsgi.sh' + - 'Docker/run-uwsgilog.sh' + - 'Docker/run-celery.sh' + - 'Docker/run-celery-single.sh' + - 'Docker/run-websockets.sh' + - 'Docker/run-cron.sh' + - 'Docker/cron/**' + - 'Docker/sync.sh' + - 'Docker/process-email.sh' + - 'Docker/initialize.sh' + - 'Dockerfile' + - '.dockerignore' + - 'docassemble_webapp/docassemble/webapp/log_initialize.py' + - 'docassemble_webapp/docassemble/webapp/privacy_logging.py' + - 'docassemble_webapp/docassemble/webapp/process_email.py' + - 'tests/privacy_*/**' + - 'tests/verify_privacy.sh' + - 'docs/privacy/install-catalog.json' + - '.github/workflows/privacy-candidate.yml' + pull_request: + branches: [jacob/maintained] + +permissions: + contents: read + +concurrency: + group: privacy-${{ github.event.pull_request.head.repo.full_name || github.repository }}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true + +jobs: + populated-volume-install: + runs-on: ubuntu-24.04 + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-go@v5 + with: + go-version-file: Docker/privacy-diagnostic/go.mod + cache: false + - uses: actions/setup-python@v5 + with: + python-version: '3.14' + - name: Rehearse non-mail installation and rollback on the pinned target image + run: bash tests/privacy_native/test_install_image.sh + privacy-candidate: + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + GOCACHE: ${{ github.workspace }}/tests/.privacy-build/go-cache + GOMODCACHE: ${{ github.workspace }}/tests/.privacy-build/go-mod + GOTMPDIR: ${{ github.workspace }}/tests/.privacy-build/go-tmp + TEST_TELEMETRY_DIR: ${{ github.workspace }}/tests/.privacy-build/telemetry + XDG_CONFIG_HOME: ${{ github.workspace }}/tests/.privacy-build/config + GOTOOLCHAIN: local + steps: + - uses: actions/checkout@v4 + - name: Create Go working directories before toolchain setup + run: mkdir -p "$GOCACHE" "$GOMODCACHE" "$GOTMPDIR" "$TEST_TELEMETRY_DIR" "$XDG_CONFIG_HOME" + - uses: actions/setup-go@v5 + with: + go-version-file: Docker/privacy-diagnostic/go.mod + cache: false + - uses: actions/setup-python@v5 + with: + python-version: '3.14' + - name: Check diagnostics, aggregation, native lifecycle, and launchers + run: bash tests/verify_privacy.sh + - name: Build disposable native-protocol test image + run: docker build -t privacy-candidate-check -f tests/privacy_native/fixtures/privacy-check.Dockerfile tests/privacy_native/fixtures + - name: Check compiled preflight and monitor against nginx and Supervisor + run: | + bash tests/privacy_native/test_nginx_preflight_image.sh privacy-candidate-check + bash tests/privacy_native/test_supervisor_monitor_image.sh privacy-candidate-check + - name: Build pinned real uWSGI test runtime + run: docker build -t privacy-uwsgi-check -f tests/privacy_native/fixtures/uwsgi-check.Dockerfile tests/privacy_native/fixtures + - name: Check real uWSGI requests through Go capture + run: bash tests/privacy_native/test_uwsgi_image.sh privacy-uwsgi-check + - name: Build native log rotation and forwarding fixture + run: docker build -t privacy-rotation-check -f tests/privacy_native/fixtures/rotation-check.Dockerfile tests/privacy_native/fixtures + - name: Check exclusive rotation ownership and continued forwarding + run: bash tests/privacy_native/test_rotation_image.sh privacy-rotation-check + - name: Scan Go packages and reachable standard library for vulnerabilities + run: GOPROXY=https://proxy.golang.org go -C Docker/privacy-diagnostic run golang.org/x/vuln/cmd/govulncheck@v1.7.0 ./... diff --git a/.gitignore b/.gitignore index ad549198f..f47846c7e 100644 --- a/.gitignore +++ b/.gitignore @@ -76,6 +76,8 @@ CLAUDE.md .claude/ .claudeignore __pycache__ +tests/.privacy-build/ +tests/.aggregate-review/ pyproject.toml autoimport.py -runpylint.py \ No newline at end of file +runpylint.py diff --git a/Docker/config/docassemble-expose-uwsgi.ini b/Docker/config/docassemble-expose-uwsgi.ini index e36526126..b026d0dfb 100644 --- a/Docker/config/docassemble-expose-uwsgi.ini +++ b/Docker/config/docassemble-expose-uwsgi.ini @@ -10,4 +10,8 @@ mount = /=docassemble.webapp.run:application venv = /usr/share/docassemble/local3.14 pidfile = /var/run/uwsgi/uwsgi.pid buffer-size = 32768 -py-executable = /usr/share/docassemble/local3.14/bin/python \ No newline at end of file +py-executable = /usr/share/docassemble/local3.14/bin/python + +# Foreground request diagnostics are consumed by the privacy wrapper. +die-on-term = true +log-format = PRIVACY_REQUEST status=%(status) msecs=%(msecs) diff --git a/Docker/config/docassemble.ini.dist b/Docker/config/docassemble.ini.dist index a4a9a155e..2626864b2 100644 --- a/Docker/config/docassemble.ini.dist +++ b/Docker/config/docassemble.ini.dist @@ -13,3 +13,7 @@ buffer-size = 32768 touch-reload = {{DA_ROOT}}/webapp/docassemble.wsgi py-executable = {{DA_PYTHON}}/bin/python max-fd = 1048576 + +# Foreground request diagnostics are consumed by the privacy wrapper. +die-on-term = true +log-format = PRIVACY_REQUEST status=%(status) msecs=%(msecs) diff --git a/Docker/config/docassemblelog-expose-uwsgi.ini b/Docker/config/docassemblelog-expose-uwsgi.ini index b1261c3bb..069226639 100644 --- a/Docker/config/docassemblelog-expose-uwsgi.ini +++ b/Docker/config/docassemblelog-expose-uwsgi.ini @@ -9,3 +9,7 @@ venv = /usr/share/docassemble/local3.14 module = docassemble.webapp.listlog pidfile = /var/run/uwsgi/uwsgilog.pid callable = app + +# Foreground request diagnostics are consumed by the privacy wrapper. +die-on-term = true +log-format = PRIVACY_REQUEST status=%(status) msecs=%(msecs) diff --git a/Docker/config/docassemblelog.ini.dist b/Docker/config/docassemblelog.ini.dist index fca4c5407..53759f39f 100644 --- a/Docker/config/docassemblelog.ini.dist +++ b/Docker/config/docassemblelog.ini.dist @@ -9,3 +9,7 @@ venv = {{DA_PYTHON}} module = docassemble.webapp.listlog pidfile = /var/run/uwsgi/uwsgilog.pid callable = app + +# Foreground request diagnostics are consumed by the privacy wrapper. +die-on-term = true +log-format = PRIVACY_REQUEST status=%(status) msecs=%(msecs) diff --git a/Docker/cron/docassemble-cron-daily.sh b/Docker/cron/docassemble-cron-daily.sh index 3fb6a53d3..a6cd85c6e 100755 --- a/Docker/cron/docassemble-cron-daily.sh +++ b/Docker/cron/docassemble-cron-daily.sh @@ -1,6 +1,20 @@ #! /bin/bash export DA_ROOT="${DA_ROOT:-/usr/share/docassemble}" +# Internal handoff only; normal invocations always enter native capture. +if [ "${1:-}" != "--privacy-captured" ]; then + exec 3>&1 + exec >/dev/null 2>&1 + [ -x "${DA_ROOT}/webapp/privacy-diagnostic" ] || exit 69 + shopt -s execfail + exec "${DA_ROOT}/webapp/privacy-process" --component maintenance -- \ + /bin/bash "${BASH_SOURCE[0]}" --privacy-captured "$@" >&3 3>&- + exec 3>&1 >/dev/null + "${DA_ROOT}/webapp/privacy-diagnostic" maintenance launch >&3 3>&- + exit $? +fi +shift + export DA_DEFAULT_LOCAL="local3.14" export DA_ACTIVATE="${DA_PYTHON:-${DA_ROOT}/${DA_DEFAULT_LOCAL}}/bin/activate" diff --git a/Docker/cron/docassemble-cron-hourly.sh b/Docker/cron/docassemble-cron-hourly.sh index c305b247f..7275872d2 100755 --- a/Docker/cron/docassemble-cron-hourly.sh +++ b/Docker/cron/docassemble-cron-hourly.sh @@ -1,6 +1,20 @@ #! /bin/bash export DA_ROOT="${DA_ROOT:-/usr/share/docassemble}" +# Internal handoff only; normal invocations always enter native capture. +if [ "${1:-}" != "--privacy-captured" ]; then + exec 3>&1 + exec >/dev/null 2>&1 + [ -x "${DA_ROOT}/webapp/privacy-diagnostic" ] || exit 69 + shopt -s execfail + exec "${DA_ROOT}/webapp/privacy-process" --component maintenance -- \ + /bin/bash "${BASH_SOURCE[0]}" --privacy-captured "$@" >&3 3>&- + exec 3>&1 >/dev/null + "${DA_ROOT}/webapp/privacy-diagnostic" maintenance launch >&3 3>&- + exit $? +fi +shift + export DA_DEFAULT_LOCAL="local3.14" export DA_ACTIVATE="${DA_PYTHON:-${DA_ROOT}/${DA_DEFAULT_LOCAL}}/bin/activate" diff --git a/Docker/cron/docassemble-cron-monthly.sh b/Docker/cron/docassemble-cron-monthly.sh index 149ea1c60..a985ae397 100755 --- a/Docker/cron/docassemble-cron-monthly.sh +++ b/Docker/cron/docassemble-cron-monthly.sh @@ -1,6 +1,20 @@ #! /bin/bash export DA_ROOT="${DA_ROOT:-/usr/share/docassemble}" +# Internal handoff only; normal invocations always enter native capture. +if [ "${1:-}" != "--privacy-captured" ]; then + exec 3>&1 + exec >/dev/null 2>&1 + [ -x "${DA_ROOT}/webapp/privacy-diagnostic" ] || exit 69 + shopt -s execfail + exec "${DA_ROOT}/webapp/privacy-process" --component maintenance -- \ + /bin/bash "${BASH_SOURCE[0]}" --privacy-captured "$@" >&3 3>&- + exec 3>&1 >/dev/null + "${DA_ROOT}/webapp/privacy-diagnostic" maintenance launch >&3 3>&- + exit $? +fi +shift + export DA_DEFAULT_LOCAL="local3.14" export DA_ACTIVATE="${DA_PYTHON:-${DA_ROOT}/${DA_DEFAULT_LOCAL}}/bin/activate" diff --git a/Docker/cron/docassemble-cron-weekly.sh b/Docker/cron/docassemble-cron-weekly.sh index f4e3c1ae7..6a879d704 100755 --- a/Docker/cron/docassemble-cron-weekly.sh +++ b/Docker/cron/docassemble-cron-weekly.sh @@ -1,6 +1,20 @@ #! /bin/bash export DA_ROOT="${DA_ROOT:-/usr/share/docassemble}" +# Internal handoff only; normal invocations always enter native capture. +if [ "${1:-}" != "--privacy-captured" ]; then + exec 3>&1 + exec >/dev/null 2>&1 + [ -x "${DA_ROOT}/webapp/privacy-diagnostic" ] || exit 69 + shopt -s execfail + exec "${DA_ROOT}/webapp/privacy-process" --component maintenance -- \ + /bin/bash "${BASH_SOURCE[0]}" --privacy-captured "$@" >&3 3>&- + exec 3>&1 >/dev/null + "${DA_ROOT}/webapp/privacy-diagnostic" maintenance launch >&3 3>&- + exit $? +fi +shift + export DA_DEFAULT_LOCAL="local3.14" export DA_ACTIVATE="${DA_PYTHON:-${DA_ROOT}/${DA_DEFAULT_LOCAL}}/bin/activate" diff --git a/Docker/docassemble-supervisor.conf b/Docker/docassemble-supervisor.conf index b909dd815..26628d7d2 100644 --- a/Docker/docassemble-supervisor.conf +++ b/Docker/docassemble-supervisor.conf @@ -2,6 +2,24 @@ loglevel=%(ENV_SUPERVISORLOGLEVEL)s directory=/tmp +[eventlistener:privacy-monitor] +command=/usr/share/docassemble/webapp/privacy-monitor +user=www-data +events=PROCESS_STATE_EXITED,PROCESS_STATE_BACKOFF,PROCESS_STATE_FATAL,TICK_60 +buffer_size=256 +numprocs=1 +autostart=true +autorestart=true +startsecs=1 +startretries=3 +priority=250 +stopwaitsecs=5 +stdout_logfile=NONE +redirect_stderr=false +stderr_logfile=/usr/share/docassemble/log/privacy-monitor.log +stderr_logfile_maxbytes=5MB +stderr_logfile_backups=7 + [inet_http_server] port = *:9001 username = %(ENV_DASUPERVISORUSERNAME)s @@ -86,12 +104,14 @@ command=bash /usr/share/docassemble/webapp/run-celery.sh directory=/tmp user=www-data numprocs=1 -stdout_logfile=/usr/share/docassemble/log/worker.log -stderr_logfile=/usr/share/docassemble/log/worker.log +stdout_logfile=/usr/share/docassemble/log/privacy-celery.log +stdout_logfile_maxbytes=5MB +stdout_logfile_backups=7 +redirect_stderr=true autostart=false autorestart=true startsecs=5 -stopwaitsecs=60 +stopwaitsecs=70 killasgroup=true priority=500 @@ -100,12 +120,14 @@ command=bash /usr/share/docassemble/webapp/run-celery-single.sh directory=/tmp user=www-data numprocs=1 -stdout_logfile=/usr/share/docassemble/log/single_worker.log -stderr_logfile=/usr/share/docassemble/log/single_worker.log +stdout_logfile=/usr/share/docassemble/log/privacy-celerysingle.log +stdout_logfile_maxbytes=5MB +stdout_logfile_backups=7 +redirect_stderr=true autostart=false autorestart=true startsecs=5 -stopwaitsecs=60 +stopwaitsecs=70 killasgroup=true priority=500 @@ -145,8 +167,27 @@ killasgroup=true stopasgroup=true stopwaitsecs=20 priority=500 -stdout_logfile=/usr/share/docassemble/log/uwsgi.log -stderr_logfile=/usr/share/docassemble/log/uwsgi.log +stdout_logfile=/usr/share/docassemble/log/privacy-uwsgi.log +stdout_logfile_maxbytes=5MB +stdout_logfile_backups=7 +redirect_stderr=true + +[program:uwsgilog] +command=bash /usr/share/docassemble/webapp/run-uwsgilog.sh +user=www-data +autostart=false +autorestart=true +startretries=0 +numprocs=1 +startsecs=5 +killasgroup=true +stopasgroup=true +stopwaitsecs=20 +priority=500 +stdout_logfile=/usr/share/docassemble/log/uwsgilog.log +stdout_logfile_maxbytes=5MB +stdout_logfile_backups=7 +redirect_stderr=true [program:nginx] command=bash /usr/share/docassemble/webapp/run-nginx.sh @@ -159,6 +200,10 @@ killasgroup=true stopasgroup=true stopwaitsecs=20 priority=600 +stdout_logfile=/usr/share/docassemble/log/nginx-safe.log +stdout_logfile_maxbytes=5MB +stdout_logfile_backups=7 +redirect_stderr=true [program:nascent] command=bash /usr/share/docassemble/webapp/run-nascent.sh @@ -180,10 +225,12 @@ autorestart=true startretries=1 startsecs=1 killasgroup=true -stopwaitsecs=20 +stopwaitsecs=30 priority=600 -stdout_logfile=/usr/share/docassemble/log/websockets.log -stderr_logfile=/usr/share/docassemble/log/websockets.log +stdout_logfile=/usr/share/docassemble/log/privacy-websockets.log +stdout_logfile_maxbytes=5MB +stdout_logfile_backups=7 +redirect_stderr=true [program:initialize] command=bash /usr/share/docassemble/webapp/initialize.sh @@ -193,7 +240,7 @@ autostart=true autorestart=false exitcodes=0 startsecs=0 -stopwaitsecs=600 +stopwaitsecs=630 priority=400 [program:sync] @@ -204,6 +251,7 @@ autorestart=false exitcodes=0 stopasgroup=true killasgroup=true +stopwaitsecs=40 startsecs=0 priority=600 diff --git a/Docker/docassemble-syslog-ng.conf b/Docker/docassemble-syslog-ng.conf index f743ff0c6..c51c67992 100644 --- a/Docker/docassemble-syslog-ng.conf +++ b/Docker/docassemble-syslog-ng.conf @@ -1,9 +1,9 @@ source s_docassemble { file("/usr/share/docassemble/log/docassemble.log" flags(no-parse) follow-freq(2) program-override("docassemble") default-priority(debug)); - file("/usr/share/docassemble/log/websockets.log" flags(no-parse) follow-freq(2) program-override("websockets") default-priority(debug)); - file("/usr/share/docassemble/log/worker.log" flags(no-parse) follow-freq(2) program-override("celery") default-priority(debug)); - file("/usr/share/docassemble/log/single_worker.log" flags(no-parse) follow-freq(2) program-override("celerysingle") default-priority(debug)); - file("/usr/share/docassemble/log/uwsgi.log" flags(no-parse) follow-freq(2) program-override("uwsgi") default-priority(debug)); + file("/usr/share/docassemble/log/privacy-websockets.log" flags(no-parse) follow-freq(2) program-override("websockets") default-priority(debug)); + file("/usr/share/docassemble/log/privacy-celery.log" flags(no-parse) follow-freq(2) program-override("celery") default-priority(debug)); + file("/usr/share/docassemble/log/privacy-celerysingle.log" flags(no-parse) follow-freq(2) program-override("celerysingle") default-priority(debug)); + file("/usr/share/docassemble/log/privacy-uwsgi.log" flags(no-parse) follow-freq(2) program-override("uwsgi") default-priority(debug)); file("/var/log/apache2/access.log" flags(no-parse) follow-freq(2) program-override("apache") default-priority(debug)); file("/var/log/apache2/error.log" flags(no-parse) follow-freq(2) program-override("apache") default-priority(error)); file("/var/log/nginx/access.log" flags(no-parse) follow-freq(2) program-override("nginx") default-priority(debug)); diff --git a/Docker/docassemble.logrotate b/Docker/docassemble.logrotate index f7eae369c..c3b04bc34 100644 --- a/Docker/docassemble.logrotate +++ b/Docker/docassemble.logrotate @@ -15,6 +15,7 @@ daily missingok notifempty + sharedscripts postrotate /usr/share/docassemble/webapp/restart-post-logrotate.sh endscript diff --git a/Docker/initialize.sh b/Docker/initialize.sh index f906b6075..a7c730db3 100755 --- a/Docker/initialize.sh +++ b/Docker/initialize.sh @@ -2,6 +2,20 @@ export HOME=/root export DA_ROOT="${DA_ROOT:-/usr/share/docassemble}" +# Internal handoff only; normal invocations always enter native capture. +if [ "${1:-}" != "--privacy-captured" ]; then + exec 3>&1 + exec >/dev/null 2>&1 + [ -x "${DA_ROOT}/webapp/privacy-diagnostic" ] || exit 69 + shopt -s execfail + exec "${DA_ROOT}/webapp/privacy-process" --component initialize -- \ + /bin/bash "${BASH_SOURCE[0]}" --privacy-captured "$@" >&3 3>&- + exec 3>&1 >/dev/null + "${DA_ROOT}/webapp/privacy-diagnostic" initialize launch >&3 3>&- + exit $? +fi +shift + export DA_DEFAULT_LOCAL="local3.14" export DA_ACTIVATE="${DA_PYTHON:-${DA_ROOT}/${DA_DEFAULT_LOCAL}}/bin/activate" @@ -708,7 +722,7 @@ echo "initialize: Running start hook" >&2 python -m docassemble.webapp.starthook "${DA_CONFIG_FILE}" -if [ "${DAWEBSERVER:-nginx}" = "nginx" ]; then +if [ "${DAWEBSERVER:-nginx}" = "nginx" ] || { [ "${DAWEBSERVER:-nginx}" = "none" ] && [[ $CONTAINERROLE =~ .*:log:.* ]]; }; then echo "initialize: Setting up NGINX basic configuration and uwsgi directory" >&2 if [ "${DAREADONLYFILESYSTEM:-false}" == "false" ]; then sed -e 's@{{DA_PYTHON}}@'"${DA_PYTHON:-${DA_ROOT}/${DA_DEFAULT_LOCAL}}"'@' \ @@ -1349,6 +1363,10 @@ touch /usr/share/docassemble/log/worker.log \ && touch /usr/share/docassemble/log/single_worker.log \ && touch /usr/share/docassemble/log/uwsgi.log \ && touch /usr/share/docassemble/log/websockets.log \ + && touch /usr/share/docassemble/log/privacy-celery.log \ + && touch /usr/share/docassemble/log/privacy-celerysingle.log \ + && touch /usr/share/docassemble/log/privacy-uwsgi.log \ + && touch /usr/share/docassemble/log/privacy-websockets.log \ && chown -R www-data:www-data /usr/share/docassemble/log if [ "${DAWEBSERVER:-nginx}" = "none" ]; then @@ -1366,6 +1384,7 @@ if [ "${DAWEBSERVER:-nginx}" = "none" ]; then ${SUPERVISORCMD} start uwsgi fi if [[ $CONTAINERROLE =~ .*:log:.* ]]; then + ${SUPERVISORCMD} start uwsgilog || exit 1 echo "initialize: Starting NGINX" >&2 ${SUPERVISORCMD} start nginx fi @@ -1456,6 +1475,9 @@ if [ "${DAWEBSERVER:-nginx}" = "nginx" ]; then echo "initialize: Starting uwsgi" >&2 ${SUPERVISORCMD} start uwsgi fi + if [[ $CONTAINERROLE =~ .*:log:.* ]]; then + ${SUPERVISORCMD} start uwsgilog || exit 1 + fi if [[ $CONTAINERROLE =~ .*:(all|web|log):.* ]]; then if [ "$NGINXRUNNING" == "false" ]; then if [ "$NASCENTRUNNING" == "true" ]; then diff --git a/Docker/nginx.conf b/Docker/nginx.conf index e69de29bb..a8bdd1fea 100644 --- a/Docker/nginx.conf +++ b/Docker/nginx.conf @@ -0,0 +1,26 @@ +# Source-owned nginx configuration for the maintained privacy profile. +user www-data; +worker_processes auto; +worker_cpu_affinity auto; +pid /run/nginx.pid; +error_log stderr; +include /etc/nginx/modules-enabled/*.conf; +include /usr/local/lib/docassemble-privacy/nginx-lifecycle.conf; + +events { worker_connections 768; } + +http { + sendfile on; + tcp_nopush on; + types_hash_max_size 2048; + server_tokens off; + include /etc/nginx/mime.types; + default_type application/octet-stream; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_prefer_server_ciphers off; + log_format privacy_counts 'PRIVACY_REQUEST status=$status seconds=$request_time'; + access_log /dev/stdout privacy_counts; + gzip on; + include /etc/nginx/conf.d/*.conf; + include /etc/nginx/sites-enabled/*; +} diff --git a/Docker/privacy-diagnostic/cmd/privacy-monitor/main.go b/Docker/privacy-diagnostic/cmd/privacy-monitor/main.go new file mode 100644 index 000000000..a665d89c3 --- /dev/null +++ b/Docker/privacy-diagnostic/cmd/privacy-monitor/main.go @@ -0,0 +1,10 @@ +// privacy-monitor consumes Supervisor state events and emits fixed failure records. +package main + +import ( + "os" + + "github.com/jacobyoby/docassemble/privacy-diagnostic/internal/monitor" +) + +func main() { os.Exit(monitor.Run(os.Args[1:])) } diff --git a/Docker/privacy-diagnostic/cmd/privacy-preflight/main.go b/Docker/privacy-diagnostic/cmd/privacy-preflight/main.go new file mode 100644 index 000000000..96ed57387 --- /dev/null +++ b/Docker/privacy-diagnostic/cmd/privacy-preflight/main.go @@ -0,0 +1,9 @@ +package main + +import ( + "os" + + "github.com/jacobyoby/docassemble/privacy-diagnostic/internal/preflight" +) + +func main() { os.Exit(preflight.Run(os.Args[1:])) } diff --git a/Docker/privacy-diagnostic/cmd/privacy-preflight/main_linux_test.go b/Docker/privacy-diagnostic/cmd/privacy-preflight/main_linux_test.go new file mode 100644 index 000000000..0d2642824 --- /dev/null +++ b/Docker/privacy-diagnostic/cmd/privacy-preflight/main_linux_test.go @@ -0,0 +1,69 @@ +package main + +import ( + "context" + "errors" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "testing" + "time" + + "github.com/jacobyoby/docassemble/privacy-diagnostic/internal/preflight" +) + +func TestMain(m *testing.M) { + if os.Getenv("PRIVACY_PREFLIGHT_CLI_TEST") == "1" { + main() + os.Exit(99) + } + os.Exit(m.Run()) +} + +func TestFileChecksAndInvalidArgumentsRemainSilent(t *testing.T) { + root := t.TempDir() + valid := filepath.Join(root, "valid.ini") + invalid := filepath.Join(root, "SYNTHETIC_PRIVATE.ini") + large := filepath.Join(root, "large.ini") + fifo := filepath.Join(root, "fifo") + base := "[uwsgi]\nmaster=true\ndie-on-term=true\nlog-format=" + preflight.UwsgiFormat + "\n" + for path, data := range map[string]string{valid: base, invalid: base + "logto=/tmp/private\n", large: strings.Repeat("#", preflight.MaxBytes+1)} { + if os.WriteFile(path, []byte(data), 0600) != nil { + t.Fatal("fixture creation failed") + } + } + if err := syscall.Mkfifo(fifo, 0600); err != nil { + t.Fatal(err) + } + self, err := os.Executable() + if err != nil { + t.Fatal(err) + } + for _, item := range []struct { + args []string + code int + }{{[]string{"uwsgi", valid}, 0}, {[]string{"uwsgi", invalid}, 70}, + {[]string{"uwsgi", large}, 70}, {[]string{"uwsgi", fifo}, 70}, + {[]string{"uwsgi", root}, 70}, {[]string{"uwsgi", valid + ".missing"}, 70}, + {nil, 70}, {[]string{"nginx", "extra"}, 70}, {[]string{"uwsgi"}, 70}, + {[]string{"SYNTHETIC_PRIVATE"}, 70}} { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + cmd := exec.CommandContext(ctx, self, item.args...) + cmd.Env = append(os.Environ(), "PRIVACY_PREFLIGHT_CLI_TEST=1") + output, err := cmd.CombinedOutput() + cancel() + code := 0 + if err != nil { + var exit *exec.ExitError + if !errors.As(err, &exit) { + t.Fatal(err) + } + code = exit.ExitCode() + } + if code != item.code || len(output) != 0 { + t.Fatalf("preflight leaked or returned wrong exit: got=%d want=%d output=%q", code, item.code, output) + } + } +} diff --git a/Docker/privacy-diagnostic/cmd/privacy-process/application_linux_test.go b/Docker/privacy-diagnostic/cmd/privacy-process/application_linux_test.go new file mode 100644 index 000000000..333a8390d --- /dev/null +++ b/Docker/privacy-diagnostic/cmd/privacy-process/application_linux_test.go @@ -0,0 +1,95 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + "github.com/jacobyoby/docassemble/privacy-diagnostic/internal/aggregate" +) + +func TestApplicationLaunchersReachGoCapture(t *testing.T) { + for _, item := range []struct{ script, component string }{ + {"run-celery.sh", "celery"}, {"run-celery-single.sh", "celerysingle"}, + {"run-websockets.sh", "websockets"}, + } { + t.Run(item.component, func(t *testing.T) { + root := t.TempDir() + for _, dir := range []string{"runtime/bin", "webapp"} { + if err := os.MkdirAll(filepath.Join(root, dir), 0700); err != nil { + t.Fatal(err) + } + } + native := `printf '%s\n' 'SYNTHETIC_PRIVATE_NATIVE' 'PRIVACY_REQUEST status=200 msecs=0' +printf '%s\n' 'SYNTHETIC_PRIVATE_NATIVE' 'PRIVACY_REQUEST status=500 msecs=0' >&2 +exit 17 +` + files := map[string]string{ + "runtime/bin/activate": "printf 'SYNTHETIC_PRIVATE_BOOTSTRAP\\n' >&2\n", + "runtime/bin/python": `#!/bin/sh +if [ "$1" = "-m" ]; then + printf '%s\n' 'export LOCALE="C.UTF-8 UTF-8"' 'DAMAXCELERYWORKERS=4' 'DACELERYWORKERS=3' + printf 'SYNTHETIC_PRIVATE_BOOTSTRAP\n' >&2 + exit 0 +fi +` + native, + "runtime/bin/celery": "#!/bin/sh\n" + native, + "webapp/privacy-diagnostic": "#!/bin/sh\nexit 70\n", + } + for path, content := range files { + if err := os.WriteFile(filepath.Join(root, path), []byte(content), 0700); err != nil { + t.Fatal(err) + } + } + self, err := os.Executable() + if err != nil { + t.Fatal(err) + } + binary, err := os.ReadFile(self) + if err != nil || os.WriteFile(filepath.Join(root, "webapp/privacy-process"), binary, 0700) != nil { + t.Fatal("cannot prepare actual Go CLI") + } + script, err := filepath.Abs("../../../../Docker/" + item.script) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, "bash", script) + cmd.Env = append(os.Environ(), "PRIVACY_PROCESS_CLI_TEST=1", "DA_ROOT="+root, "DA_PYTHON="+root+"/runtime") + cmd.WaitDelay = time.Second + var stderr bytes.Buffer + cmd.Stderr = &stderr + data, err := cmd.Output() + var exit *exec.ExitError + if !errors.As(err, &exit) || exit.ExitCode() != 17 || stderr.Len() != 0 || bytes.Contains(data, []byte("SYNTHETIC_PRIVATE")) { + t.Fatalf("application capture failed: %v stdout=%q stderr=%q", err, data, stderr.Bytes()) + } + var last aggregate.Snapshot + decoder := json.NewDecoder(bytes.NewReader(data)) + for { + var value aggregate.Snapshot + if err := decoder.Decode(&value); err != nil { + if err == io.EOF { + break + } + t.Fatal(err) + } + if _, err := json.Marshal(value); err != nil { + t.Fatal(err) + } + last = value + } + if last.Component != item.component || last.Unclassified != 4 || last.Status2xx != 0 || last.Status5xx != 0 { + t.Fatalf("wrong final application counters: %#v", last) + } + }) + } +} diff --git a/Docker/privacy-diagnostic/cmd/privacy-process/cron_linux_test.go b/Docker/privacy-diagnostic/cmd/privacy-process/cron_linux_test.go new file mode 100644 index 000000000..4ffcb452f --- /dev/null +++ b/Docker/privacy-diagnostic/cmd/privacy-process/cron_linux_test.go @@ -0,0 +1,49 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "os" + "os/exec" + "testing" + "time" + + "github.com/jacobyoby/docassemble/privacy-diagnostic/internal/aggregate" +) + +func TestFiniteCommandsReportOnlyAtExitWithoutMailInputOrRetrySemantics(t *testing.T) { + for _, profile := range []string{"cron", "maintenance"} { + t.Run(profile, func(t *testing.T) { + self, err := os.Executable() + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 8*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, self, "--component", profile, "--", "/bin/sh", "-c", + "read value && exit 99; printf 'SYNTHETIC_PRIVATE_CRON\\n'; printf 'SYNTHETIC_PRIVATE_CRON\\n' >&2; sleep 1.2; exit 17") + cmd.Env = append(os.Environ(), "PRIVACY_PROCESS_CLI_TEST=1") + cmd.Stdin = bytes.NewBufferString("must not reach the cron child\n") + cmd.WaitDelay = time.Second + var stderr bytes.Buffer + cmd.Stderr = &stderr + output, err := cmd.Output() + var failure *exec.ExitError + if !errors.As(err, &failure) || failure.ExitCode() != 17 || stderr.Len() != 0 { + t.Fatalf("cron exit/input semantics changed: %v %q", err, stderr.Bytes()) + } + if bytes.Count(output, []byte("\n")) != 1 || bytes.Contains(output, []byte("SYNTHETIC_PRIVATE")) { + t.Fatalf("cron emitted periodic or raw output: %q", output) + } + var record aggregate.Snapshot + if err := json.Unmarshal(output, &record); err != nil { + t.Fatal(err) + } + if _, err := json.Marshal(record); err != nil || record.Component != profile || record.Unclassified != 2 { + t.Fatalf("cron counter contract changed: %#v %v", record, err) + } + }) + } +} diff --git a/Docker/privacy-diagnostic/cmd/privacy-process/logger_linux_test.go b/Docker/privacy-diagnostic/cmd/privacy-process/logger_linux_test.go new file mode 100644 index 000000000..b8367769a --- /dev/null +++ b/Docker/privacy-diagnostic/cmd/privacy-process/logger_linux_test.go @@ -0,0 +1,76 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "io" + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + "github.com/jacobyoby/docassemble/privacy-diagnostic/internal/aggregate" +) + +func TestApplicationLoggerUsesOnlyCapturedStreams(t *testing.T) { + python, err := exec.LookPath("python3.14") + if err != nil { + t.Fatal("Python 3.14 required for real application logger fixture") + } + root, err := filepath.Abs("../../../..") + if err != nil { + t.Fatal(err) + } + self, err := os.Executable() + if err != nil { + t.Fatal(err) + } + for _, profile := range []string{"uwsgi", "celery", "celerysingle", "websockets", "mail"} { + for _, appContext := range []string{"web", "std", "celery", "cron"} { + for _, server := range []string{"none", "synthetic"} { + for _, debug := range []string{"info", "debug"} { + t.Run(profile+"/"+appContext+"/"+server+"/"+debug, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + directory := t.TempDir() + cmd := exec.CommandContext(ctx, self, "--component", profile, "--", python, + "-B", root+"/tests/privacy_native/fixtures/application-logger.py", root, + directory, appContext, server, debug, "--die-on-term") + cmd.Env = append(os.Environ(), "PRIVACY_PROCESS_CLI_TEST=1", "PYTHONDONTWRITEBYTECODE=1") + cmd.WaitDelay = time.Second + var stderr bytes.Buffer + cmd.Stderr = &stderr + data, err := cmd.Output() + if err != nil || stderr.Len() != 0 || bytes.Contains(data, []byte("SYNTHETIC_PRIVATE")) { + t.Fatalf("application logger capture failed: %v stdout=%q stderr=%q", err, data, stderr.Bytes()) + } + entries, err := os.ReadDir(directory) + if err != nil || len(entries) != 0 { + t.Fatal("application logger opened an independent retained file") + } + decoder := json.NewDecoder(bytes.NewReader(data)) + var last aggregate.Snapshot + for { + var value aggregate.Snapshot + if err := decoder.Decode(&value); err != nil { + if err == io.EOF { + break + } + t.Fatal(err) + } + if _, err := json.Marshal(value); err != nil { + t.Fatal(err) + } + last = value + } + if last.Component != profile || last.Unclassified != 6 || last.Rejected != 0 || last.Dropped != 0 { + t.Fatalf("application lines did not reach capture: %#v", last) + } + }) + } + } + } + } +} diff --git a/Docker/privacy-diagnostic/cmd/privacy-process/mail_contract_linux_test.go b/Docker/privacy-diagnostic/cmd/privacy-process/mail_contract_linux_test.go new file mode 100644 index 000000000..14043c655 --- /dev/null +++ b/Docker/privacy-diagnostic/cmd/privacy-process/mail_contract_linux_test.go @@ -0,0 +1,150 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "os" + "os/exec" + "path/filepath" + "syscall" + "testing" + "time" + + "github.com/jacobyoby/docassemble/privacy-diagnostic/internal/aggregate" + "github.com/jacobyoby/docassemble/privacy-diagnostic/internal/runner" +) + +func mailCLI(t *testing.T, script string, args ...string) *exec.Cmd { + t.Helper() + self, err := os.Executable() + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 8*time.Second) + t.Cleanup(cancel) + command := []string{"--component", "mail", "--", "/bin/sh", "-c", script, "synthetic-mail"} + cmd := exec.CommandContext(ctx, self, append(command, args...)...) + cmd.Env = append(os.Environ(), "PRIVACY_PROCESS_CLI_TEST=1") + cmd.WaitDelay = time.Second + t.Cleanup(func() { + if cmd.Process != nil && cmd.ProcessState == nil { + cmd.Process.Kill() + cmd.Wait() + } + }) + return cmd +} + +func temporaryMailExit(t *testing.T, err error) { + t.Helper() + var exit *exec.ExitError + if !errors.As(err, &exit) || exit.ExitCode() != runner.TemporaryFailure { + t.Fatalf("mail failure did not defer delivery: %v", err) + } +} + +func TestMailCountersHaveOneFinalRecordAcrossReportingIntervals(t *testing.T) { + // More than 20 KiB of child output and multiple normal reporting intervals + // must still produce one record below Exim's default total-output limit. + cmd := mailCLI(t, `j=0; while [ "$j" -lt 3 ]; do +i=0; while [ "$i" -lt 1000 ]; do +printf 'SYNTHETIC_PRIVATE_MAIL\n'; i=$((i+1)); done +sleep 1; j=$((j+1)); done`) + var stderr bytes.Buffer + cmd.Stderr = &stderr + data, err := cmd.Output() + if err != nil || stderr.Len() != 0 || len(data) > 8191 || bytes.Count(data, []byte{'\n'}) != 1 { + t.Fatalf("mail output is not bounded to one record: exit=%v bytes=%d records=%d", err, len(data), bytes.Count(data, []byte{'\n'})) + } + var value aggregate.Snapshot + if json.Unmarshal(data, &value) != nil || value.Component != "mail" || value.Unclassified != 3000 { + t.Fatalf("mail final counters lost output: %q", data) + } + if _, err := json.Marshal(value); err != nil { + t.Fatal(err) + } +} + +func TestMailChildSignalDefersDelivery(t *testing.T) { + cmd := mailCLI(t, "kill -KILL $$") + data, err := cmd.CombinedOutput() + temporaryMailExit(t, err) + var value aggregate.Snapshot + if json.Unmarshal(data, &value) != nil || value.Component != "mail" { + t.Fatal("child signal lost the fixed final counters") + } +} + +func TestMailInterruptedSuccessfulChildStillDefersDelivery(t *testing.T) { + for _, stop := range []syscall.Signal{syscall.SIGINT, syscall.SIGTERM} { + t.Run(stop.String(), func(t *testing.T) { + ready := filepath.Join(t.TempDir(), "ready") + cmd := mailCLI(t, `trap 'exit 0' TERM; : > "$1"; while :; do sleep 0.05; done`, ready) + var output bytes.Buffer + cmd.Stdout, cmd.Stderr = &output, &output + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(2 * time.Second) + for { + if _, err := os.Stat(ready); err == nil { + break + } + if time.Now().After(deadline) { + t.Fatal("synthetic mail child did not become ready") + } + time.Sleep(5 * time.Millisecond) + } + if err := cmd.Process.Signal(stop); err != nil { + t.Fatal(err) + } + temporaryMailExit(t, cmd.Wait()) + if bytes.Contains(output.Bytes(), []byte("SYNTHETIC_PRIVATE")) { + t.Fatal("interruption exposed child output") + } + }) + } +} + +func TestMailBrokenAndFullOutputPipesDeferWithinDeadline(t *testing.T) { + for _, mode := range []string{"broken", "full"} { + t.Run(mode, func(t *testing.T) { + reader, writer, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + defer reader.Close() + defer writer.Close() + if mode == "broken" { + reader.Close() + } else { + fd := int(writer.Fd()) + if err := syscall.SetNonblock(fd, true); err != nil { + t.Fatal(err) + } + for { + _, err := syscall.Write(fd, bytes.Repeat([]byte{'x'}, 4096)) + if err == syscall.EAGAIN { + break + } + if err != nil { + t.Fatal(err) + } + } + if err := syscall.SetNonblock(fd, false); err != nil { + t.Fatal(err) + } + } + cmd := mailCLI(t, "printf 'SYNTHETIC_PRIVATE_MAIL\\n'") + var stderr bytes.Buffer + cmd.Stdout, cmd.Stderr = writer, &stderr + started := time.Now() + temporaryMailExit(t, cmd.Run()) + if time.Since(started) > 7*time.Second || stderr.Len() != 0 { + t.Fatal("mail output failure was unbounded or exposed diagnostics") + } + }) + } +} diff --git a/Docker/privacy-diagnostic/cmd/privacy-process/mail_linux_test.go b/Docker/privacy-diagnostic/cmd/privacy-process/mail_linux_test.go new file mode 100644 index 000000000..276eb9481 --- /dev/null +++ b/Docker/privacy-diagnostic/cmd/privacy-process/mail_linux_test.go @@ -0,0 +1,154 @@ +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "io" + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + "github.com/jacobyoby/docassemble/privacy-diagnostic/internal/aggregate" + "github.com/jacobyoby/docassemble/privacy-diagnostic/internal/runner" +) + +func TestMailLauncherStreamsInputAndReportsProcessingFailures(t *testing.T) { + root, err := filepath.Abs("../../../..") + if err != nil { + t.Fatal(err) + } + python, err := exec.LookPath("python3.14") + if err != nil { + t.Fatal(err) + } + inputCommand := exec.Command(python, "-B", "-c", "from mail_support import MESSAGE; import sys; sys.stdout.buffer.write(MESSAGE)") + inputCommand.Env = append(os.Environ(), "PYTHONPATH="+root+"/tests/privacy_native") + message, err := inputCommand.Output() + if err != nil { + t.Fatal("cannot load synthetic message", err) + } + for _, item := range []struct { + mode, nativeExit string + want int + input []byte + }{ + {"input", "0", 0, bytes.Repeat([]byte{0, 255, '\r', '\n', 'x'}, 256*1024)}, + {"input", "17", 75, message}, + {"success", "0", 0, message}, + {"read", "0", 75, message}, + {"unknown", "0", 75, message}, + {"database", "0", 75, message}, + {"broker", "0", 75, message}, + } { + t.Run(item.mode+"/"+item.nativeExit, func(t *testing.T) { + directory := t.TempDir() + for _, path := range []string{"runtime/bin", "webapp", "message-files"} { + if err := os.MkdirAll(filepath.Join(directory, path), 0700); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(directory+"/runtime/bin/activate", []byte("printf 'SYNTHETIC_PRIVATE_BOOTSTRAP\\n' >&2\n"), 0600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(root+"/tests/privacy_native/fixtures/mail-runtime.py", directory+"/runtime/bin/python"); err != nil { + t.Fatal(err) + } + self, err := os.Executable() + if err != nil { + t.Fatal(err) + } + binary, err := os.ReadFile(self) + if err != nil || os.WriteFile(directory+"/webapp/privacy-process", binary, 0700) != nil { + t.Fatal("cannot prepare actual Go capture CLI") + } + if err := os.WriteFile(directory+"/webapp/privacy-diagnostic", []byte("#!/bin/sh\nexit 70\n"), 0700); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, "bash", root+"/Docker/process-email.sh") + digest := sha256.Sum256(item.input) + cmd.Env = append(os.Environ(), "PRIVACY_PROCESS_CLI_TEST=1", "DA_ROOT="+directory, "DA_PYTHON="+directory+"/runtime", + "SYNTHETIC_MAIL_MODE="+item.mode, "SYNTHETIC_MAIL_EXIT="+item.nativeExit, + "SYNTHETIC_MAIL_SHA256="+hex.EncodeToString(digest[:]), "SYNTHETIC_MAIL_DIRECTORY="+directory+"/message-files") + cmd.Stdin = bytes.NewReader(item.input) + cmd.WaitDelay = time.Second + var stderr bytes.Buffer + cmd.Stderr = &stderr + data, err := cmd.Output() + code := 0 + if err != nil { + var exit *exec.ExitError + if !errors.As(err, &exit) { + t.Fatal(err) + } + code = exit.ExitCode() + } + if code != item.want || stderr.Len() != 0 || bytes.Contains(data, []byte("SYNTHETIC_PRIVATE")) { + t.Fatalf("mail capture/exit failed: code=%d stdout=%q stderr=%q", code, data, stderr.Bytes()) + } + decoder := json.NewDecoder(bytes.NewReader(data)) + var last aggregate.Snapshot + for { + var value aggregate.Snapshot + if err := decoder.Decode(&value); err != nil { + if err == io.EOF { + break + } + t.Fatal(err) + } + if _, err := json.Marshal(value); err != nil { + t.Fatal(err) + } + last = value + } + if last.Component != "mail" || last.Unclassified == 0 { + t.Fatal("missing mail counters") + } + if (item.mode == "input" || item.mode == "success") && last.Unclassified != 3 { + t.Fatalf("mail output lost or became request metrics: %#v", last) + } + entries, err := os.ReadDir(directory + "/message-files") + if err != nil || len(entries) != 0 { + t.Fatal("mail processor created an independent message/log file") + } + }) + } +} + +func TestMailSetupFailuresDeferDelivery(t *testing.T) { + for _, args := range [][]string{ + {"--component", "mail"}, + {"--component", "mail", "--", "relative"}, + {"--component", "mail", "--", "/missing-synthetic-mail-runtime"}, + {"--component", "mail", "--", "/bin/true"}, // Unsupported regular-file sink. + } { + self, err := os.Executable() + if err != nil { + t.Fatal(err) + } + cmd := exec.Command(self, args...) + cmd.Env = append(os.Environ(), "PRIVACY_PROCESS_CLI_TEST=1") + output, err := os.CreateTemp(t.TempDir(), "sink") + if err != nil { + t.Fatal(err) + } + cmd.Stdout, cmd.Stderr = output, output + err = cmd.Run() + output.Close() + var exit *exec.ExitError + if !errors.As(err, &exit) || exit.ExitCode() != runner.TemporaryFailure { + t.Fatalf("mail setup did not defer: %v", err) + } + data, err := os.ReadFile(output.Name()) + if err != nil || len(data) != 0 { + t.Fatal("mail setup emitted raw diagnostics") + } + } +} diff --git a/Docker/privacy-diagnostic/cmd/privacy-process/main.go b/Docker/privacy-diagnostic/cmd/privacy-process/main.go new file mode 100644 index 000000000..269d1514f --- /dev/null +++ b/Docker/privacy-diagnostic/cmd/privacy-process/main.go @@ -0,0 +1,10 @@ +// privacy-process wraps one foreground service with bounded counter capture. +package main + +import ( + "os" + + "github.com/jacobyoby/docassemble/privacy-diagnostic/internal/runner" +) + +func main() { os.Exit(runner.Run(os.Args[1:])) } diff --git a/Docker/privacy-diagnostic/cmd/privacy-process/main_linux_test.go b/Docker/privacy-diagnostic/cmd/privacy-process/main_linux_test.go new file mode 100644 index 000000000..beb5a59c2 --- /dev/null +++ b/Docker/privacy-diagnostic/cmd/privacy-process/main_linux_test.go @@ -0,0 +1,153 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + "github.com/jacobyoby/docassemble/privacy-diagnostic/internal/aggregate" + "github.com/jacobyoby/docassemble/privacy-diagnostic/internal/preflight" + "github.com/jacobyoby/docassemble/privacy-diagnostic/internal/runner" +) + +func TestMain(m *testing.M) { + if os.Getenv("PRIVACY_PROCESS_CLI_TEST") == "1" { + if filepath.Base(os.Args[0]) == "privacy-preflight" { + os.Exit(preflight.Run(os.Args[1:])) + } + main() + os.Exit(99) + } + os.Exit(m.Run()) +} + +func TestLaunchersReachGoPreflightAndCapture(t *testing.T) { + for _, item := range []struct{ name, mode, ini, fault string }{ + {"run-uwsgi.sh", "nginx", "docassemble.ini", ""}, + {"run-uwsgi.sh", "none", "docassemble-expose-uwsgi.ini", ""}, + {"run-uwsgilog.sh", "nginx", "docassemblelog.ini", ""}, + {"run-uwsgi.sh", "nginx", "docassemble.ini", "unsafe"}, + {"run-uwsgilog.sh", "nginx", "docassemblelog.ini", "unsafe"}, + {"run-uwsgi.sh", "nginx", "docassemble.ini", "missing-preflight"}, + {"run-uwsgilog.sh", "nginx", "docassemblelog.ini", "missing-preflight"}, + } { + t.Run(item.name+"/"+item.mode+"/"+item.fault, func(t *testing.T) { + root := t.TempDir() + for _, dir := range []string{"runtime/bin", "webapp", "config"} { + if err := os.MkdirAll(filepath.Join(root, dir), 0700); err != nil { + t.Fatal(err) + } + } + files := map[string]string{ + "runtime/bin/activate": "true\n", + "runtime/bin/python": `#!/bin/sh +if [ "$1" = "-m" ]; then + printf '%s\n' 'export LOCALE="C.UTF-8 UTF-8"' 'export DAREADONLYFILESYSTEM="true"' + exit 0 +fi +exit 99 +`, + "runtime/bin/uwsgi": `#!/bin/sh +printf 'started\n' > "$DA_ROOT/native-started" || exit 98 +printf '%s\n' 'SYNTHETIC_PRIVATE_NATIVE' 'PRIVACY_REQUEST status=200 msecs=99' +printf '%s\n' 'PRIVACY_REQUEST status=503 msecs=30000' 'SYNTHETIC_PRIVATE_NATIVE' >&2 +exit 17 +`, + "webapp/privacy-diagnostic": "#!/bin/sh\nprintf '%s:%s\\n' \"$1\" \"$2\"\nexit 70\n", + } + base := "[uwsgi]\nmaster=true\ndie-on-term=true\nlog-format=" + preflight.UwsgiFormat + "\n" + for _, ini := range []string{"docassemble.ini", "docassemble-expose-uwsgi.ini", "docassemblelog.ini"} { + files["config/"+ini] = base + "logto=/tmp/SYNTHETIC_PRIVATE\n" + } + if item.fault != "unsafe" { + files["config/"+item.ini] = base + } + for path, content := range files { + if err := os.WriteFile(filepath.Join(root, path), []byte(content), 0700); err != nil { + t.Fatal(err) + } + } + self, err := os.Executable() + if err != nil { + t.Fatal(err) + } + binary, err := os.ReadFile(self) + if err != nil { + t.Fatal(err) + } + for _, executable := range []string{"privacy-process", "privacy-preflight"} { + if executable == "privacy-preflight" && item.fault == "missing-preflight" { + continue + } + if os.WriteFile(filepath.Join(root, "webapp/"+executable), binary, 0700) != nil { + t.Fatal("cannot install Go CLI fixture") + } + } + script, err := filepath.Abs("../../../../Docker/" + item.name) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, "bash", script) + cmd.Env = append(os.Environ(), "PRIVACY_PROCESS_CLI_TEST=1", "DA_ROOT="+root, "DA_PYTHON="+root+"/runtime", "DAWEBSERVER="+item.mode) + cmd.WaitDelay = time.Second + var stderr bytes.Buffer + cmd.Stderr = &stderr + data, err := cmd.Output() + var exit *exec.ExitError + if item.fault != "" { + component := "uwsgi" + if item.name == "run-uwsgilog.sh" { + component = "uwsgilog" + } + if !errors.As(err, &exit) || exit.ExitCode() != 70 || string(data) != component+":preflight\n" || stderr.Len() != 0 { + t.Fatalf("unsafe setup did not fail at preflight: %v stdout=%q stderr=%q", err, data, stderr.Bytes()) + } + if _, err := os.Stat(filepath.Join(root, "native-started")); !errors.Is(err, os.ErrNotExist) { + t.Fatal("native service started after rejected preflight") + } + return + } + if !errors.As(err, &exit) || exit.ExitCode() != 17 || stderr.Len() != 0 || bytes.Contains(data, []byte("SYNTHETIC_PRIVATE")) { + t.Fatalf("handoff/capture failed: %v stdout=%q stderr=%q", err, data, stderr.Bytes()) + } + decoder := json.NewDecoder(bytes.NewReader(data)) + var last aggregate.Snapshot + for { + var value aggregate.Snapshot + if err := decoder.Decode(&value); err != nil { + if err == io.EOF { + break + } + t.Fatal(err) + } + last = value + } + if last.Component != "uwsgi" || last.Status2xx != 1 || last.Status5xx != 1 || last.Unclassified != 2 { + t.Fatalf("final native counters missing: %#v", last) + } + }) + } +} + +func TestCLIRejectsInvalidInvocation(t *testing.T) { + self, err := os.Executable() + if err != nil { + t.Fatal(err) + } + cmd := exec.Command(self, "SYNTHETIC_PRIVATE_NATIVE") + cmd.Env = append(os.Environ(), "PRIVACY_PROCESS_CLI_TEST=1") + data, err := cmd.CombinedOutput() + var exit *exec.ExitError + if !errors.As(err, &exit) || exit.ExitCode() != runner.UsageFailure || len(data) != 0 { + t.Fatalf("invalid CLI did not fail closed: %v %q", err, data) + } +} diff --git a/Docker/privacy-diagnostic/go.mod b/Docker/privacy-diagnostic/go.mod new file mode 100644 index 000000000..4aa210be6 --- /dev/null +++ b/Docker/privacy-diagnostic/go.mod @@ -0,0 +1,3 @@ +module github.com/jacobyoby/docassemble/privacy-diagnostic + +go 1.27.0 diff --git a/Docker/privacy-diagnostic/internal/aggregate/aggregate.go b/Docker/privacy-diagnostic/internal/aggregate/aggregate.go new file mode 100644 index 000000000..bead99909 --- /dev/null +++ b/Docker/privacy-diagnostic/internal/aggregate/aggregate.go @@ -0,0 +1,158 @@ +package aggregate + +import ( + "bytes" + "unicode/utf8" +) + +// Feed borrows chunk only for this call. Partial input is copied into a fixed +// buffer; oversized lines are discarded and counted exactly once, immediately. +func (a *Aggregator) Feed(stream Stream, chunk []byte) error { + s, err := a.state(stream) + if err != nil { + return err + } + for len(chunk) > 0 { + newline := bytes.IndexByte(chunk, '\n') + if s.dropping { + if newline < 0 { + return nil + } + s.dropping = false + chunk = chunk[newline+1:] + continue + } + if newline < 0 { + if len(chunk) > MaxLine-s.used { + s.erase() + s.dropping = true + increment(&a.counts.dropped) + } else { + s.used += copy(s.buffer[s.used:], chunk) + } + return nil + } + if newline > MaxLine-s.used { + s.erase() + increment(&a.counts.dropped) + } else if s.used == 0 { + a.line(chunk[:newline]) + } else { + s.used += copy(s.buffer[s.used:], chunk[:newline]) + a.line(s.buffer[:s.used]) + s.erase() + } + chunk = chunk[newline+1:] + } + return nil +} + +// Finish processes one unterminated tail and resets only the requested stream. +// Repeated calls are harmless. Overflow was already counted by Feed. +func (a *Aggregator) Finish(stream Stream) error { + s, err := a.state(stream) + if err != nil { + return err + } + if !s.dropping && s.used > 0 { + a.line(s.buffer[:s.used]) + } + s.erase() + s.dropping = false + return nil +} + +func (a *Aggregator) line(line []byte) { + if len(line) > 0 && line[len(line)-1] == '\r' { + line = line[:len(line)-1] + } + if !utf8.Valid(line) || bytes.IndexByte(line, 0) >= 0 || bytes.IndexByte(line, '\r') >= 0 { + increment(&a.counts.rejected) + return + } + if a.component.application() { + increment(&a.counts.unclassified) + return + } + first := bytes.IndexByte(line, ' ') + prefix := line + if first >= 0 { + prefix = line[:first] + } + if !bytes.Equal(prefix, []byte("PRIVACY_REQUEST")) { + increment(&a.counts.unclassified) + return + } + if first < 0 { + increment(&a.counts.rejected) + return + } + rest := line[first+1:] + second := bytes.IndexByte(rest, ' ') + if second < 0 || bytes.IndexByte(rest[second+1:], ' ') >= 0 { + increment(&a.counts.rejected) + return + } + status, goodStatus := requestStatus(rest[:second]) + milliseconds, goodLatency := requestLatency(rest[second+1:], a.component) + if !goodStatus || !goodLatency { + increment(&a.counts.rejected) + return + } + increment(&a.counts.status[status/100-1]) + bucket := 3 + switch { + case milliseconds < 100: + bucket = 0 + case milliseconds < 1000: + bucket = 1 + case milliseconds < 30000: + bucket = 2 + } + increment(&a.counts.latency[bucket]) +} + +func requestStatus(token []byte) (uint32, bool) { + data, ok := bytes.CutPrefix(token, []byte("status=")) + if !ok || len(data) != 3 || data[0] < '1' || data[0] > '5' { + return 0, false + } + return digits(data) +} + +func requestLatency(token []byte, component Component) (uint32, bool) { + if data, ok := bytes.CutPrefix(token, []byte("msecs=")); ok { + value, valid := canonicalNumber(data, 6) + return value, valid && value <= 600000 + } + data, ok := bytes.CutPrefix(token, []byte("seconds=")) + if !ok || component != Nginx { + return 0, false + } + dot := bytes.IndexByte(data, '.') + if dot < 0 || len(data)-dot-1 != 3 { + return 0, false + } + seconds, validSeconds := canonicalNumber(data[:dot], 3) + fraction, validFraction := digits(data[dot+1:]) + value := seconds*1000 + fraction + return value, validSeconds && validFraction && value <= 600000 +} + +func canonicalNumber(data []byte, maxDigits int) (uint32, bool) { + if len(data) == 0 || len(data) > maxDigits || (len(data) > 1 && data[0] == '0') { + return 0, false + } + return digits(data) +} + +func digits(data []byte) (uint32, bool) { + var value uint32 + for _, digit := range data { + if digit < '0' || digit > '9' { + return 0, false + } + value = value*10 + uint32(digit-'0') + } + return value, true +} diff --git a/Docker/privacy-diagnostic/internal/aggregate/aggregate_test.go b/Docker/privacy-diagnostic/internal/aggregate/aggregate_test.go new file mode 100644 index 000000000..58a553a03 --- /dev/null +++ b/Docker/privacy-diagnostic/internal/aggregate/aggregate_test.go @@ -0,0 +1,303 @@ +package aggregate + +import ( + "bytes" + "encoding/json" + "errors" + "reflect" + "strconv" + "strings" + "testing" +) + +func newAggregate(t *testing.T, component Component) *Aggregator { + t.Helper() + a, err := New(component) + if err != nil { + t.Fatal(err) + } + return a +} + +func snapshot(t *testing.T, a *Aggregator) Snapshot { + t.Helper() + s, err := a.Snapshot() + if err != nil { + t.Fatal(err) + } + return s +} + +func feed(t *testing.T, a *Aggregator, stream Stream, data []byte) { + t.Helper() + if err := a.Feed(stream, data); err != nil { + t.Fatal(err) + } +} + +func finish(t *testing.T, a *Aggregator, stream Stream) { + t.Helper() + if err := a.Finish(stream); err != nil { + t.Fatal(err) + } +} + +func TestStatusAndLatencyBoundaries(t *testing.T) { + statuses := []struct { + text string + family int + }{ + {"100", 0}, {"199", 0}, {"200", 1}, {"299", 1}, {"300", 2}, + {"399", 2}, {"400", 3}, {"499", 3}, {"500", 4}, {"599", 4}, + } + latencies := []struct { + millis string + seconds string + bucket int + }{ + {"0", "0.000", 0}, {"99", "0.099", 0}, {"100", "0.100", 1}, + {"999", "0.999", 1}, {"1000", "1.000", 2}, {"29999", "29.999", 2}, + {"30000", "30.000", 3}, {"600000", "600.000", 3}, + } + for _, component := range []Component{Nginx, UWsgi} { + for _, status := range statuses { + for _, latency := range latencies { + formats := []string{"msecs=" + latency.millis} + if component == Nginx { + formats = append(formats, "seconds="+latency.seconds) + } + for _, timing := range formats { + t.Run(component.name()+"/"+status.text+"/"+timing, func(t *testing.T) { + a := newAggregate(t, component) + line := []byte("PRIVACY_REQUEST status=" + status.text + " " + timing + "\r\n") + for _, b := range line { + feed(t, a, Stdout, []byte{b}) + } + var expected counts + expected.status[status.family] = 1 + expected.latency[latency.bucket] = 1 + if a.counts != expected { + t.Fatalf("counts: %#v; want %#v", a.counts, expected) + } + }) + } + } + } + } +} + +func TestMalformedLinesCannotBecomeRequests(t *testing.T) { + for _, value := range []string{"0", "99", "600", "0200", "+200", "-200", "2_00", "2000", "２００"} { + a := newAggregate(t, Nginx) + feed(t, a, Stdout, []byte("PRIVACY_REQUEST status="+value+" msecs=0\n")) + if a.counts != (counts{rejected: 1}) { + t.Fatalf("accepted malformed status %q", value) + } + } + for _, value := range []string{"", "00", "01", "-1", "+0", "1_0", "0x10", "1.0", "600001", strings.Repeat("9", 3000), "١"} { + a := newAggregate(t, UWsgi) + feed(t, a, Stdout, []byte("PRIVACY_REQUEST status=200 msecs="+value+"\n")) + if a.counts != (counts{rejected: 1}) { + t.Fatalf("accepted malformed milliseconds of length %d", len(value)) + } + } + for _, value := range []string{"00.001", "+0.001", "-0.001", "0.1", "0.0000", "1e3", "600.001", "1000.000", "0.000 SYNTHETIC_PRIVATE", "nan", "١.000"} { + a := newAggregate(t, Nginx) + feed(t, a, Stderr, []byte("PRIVACY_REQUEST status=200 seconds="+value+"\n")) + if a.counts != (counts{rejected: 1}) { + t.Fatalf("accepted malformed seconds %q", value) + } + } + a := newAggregate(t, UWsgi) + feed(t, a, Stderr, []byte("PRIVACY_REQUEST status=200 seconds=0.000\n")) + if a.counts != (counts{rejected: 1}) { + t.Fatal("uWSGI accepted nginx timing") + } +} + +func TestClassificationAndExactSchema(t *testing.T) { + cases := []struct { + line []byte + rejected bool + }{ + {[]byte(""), false}, {[]byte("ordinary SYNTHETIC_PRIVATE"), false}, + {[]byte("PRIVACY_REQUESTx status=200 msecs=0"), false}, + {[]byte(" PRIVACY_REQUEST status=200 msecs=0"), false}, + {[]byte("PRIVACY_REQUEST\tstatus=200 msecs=0"), false}, + {[]byte("PRIVACY_REQUEST"), true}, + {[]byte("PRIVACY_REQUEST status=200 msecs=0"), true}, + {[]byte("PRIVACY_REQUEST status=200 msecs=0 "), true}, + {[]byte("PRIVACY_REQUEST msecs=0 status=200"), true}, + {[]byte("PRIVACY_REQUEST status=200 msecs=0\rhidden"), true}, + {[]byte("ordinary\x00private"), true}, {[]byte{0xff}, true}, + } + keys := map[string]bool{"schema": true, "component": true, "status_1xx": true, "status_2xx": true, "status_3xx": true, "status_4xx": true, "status_5xx": true, "latency_fast": true, "latency_medium": true, "latency_slow": true, "latency_timeout": true, "unclassified": true, "rejected": true, "dropped": true} + for index, item := range cases { + t.Run(strconv.Itoa(index), func(t *testing.T) { + a := newAggregate(t, Nginx) + feed(t, a, Stdout, append(bytes.Clone(item.line), '\n')) + want := counts{unclassified: 1} + if item.rejected { + want = counts{rejected: 1} + } + if a.counts != want { + t.Fatalf("classification: %#v", a.counts) + } + data, err := json.Marshal(snapshot(t, a)) + if err != nil { + t.Fatal(err) + } + if bytes.Contains(data, []byte("SYNTHETIC_PRIVATE")) { + t.Fatal("raw marker reached output") + } + var record map[string]any + if err := json.Unmarshal(data, &record); err != nil { + t.Fatal(err) + } + if len(record) != len(keys) { + t.Fatalf("unexpected fields: %#v", record) + } + for key, value := range record { + if !keys[key] { + t.Fatalf("unexpected field %q", key) + } + if key == "component" { + if value != "nginx" { + t.Fatal("component changed") + } + } else if number, ok := value.(float64); !ok || number < 0 || number > float64(CounterMax) { + t.Fatalf("invalid count %q: %v", key, value) + } + } + if record["schema"] != float64(1) { + t.Fatal("wrong schema") + } + }) + } +} + +func TestStreamIsolationAndRepeatedEOF(t *testing.T) { + a := newAggregate(t, Nginx) + feed(t, a, Stdout, []byte("PRIVACY_REQUEST status=2")) + feed(t, a, Stderr, []byte("PRIVACY_REQUEST status=503 msecs=30000\n")) + feed(t, a, Stdout, []byte("04 msecs=50")) + finish(t, a, Stderr) + finish(t, a, Stderr) + if snapshot(t, a).Status2xx != 0 { + t.Fatal("other stream's EOF consumed a partial line") + } + finish(t, a, Stdout) + finish(t, a, Stdout) + want := counts{status: [5]uint32{0, 1, 0, 0, 1}, latency: [4]uint32{1, 0, 0, 1}} + if a.counts != want { + t.Fatalf("counts: %#v", a.counts) + } + feed(t, a, Stdout, []byte("PRIVACY_REQUEST status=201 msecs=0\n")) + if snapshot(t, a).Status2xx != 2 { + t.Fatal("stream did not resume after EOF") + } +} + +func TestExactLengthOverflowAndRecovery(t *testing.T) { + for _, size := range []int{MaxLine - 1, MaxLine, MaxLine + 1, MaxLine * 2} { + for _, delimited := range []bool{false, true} { + a := newAggregate(t, Nginx) + input := bytes.Repeat([]byte("x"), size) + if delimited { + input = append(input, '\n') + } + feed(t, a, Stdout, input[:17]) + feed(t, a, Stdout, input[17:]) + if size > MaxLine && snapshot(t, a).Dropped != 1 { + t.Fatal("overflow not counted immediately") + } + finish(t, a, Stdout) + finish(t, a, Stdout) + if size <= MaxLine && a.counts != (counts{unclassified: 1}) { + t.Fatalf("valid length %d was dropped", size) + } + if size > MaxLine && a.counts != (counts{dropped: 1}) { + t.Fatalf("overflow counted incorrectly: %#v", a.counts) + } + feed(t, a, Stdout, []byte("PRIVACY_REQUEST status=200 msecs=0\n")) + if snapshot(t, a).Status2xx != 1 { + t.Fatal("stream did not recover") + } + } + } +} + +func TestInputsCopiedAndConsumedBufferErased(t *testing.T) { + a := newAggregate(t, Nginx) + input := []byte("PRIVACY_REQUEST status=200 msecs=0") + feed(t, a, Stdout, input) + clear(input) + finish(t, a, Stdout) + if snapshot(t, a).Status2xx != 1 { + t.Fatal("partial input was borrowed after Feed returned") + } + feed(t, a, Stderr, []byte("SYNTHETIC_PRIVATE")) + feed(t, a, Stderr, bytes.Repeat([]byte("x"), MaxLine+1)) + for _, stream := range a.streams { + if stream.used != 0 || stream.buffer != ([MaxLine]byte{}) { + t.Fatal("consumed bytes remain in owned buffer") + } + } +} + +func TestFixedStorageAndNoPerChunkAllocation(t *testing.T) { + a := newAggregate(t, Nginx) + if size := reflect.TypeOf(*a).Size(); size > 8500 { + t.Fatalf("unbounded state size: %d", size) + } + input := bytes.Repeat([]byte("x"), 1024*1024) + allocations := testing.AllocsPerRun(20, func() { + if err := a.Feed(Stdout, input); err != nil { + panic(err) + } + if err := a.Finish(Stdout); err != nil { + panic(err) + } + }) + if allocations != 0 { + t.Fatalf("allocated %v times per oversized chunk", allocations) + } +} + +func TestInvalidInputsDoNotMutateState(t *testing.T) { + for _, value := range []Component{0, 3, 255} { + if a, err := New(value); a != nil || !errors.Is(err, ErrComponent) { + t.Fatal("invalid component accepted") + } + } + for _, text := range []string{"", "NGINX", "uwsgilog", "SYNTHETIC_PRIVATE"} { + if _, err := ParseComponent(text); !errors.Is(err, ErrComponent) || strings.Contains(err.Error(), text) && text != "" { + t.Fatal("invalid component message leaked input") + } + } + for _, a := range []*Aggregator{nil, {}} { + if err := a.Feed(Stdout, nil); !errors.Is(err, ErrComponent) { + t.Fatal("invalid state accepted") + } + if err := a.Finish(Stdout); !errors.Is(err, ErrComponent) { + t.Fatal("invalid state accepted") + } + if _, err := a.Snapshot(); !errors.Is(err, ErrComponent) { + t.Fatal("invalid state accepted") + } + } + a := newAggregate(t, Nginx) + feed(t, a, Stdout, []byte("retained partial")) + before := *a + for _, stream := range []Stream{0, 3, 255} { + if err := a.Feed(stream, []byte("private")); !errors.Is(err, ErrStream) { + t.Fatal("invalid stream accepted") + } + if err := a.Finish(stream); !errors.Is(err, ErrStream) { + t.Fatal("invalid stream accepted") + } + if *a != before { + t.Fatal("invalid operation mutated state") + } + } +} diff --git a/Docker/privacy-diagnostic/internal/aggregate/application_test.go b/Docker/privacy-diagnostic/internal/aggregate/application_test.go new file mode 100644 index 000000000..173502f36 --- /dev/null +++ b/Docker/privacy-diagnostic/internal/aggregate/application_test.go @@ -0,0 +1,38 @@ +package aggregate + +import ( + "bytes" + "encoding/json" + "errors" + "reflect" + "testing" +) + +func TestApplicationOutputCannotBecomeRequestMetrics(t *testing.T) { + for _, name := range []string{"celery", "celerysingle", "websockets", "mail", "cron", "initialize", "maintenance"} { + component, err := ParseComponent(name) + if err != nil { + t.Fatalf("application profile %s unavailable: %v", name, err) + } + a := newAggregate(t, component) + feed(t, a, Stdout, []byte("SYNTHETIC_PRIVATE\nPRIVACY_REQUEST status=200 msecs=0\n")) + feed(t, a, Stderr, []byte("PRIVACY_REQUEST status=500 seconds=0.001\n\xff\n")) + feed(t, a, Stderr, bytes.Repeat([]byte{'x'}, MaxLine+1)) + finish(t, a, Stderr) + s := snapshot(t, a) + if s.Component != name || s.Unclassified != 3 || s.Rejected != 1 || s.Dropped != 1 || s.Status2xx != 0 || s.Status5xx != 0 || s.LatencyFast != 0 { + t.Fatalf("application stream misclassified: %#v", s) + } + encoded, err := json.Marshal(s) + if err != nil || bytes.Contains(encoded, []byte("SYNTHETIC_PRIVATE")) { + t.Fatalf("invalid application serialization: %s %v", encoded, err) + } + for _, field := range []string{"Status1xx", "Status2xx", "Status3xx", "Status4xx", "Status5xx", "LatencyFast", "LatencyMedium", "LatencySlow", "LatencyTimeout"} { + forged := s + reflect.ValueOf(&forged).Elem().FieldByName(field).SetUint(1) + if data, err := json.Marshal(forged); !errors.Is(err, ErrSnapshot) || len(data) != 0 { + t.Fatalf("application request counters accepted: %s", field) + } + } + } +} diff --git a/Docker/privacy-diagnostic/internal/aggregate/reference_test.go b/Docker/privacy-diagnostic/internal/aggregate/reference_test.go new file mode 100644 index 000000000..615035f2e --- /dev/null +++ b/Docker/privacy-diagnostic/internal/aggregate/reference_test.go @@ -0,0 +1,137 @@ +package aggregate + +import ( + "bytes" + "context" + "encoding/json" + "math/rand" + "os/exec" + "path/filepath" + "strconv" + "testing" + "time" +) + +type operation struct { + Stream string `json:"stream"` + Data []byte `json:"data"` + Finish bool `json:"finish"` +} + +type scenario struct { + Component string `json:"component"` + Operations []operation `json:"operations"` +} + +func referenceScenarios() []scenario { + rng := rand.New(rand.NewSource(81)) + lines := [][]byte{ + []byte("PRIVACY_REQUEST status=200 msecs=99\n"), + []byte("PRIVACY_REQUEST status=503 seconds=30.000\n"), + []byte("ordinary SYNTHETIC_PRIVATE\r\n"), + []byte("PRIVACY_REQUEST status=200 msecs=01\n"), + []byte("PRIVACY_REQUEST status=099 msecs=0\n"), + []byte("PRIVACY_REQUEST status=599 seconds=600.001\n"), + []byte("PRIVACY_REQUEST status=200 msecs=0\n"), + {0xff, 0, '\n'}, []byte("\n\r\n"), + append(bytes.Repeat([]byte("x"), MaxLine), '\n'), + append(bytes.Repeat([]byte("x"), MaxLine+1), '\n'), + bytes.Repeat([]byte("x"), MaxLine+1), + } + cases := make([]scenario, 0, 240) + for index := 0; index < 240; index++ { + comp := "nginx" + if index%2 != 0 { + comp = "uwsgi" + } + item := scenario{Component: comp} + for count := 0; count < 3; count++ { + stream := "stdout" + if rng.Intn(2) == 1 { + stream = "stderr" + } + line := lines[rng.Intn(len(lines))] + if index%5 == 0 { + line = []byte("PRIVACY_REQUEST status=" + strconv.Itoa(80+rng.Intn(550)) + " msecs=" + strconv.Itoa(rng.Intn(610000)) + "\n") + } + for start := 0; start < len(line); { + end := min(len(line), start+1+rng.Intn(1500)) + item.Operations = append(item.Operations, operation{Stream: stream, Data: bytes.Clone(line[start:end])}) + start = end + if rng.Intn(5) == 0 { + item.Operations = append(item.Operations, operation{Stream: stream, Finish: true}) + } + } + item.Operations = append(item.Operations, operation{Stream: stream, Data: nil}) + } + item.Operations = append(item.Operations, operation{Stream: "stdout", Finish: true}, operation{Stream: "stderr", Finish: true}, operation{Stream: "stdout", Finish: true}) + cases = append(cases, item) + } + return cases +} + +func TestSnapshotsMatchFrozenPythonReference(t *testing.T) { + cases := referenceScenarios() + input, err := json.Marshal(cases) + if err != nil || len(input) > 8*1024*1024 { + t.Fatal("invalid synthetic corpus", err) + } + script, err := filepath.Abs("../../../../tests/privacy_native/aggregate_reference.py") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + command := exec.CommandContext(ctx, "python3.14", script) + command.Stdin = bytes.NewReader(input) + output, err := command.Output() + if err != nil { + if failure, ok := err.(*exec.ExitError); ok { + t.Fatalf("reference failed: %v\n%s", err, failure.Stderr) + } + t.Fatal(err) + } + var result struct { + SourceSHA256 string `json:"source_sha256"` + Snapshots [][]Snapshot `json:"snapshots"` + } + if err := json.Unmarshal(output, &result); err != nil { + t.Fatal(err) + } + const reviewedReference = "582582eacbb29d486e1a9b8205577664d5eb239f85493441970197b53f31951f" + if result.SourceSHA256 != reviewedReference { + t.Fatal("reference source changed; review semantic differences before updating the pinned hash") + } + if len(result.Snapshots) != len(cases) { + t.Fatal("missing reference cases") + } + for index, item := range cases { + comp, err := ParseComponent(item.Component) + if err != nil { + t.Fatal(err) + } + a := newAggregate(t, comp) + expected := result.Snapshots[index] + if len(expected) != len(item.Operations)+1 { + t.Fatal("missing reference steps") + } + if snapshot(t, a) != expected[0] { + t.Fatal("initial snapshot differs") + } + for step, op := range item.Operations { + stream := Stdout + if op.Stream == "stderr" { + stream = Stderr + } + if op.Finish { + finish(t, a, stream) + } else { + feed(t, a, stream, op.Data) + } + if got := snapshot(t, a); got != expected[step+1] { + t.Fatalf("case %d step %d: Go=%#v Python=%#v", index, step, got, expected[step+1]) + } + } + } + t.Logf("matched %d synthetic scenarios (%d input bytes), including intermediate snapshots", len(cases), len(input)) +} diff --git a/Docker/privacy-diagnostic/internal/aggregate/snapshot_test.go b/Docker/privacy-diagnostic/internal/aggregate/snapshot_test.go new file mode 100644 index 000000000..7b86d1fdd --- /dev/null +++ b/Docker/privacy-diagnostic/internal/aggregate/snapshot_test.go @@ -0,0 +1,140 @@ +package aggregate + +import ( + "bytes" + "encoding/json" + "errors" + "reflect" + "strconv" + "strings" + "testing" +) + +func TestEveryCounterSaturatesThroughFeed(t *testing.T) { + a := newAggregate(t, Nginx) + for i := range a.counts.status { + a.counts.status[i] = CounterMax - 1 + } + for i := range a.counts.latency { + a.counts.latency[i] = CounterMax - 1 + } + a.counts.rejected = CounterMax - 1 + a.counts.unclassified = CounterMax - 1 + a.counts.dropped = CounterMax - 1 + for repeat := 0; repeat < 3; repeat++ { + for _, status := range []int{100, 200, 300, 400, 500} { + for _, latency := range []int{0, 100, 1000, 30000} { + feed(t, a, Stdout, []byte("PRIVACY_REQUEST status="+strconv.Itoa(status)+" msecs="+strconv.Itoa(latency)+"\n")) + } + } + feed(t, a, Stderr, []byte("PRIVACY_REQUEST\nordinary\n")) + feed(t, a, Stderr, bytes.Repeat([]byte("x"), MaxLine+1)) + finish(t, a, Stderr) + } + s := snapshot(t, a) + data, err := json.Marshal(s) + if err != nil { + t.Fatal(err) + } + var fields map[string]any + if err := json.Unmarshal(data, &fields); err != nil { + t.Fatal(err) + } + for name, value := range fields { + if name != "schema" && name != "component" && value != float64(CounterMax) { + t.Fatalf("counter %s did not saturate: %v", name, value) + } + } + s.Status2xx = 0 + s.Component = "SYNTHETIC_PRIVATE" + if next := snapshot(t, a); next.Status2xx != CounterMax || next.Component != "nginx" { + t.Fatal("snapshot aliases mutable state") + } +} + +func TestForgedSnapshotsCannotSerializePrivateOrOutOfRangeFields(t *testing.T) { + valid := Snapshot{Schema: 1, Component: "uwsgi"} + bad := []Snapshot{{}, {Schema: 2, Component: "uwsgi"}, {Schema: 1, Component: "SYNTHETIC_PRIVATE"}} + for _, field := range []string{"Status1xx", "Status2xx", "Status3xx", "Status4xx", "Status5xx", "LatencyFast", "LatencyMedium", "LatencySlow", "LatencyTimeout", "Unclassified", "Rejected", "Dropped"} { + copy := valid + reflect.ValueOf(©).Elem().FieldByName(field).SetUint(uint64(CounterMax) + 1) + bad = append(bad, copy) + } + for _, value := range bad { + data, err := json.Marshal(value) + if !errors.Is(err, ErrSnapshot) || len(data) != 0 { + t.Fatalf("invalid snapshot accepted: %s %v", data, err) + } + if strings.Contains(err.Error(), "SYNTHETIC_PRIVATE") { + t.Fatal("private field leaked into error") + } + } + if _, err := json.Marshal(valid); err != nil { + t.Fatal("valid control rejected", err) + } +} + +func FuzzFragmentationPreservesCounters(f *testing.F) { + f.Add([]byte("PRIVACY_REQUEST status=200 msecs=0\n"), uint16(1), uint8(0)) + f.Add(append(bytes.Repeat([]byte("x"), MaxLine+1), []byte("\nPRIVACY_REQUEST status=503 seconds=30.000")...), uint16(4096), uint8(0)) + f.Add([]byte{0xff, '\n', '\r', '\n'}, uint16(2), uint8(1)) + f.Fuzz(func(t *testing.T, data []byte, split uint16, rawComponent uint8) { + // Limit fuzz-case resource use; ordinary unit tests separately cover 1 MiB. + if len(data) > 64*1024 { + data = data[:64*1024] + } + components := [...]Component{Nginx, UWsgi, Celery, CelerySingle, Websockets, Mail, Cron, Initialize, Maintenance} + component := components[int(rawComponent)%len(components)] + whole, parts := newAggregate(t, component), newAggregate(t, component) + feed(t, whole, Stdout, data) + finish(t, whole, Stdout) + step := 1 + int(split)%8192 + for start := 0; start < len(data); start += step { + end := min(start+step, len(data)) + // Avoid testing.Helper stack bookkeeping for each one-byte fragment. + if err := parts.Feed(Stdout, data[start:end]); err != nil { + t.Fatal(err) + } + } + finish(t, parts, Stdout) + if snapshot(t, whole) != snapshot(t, parts) { + t.Fatal("fragmentation changed counters") + } + if _, err := json.Marshal(snapshot(t, parts)); err != nil { + t.Fatal(err) + } + for _, stream := range parts.streams { + if stream.used != 0 || stream.dropping || stream.buffer != ([MaxLine]byte{}) { + t.Fatal("EOF retained bytes or drop state") + } + } + }) +} + +func BenchmarkFragmentedOversizedLine(b *testing.B) { + input := bytes.Repeat([]byte("x"), 64*1024) + for _, step := range []int{1, 4096} { + b.Run(strconv.Itoa(step), func(b *testing.B) { + a, err := New(Nginx) + if err != nil { + b.Fatal(err) + } + b.SetBytes(int64(len(input))) + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + for start := 0; start < len(input); start += step { + if err := a.Feed(Stdout, input[start:min(start+step, len(input))]); err != nil { + b.Fatal(err) + } + } + if err := a.Finish(Stdout); err != nil { + b.Fatal(err) + } + } + if a.counts != (counts{dropped: uint32(min(b.N, int(CounterMax)))}) { + b.Fatal("oversized lines counted incorrectly") + } + }) + } +} diff --git a/Docker/privacy-diagnostic/internal/aggregate/types.go b/Docker/privacy-diagnostic/internal/aggregate/types.go new file mode 100644 index 000000000..92f1387f2 --- /dev/null +++ b/Docker/privacy-diagnostic/internal/aggregate/types.go @@ -0,0 +1,206 @@ +// Package aggregate reduces captured output to fixed, bounded counters. +// Callers must serialize Feed, Finish and Snapshot calls. No file or network I/O +// occurs here; the process runner remains responsible for capture and delivery. +package aggregate + +import ( + "encoding/json" + "errors" +) + +const ( + MaxLine = 4096 + CounterMax uint32 = 1<<31 - 1 +) + +type Component uint8 + +const ( + Nginx Component = iota + 1 + UWsgi +) + +// Application profiles have no native request-count interpretation. Keep the +// original native value range separate, including its invalid sentinels. +const ( + Celery Component = 16 + iota + CelerySingle + Websockets + Mail + Cron + Initialize + Maintenance +) + +type Stream uint8 + +const ( + Stdout Stream = iota + 1 + Stderr +) + +var ( + ErrComponent = errors.New("invalid aggregate component") + ErrStream = errors.New("invalid aggregate stream") + ErrSnapshot = errors.New("invalid aggregate snapshot") +) + +func ParseComponent(name string) (Component, error) { + switch name { + case "nginx": + return Nginx, nil + case "uwsgi": + return UWsgi, nil + case "celery": + return Celery, nil + case "celerysingle": + return CelerySingle, nil + case "websockets": + return Websockets, nil + case "mail": + return Mail, nil + case "cron": + return Cron, nil + case "initialize": + return Initialize, nil + case "maintenance": + return Maintenance, nil + default: + return 0, ErrComponent + } +} + +func (c Component) application() bool { return c >= Celery && c <= Maintenance } + +func (c Component) valid() bool { return c == Nginx || c == UWsgi || c.application() } + +func (c Component) name() string { + if c == Nginx { + return "nginx" + } + if c == UWsgi { + return "uwsgi" + } + switch c { + case Celery: + return "celery" + case CelerySingle: + return "celerysingle" + case Websockets: + return "websockets" + case Mail: + return "mail" + case Cron: + return "cron" + case Initialize: + return "initialize" + case Maintenance: + return "maintenance" + } + return "" +} + +// Snapshot is a value copy. MarshalJSON rejects forged fields before encoding. +type Snapshot struct { + Schema uint8 `json:"schema"` + Component string `json:"component"` + Status1xx uint32 `json:"status_1xx"` + Status2xx uint32 `json:"status_2xx"` + Status3xx uint32 `json:"status_3xx"` + Status4xx uint32 `json:"status_4xx"` + Status5xx uint32 `json:"status_5xx"` + LatencyFast uint32 `json:"latency_fast"` + LatencyMedium uint32 `json:"latency_medium"` + LatencySlow uint32 `json:"latency_slow"` + LatencyTimeout uint32 `json:"latency_timeout"` + Unclassified uint32 `json:"unclassified"` + Rejected uint32 `json:"rejected"` + Dropped uint32 `json:"dropped"` +} + +func (s Snapshot) MarshalJSON() ([]byte, error) { + component, err := ParseComponent(s.Component) + if s.Schema != 1 || err != nil { + return nil, ErrSnapshot + } + counts := [...]uint32{s.Status1xx, s.Status2xx, s.Status3xx, s.Status4xx, + s.Status5xx, s.LatencyFast, s.LatencyMedium, s.LatencySlow, + s.LatencyTimeout, s.Unclassified, s.Rejected, s.Dropped} + for _, count := range counts { + if count > CounterMax { + return nil, ErrSnapshot + } + } + if component.application() { + for _, count := range counts[:9] { + if count != 0 { + return nil, ErrSnapshot + } + } + } + type wire Snapshot + return json.Marshal(wire(s)) +} + +type streamState struct { + buffer [MaxLine]byte + used int + dropping bool +} + +func (s *streamState) erase() { + clear(s.buffer[:s.used]) + s.used = 0 +} + +type counts struct { + status [5]uint32 + latency [4]uint32 + unclassified uint32 + rejected uint32 + dropped uint32 +} + +// Aggregator owns exactly two fixed input buffers and bounded numeric state. +// Its zero value and nil pointers reject operations; construct it with New. +type Aggregator struct { + component Component + streams [2]streamState + counts counts +} + +func New(component Component) (*Aggregator, error) { + if !component.valid() { + return nil, ErrComponent + } + return &Aggregator{component: component}, nil +} + +func (a *Aggregator) state(stream Stream) (*streamState, error) { + if a == nil || !a.component.valid() { + return nil, ErrComponent + } + if stream != Stdout && stream != Stderr { + return nil, ErrStream + } + return &a.streams[stream-1], nil +} + +func (a *Aggregator) Snapshot() (Snapshot, error) { + if a == nil || !a.component.valid() { + return Snapshot{}, ErrComponent + } + c := a.counts + return Snapshot{Schema: 1, Component: a.component.name(), + Status1xx: c.status[0], Status2xx: c.status[1], Status3xx: c.status[2], + Status4xx: c.status[3], Status5xx: c.status[4], + LatencyFast: c.latency[0], LatencyMedium: c.latency[1], + LatencySlow: c.latency[2], LatencyTimeout: c.latency[3], + Unclassified: c.unclassified, Rejected: c.rejected, Dropped: c.dropped}, nil +} + +func increment(value *uint32) { + if *value < CounterMax { + *value++ + } +} diff --git a/Docker/privacy-diagnostic/internal/monitor/pipe_linux.go b/Docker/privacy-diagnostic/internal/monitor/pipe_linux.go new file mode 100644 index 000000000..9909b3826 --- /dev/null +++ b/Docker/privacy-diagnostic/internal/monitor/pipe_linux.go @@ -0,0 +1,62 @@ +package monitor + +import ( + "os" + "syscall" + "time" +) + +type pipe struct { + file *os.File + flags uintptr +} + +func openPipe(source int) (*pipe, error) { + fd, err := syscall.Dup(source) + if err != nil { + return nil, errProtocol + } + syscall.CloseOnExec(fd) + var stat syscall.Stat_t + if syscall.Fstat(fd, &stat) != nil || stat.Mode&syscall.S_IFMT != syscall.S_IFIFO && stat.Mode&syscall.S_IFMT != syscall.S_IFSOCK { + syscall.Close(fd) + return nil, errProtocol + } + flags, _, errno := syscall.Syscall(syscall.SYS_FCNTL, uintptr(fd), syscall.F_GETFL, 0) + if errno != 0 { + syscall.Close(fd) + return nil, errProtocol + } + _, _, errno = syscall.Syscall(syscall.SYS_FCNTL, uintptr(fd), syscall.F_SETFL, flags|syscall.O_NONBLOCK) + if errno != 0 { + syscall.Close(fd) + return nil, errProtocol + } + file := os.NewFile(uintptr(fd), "monitor-pipe") + if file == nil { + syscall.Syscall(syscall.SYS_FCNTL, uintptr(fd), syscall.F_SETFL, flags) + syscall.Close(fd) + return nil, errProtocol + } + return &pipe{file, flags}, nil +} + +func (p *pipe) close() error { + _, _, errno := syscall.Syscall(syscall.SYS_FCNTL, p.file.Fd(), syscall.F_SETFL, p.flags) + err := p.file.Close() + if errno != 0 || err != nil { + return errProtocol + } + return nil +} + +func (p *pipe) write(data []byte, timeout time.Duration) error { + if timeout <= 0 || p.file.SetWriteDeadline(time.Now().Add(timeout)) != nil { + return errProtocol + } + n, err := p.file.Write(data) + if err != nil || n != len(data) { + return errProtocol + } + return nil +} diff --git a/Docker/privacy-diagnostic/internal/monitor/protocol.go b/Docker/privacy-diagnostic/internal/monitor/protocol.go new file mode 100644 index 000000000..db5e05d60 --- /dev/null +++ b/Docker/privacy-diagnostic/internal/monitor/protocol.go @@ -0,0 +1,143 @@ +// Package monitor consumes Supervisor state events without retaining process +// output, arguments, identifiers, or arbitrary event fields in its records. +package monitor + +import ( + "encoding/json" + "errors" + "strconv" + "strings" +) + +const MaxFrame = 1024 + +var errProtocol = errors.New("monitor protocol unavailable") + +type header struct { + event string + size int +} + +func fields(data []byte) (map[string]string, error) { + if len(data) == 0 || len(data) > MaxFrame { + return nil, errProtocol + } + for _, value := range data { + if value < 32 || value > 126 { + return nil, errProtocol + } + } + result := map[string]string{} + for _, field := range strings.Split(string(data), " ") { + key, value, found := strings.Cut(field, ":") + if !found || key == "" || value == "" || result[key] != "" || len(result) >= 16 { + return nil, errProtocol + } + result[key] = value + } + return result, nil +} + +func number(value string) (int, bool) { + if value == "" || len(value) > 10 || len(value) > 1 && value[0] == '0' { + return 0, false + } + for _, digit := range value { + if digit < '0' || digit > '9' { + return 0, false + } + } + n, err := strconv.ParseUint(value, 10, 31) + return int(n), err == nil +} + +func parseHeader(data []byte) (header, error) { + values, err := fields(data) + if err != nil || values["ver"] != "3.0" || values["server"] == "" || values["pool"] == "" || values["eventname"] == "" { + return header{}, errProtocol + } + for _, key := range []string{"serial", "poolserial"} { + if _, ok := number(values[key]); !ok { + return header{}, errProtocol + } + } + size, ok := number(values["len"]) + if !ok || size > MaxFrame { + return header{}, errProtocol + } + return header{values["eventname"], size}, nil +} + +type record struct { + Schema int `json:"schema"` + Component string `json:"component"` + Event string `json:"event"` + State string `json:"state"` +} + +func fixed(component, event, state string) []byte { + data, _ := json.Marshal(record{1, component, event, state}) // Fixed string/integer fields cannot fail. + return append(data, '\n') +} + +func eventRecord(event string, payload []byte) ([]byte, error) { + switch event { + case "PROCESS_STATE_EXITED", "PROCESS_STATE_BACKOFF", "PROCESS_STATE_FATAL", "TICK_60": + default: + return nil, nil // Unsubscribed events are acknowledged without inspecting their body. + } + values, err := fields(payload) + if err != nil { + return nil, err + } + if event == "TICK_60" { + if _, ok := number(values["when"]); !ok { + return nil, errProtocol + } + return fixed("monitor", "monitor_alive", "ready"), nil + } + component := "" + switch values["processname"] { + case "nginx": + component = "nginx" + case "uwsgi": + component = "uwsgi" + case "uwsgilog": + component = "uwsgilog" + case "celery": + component = "celery" + case "celerysingle": + component = "celerysingle" + case "websockets": + component = "websockets" + case "initialize": + component = "initialize" + default: + return nil, nil // Other services and the listener itself are outside this subscription's report scope. + } + group := component + if component == "initialize" { + group = "main" // Keep the initializer's existing concurrent shutdown group. + } + if values["groupname"] != group || values["from_state"] == "" { + return nil, errProtocol + } + switch event { + case "PROCESS_STATE_EXITED": + pid, ok := number(values["pid"]) + if !ok || pid == 0 || values["from_state"] != "RUNNING" || values["expected"] != "0" && values["expected"] != "1" { + return nil, errProtocol + } + if values["expected"] == "1" { + return nil, nil + } + return fixed(component, "process_failed", "exited"), nil + case "PROCESS_STATE_BACKOFF": + if _, ok := number(values["tries"]); !ok { + return nil, errProtocol + } + return fixed(component, "process_failed", "backoff"), nil + default: + return fixed(component, "process_failed", "fatal"), nil + } +} diff --git a/Docker/privacy-diagnostic/internal/monitor/protocol_test.go b/Docker/privacy-diagnostic/internal/monitor/protocol_test.go new file mode 100644 index 000000000..19104069b --- /dev/null +++ b/Docker/privacy-diagnostic/internal/monitor/protocol_test.go @@ -0,0 +1,124 @@ +package monitor + +import ( + "bytes" + "encoding/json" + "strings" + "testing" +) + +func TestStateEventsContainOnlyFixedFields(t *testing.T) { + for _, component := range []string{"nginx", "uwsgi", "uwsgilog", "celery", "celerysingle", "websockets", "initialize"} { + group := component + if component == "initialize" { + group = "main" + } + for _, item := range []struct{ event, details, state string }{ + {"PROCESS_STATE_EXITED", "from_state:RUNNING expected:0 pid:42", "exited"}, + {"PROCESS_STATE_BACKOFF", "from_state:STARTING tries:1", "backoff"}, + {"PROCESS_STATE_FATAL", "from_state:BACKOFF", "fatal"}, + } { + payload := "processname:" + component + " groupname:" + group + " " + item.details + " ignored:SYNTHETIC_PRIVATE" + data, err := eventRecord(item.event, []byte(payload)) + if err != nil { + t.Fatal(err) + } + assertRecord(t, data, component, "process_failed", item.state) + } + } + data, err := eventRecord("TICK_60", []byte("when:1788654000")) + if err != nil { + t.Fatal(err) + } + assertRecord(t, data, "monitor", "monitor_alive", "ready") +} + +func assertRecord(t *testing.T, data []byte, component, event, state string) { + t.Helper() + var value map[string]any + if len(data) > 256 || !bytes.HasSuffix(data, []byte{'\n'}) || json.Unmarshal(data, &value) != nil || len(value) != 4 || value["schema"] != float64(1) || value["component"] != component || value["event"] != event || value["state"] != state { + t.Fatalf("invalid fixed record: %q", data) + } + if bytes.Contains(data, []byte("SYNTHETIC_PRIVATE")) { + t.Fatal("private marker retained") + } +} + +func TestExpectedAndUnrelatedEventsProduceNoRecord(t *testing.T) { + for _, item := range []struct{ event, payload string }{ + {"PROCESS_STATE_EXITED", "processname:nginx groupname:nginx from_state:RUNNING expected:1 pid:42"}, + {"PROCESS_STATE_FATAL", "processname:SYNTHETIC_PRIVATE groupname:SYNTHETIC_PRIVATE from_state:BACKOFF"}, + {"PROCESS_STATE_FATAL", "processname:privacy-monitor groupname:privacy-monitor from_state:BACKOFF"}, + {"PROCESS_LOG_STDOUT", "SYNTHETIC_PRIVATE\n192.0.2.1 /interview?i=synthetic.yml"}, + {"PROCESS_STATE_STOPPED", "processname:uwsgi groupname:uwsgi from_state:STOPPING pid:42"}, + } { + data, err := eventRecord(item.event, []byte(item.payload)) + if err != nil || len(data) != 0 { + t.Fatalf("unrelated/expected event was reported: %q %v", data, err) + } + } +} + +func TestMalformedFramesRejectWithoutInputInErrors(t *testing.T) { + valid := "ver:3.0 server:SYNTHETIC_PRIVATE serial:12 pool:privacy-monitor poolserial:3 eventname:TICK_60 len:15" + if value, err := parseHeader([]byte(valid)); err != nil || value.size != 15 || value.event != "TICK_60" { + t.Fatal("valid Supervisor header rejected") + } + for _, input := range []string{ + "", valid + " len:20", strings.Replace(valid, "ver:3.0", "ver:4.0", 1), + strings.Replace(valid, "len:15", "len:1025", 1), strings.Replace(valid, "len:15", "len:-1", 1), + strings.Replace(valid, "len:15", "len:01", 1), strings.Replace(valid, "serial:12 ", "", 1), + valid + "\nSYNTHETIC_PRIVATE", valid + "\x00", valid + "\t", strings.Repeat("a", MaxFrame+1), + } { + if _, err := parseHeader([]byte(input)); err != errProtocol { + t.Fatalf("malformed header accepted or error varied: %v", err) + } + } + for _, payload := range []string{ + "processname:uwsgi groupname:uwsgi from_state:RUNNING expected:maybe pid:42", + "processname:uwsgi groupname:uwsgi from_state:RUNNING expected:0 pid:0", + "processname:uwsgi groupname:other from_state:RUNNING expected:0 pid:42", + "processname:initialize groupname:initialize from_state:RUNNING expected:0 pid:42", + "processname:initialize groupname:other from_state:RUNNING expected:0 pid:42", + "processname:uwsgi groupname:main from_state:RUNNING expected:0 pid:42", + "processname:uwsgi groupname:uwsgi from_state:RUNNING expected:0 pid:42 expected:1", + "processname:uwsgi groupname:uwsgi from_state:SYNTHETIC_PRIVATE expected:0 pid:42", + } { + if _, err := eventRecord("PROCESS_STATE_EXITED", []byte(payload)); err != errProtocol { + t.Fatalf("malformed payload accepted or error varied: %v", err) + } + } +} + +func FuzzEventRecordsHaveOnlyFixedOutput(f *testing.F) { + f.Add("PROCESS_STATE_EXITED", []byte("processname:uwsgi groupname:uwsgi from_state:RUNNING expected:0 pid:42")) + f.Add("TICK_60", []byte("when:1788654000")) + f.Add("PROCESS_STATE_FATAL", []byte("processname:initialize groupname:main from_state:BACKOFF")) + f.Add("PROCESS_STATE_FATAL", []byte("SYNTHETIC_PRIVATE")) + f.Fuzz(func(t *testing.T, event string, payload []byte) { + if len(payload) > MaxFrame+1 || len(event) > MaxFrame+1 { + return + } + _, err := parseHeader(payload) + if err != nil && err != errProtocol { + t.Fatal("header error contains nonconstant data") + } + data, err := eventRecord(event, payload) + if err != nil && err != errProtocol { + t.Fatal("event error contains nonconstant data") + } + if len(data) == 0 { + return + } + allowed := false + for _, component := range []string{"nginx", "uwsgi", "uwsgilog", "celery", "celerysingle", "websockets", "initialize"} { + for _, state := range []string{"exited", "backoff", "fatal"} { + allowed = allowed || bytes.Equal(data, fixed(component, "process_failed", state)) + } + } + allowed = allowed || bytes.Equal(data, fixed("monitor", "monitor_alive", "ready")) + if !allowed || len(data) > 256 { + t.Fatalf("unapproved output: %q", data) + } + }) +} diff --git a/Docker/privacy-diagnostic/internal/monitor/run_linux.go b/Docker/privacy-diagnostic/internal/monitor/run_linux.go new file mode 100644 index 000000000..702ca8e3f --- /dev/null +++ b/Docker/privacy-diagnostic/internal/monitor/run_linux.go @@ -0,0 +1,112 @@ +package monitor + +import ( + "io" + "syscall" + "time" + + "github.com/jacobyoby/docassemble/privacy-diagnostic/internal/nativeguard" +) + +const ( + UsageFailure = 64 + ProtocolFailure = 70 + SinkFailure = 74 +) + +// Run is for a dedicated Supervisor event-listener process. Stdout carries +// protocol tokens only; stderr carries fixed-schema records only. No network +// calls, output replay, service changes, or direct file writes are performed. +func Run(args []string) int { + if len(args) != 0 { + return UsageFailure + } + if syscall.Setrlimit(syscall.RLIMIT_CORE, &syscall.Rlimit{}) != nil || nativeguard.SealDescriptors() != nil { + return ProtocolFailure + } + return run([3]int{0, 1, 2}, 5*time.Second) +} + +func run(descriptors [3]int, timeout time.Duration) (code int) { + var streams []*pipe + defer func() { + // Reverse order also restores flags correctly if stdio shares an open file description. + for i := len(streams) - 1; i >= 0; i-- { + if streams[i].close() != nil { + code = SinkFailure + } + } + }() + for _, fd := range descriptors { + stream, err := openPipe(fd) + if err != nil { + return SinkFailure + } + streams = append(streams, stream) + } + input, protocol, output := streams[0], streams[1], streams[2] + if output.write(fixed("monitor", "monitor_ready", "ready"), timeout) != nil { + return SinkFailure + } + var storage [MaxFrame]byte + defer clear(storage[:]) + for { + if protocol.write([]byte("READY\n"), timeout) != nil { + return SinkFailure + } + headerBytes, err := readHeader(input, storage[:], timeout) + if err == io.EOF { + return 0 // Supervisor closed its input while no event was in flight. + } + info, parsed := parseHeader(headerBytes) + clear(storage[:]) + if err != nil || parsed != nil { + return protocolFailure(output, timeout) + } + _, err = io.ReadFull(input.file, storage[:info.size]) + if err != nil { + return protocolFailure(output, timeout) + } + data, err := eventRecord(info.event, storage[:info.size]) + clear(storage[:]) + if err != nil || input.file.SetReadDeadline(time.Time{}) != nil { + return protocolFailure(output, timeout) + } + if len(data) != 0 && output.write(data, timeout) != nil { + return SinkFailure // Do not acknowledge undelivered records: Supervisor requeues them. + } + if protocol.write([]byte("RESULT 2\nOK"), timeout) != nil { + return SinkFailure + } + } +} + +func readHeader(input *pipe, buffer []byte, timeout time.Duration) ([]byte, error) { + // Idle waits have no deadline. Once a frame starts, one deadline bounds its + // complete header and payload so a partial sender cannot stall processing. + if _, err := io.ReadFull(input.file, buffer[:1]); err != nil { + return nil, err + } + if input.file.SetReadDeadline(time.Now().Add(timeout)) != nil { + return nil, errProtocol + } + for size := 1; size <= len(buffer); size++ { + if buffer[size-1] == '\n' { + return buffer[:size-1], nil + } + if size == len(buffer) { + break + } + if _, err := io.ReadFull(input.file, buffer[size:size+1]); err != nil { + return nil, errProtocol + } + } + return nil, errProtocol +} + +func protocolFailure(output *pipe, timeout time.Duration) int { + if output.write(fixed("monitor", "protocol_failed", "invalid"), timeout) != nil { + return SinkFailure + } + return ProtocolFailure +} diff --git a/Docker/privacy-diagnostic/internal/monitor/run_linux_test.go b/Docker/privacy-diagnostic/internal/monitor/run_linux_test.go new file mode 100644 index 000000000..c5fa64266 --- /dev/null +++ b/Docker/privacy-diagnostic/internal/monitor/run_linux_test.go @@ -0,0 +1,179 @@ +package monitor + +import ( + "bufio" + "bytes" + "fmt" + "io" + "os" + "strings" + "syscall" + "testing" + "time" +) + +type fixture struct { + input, send, protocol, replies, output, reports *os.File + descriptors [3]int +} + +func newFixture(t *testing.T) *fixture { + t.Helper() + f := &fixture{} + for _, pair := range [][2]**os.File{{&f.input, &f.send}, {&f.replies, &f.protocol}, {&f.reports, &f.output}} { + read, write, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + *pair[0], *pair[1] = read, write + t.Cleanup(func() { read.Close(); write.Close() }) + } + f.descriptors = [3]int{int(f.input.Fd()), int(f.protocol.Fd()), int(f.output.Fd())} + return f +} + +func frame(event, payload string) string { + return fmt.Sprintf("ver:3.0 server:SYNTHETIC_PRIVATE serial:12 pool:privacy-monitor poolserial:3 eventname:%s len:%d\n%s", event, len(payload), payload) +} + +func collect(file *os.File) <-chan []byte { + result := make(chan []byte, 1) + go func() { data, _ := io.ReadAll(file); result <- data }() + return result +} + +func TestProtocolAcknowledgesOnlyAfterReportAndPreservesFlags(t *testing.T) { + f := newFixture(t) + flags := [3]uintptr{} + for i, fd := range f.descriptors { + flags[i], _, _ = syscall.Syscall(syscall.SYS_FCNTL, uintptr(fd), syscall.F_GETFL, 0) + } + protocol, reports := collect(f.replies), collect(f.reports) + input := frame("PROCESS_STATE_EXITED", "processname:uwsgi groupname:uwsgi from_state:RUNNING expected:0 pid:42") + + frame("PROCESS_STATE_EXITED", "processname:nginx groupname:nginx from_state:RUNNING expected:1 pid:43") + + frame("TICK_60", "when:1788654000") + if _, err := f.send.WriteString(input); err != nil { + t.Fatal(err) + } + f.send.Close() + if status := run(f.descriptors, time.Second); status != 0 { + t.Fatalf("valid event stream failed: %d", status) + } + for i, fd := range f.descriptors { + after, _, errno := syscall.Syscall(syscall.SYS_FCNTL, uintptr(fd), syscall.F_GETFL, 0) + if errno != 0 || after != flags[i] { + t.Fatal("shared descriptor flags changed") + } + } + f.protocol.Close() + f.output.Close() + if got := string(<-protocol); got != "READY\n"+strings.Repeat("RESULT 2\nOKREADY\n", 3) { + t.Fatalf("wrong Supervisor protocol: %q", got) + } + rows := bytes.Split(bytes.TrimSpace(<-reports), []byte{'\n'}) + if len(rows) != 3 { + t.Fatalf("expected readiness, failure, heartbeat; got %d", len(rows)) + } + assertRecord(t, append(rows[0], '\n'), "monitor", "monitor_ready", "ready") + assertRecord(t, append(rows[1], '\n'), "uwsgi", "process_failed", "exited") + assertRecord(t, append(rows[2], '\n'), "monitor", "monitor_alive", "ready") +} + +func TestTruncatedOversizedAndStalledFramesAreNotAcknowledged(t *testing.T) { + for _, item := range []struct { + input string + close bool + }{ + {"SYNTHETIC_PRIVATE\n", false}, {strings.Repeat("x", MaxFrame+1), false}, + {frame("PROCESS_STATE_EXITED", "processname:uwsgi") + "SYNTHETIC_PRIVATE", false}, + {"ver:3.0 server:test serial:1 pool:test poolserial:1 eventname:TICK_60 len:1025\n", false}, + {"ver:3.0", false}, // A stalled header must hit its deadline. + {"ver:3.0", true}, // EOF in a header is not a clean shutdown. + {"ver:3.0 server:test serial:1 pool:test poolserial:1 eventname:TICK_60 len:15\nwhen:1", false}, + {"ver:3.0 server:test serial:1 pool:test poolserial:1 eventname:TICK_60 len:15\nwhen:1", true}, + } { + f := newFixture(t) + protocol, reports := collect(f.replies), collect(f.reports) + if _, err := f.send.WriteString(item.input); err != nil { + t.Fatal(err) + } + if item.close { + f.send.Close() + } + started := time.Now() + if status := run(f.descriptors, 40*time.Millisecond); status != ProtocolFailure || time.Since(started) > time.Second { + t.Fatalf("malformed/stalled input did not fail promptly: %d", status) + } + f.protocol.Close() + f.output.Close() + if string(<-protocol) != "READY\n" { + t.Fatal("invalid input was acknowledged") + } + data := <-reports + if bytes.Contains(data, []byte("SYNTHETIC_PRIVATE")) || !bytes.Contains(data, []byte(`"event":"protocol_failed"`)) { + t.Fatalf("unsafe or missing failure signal: %q", data) + } + } +} + +func TestFailedRecordSinkLeavesEventUnacknowledged(t *testing.T) { + f := newFixture(t) + protocol := collect(f.replies) + done := make(chan int, 1) + go func() { done <- run(f.descriptors, time.Second) }() + ready, err := bufio.NewReader(f.reports).ReadBytes('\n') + if err != nil { + t.Fatal(err) + } + assertRecord(t, ready, "monitor", "monitor_ready", "ready") + f.reports.Close() + f.send.WriteString(frame("PROCESS_STATE_FATAL", "processname:uwsgi groupname:uwsgi from_state:BACKOFF")) + select { + case code := <-done: + if code != SinkFailure { + t.Fatalf("broken sink returned %d", code) + } + case <-time.After(2 * time.Second): + t.Fatal("broken sink blocked") + } + f.protocol.Close() + if got := string(<-protocol); got != "READY\n" { + t.Fatalf("undelivered event was acknowledged: %q", got) + } +} + +func TestRegularAndFullSinksFail(t *testing.T) { + for _, kind := range []string{"regular", "full", "protocol-full"} { + f := newFixture(t) + if kind == "regular" { + file, err := os.CreateTemp(t.TempDir(), "synthetic") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { file.Close() }) + f.descriptors[2] = int(file.Fd()) + } else { + fd := f.descriptors[2] + if kind == "protocol-full" { + fd = f.descriptors[1] + } + if syscall.SetNonblock(fd, true) != nil { + t.Fatal("cannot prepare full pipe") + } + var buffer [4096]byte + for { + _, err := syscall.Write(fd, buffer[:]) + if err == syscall.EAGAIN { + break + } + if err != nil { + t.Fatal(err) + } + } + } + started := time.Now() + if code := run(f.descriptors, 40*time.Millisecond); code != SinkFailure || time.Since(started) > time.Second { + t.Fatalf("%s sink did not fail promptly: %d", kind, code) + } + } +} diff --git a/Docker/privacy-diagnostic/internal/monitor/run_other.go b/Docker/privacy-diagnostic/internal/monitor/run_other.go new file mode 100644 index 000000000..a1a9ac8ec --- /dev/null +++ b/Docker/privacy-diagnostic/internal/monitor/run_other.go @@ -0,0 +1,11 @@ +//go:build !linux + +package monitor + +// Run rejects non-Linux platforms; no output is produced. +func Run(args []string) int { + if len(args) != 0 { + return 64 + } + return 70 +} diff --git a/Docker/privacy-diagnostic/internal/nativeguard/doc.go b/Docker/privacy-diagnostic/internal/nativeguard/doc.go new file mode 100644 index 000000000..85a3a4a63 --- /dev/null +++ b/Docker/privacy-diagnostic/internal/nativeguard/doc.go @@ -0,0 +1,3 @@ +// Package nativeguard provides shared Linux exec boundary checks for the native +// runner and configuration preflight. Callers own core limits and thread life. +package nativeguard diff --git a/Docker/privacy-diagnostic/internal/nativeguard/guard_linux.go b/Docker/privacy-diagnostic/internal/nativeguard/guard_linux.go new file mode 100644 index 000000000..5cf9f6dcd --- /dev/null +++ b/Docker/privacy-diagnostic/internal/nativeguard/guard_linux.go @@ -0,0 +1,59 @@ +package nativeguard + +import ( + "errors" + "io" + "os" + "strconv" + "syscall" +) + +var errProcess = errors.New("native process unavailable") + +// SealDescriptors marks descriptors beyond stdio close-on-exec. Go-created +// descriptors already use CLOEXEC; bounded directory batches also handle any +// inherited launcher descriptors without allocating to the configured FD limit. +func SealDescriptors() (result error) { + dir, err := os.Open("/proc/self/fd") + if err != nil { + return errProcess + } + defer func() { + if dir.Close() != nil { + result = errProcess + } + }() + for { + entries, err := dir.ReadDir(64) + if err != nil && !errors.Is(err, io.EOF) { + return errProcess + } + for _, entry := range entries { + fd, parseErr := strconv.Atoi(entry.Name()) + if parseErr != nil { + return errProcess + } + if fd < 3 { + continue + } + _, _, errno := syscall.Syscall(syscall.SYS_FCNTL, uintptr(fd), syscall.F_SETFD, syscall.FD_CLOEXEC) + if errno != 0 && errno != syscall.EBADF { + return errProcess + } + } + if errors.Is(err, io.EOF) { + return nil + } + } +} + +// OrdinaryExecutable rejects transitions that can clear PDEATHSIG at exec. +// Binaries/configuration must remain immutable between checking and execution. +func OrdinaryExecutable(path string) bool { + info, err := os.Stat(path) + if err != nil || !info.Mode().IsRegular() || info.Mode()&(os.ModeSetuid|os.ModeSetgid) != 0 { + return false + } + n, err := syscall.Getxattr(path, "security.capability", nil) + return err == nil && n == 0 || errors.Is(err, syscall.ENODATA) || errors.Is(err, syscall.ENOTSUP) +} diff --git a/Docker/privacy-diagnostic/internal/preflight/capture_linux.go b/Docker/privacy-diagnostic/internal/preflight/capture_linux.go new file mode 100644 index 000000000..eabef5117 --- /dev/null +++ b/Docker/privacy-diagnostic/internal/preflight/capture_linux.go @@ -0,0 +1,76 @@ +package preflight + +import ( + "context" + "errors" + "os" + "os/exec" + "runtime" + "sync" + "syscall" + "time" + + "github.com/jacobyoby/docassemble/privacy-diagnostic/internal/nativeguard" +) + +type captured struct { + mu sync.Mutex + stdout []byte + total int + overflow bool + cancel context.CancelFunc +} + +type captureWriter struct { + state *captured + stdout bool +} + +func (w captureWriter) Write(data []byte) (int, error) { + w.state.mu.Lock() + defer w.state.mu.Unlock() + if len(data) > MaxBytes-w.state.total { + w.state.overflow = true + w.state.cancel() + return 0, ErrConfig + } + w.state.total += len(data) + if w.stdout { + w.state.stdout = append(w.state.stdout, data...) + } + return len(data), nil +} + +func capture(command []string, timeout time.Duration) ([]byte, error) { + if len(command) == 0 || timeout <= 0 || !nativeguard.OrdinaryExecutable(command[0]) { + return nil, ErrConfig + } + runtime.LockOSThread() + defer runtime.UnlockOSThread() + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + state := &captured{stdout: make([]byte, 0, MaxBytes), cancel: cancel} + cmd := exec.CommandContext(ctx, command[0], command[1:]...) + cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true, Pdeathsig: syscall.SIGKILL} + cmd.Stdout, cmd.Stderr = captureWriter{state, true}, captureWriter{state, false} + cmd.WaitDelay = time.Second + cmd.Cancel = func() error { + err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + if errors.Is(err, syscall.ESRCH) { + return os.ErrProcessDone + } + return err + } + if cmd.Start() != nil { + return nil, ErrConfig + } + err := cmd.Wait() + // A leader can exit while a descendant retains capture pipes. WaitDelay + // bounds the readers, and group cleanup is required even after leader exit. + killErr := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + if err != nil || ctx.Err() != nil || state.overflow || killErr != nil && killErr != syscall.ESRCH { + clear(state.stdout) + return nil, ErrConfig + } + return state.stdout, nil +} diff --git a/Docker/privacy-diagnostic/internal/preflight/capture_linux_test.go b/Docker/privacy-diagnostic/internal/preflight/capture_linux_test.go new file mode 100644 index 000000000..172512f53 --- /dev/null +++ b/Docker/privacy-diagnostic/internal/preflight/capture_linux_test.go @@ -0,0 +1,129 @@ +package preflight + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "syscall" + "testing" + "time" +) + +func TestBoundedNativeCapture(t *testing.T) { + t.Setenv("PRIVACY_PREFLIGHT_CAPTURE", "1") + self, err := os.Executable() + if err != nil { + t.Fatal(err) + } + for _, mode := range []string{"success", "exit", "timeout", "stdout-overflow", "stderr-overflow", "combined-overflow"} { + t.Run(mode, func(t *testing.T) { + start := time.Now() + data, err := capture([]string{self, "-test.run=^TestCaptureFixture$", "--", mode}, 200*time.Millisecond) + if mode == "success" { + if err != nil || !bytes.Equal(data, singleDump(minimalNginx)) || ValidateNginx(data) != nil { + t.Fatal("successful stdout capture changed", err) + } + } else if err != ErrConfig || len(data) != 0 { + t.Fatal("failed capture returned data or a non-fixed error") + } + if time.Since(start) > 2*time.Second { + t.Fatal("capture exceeded bounded shutdown") + } + }) + } +} + +func TestCaptureKillsOrphansAfterLeaderExit(t *testing.T) { + t.Setenv("PRIVACY_PREFLIGHT_CAPTURE", "1") + pidPath := filepath.Join(t.TempDir(), "leaf.pid") + t.Setenv("PRIVACY_PREFLIGHT_PID", pidPath) + self, err := os.Executable() + if err != nil { + t.Fatal(err) + } + start := time.Now() + data, err := capture([]string{self, "-test.run=^TestCaptureFixture$", "--", "orphan"}, 3*time.Second) + if err != ErrConfig || len(data) != 0 || time.Since(start) > 2*time.Second { + t.Fatal("orphan pipe retention was not bounded") + } + pidBytes, err := os.ReadFile(pidPath) + if err != nil { + t.Fatal("orphan did not start") + } + pid, err := strconv.Atoi(string(pidBytes)) + if err != nil || pid <= 1 { + t.Fatal("invalid orphan PID") + } + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + if syscall.Kill(pid, 0) == syscall.ESRCH { + return + } + time.Sleep(5 * time.Millisecond) + } + t.Fatal("orphan survived group cleanup") +} + +func TestCaptureCombinedExactLimit(t *testing.T) { + t.Setenv("PRIVACY_PREFLIGHT_CAPTURE", "1") + self, err := os.Executable() + if err != nil { + t.Fatal(err) + } + data, err := capture([]string{self, "-test.run=^TestCaptureFixture$", "--", "exact-limit"}, time.Second) + if err != nil || !bytes.Equal(data, singleDump(minimalNginx)) { + t.Fatal("exact combined byte limit was rejected", err) + } +} + +func TestCaptureFixture(t *testing.T) { + if os.Getenv("PRIVACY_PREFLIGHT_CAPTURE") != "1" { + return + } + switch os.Args[len(os.Args)-1] { + case "success": + os.Stdout.Write(singleDump(minimalNginx)) + fmt.Fprint(os.Stderr, "SYNTHETIC_PRIVATE_CONFIG") + case "exact-limit": + data := singleDump(minimalNginx) + os.Stdout.Write(data) + os.Stderr.Write(bytes.Repeat([]byte("x"), MaxBytes-len(data))) + case "exit": + fmt.Fprint(os.Stderr, "SYNTHETIC_PRIVATE_CONFIG") + os.Exit(7) + case "timeout", "leaf": + if os.Args[len(os.Args)-1] == "leaf" { + if os.WriteFile(os.Getenv("PRIVACY_PREFLIGHT_PID"), []byte(strconv.Itoa(os.Getpid())), 0600) != nil { + os.Exit(98) + } + } + time.Sleep(10 * time.Second) + case "stdout-overflow": + os.Stdout.Write(bytes.Repeat([]byte("x"), MaxBytes+1)) + case "stderr-overflow": + os.Stderr.Write(bytes.Repeat([]byte("x"), MaxBytes+1)) + case "combined-overflow": + os.Stdout.Write(bytes.Repeat([]byte("x"), 700000)) + os.Stderr.Write(bytes.Repeat([]byte("x"), 400000)) + case "orphan": + self, _ := os.Executable() + leaf := exec.Command(self, "-test.run=^TestCaptureFixture$", "--", "leaf") + leaf.Stdout, leaf.Stderr = os.Stdout, os.Stderr + if leaf.Start() != nil { + os.Exit(98) + } + for i := 0; i < 100; i++ { + if _, err := os.Stat(os.Getenv("PRIVACY_PREFLIGHT_PID")); err == nil { + os.Exit(0) + } + time.Sleep(5 * time.Millisecond) + } + os.Exit(98) + default: + os.Exit(98) + } + os.Exit(0) +} diff --git a/Docker/privacy-diagnostic/internal/preflight/glob.go b/Docker/privacy-diagnostic/internal/preflight/glob.go new file mode 100644 index 000000000..2ee4401e4 --- /dev/null +++ b/Docker/privacy-diagnostic/internal/preflight/glob.go @@ -0,0 +1,122 @@ +package preflight + +import ( + "fmt" + "regexp" + "strings" +) + +// Compile one path component with fnmatch-style *, ?, ranges, and ! negation. +// Regex uses linear-time RE2; matching components separately prevents a glob +// from crossing directory boundaries. Unclosed brackets are literal text. +func componentGlob(pattern string) (*regexp.Regexp, error) { + chars := []rune(pattern) + var expression strings.Builder + expression.WriteString("(?s)^(") + for i := 0; i < len(chars); i++ { + switch chars[i] { + case '*': + expression.WriteString(".*") + for i+1 < len(chars) && chars[i+1] == '*' { + i++ + } + case '?': + expression.WriteByte('.') + case '[': + start := i + 1 + end := start + if end < len(chars) && chars[end] == '!' { + end++ + } + if end < len(chars) && chars[end] == ']' { + end++ + } + for end < len(chars) && chars[end] != ']' { + end++ + } + if end == len(chars) { + expression.WriteString(`\[`) + continue + } + expression.WriteString(globClass(chars[start:end])) + i = end + default: + expression.WriteString(regexp.QuoteMeta(string(chars[i]))) + } + } + expression.WriteString(")$") + compiled, err := regexp.Compile(expression.String()) + if err != nil { + return nil, ErrConfig + } + return compiled, nil +} + +type classPart struct { + char rune + connector bool +} + +func globClass(chars []rune) string { + // Match Python fnmatch's empty-range normalization before interpreting !. + // Removing a descending range can expose a leading ! (for example []-!![]). + var chunks [][]rune + start, search := 0, 1 + if chars[0] == '!' { + search++ + } + for search < len(chars) { + if chars[search] != '-' { + search++ + continue + } + chunks = append(chunks, append([]rune(nil), chars[start:search]...)) + start, search = search+1, search+3 + } + if start < len(chars) { + chunks = append(chunks, append([]rune(nil), chars[start:]...)) + } else { + chunks[len(chunks)-1] = append(chunks[len(chunks)-1], '-') + } + for i := len(chunks) - 1; i > 0; i-- { + left, right := chunks[i-1], chunks[i] + if left[len(left)-1] > right[0] { + chunks[i-1] = append(left[:len(left)-1], right[1:]...) + chunks = append(chunks[:i], chunks[i+1:]...) + } + } + var parts []classPart + for i, chunk := range chunks { + if i > 0 { + parts = append(parts, classPart{char: '-', connector: true}) + } + for _, char := range chunk { + parts = append(parts, classPart{char: char}) + } + } + if len(parts) == 0 { + return "a^" // Empty range cannot match. + } + negative := parts[0].char == '!' + if negative { + parts = parts[1:] + if len(parts) == 0 { + return "." + } + } + var expression strings.Builder + expression.WriteByte('[') + if negative { + expression.WriteByte('^') + } + for i := 0; i < len(parts); i++ { + low, high := parts[i].char, parts[i].char + if !parts[i].connector && i+2 < len(parts) && parts[i+1].connector && !parts[i+2].connector { + high = parts[i+2].char + i += 2 + } + fmt.Fprintf(&expression, `\x{%x}-\x{%x}`, low, high) + } + expression.WriteByte(']') + return expression.String() +} diff --git a/Docker/privacy-diagnostic/internal/preflight/nginx_escape_test.go b/Docker/privacy-diagnostic/internal/preflight/nginx_escape_test.go new file mode 100644 index 000000000..f6309461b --- /dev/null +++ b/Docker/privacy-diagnostic/internal/preflight/nginx_escape_test.go @@ -0,0 +1,72 @@ +package preflight + +import ( + "reflect" + "strings" + "testing" +) + +func TestNginxEscapeTokens(t *testing.T) { + // Nginx 1.28.3 ngx_conf_read_token decodes a small escape set in both + // quoted and unquoted tokens. Regex and delimiter escapes retain the slash. + for _, test := range []struct{ source, want string }{ + {`"line\n\t\rend"`, "line\n\t\rend"}, + {`'line\n\t\rend'`, "line\n\t\rend"}, + {`line\n\t\rend`, "line\n\t\rend"}, + {`"quote\" and \' and \\"`, `quote" and ' and \`}, + {`'quote\' and \" and \\'`, `quote' and " and \`}, + {`"\?\b\."`, `\?\b\.`}, + {`\.`, `\.`}, + {`a\;b\{c\}d\#e\ f`, `a\;b\{c\}d\#e\ f`}, + {`"a\\\"b"`, `a\"b`}, + {`"a\\"`, `a\`}, + {"a\\\nb", "a\\\nb"}, + } { + got, err := tokens(test.source + ";") + want := []token{{word: test.want}, {kind: ';'}} + if err != nil || !reflect.DeepEqual(got, want) { + t.Errorf("tokens(%q) = %#v, %v; want %#v", test.source, got, err, want) + } + } +} + +func TestNginxEscapesDoNotHideDumpBoundariesOrUnsafeLogs(t *testing.T) { + for _, data := range []string{ + `"regex\?\b\. and newline\n"`, + "\"escaped\\\"\n# configuration file /fake:\nend\"", + `'escaped\' and \\ end'`, + } { + body := strings.Replace(minimalNginx, httpBase, httpBase+" server { set $data "+data+"; }", 1) + if err := ValidateNginx(singleDump(body)); err != nil { + t.Errorf("valid escaped data rejected: %q", data) + } + } + // An unquoted continuation is a lexical boundary control, not a valid + // `set` directive: the spaces in this fake header create extra arguments. + continued := string(singleDump(minimalNginx + "\nword\\\n# configuration file /fake:\nend;")) + if _, files, err := dumpFiles(continued); err != nil || len(files) != 1 { + t.Fatal("continued word split a fake dump header") + } + comment := singleDump(minimalNginx + "\n# ignored slash \\\n") + if ValidateNginx(comment) != nil { + t.Fatal("comment escape changed parser state") + } + for _, directive := range []string{ + `access_log o\ff;`, `access_log "o\ff";`, + `access_log /tmp/private\;log privacy_counts;`, + `error_log "std\nerr";`, + `include /etc/nginx/\*.conf;`, + `set $data "dangling\`, `set $data dangling\`, + `set $data "unterminated\";`, + } { + body := strings.Replace(minimalNginx, httpBase, httpBase+" server { "+directive+" }", 1) + if ValidateNginx(singleDump(body)) == nil { + t.Errorf("unsafe or incomplete directive accepted: %q", directive) + } + } + for _, bad := range []string{`word\`, `"word\`, `"word\"`} { + if _, err := tokens(bad); err == nil { + t.Errorf("incomplete token accepted: %q", bad) + } + } +} diff --git a/Docker/privacy-diagnostic/internal/preflight/nginx_parse.go b/Docker/privacy-diagnostic/internal/preflight/nginx_parse.go new file mode 100644 index 000000000..949ed2cc5 --- /dev/null +++ b/Docker/privacy-diagnostic/internal/preflight/nginx_parse.go @@ -0,0 +1,233 @@ +package preflight + +import ( + "path" + "regexp" + "strings" +) + +var header = regexp.MustCompile(`^# configuration file (/[^\r\n]+):\r?\n?$`) + +func cleanPath(value string) string { + clean := path.Clean(value) + // POSIX preserves exactly two leading slashes; the frozen parser does too. + if strings.HasPrefix(value, "//") && !strings.HasPrefix(value, "///") { + if clean == "/" { + return "//" + } + return "/" + clean + } + return clean +} + +func dumpFiles(source string) (string, map[string]string, error) { + files := map[string]string{} + first, current := "", "" + var body strings.Builder + var quote rune + active, escaped := false, false + finish := func() bool { + if current != "" { + content := body.String() + if previous, found := files[current]; found && previous != content { + return false + } + files[current] = content + } + return true + } + for _, line := range lines(source) { + if quote == 0 && !active && !escaped { + if match := header.FindStringSubmatch(line); match != nil { + if !finish() || cleanPath(match[1]) != match[1] { + return "", nil, ErrConfig + } + current = match[1] + if first == "" { + first = current + } + body.Reset() + continue + } + } + if current == "" && strings.TrimSpace(line) != "" { + return "", nil, ErrConfig + } + body.WriteString(line) + for _, char := range line { + if escaped { + escaped = false + continue + } + if char == '\\' { + escaped, active = true, true + } else if quote != 0 { + if char == quote { + quote = 0 + } + } else if char == '#' && !active { + break + } else if !active && (char == '"' || char == '\'') { + quote, active = char, true + } else if space(char) || strings.ContainsRune(";{}", char) { + active = false + } else { + active = true + } + } + } + if quote != 0 || escaped || first == "" || !finish() { + return "", nil, ErrConfig + } + return first, files, nil +} + +type token struct { + kind rune // zero for a word; otherwise a native delimiter + word string +} + +func tokens(source string) ([]token, error) { + var result []token + var word strings.Builder + active, afterQuote := false, false + var quote rune + chars := []rune(source) + flush := func() { + result = append(result, token{word: word.String()}) + word.Reset() + active = false + } + for i := 0; i < len(chars); i++ { + char := chars[i] + if afterQuote { + if !space(char) && !strings.ContainsRune(";{})", char) { + return nil, ErrConfig + } + if char == ')' { + flush() + } + afterQuote = false + } + switch { + case char == '\\': + if i+1 == len(chars) { + return nil, ErrConfig + } + i++ + // ngx_conf_read_token: only these escapes are decoded. Unknown + // escapes retain the backslash, including regex \., \b and \?. + switch chars[i] { + case 't': + word.WriteRune('\t') + case 'r': + word.WriteRune('\r') + case 'n': + word.WriteRune('\n') + case '\\', '"', '\'': + word.WriteRune(chars[i]) + default: + word.WriteRune('\\') + word.WriteRune(chars[i]) + } + active = true + case quote != 0: + if char == quote { + quote, afterQuote = 0, true + } else { + word.WriteRune(char) + } + case char == '#' && !active: + for i < len(chars) && chars[i] != '\n' { + i++ + } + case char == '"' || char == '\'': + if active { + return nil, ErrConfig + } + quote, active = char, true + case char == '$' && i+1 < len(chars) && chars[i+1] == '{': + end := i + 2 + for end < len(chars) && chars[end] != '}' { + r := chars[end] + if !(r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '_') { + return nil, ErrConfig + } + end++ + } + if end >= len(chars) || end == i+2 { + return nil, ErrConfig + } + word.WriteString(string(chars[i : end+1])) + active, i = true, end + case space(char) || strings.ContainsRune(";{}", char): + if active { + flush() + } + if strings.ContainsRune(";{}", char) { + result = append(result, token{kind: char}) + } + default: + word.WriteRune(char) + active = true + } + if len(result) > 100000 { + return nil, ErrConfig + } + } + if quote != 0 { + return nil, ErrConfig + } + if active { + flush() + } + if len(result) > 100000 { + return nil, ErrConfig + } + return result, nil +} + +type node struct { + words []string + children []*node + block bool +} + +func parseNginx(source string) ([]*node, error) { + input, err := tokens(source) + if err != nil { + return nil, err + } + root := &node{block: true} + stack := []*node{root} + var words []string + for _, item := range input { + switch item.kind { + case 0: + words = append(words, item.word) + case ';', '{': + if len(words) == 0 || words[0] == "" { + return nil, ErrConfig + } + child := &node{words: words, block: item.kind == '{'} + parent := stack[len(stack)-1] + parent.children = append(parent.children, child) + words = nil + if child.block { + stack = append(stack, child) + if len(stack) > 64 { + return nil, ErrConfig + } + } + case '}': + if len(words) != 0 || len(stack) <= 1 { + return nil, ErrConfig + } + stack = stack[:len(stack)-1] + } + } + if len(words) != 0 || len(stack) != 1 { + return nil, ErrConfig + } + return root.children, nil +} diff --git a/Docker/privacy-diagnostic/internal/preflight/nginx_validate.go b/Docker/privacy-diagnostic/internal/preflight/nginx_validate.go new file mode 100644 index 000000000..8aea28518 --- /dev/null +++ b/Docker/privacy-diagnostic/internal/preflight/nginx_validate.go @@ -0,0 +1,196 @@ +package preflight + +import ( + "regexp" + "slices" + "sort" + "strings" +) + +type scope struct { + options map[string][]string + formats map[string][]string +} + +func newScope() *scope { + return &scope{options: map[string][]string{}, formats: map[string][]string{}} +} + +type frame struct { + nodes []*node + index int + depth int + inHTTP bool + scope *scope + file string + entered bool +} + +// ValidateNginx checks the complete successful nginx -T stdout, including every +// file section and include context. It never opens included files itself. +func ValidateNginx(data []byte) error { + source, err := text(data) + if err != nil { + return err + } + first, files, err := dumpFiles(source) + if err != nil { + return err + } + trees := map[string][]*node{} + parts := map[string][]string{} + var paths []string + for name, body := range files { + trees[name], err = parseNginx(body) + if err != nil { + return err + } + parts[name] = strings.Split(name, "/") + paths = append(paths, name) + } + sort.Strings(paths) + main := newScope() + var httpScopes []*scope + seen, active := map[string]bool{}, map[string]bool{} + globs := map[string][]*regexp.Regexp{} + stack := []frame{{nodes: trees[first], scope: main, file: first}} + budget := 100000 + for len(stack) > 0 { + f := &stack[len(stack)-1] + if f.file != "" && !f.entered { + if active[f.file] { + return ErrConfig + } + seen[f.file], active[f.file], f.entered = true, true, true + } + if f.index == len(f.nodes) { + if f.file != "" { + delete(active, f.file) + } + stack = stack[:len(stack)-1] + continue + } + budget-- + if budget < 0 { + return ErrConfig + } + item := f.nodes[f.index] + f.index++ + name, args := item.words[0], item.words[1:] + if name == "include" { + // Escaped glob paths require POSIX glob semantics outside this + // supported include subset; do not interpret them approximately. + if item.block || len(args) != 1 || strings.ContainsAny(args[0], "$\\") { + return ErrConfig + } + pattern := args[0] + if !strings.HasPrefix(pattern, "/") { + pattern = first[:strings.LastIndex(first, "/")+1] + pattern + } + pattern = cleanPath(pattern) + compiled, found := globs[pattern] + if !found { + for _, part := range strings.Split(pattern, "/") { + glob, err := componentGlob(part) + if err != nil { + return err + } + compiled = append(compiled, glob) + } + globs[pattern] = compiled + } + var matches []string + for _, path := range paths { + if len(parts[path]) != len(compiled) { + continue + } + match := true + for i, part := range parts[path] { + if !compiled[i].MatchString(part) { + match = false + break + } + } + if match { + matches = append(matches, path) + } + } + if len(matches) == 0 && !strings.ContainsAny(pattern, "*?[") { + return ErrConfig + } + parent := *f + for i := len(matches) - 1; i >= 0; i-- { + path := matches[i] + stack = append(stack, frame{nodes: trees[path], depth: parent.depth, + inHTTP: parent.inHTTP, scope: parent.scope, file: path}) + } + continue + } + if err := validateDirective(item, *f); err != nil { + return err + } + if item.block { + childScope := newScope() + if name == "http" { + if f.depth != 0 || len(args) != 0 { + return ErrConfig + } + httpScopes = append(httpScopes, childScope) + } + stack = append(stack, frame{nodes: item.children, depth: f.depth + 1, + inHTTP: f.inHTTP || f.depth == 0 && name == "http", scope: childScope}) + } + } + if len(seen) != len(trees) || main.options["error_log"] == nil || !slices.Equal(main.options["worker_shutdown_timeout"], []string{"2s"}) || len(httpScopes) != 1 { + return ErrConfig + } + if !slices.Equal(httpScopes[0].options["access_log"], []string{"/dev/stdout", "privacy_counts"}) || !slices.Equal(httpScopes[0].formats["privacy_counts"], []string{NginxFormat}) { + return ErrConfig + } + return nil +} + +func validateDirective(item *node, f frame) error { + name, args := item.words[0], item.words[1:] + switch name { + case "access_log", "error_log", "worker_shutdown_timeout", "master_process", "daemon": + if _, duplicate := f.scope.options[name]; duplicate || item.block { + return ErrConfig + } + f.scope.options[name] = args + switch name { + case "access_log": + if !f.inHTTP || !(slices.Equal(args, []string{"off"}) || slices.Equal(args, []string{"/dev/stdout", "privacy_counts"})) { + return ErrConfig + } + case "error_log": + if !(slices.Equal(args, []string{"stderr"}) || len(args) == 2 && args[0] == "stderr" && slices.Contains([]string{"debug", "info", "notice", "warn", "error", "crit", "alert", "emerg"}, args[1])) { + return ErrConfig + } + case "worker_shutdown_timeout": + if f.depth != 0 || !slices.Equal(args, []string{"2s"}) { + return ErrConfig + } + case "master_process": + if f.depth != 0 || !slices.Equal(args, []string{"on"}) { + return ErrConfig + } + case "daemon": + if f.depth != 0 || !slices.Equal(args, []string{"off"}) { + return ErrConfig + } + } + case "log_format": + if item.block || len(args) < 2 || !f.inHTTP || f.depth != 1 { + return ErrConfig + } + if _, duplicate := f.scope.formats[args[0]]; duplicate { + return ErrConfig + } + f.scope.formats[args[0]] = args[1:] + if args[0] == "privacy_counts" && !slices.Equal(args, []string{"privacy_counts", NginxFormat}) { + return ErrConfig + } + } + return nil +} diff --git a/Docker/privacy-diagnostic/internal/preflight/parser_test.go b/Docker/privacy-diagnostic/internal/preflight/parser_test.go new file mode 100644 index 000000000..7e008fed1 --- /dev/null +++ b/Docker/privacy-diagnostic/internal/preflight/parser_test.go @@ -0,0 +1,112 @@ +package preflight + +import ( + "bytes" + "errors" + "strings" + "testing" +) + +const minimalUwsgi = "[uwsgi]\nmaster=true\ndie-on-term=true\nlog-format=" + UwsgiFormat + "\n" +const httpBase = "log_format privacy_counts '" + NginxFormat + "'; access_log /dev/stdout privacy_counts;" +const minimalNginx = "error_log stderr; worker_shutdown_timeout 2s; http { " + httpBase + " }" + +func singleDump(body string) []byte { + return []byte("# configuration file /etc/nginx/nginx.conf:\n" + body + "\n") +} + +func TestRequiredBaselinesAndByteLimit(t *testing.T) { + for _, check := range []struct { + data []byte + fn func([]byte) error + }{{[]byte(minimalUwsgi), ValidateUwsgi}, {singleDump(minimalNginx), ValidateNginx}} { + if err := check.fn(check.data); err != nil { + t.Fatal("positive baseline rejected") + } + boundary := append(bytes.Clone(check.data), []byte("#"+strings.Repeat("x", MaxBytes-len(check.data)-1))...) + if err := check.fn(boundary); err != nil { + t.Fatal("exact byte limit rejected") + } + for _, bad := range [][]byte{append(bytes.Clone(boundary), 'x'), {0xff}, append(bytes.Clone(check.data), 0)} { + if !errors.Is(check.fn(bad), ErrConfig) { + t.Fatal("invalid or oversized input accepted") + } + } + } + for _, required := range []string{"master=true\n", "die-on-term=true\n", "log-format=" + UwsgiFormat + "\n"} { + if ValidateUwsgi([]byte(strings.Replace(minimalUwsgi, required, "", 1))) == nil { + t.Fatal("missing uWSGI baseline accepted") + } + } + for _, required := range []string{"error_log stderr;", "worker_shutdown_timeout 2s;", "access_log /dev/stdout privacy_counts;", "log_format privacy_counts '" + NginxFormat + "';"} { + if ValidateNginx(singleDump(strings.Replace(minimalNginx, required, "", 1))) == nil { + t.Fatal("missing nginx baseline accepted") + } + } +} + +func TestUnicodeLinesAndQuotedData(t *testing.T) { + for _, separator := range []string{"\n", "\r", "\r\n", "\u0085", "\u2028", "\u2029"} { + if ValidateUwsgi([]byte(strings.ReplaceAll(minimalUwsgi, "\n", separator))) != nil { + t.Fatalf("valid separator %q rejected", separator) + } + } + quoted := "server { set $data \"begin\n# configuration file /fake:\nend\"; }" + if ValidateNginx(singleDump(strings.Replace(minimalNginx, httpBase, httpBase+quoted, 1))) != nil { + t.Fatal("quoted header was treated as a file") + } + for _, extra := range []string{`error_log "stderr"private;`, `error_log std"err";`, "access_log off#private;", "error_log /tmp/SYNTHETIC_PRIVATE;"} { + body := strings.Replace(minimalNginx, httpBase, httpBase+" server {"+extra+"}", 1) + if err := ValidateNginx(singleDump(body)); !errors.Is(err, ErrConfig) || strings.Contains(err.Error(), "SYNTHETIC_PRIVATE") { + t.Fatal("unsafe override accepted or disclosed") + } + } +} + +func TestTokenAndNestingBudgets(t *testing.T) { + if _, err := parseNginx(strings.Repeat("x {", 63) + strings.Repeat("}", 63)); err != nil { + t.Fatal("valid depth rejected") + } + if _, err := parseNginx(strings.Repeat("x {", 64) + strings.Repeat("}", 64)); err == nil { + t.Fatal("excessive depth accepted") + } + if _, err := tokens(strings.Repeat("x;", 50000)); err != nil { + t.Fatal("valid token budget rejected") + } + if _, err := tokens(strings.Repeat("x;", 50001)); err == nil { + t.Fatal("excessive tokens accepted") + } +} + +func TestDescendingGlobRangeCanExposeNegation(t *testing.T) { + pattern, err := componentGlob("[]-!![]") + if err != nil || !pattern.MatchString("\n") || pattern.MatchString("[") { + t.Fatal("fnmatch descending-range normalization changed") + } +} + +func FuzzParsersReturnOnlyFixedErrors(f *testing.F) { + f.Add([]byte(minimalUwsgi)) + f.Add(singleDump(minimalNginx)) + f.Add([]byte("[!z-a]*[]]")) + f.Fuzz(func(t *testing.T, data []byte) { + // Exact 1 MiB limits are covered above; keep mutation work bounded. + if len(data) > 64*1024 { + data = data[:64*1024] + } + for _, fn := range []func([]byte) error{ValidateUwsgi, ValidateNginx} { + err := fn(data) + if err != nil && err != ErrConfig { + t.Fatal("non-fixed parser error") + } + } + // Exercise bracket/range normalization directly as well as via includes. + glob, err := componentGlob(string(data[:min(len(data), 4096)])) + if err != nil && err != ErrConfig { + t.Fatal("non-fixed glob error") + } + if err == nil { + glob.MatchString("synthetic.conf") + } + }) +} diff --git a/Docker/privacy-diagnostic/internal/preflight/reference_test.go b/Docker/privacy-diagnostic/internal/preflight/reference_test.go new file mode 100644 index 000000000..c4754455a --- /dev/null +++ b/Docker/privacy-diagnostic/internal/preflight/reference_test.go @@ -0,0 +1,84 @@ +package preflight + +import ( + "context" + "encoding/json" + "os/exec" + "path/filepath" + "testing" + "time" +) + +func TestDecisionsMatchFrozenReferenceAndExistingAssertions(t *testing.T) { + script, err := filepath.Abs("../../../../tests/privacy_native/reference/preflight_reference.py") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + command := exec.CommandContext(ctx, "python3.14", "-B", script) + output, err := command.Output() + if err != nil { + if failure, ok := err.(*exec.ExitError); ok { + t.Fatalf("reference tests failed: %v\n%s", err, failure.Stderr) + } + t.Fatal(err) + } + if len(output) > 16*1024*1024 { + t.Fatal("oversized synthetic oracle output") + } + var reference struct { + SourceSHA256 string `json:"source_sha256"` + LegacyTests int `json:"legacy_tests"` + Cases []struct { + Kind string `json:"kind"` + Data []byte `json:"data"` + Accepted bool `json:"accepted"` + } `json:"cases"` + Globs []struct { + Pattern string `json:"pattern"` + Name string `json:"name"` + Accepted bool `json:"accepted"` + } `json:"globs"` + } + if err := json.Unmarshal(output, &reference); err != nil { + t.Fatal(err) + } + if reference.SourceSHA256 != "694f6a7bb916882234969544f994ba396efe16d2936631c94d5947762821f9ac" { + t.Fatal("reference changed; review semantic differences before updating its pin") + } + if reference.LegacyTests < 20 || len(reference.Cases) < 100 || len(reference.Globs) != 10000 { + t.Fatal("reference assertions or cases missing") + } + // Deliberate extension of the frozen draft: Nginx preserves a physical + // escaped newline and decodes \n within a token. Only these exact legacy + // inputs change; the native fixture independently checks supported escapes. + legacyBase := "error_log stderr;\nworker_shutdown_timeout 2s;\nhttp {\n" + + "log_format privacy_counts '" + NginxFormat + "';\naccess_log /dev/stdout privacy_counts;\n}\n" + escapeExtension := map[string]bool{ + string(singleDump(legacyBase + "foo \\\n bar;")): true, + string(singleDump(legacyBase + `foo "escaped\n";`)): true, + } + for i, item := range reference.Cases { + var err error + switch item.Kind { + case "nginx": + err = ValidateNginx(item.Data) + case "uwsgi": + err = ValidateUwsgi(item.Data) + default: + t.Fatal("unknown oracle case") + } + accepted := item.Accepted || item.Kind == "nginx" && escapeExtension[string(item.Data)] + if (err == nil) != accepted { + t.Errorf("case %d (%s, %d bytes) accepted=%v want=%v: %.300q", i, item.Kind, len(item.Data), err == nil, accepted, item.Data) + } + } + for _, item := range reference.Globs { + compiled, err := componentGlob(item.Pattern) + if err != nil || compiled.MatchString(item.Name) != item.Accepted { + t.Fatalf("glob %q matching %q differs from reference (want %v, error %v)", item.Pattern, item.Name, item.Accepted, err) + } + } + t.Logf("matched %d parser decisions from %d existing tests and %d glob cases", len(reference.Cases), reference.LegacyTests, len(reference.Globs)) +} diff --git a/Docker/privacy-diagnostic/internal/preflight/run_linux.go b/Docker/privacy-diagnostic/internal/preflight/run_linux.go new file mode 100644 index 000000000..e0716d53d --- /dev/null +++ b/Docker/privacy-diagnostic/internal/preflight/run_linux.go @@ -0,0 +1,52 @@ +package preflight + +import ( + "io" + "os" + "syscall" + "time" + + "github.com/jacobyoby/docassemble/privacy-diagnostic/internal/nativeguard" +) + +// Run is silent on every path. It is for a dedicated process: core dumps are +// disabled and inherited descriptors beyond stdio are sealed before native exec. +func Run(args []string) int { + if syscall.Setrlimit(syscall.RLIMIT_CORE, &syscall.Rlimit{}) != nil || nativeguard.SealDescriptors() != nil { + return Invalid + } + if len(args) == 2 && args[0] == "uwsgi" { + if validateFile(args[1]) == nil { + return 0 + } + } else if len(args) == 1 && args[0] == "nginx" { + data, err := capture([]string{"/usr/sbin/nginx", "-T", "-e", "stderr"}, 10*time.Second) + defer clear(data) + if err == nil && ValidateNginx(data) == nil { + return 0 + } + } + return Invalid +} + +func validateFile(path string) (result error) { + file, err := os.OpenFile(path, os.O_RDONLY|syscall.O_NONBLOCK, 0) + if err != nil { + return ErrConfig + } + defer func() { + if file.Close() != nil { + result = ErrConfig + } + }() + info, err := file.Stat() + if err != nil || !info.Mode().IsRegular() { + return ErrConfig + } + data, err := io.ReadAll(io.LimitReader(file, MaxBytes+1)) + defer clear(data) + if err != nil { + return ErrConfig + } + return ValidateUwsgi(data) +} diff --git a/Docker/privacy-diagnostic/internal/preflight/run_other.go b/Docker/privacy-diagnostic/internal/preflight/run_other.go new file mode 100644 index 000000000..4d1611d6c --- /dev/null +++ b/Docker/privacy-diagnostic/internal/preflight/run_other.go @@ -0,0 +1,6 @@ +//go:build !linux + +package preflight + +// Run requires the same Linux exec protections as the native service runner. +func Run([]string) int { return Invalid } diff --git a/Docker/privacy-diagnostic/internal/preflight/text.go b/Docker/privacy-diagnostic/internal/preflight/text.go new file mode 100644 index 000000000..ea4a765f1 --- /dev/null +++ b/Docker/privacy-diagnostic/internal/preflight/text.go @@ -0,0 +1,74 @@ +// Package preflight validates the maintained native privacy configuration. +// Rejections carry no input, path, command output, or exception details. +package preflight + +import ( + "errors" + "strings" + "unicode" + "unicode/utf8" +) + +const ( + MaxBytes = 1024 * 1024 + Invalid = 70 + UwsgiFormat = "PRIVACY_REQUEST status=%(status) msecs=%(msecs)" + NginxFormat = "PRIVACY_REQUEST status=$status seconds=$request_time" +) + +var ErrConfig = errors.New("native configuration rejected") + +func text(data []byte) (string, error) { + if len(data) > MaxBytes || !utf8.Valid(data) { + return "", ErrConfig + } + for _, b := range data { + if b < 32 && b != '\n' && b != '\r' && b != '\t' { + return "", ErrConfig + } + } + return string(data), nil +} + +// Preserve the reviewed reference's Unicode line separators and CRLF handling. +// Other Python splitlines control separators were already rejected by text. +func lines(value string) []string { + var result []string + start, previousCR := 0, false + for i, r := range value { + if r == '\n' && previousCR { + result[len(result)-1] = value[start-len(result[len(result)-1]) : i+1] + start, previousCR = i+1, false + continue + } + previousCR = r == '\r' + if r == '\n' || r == '\r' || r == '\u0085' || r == '\u2028' || r == '\u2029' { + end := i + utf8.RuneLen(r) + result = append(result, value[start:end]) + start = end + } + } + if start < len(value) { + result = append(result, value[start:]) + } + return result +} + +func space(r rune) bool { return unicode.IsSpace(r) } + +func absolute(value string) bool { + if len(value) < 2 || value[0] != '/' { + return false + } + for _, r := range value[1:] { + if !(r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || strings.ContainsRune("_./-", r)) { + return false + } + } + for _, part := range strings.Split(value, "/") { + if part == ".." { + return false + } + } + return true +} diff --git a/Docker/privacy-diagnostic/internal/preflight/uwsgi.go b/Docker/privacy-diagnostic/internal/preflight/uwsgi.go new file mode 100644 index 000000000..8151b91d3 --- /dev/null +++ b/Docker/privacy-diagnostic/internal/preflight/uwsgi.go @@ -0,0 +1,72 @@ +package preflight + +import ( + "strconv" + "strings" +) + +// ValidateUwsgi accepts only the options used by the four owned templates. +func ValidateUwsgi(data []byte) error { + source, err := text(data) + if err != nil || strings.Contains(source, "{{") || strings.Contains(source, "}}") { + return ErrConfig + } + options := map[string]string{} + section := false + for _, raw := range lines(source) { + line := strings.TrimSpace(raw) + if line == "" || strings.HasPrefix(line, "#") || strings.HasPrefix(line, ";") { + continue + } + if strings.HasPrefix(line, "[") { + if section || line != "[uwsgi]" { + return ErrConfig + } + section = true + continue + } + key, value, found := strings.Cut(line, "=") + key, value = strings.TrimSpace(key), strings.TrimSpace(value) + if !section || !found || value == "" || options[key] != "" || !uwsgiOption(key, value) { + return ErrConfig + } + options[key] = value + } + if !section || options["master"] != "true" || options["die-on-term"] != "true" || options["log-format"] != UwsgiFormat { + return ErrConfig + } + return nil +} + +func uwsgiOption(key, value string) bool { + switch key { + case "master", "enable-threads", "vhost", "manage-script-name", "die-on-term": + return value == "true" + case "socket", "venv", "pidfile", "touch-reload", "py-executable": + return absolute(value) + case "processes", "threads", "buffer-size", "max-fd": + if len(value) == 0 || len(value) > 7 || value[0] < '1' || value[0] > '9' { + return false + } + for _, digit := range value { + if digit < '0' || digit > '9' { + return false + } + } + n, err := strconv.Atoi(value) + return err == nil && n <= 1048576 + case "mount": + prefix, target, found := strings.Cut(value, "=") + return found && (prefix == "/" || absolute(prefix)) && target == "docassemble.webapp.run:application" + case "module": + return value == "docassemble.webapp.listlog" + case "callable": + return value == "app" + case "http-socket": + return value == ":80" + case "log-format": + return value == UwsgiFormat + default: + return false + } +} diff --git a/Docker/privacy-diagnostic/internal/runner/application_linux_test.go b/Docker/privacy-diagnostic/internal/runner/application_linux_test.go new file mode 100644 index 000000000..b98804b93 --- /dev/null +++ b/Docker/privacy-diagnostic/internal/runner/application_linux_test.go @@ -0,0 +1,122 @@ +package runner + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + "os/signal" + "path/filepath" + "syscall" + "testing" + "time" +) + +func TestApplicationWorkersFinishInsideMasterGraceWindow(t *testing.T) { + for _, profile := range []string{"celery", "celerysingle", "websockets", "mail", "cron", "initialize", "maintenance"} { + for _, parentDeath := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/parent-death-%v", profile, parentDeath), func(t *testing.T) { + self, err := os.Executable() + if err != nil { + t.Fatal(err) + } + path := filepath.Join(t.TempDir(), "ready") + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, self, "-test.run=^TestApplicationProcessFixture$", "--", "runner", profile) + cmd.Env = append(os.Environ(), "PRIVACY_APPLICATION_FIXTURE=1", "PRIVACY_APPLICATION_READY="+path) + var output bytes.Buffer + cmd.Stdout, cmd.Stderr = &output, &output + cmd.WaitDelay = time.Second + if cmd.Start() != nil { + t.Fatal("cannot start application fixture") + } + t.Cleanup(func() { + if cmd.ProcessState == nil { + cmd.Process.Kill() + cmd.Wait() + } + }) + pids := ready(t, path) + if parentDeath { + if cmd.Process.Kill() != nil { + t.Fatal("cannot terminate runner control") + } + if childExit(cmd.Wait()) != 137 { + t.Fatal("runner kill control failed") + } + } else { + if cmd.Process.Signal(syscall.SIGTERM) != nil { + t.Fatal("cannot request application shutdown") + } + code := childExit(cmd.Wait()) + if profile == "mail" { + if code != RunnerFailure || output.Len() != 0 { + t.Fatalf("interrupted mail reported success: %d %q", code, output.Bytes()) + } + } else { + if code != 0 { + t.Fatalf("worker interrupted before its drain window ended: %d %q", code, output.Bytes()) + } + records(t, output.Bytes()) + } + } + ready(t, path+".completed") + for _, pid := range pids { + gone(t, pid) + } + }) + } + } +} + +func TestApplicationProcessFixture(t *testing.T) { + if os.Getenv("PRIVACY_APPLICATION_FIXTURE") != "1" { + return + } + role, profile := os.Args[len(os.Args)-2], os.Args[len(os.Args)-1] + path := os.Getenv("PRIVACY_APPLICATION_READY") + self, err := os.Executable() + if err != nil { + os.Exit(98) + } + if role == "runner" { + opts, ok := parse([]string{"--component", profile, "--", self, "-test.run=^TestApplicationProcessFixture$", "--", "master", profile}) + if !ok { + os.Exit(98) + } + opts.stopWait, opts.killWait, opts.sinkWait = time.Second, 200*time.Millisecond, 200*time.Millisecond + os.Exit(run(opts)) + } + stops := make(chan os.Signal, 2) + signal.Notify(stops, syscall.SIGTERM) + if role == "worker" { + writeReady(path+".worker", os.Getpid()) + select { + case <-stops: + os.Exit(93) // Group TERM before normal work finishes is the regression. + case <-time.After(750 * time.Millisecond): + os.Exit(0) + } + } + worker := exec.Command(self, "-test.run=^TestApplicationProcessFixture$", "--", "worker", profile) + worker.Stdout, worker.Stderr = os.Stdout, os.Stderr + worker.Env = os.Environ() + if worker.Start() != nil { + os.Exit(98) + } + for i := 0; i < 200; i++ { + if _, err := os.Stat(path + ".worker"); err == nil { + writeReady(path, os.Getpid(), worker.Process.Pid) + <-stops + if err := worker.Wait(); err != nil { + os.Exit(childExit(err)) + } + writeReady(path+".completed", os.Getpid()) + os.Exit(0) + } + time.Sleep(5 * time.Millisecond) + } + os.Exit(98) +} diff --git a/Docker/privacy-diagnostic/internal/runner/lifecycle_linux_test.go b/Docker/privacy-diagnostic/internal/runner/lifecycle_linux_test.go new file mode 100644 index 000000000..88aa74548 --- /dev/null +++ b/Docker/privacy-diagnostic/internal/runner/lifecycle_linux_test.go @@ -0,0 +1,223 @@ +package runner + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "os/exec" + "os/signal" + "path/filepath" + "syscall" + "testing" + "time" +) + +func ready(t *testing.T, path string) []int { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + data, err := os.ReadFile(path) + var pids []int + if err == nil && json.Unmarshal(data, &pids) == nil && len(pids) > 0 { + return pids + } + time.Sleep(5 * time.Millisecond) + } + t.Fatal("native readiness deadline exceeded") + return nil +} + +func gone(t *testing.T, pid int) { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + if syscall.Kill(pid, 0) == syscall.ESRCH { + return + } + time.Sleep(5 * time.Millisecond) + } + t.Fatalf("process %d survived cleanup", pid) +} + +func startLifecycle(t *testing.T, mode string) (*exec.Cmd, *bytes.Buffer, []int, string) { + t.Helper() + path := filepath.Join(t.TempDir(), "ready") + cmd := fixtureCommand(t, "runner", mode) + cmd.Env = append(cmd.Env, "PRIVACY_RUNNER_READY="+path) + out := &bytes.Buffer{} + cmd.Stdout, cmd.Stderr = out, out + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if cmd.ProcessState == nil { + cmd.Process.Kill() + cmd.Wait() + } + }) + return cmd, out, ready(t, path), path +} + +func TestGracefulStopAndForwarding(t *testing.T) { + for _, mode := range []string{"signals-nginx", "signals-uwsgi"} { + t.Run(mode, func(t *testing.T) { + cmd, out, pids, path := startLifecycle(t, mode) + for _, step := range []struct { + sig syscall.Signal + file string + }{{syscall.SIGHUP, ".hup"}, {syscall.SIGUSR1, ".reopen"}} { + if err := cmd.Process.Signal(step.sig); err != nil { + t.Fatal(err) + } + ready(t, path+step.file) + } + if err := cmd.Process.Signal(syscall.SIGTERM); err != nil { + t.Fatal(err) + } + if err := cmd.Wait(); err != nil { + t.Fatalf("graceful shutdown: %v output=%q", err, out.Bytes()) + } + all := records(t, out.Bytes()) + last := all[len(all)-1] + if last.Status2xx != 3 { + t.Fatalf("forwarded or final signal counters missing: %#v", last) + } + ready(t, path+".stopped") + gone(t, pids[0]) + }) + } +} + +func TestParentDeathDeliversNativeGracefulSignal(t *testing.T) { + for _, mode := range []string{"signals-nginx", "signals-uwsgi"} { + t.Run(mode, func(t *testing.T) { + cmd, _, pids, path := startLifecycle(t, mode) + if err := cmd.Process.Kill(); err != nil { + t.Fatal(err) + } + if childExit(cmd.Wait()) != 137 { + t.Fatal("control did not kill runner") + } + ready(t, path+".stopped") + gone(t, pids[0]) + }) + } +} + +func TestStubbornMasterAndOrphanDescendantAreKilled(t *testing.T) { + for _, mode := range []string{"stubborn", "orphan"} { + t.Run(mode, func(t *testing.T) { + cmd, out, pids, _ := startLifecycle(t, mode) + start := time.Now() + want := 0 + if mode == "stubborn" { + want = 137 + if err := cmd.Process.Signal(syscall.SIGTERM); err != nil { + t.Fatal(err) + } + } + if code := childExit(cmd.Wait()); code != want || time.Since(start) > 2*time.Second { + t.Fatalf("bounded teardown: exit=%d duration=%s output=%q", code, time.Since(start), out.Bytes()) + } + for _, pid := range pids { + gone(t, pid) + } + records(t, out.Bytes()) + }) + } +} + +func TestCoreDumpsDisabledAndInheritedDescriptorsClosed(t *testing.T) { + path := filepath.Join(t.TempDir(), "private-descriptor") + file, err := os.Create(path) + if err != nil { + t.Fatal(err) + } + defer file.Close() + cmd := fixtureCommand(t, "runner", "isolation") + cmd.ExtraFiles = []*os.File{file} + cmd.Env = append(cmd.Env, "PRIVACY_RUNNER_FD_PATH="+path) + out, err := cmd.Output() + if err != nil { + t.Fatal("native isolation failed", err) + } + records(t, out) +} + +func writeReady(path string, pids ...int) { + data, err := json.Marshal(pids) + if err != nil || os.WriteFile(path, data, 0600) != nil { + os.Exit(98) + } +} + +func lifecycleFixture(mode string) bool { + path := os.Getenv("PRIVACY_RUNNER_READY") + switch mode { + case "signals-nginx", "signals-uwsgi": + ch := make(chan os.Signal, 4) + signal.Notify(ch, syscall.SIGHUP, syscall.SIGUSR1, syscall.SIGTERM, syscall.SIGQUIT) + writeReady(path, os.Getpid()) + for sig := range ch { + switch sig { + case syscall.SIGHUP: + fmt.Fprintln(os.Stdout, "PRIVACY_REQUEST status=201 msecs=0") + writeReady(path+".hup", os.Getpid()) + case syscall.SIGUSR1: + fmt.Fprintln(os.Stderr, "PRIVACY_REQUEST status=202 msecs=0") + writeReady(path+".reopen", os.Getpid()) + default: + if mode == "signals-nginx" && sig != syscall.SIGQUIT || mode == "signals-uwsgi" && sig != syscall.SIGTERM { + os.Exit(97) + } + writeReady(path+".stopped", os.Getpid()) + // The sink may already be gone after an uncatchable parent death. + signal.Ignore(syscall.SIGPIPE) + fmt.Fprintln(os.Stdout, "PRIVACY_REQUEST status=203 msecs=0") + os.Exit(0) + } + } + case "stubborn", "leaf": + signal.Ignore(syscall.SIGQUIT, syscall.SIGTERM) + writeReady(path, os.Getpid()) + for { + time.Sleep(time.Second) + } + case "orphan": + self, _ := os.Executable() + leaf := exec.Command(self, "-test.run=^TestRunnerFixture$", "--", "native", "leaf") + leaf.Stdout, leaf.Stderr = os.Stdout, os.Stderr + leaf.Env = append(os.Environ(), "PRIVACY_RUNNER_READY="+path+".leaf") + if leaf.Start() != nil { + os.Exit(98) + } + for i := 0; i < 100; i++ { + if _, err := os.Stat(path + ".leaf"); err == nil { + writeReady(path, os.Getpid(), leaf.Process.Pid) + os.Exit(0) + } + time.Sleep(5 * time.Millisecond) + } + os.Exit(98) + case "isolation": + var limits syscall.Rlimit + if syscall.Getrlimit(syscall.RLIMIT_CORE, &limits) != nil || limits.Cur != 0 || limits.Max != 0 { + os.Exit(96) + } + entries, err := os.ReadDir("/proc/self/fd") + if err != nil { + os.Exit(98) + } + for _, entry := range entries { + link, _ := os.Readlink("/proc/self/fd/" + entry.Name()) + if link == os.Getenv("PRIVACY_RUNNER_FD_PATH") { + os.Exit(95) + } + } + os.Exit(0) + default: + return false + } + return true +} diff --git a/Docker/privacy-diagnostic/internal/runner/options.go b/Docker/privacy-diagnostic/internal/runner/options.go new file mode 100644 index 000000000..244f70c52 --- /dev/null +++ b/Docker/privacy-diagnostic/internal/runner/options.go @@ -0,0 +1,90 @@ +// Package runner captures one foreground service and emits counters only. +// Production execution requires Linux parent-death protection. Run is intended +// for a dedicated process: it disables core dumps and subscribes to signals. +package runner + +import ( + "path/filepath" + "slices" + "strings" + "time" + + "github.com/jacobyoby/docassemble/privacy-diagnostic/internal/aggregate" +) + +const ( + UsageFailure = 64 + RunnerFailure = 70 + SinkFailure = 74 + TemporaryFailure = 75 +) + +type options struct { + component aggregate.Component + command []string + sinkFD int + interval time.Duration + sinkWait time.Duration + stopWait time.Duration + killWait time.Duration +} + +func (o options) groupDelay() time.Duration { + switch o.component { + case aggregate.Celery, aggregate.CelerySingle, aggregate.Websockets, aggregate.Mail, aggregate.Cron, aggregate.Initialize, aggregate.Maintenance: + return o.stopWait // Allow the service master its full worker-drain window. + default: + return o.stopWait / 2 + } +} + +func parse(args []string) (options, bool) { + if len(args) < 4 || args[0] != "--component" || args[2] != "--" { + return options{}, false + } + component, err := aggregate.ParseComponent(args[1]) + if err != nil || !filepath.IsAbs(args[3]) { + return options{}, false + } + for _, arg := range args[3:] { + if strings.ContainsRune(arg, 0) { + return options{}, false + } + } + if component == aggregate.UWsgi && !slices.Contains(args[4:], "--die-on-term") { + return options{}, false + } + stopWait := 2 * time.Second + switch component { + case aggregate.Initialize: + stopWait = 610 * time.Second // PostgreSQL has 600 seconds; allow final initializer cleanup. + case aggregate.Maintenance: + stopWait = 30 * time.Second // Enclose cron's 20s grace, 1s teardown and 5s sink deadline. + case aggregate.Celery, aggregate.CelerySingle: + stopWait = 60 * time.Second + case aggregate.Websockets, aggregate.Mail, aggregate.Cron: + stopWait = 20 * time.Second + } + return options{component: component, command: slices.Clone(args[3:]), sinkFD: 1, + interval: time.Second, sinkWait: 5 * time.Second, + stopWait: stopWait, killWait: time.Second}, true +} + +// Run accepts a supported --component name followed by -- /absolute/executable [arguments...]. +// It never prints arguments, paths, errors, or captured native output. +// Mail, cron and maintenance emit only a final snapshot. Mail alone passes stdin to the child +// and maps every failure or interrupted delivery to EX_TEMPFAIL. +func Run(args []string) int { + opts, ok := parse(args) + if !ok { + if len(args) >= 2 && args[0] == "--component" && args[1] == "mail" { + return TemporaryFailure + } + return UsageFailure + } + code := run(opts) + if opts.component == aggregate.Mail && code != 0 { + return TemporaryFailure + } + return code +} diff --git a/Docker/privacy-diagnostic/internal/runner/options_test.go b/Docker/privacy-diagnostic/internal/runner/options_test.go new file mode 100644 index 000000000..502eea901 --- /dev/null +++ b/Docker/privacy-diagnostic/internal/runner/options_test.go @@ -0,0 +1,41 @@ +package runner + +import ( + "reflect" + "testing" + "time" + + "github.com/jacobyoby/docassemble/privacy-diagnostic/internal/aggregate" +) + +func TestArgumentsAreExactAndOwned(t *testing.T) { + args := []string{"--component", "uwsgi", "--", "/test/uwsgi", "--die-on-term"} + value, ok := parse(args) + if !ok || value.component != aggregate.UWsgi || value.sinkFD != 1 { + t.Fatal("valid profile rejected") + } + args[3] = "private" + if value.command[0] != "/test/uwsgi" { + t.Fatal("borrowed caller arguments") + } + for _, bad := range [][]string{nil, {"nginx"}, {"--component", "private", "--", "/bin/true"}, + {"--component", "NGINX", "--", "/bin/true"}, {"--component", "nginx", "--", "relative"}, + {"--component", "nginx", "wrong", "/bin/true"}, {"--component", "uwsgi", "--", "/bin/true"}, + {"--component", "nginx", "--", "/bin/true", "nul\x00"}} { + if got, ok := parse(bad); ok || !reflect.DeepEqual(got, options{}) || Run(bad) != UsageFailure { + t.Fatal("invalid arguments accepted") + } + } +} + +func TestApplicationProfilesRetainServiceShutdownWindows(t *testing.T) { + for _, item := range []struct { + name string + wait time.Duration + }{{"celery", 60 * time.Second}, {"celerysingle", 60 * time.Second}, {"websockets", 20 * time.Second}, {"mail", 20 * time.Second}, {"cron", 20 * time.Second}, {"initialize", 610 * time.Second}, {"maintenance", 30 * time.Second}} { + opts, ok := parse([]string{"--component", item.name, "--", "/synthetic/service"}) + if !ok || opts.stopWait != item.wait || opts.groupDelay() != item.wait || opts.killWait != time.Second || opts.sinkWait != 5*time.Second { + t.Fatalf("application lifecycle profile unavailable: %s", item.name) + } + } +} diff --git a/Docker/privacy-diagnostic/internal/runner/process_linux.go b/Docker/privacy-diagnostic/internal/runner/process_linux.go new file mode 100644 index 000000000..10f9742bb --- /dev/null +++ b/Docker/privacy-diagnostic/internal/runner/process_linux.go @@ -0,0 +1,83 @@ +package runner + +import ( + "errors" + "io" + "os" + "os/exec" + "syscall" + + "github.com/jacobyoby/docassemble/privacy-diagnostic/internal/aggregate" +) + +func graceful(component aggregate.Component) syscall.Signal { + if component == aggregate.Nginx { + return syscall.SIGQUIT + } + return syscall.SIGTERM +} + +func signalProcess(pid int, sig syscall.Signal, group bool) error { + if group { + pid = -pid + } + err := syscall.Kill(pid, sig) + if errors.Is(err, syscall.ESRCH) { + return nil + } + return err +} + +func groupAlive(pid int) (bool, error) { + err := syscall.Kill(-pid, 0) + if errors.Is(err, syscall.ESRCH) { + return false, nil + } + return err == nil, err +} + +type outputEvent struct { + stream aggregate.Stream + data [4096]byte + n int + end bool + failed bool +} + +// Each reader and the two-slot channel have fixed storage. An acknowledged +// event is copied; buffers are cleared before reuse and on cancellation. +func readOutput(file *os.File, stream aggregate.Stream, events chan<- outputEvent, cancel <-chan struct{}, done chan<- struct{}) { + defer func() { done <- struct{}{} }() + event := outputEvent{stream: stream} + defer clear(event.data[:]) + for { + n, err := file.Read(event.data[:]) + event.n, event.end = n, err != nil || n == 0 + event.failed = err != nil && !errors.Is(err, io.EOF) && !errors.Is(err, os.ErrClosed) + select { + case events <- event: + case <-cancel: + return + } + clear(event.data[:]) + if event.end { + return + } + } +} + +func childExit(err error) int { + if err == nil { + return 0 + } + var failure *exec.ExitError + if errors.As(err, &failure) { + if status, ok := failure.Sys().(syscall.WaitStatus); ok { + if status.Signaled() { + return 128 + int(status.Signal()) + } + return status.ExitStatus() + } + } + return RunnerFailure +} diff --git a/Docker/privacy-diagnostic/internal/runner/run_linux.go b/Docker/privacy-diagnostic/internal/runner/run_linux.go new file mode 100644 index 000000000..4f02f5945 --- /dev/null +++ b/Docker/privacy-diagnostic/internal/runner/run_linux.go @@ -0,0 +1,274 @@ +package runner + +import ( + "os" + "os/exec" + "os/signal" + "runtime" + "syscall" + "time" + + "github.com/jacobyoby/docassemble/privacy-diagnostic/internal/aggregate" + "github.com/jacobyoby/docassemble/privacy-diagnostic/internal/nativeguard" +) + +type execution struct { + opts options + cmd *exec.Cmd + counts *aggregate.Aggregator + output *sink + readers [2]*os.File + events chan outputEvent + cancelReaders chan struct{} + readersDone chan struct{} + wait <-chan int + writes chan error + exited bool + exitCode int + ended [2]bool + failure int + dirty bool + writing bool + writeCanceled bool + nextWrite time.Time + stopping time.Time + groupStopped bool + killed bool + forced bool +} + +func run(opts options) (result int) { + if opts.interval <= 0 || opts.sinkWait <= 0 || opts.stopWait <= 0 || opts.killWait <= 0 { + return UsageFailure + } + counts, err := aggregate.New(opts.component) + if err != nil || len(opts.command) == 0 { + return UsageFailure + } + if !nativeguard.OrdinaryExecutable(opts.command[0]) { + return RunnerFailure + } + // PDEATHSIG is tied to the creating thread. Keep that thread alive and + // locked until the child has exited or bounded teardown has failed. + runtime.LockOSThread() + defer runtime.UnlockOSThread() + if syscall.Setrlimit(syscall.RLIMIT_CORE, &syscall.Rlimit{}) != nil || nativeguard.SealDescriptors() != nil { + return RunnerFailure + } + output, err := openSink(opts.sinkFD) + if err != nil { + return SinkFailure + } + defer func() { + if output.close() != nil { + result = SinkFailure + } + }() + x := execution{opts: opts, counts: counts, output: output, dirty: true, + events: make(chan outputEvent, 2), cancelReaders: make(chan struct{}), + readersDone: make(chan struct{}, 2), writes: make(chan error, 1)} + var writers [2]*os.File + defer func() { + for _, file := range append(x.readers[:], writers[:]...) { + if file != nil { + file.Close() // Already-returning failure, or closed by lifecycle cleanup. + } + } + }() + for i := range x.readers { + x.readers[i], writers[i], err = os.Pipe() + if err != nil { + return RunnerFailure + } + } + stops, hup, reopen := make(chan os.Signal, 2), make(chan os.Signal, 1), make(chan os.Signal, 1) + signal.Notify(stops, syscall.SIGINT, syscall.SIGTERM) + signal.Notify(hup, syscall.SIGHUP) + signal.Notify(reopen, syscall.SIGUSR1) + defer signal.Stop(stops) + defer signal.Stop(hup) + defer signal.Stop(reopen) + x.cmd = exec.Command(opts.command[0], opts.command[1:]...) + if opts.component == aggregate.Mail { + // Exim already supplies the message on stdin. Preserve that stream + // without a second retained message file or an intermediate reader. + x.cmd.Stdin = os.Stdin + } + x.cmd.Stdout, x.cmd.Stderr = writers[0], writers[1] + x.cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true, Pdeathsig: graceful(opts.component)} + if x.cmd.Start() != nil { + return RunnerFailure + } + for i, writer := range writers { + if writer.Close() != nil { + x.fail(RunnerFailure) + } + writers[i] = nil + } + wait := make(chan int, 1) + x.wait = wait + go func() { wait <- childExit(x.cmd.Wait()) }() + for i, reader := range x.readers { + go readOutput(reader, aggregate.Stream(i+1), x.events, x.cancelReaders, x.readersDone) + } + result = x.loop(stops, hup, reopen) + close(x.cancelReaders) + for i, reader := range x.readers { + if reader.Close() != nil && result == 0 { + result = RunnerFailure + } + x.readers[i] = nil + } + for range 2 { + <-x.readersDone + } + for len(x.events) > 0 { + event := <-x.events + clear(event.data[:]) + } + for _, stream := range []aggregate.Stream{aggregate.Stdout, aggregate.Stderr} { + if counts.Finish(stream) != nil && result == 0 { + result = RunnerFailure + } + } + return result +} + +func (x *execution) fail(code int) { + if x.failure == 0 { + x.failure = code + } +} + +func (x *execution) send(sig syscall.Signal, group bool) { + if signalProcess(x.cmd.Process.Pid, sig, group) != nil { + x.fail(RunnerFailure) + } +} + +func (x *execution) stop(now time.Time) { + if x.stopping.IsZero() { + x.stopping = now + x.send(graceful(x.opts.component), x.exited) + } +} + +func (x *execution) advanceStop(now time.Time, alive bool) { + if x.stopping.IsZero() { + return + } + elapsed := now.Sub(x.stopping) + if !x.groupStopped && (x.exited || elapsed >= x.opts.groupDelay()) { + x.send(graceful(x.opts.component), true) + x.groupStopped = true + } + if !x.killed && elapsed >= x.opts.stopWait { + x.send(syscall.SIGKILL, true) + x.killed = true + } + if elapsed >= x.opts.stopWait+x.opts.killWait { + if !x.exited || alive || !x.ended[0] || !x.ended[1] { + x.fail(RunnerFailure) + } + x.forced = true + } +} + +func (x *execution) consume(event outputEvent) { + defer clear(event.data[:]) + if event.failed { + x.fail(RunnerFailure) + } + if event.n > 0 && x.counts.Feed(event.stream, event.data[:event.n]) != nil { + x.fail(RunnerFailure) + } + if event.end { + x.ended[event.stream-1] = true + if x.counts.Finish(event.stream) != nil { + x.fail(RunnerFailure) + } + } + x.dirty = true +} + +func (x *execution) write(now time.Time) { + value, err := x.counts.Snapshot() + if err != nil { + x.fail(RunnerFailure) + return + } + x.writing, x.dirty = true, false + x.nextWrite = now.Add(x.opts.interval) + go func() { x.writes <- x.output.write(value, x.opts.sinkWait) }() +} + +func (x *execution) loop(stops, hup, reopen <-chan os.Signal) int { + tick := time.NewTicker(20 * time.Millisecond) + defer tick.Stop() + for { + now := time.Now() + select { + case <-stops: + if x.opts.component == aggregate.Mail { + // A clean child shutdown does not prove that an interrupted + // delivery completed. Preserve retry semantics for the caller. + x.fail(RunnerFailure) + } + x.stop(now) + default: + } + for _, item := range []struct { + channel <-chan os.Signal + signal syscall.Signal + }{{hup, syscall.SIGHUP}, {reopen, syscall.SIGUSR1}} { + select { + case <-item.channel: + if !x.exited && x.stopping.IsZero() { + x.send(item.signal, false) + } + default: + } + } + alive, err := groupAlive(x.cmd.Process.Pid) + if err != nil { + x.fail(RunnerFailure) + } + if x.failure != 0 || x.exited && (alive || !x.ended[0] || !x.ended[1]) { + x.stop(now) + } + x.advanceStop(now, alive) + final := x.forced || x.exited && !alive && x.ended[0] && x.ended[1] + // Finite mail/cron commands emit at most one final record; services + // retain periodic progress counters. Mail's delivery semantics are separate. + periodic := x.opts.component != aggregate.Mail && x.opts.component != aggregate.Cron && x.opts.component != aggregate.Maintenance + report := final || periodic && !now.Before(x.nextWrite) + if x.failure == 0 && !x.writing && x.dirty && report { + x.write(now) + } + if x.failure != 0 && x.writing && !x.writeCanceled { + if x.output.file.SetWriteDeadline(now) != nil { + x.fail(SinkFailure) + } + x.writeCanceled = true + } + if final && !x.writing && (x.failure != 0 || !x.dirty) { + if x.failure != 0 { + return x.failure + } + return x.exitCode + } + select { + case event := <-x.events: + x.consume(event) + clear(event.data[:]) + case code := <-x.wait: + x.exited, x.exitCode, x.wait = true, code, nil + case err := <-x.writes: + x.writing = false + if err != nil { + x.fail(SinkFailure) + } + case <-tick.C: + } + } +} diff --git a/Docker/privacy-diagnostic/internal/runner/run_other.go b/Docker/privacy-diagnostic/internal/runner/run_other.go new file mode 100644 index 000000000..a4fa29ae0 --- /dev/null +++ b/Docker/privacy-diagnostic/internal/runner/run_other.go @@ -0,0 +1,5 @@ +//go:build !linux + +package runner + +func run(options) int { return RunnerFailure } diff --git a/Docker/privacy-diagnostic/internal/runner/runner_linux_test.go b/Docker/privacy-diagnostic/internal/runner/runner_linux_test.go new file mode 100644 index 000000000..34c447c1f --- /dev/null +++ b/Docker/privacy-diagnostic/internal/runner/runner_linux_test.go @@ -0,0 +1,252 @@ +package runner + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "syscall" + "testing" + "time" + + "github.com/jacobyoby/docassemble/privacy-diagnostic/internal/aggregate" +) + +const testMarker = "SYNTHETIC_PRIVATE_NATIVE" + +func fixtureCommand(t *testing.T, role, mode string) *exec.Cmd { + t.Helper() + path, err := os.Executable() + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 8*time.Second) + t.Cleanup(cancel) + cmd := exec.CommandContext(ctx, path, "-test.run=^TestRunnerFixture$", "--", role, mode) + cmd.Env = append(os.Environ(), "PRIVACY_RUNNER_FIXTURE=1") + cmd.WaitDelay = time.Second + return cmd +} + +func records(t *testing.T, output []byte) []aggregate.Snapshot { + t.Helper() + if bytes.Contains(output, []byte(testMarker)) { + t.Fatal("raw output leaked") + } + var all []aggregate.Snapshot + decoder := json.NewDecoder(bytes.NewReader(output)) + for { + var fields map[string]json.RawMessage + if err := decoder.Decode(&fields); err != nil { + if err == io.EOF { + break + } + t.Fatal("non-JSON output", err) + } + if len(fields) != 14 { + t.Fatalf("unexpected field count: %d", len(fields)) + } + data, err := json.Marshal(fields) + if err != nil { + t.Fatal(err) + } + var value aggregate.Snapshot + if err := json.Unmarshal(data, &value); err != nil { + t.Fatal(err) + } + if _, err := json.Marshal(value); err != nil { + t.Fatal(err) + } + all = append(all, value) + } + if len(all) == 0 { + t.Fatal("missing final counters") + } + return all +} + +func TestCapturedStreamsFinalCountersAndExit(t *testing.T) { + cmd := fixtureCommand(t, "runner", "output") + var stderr bytes.Buffer + cmd.Stderr = &stderr + out, err := cmd.Output() + if childExit(err) != 17 || stderr.Len() != 0 { + t.Fatalf("exit/output changed: %v stderr=%q", err, stderr.String()) + } + all := records(t, out) + last := all[len(all)-1] + if last.Status2xx != 1 || last.Status5xx != 1 || last.LatencyFast != 1 || last.LatencyTimeout != 1 || last.Unclassified != 2 || last.Rejected != 1 || last.Dropped != 1 { + t.Fatalf("wrong final counts: %#v", last) + } +} + +func TestInvalidSinkAndMissingExecutableDoNotStartChild(t *testing.T) { + for _, mode := range []string{"file-sink", "missing", "setid", "directory"} { + t.Run(mode, func(t *testing.T) { + cmd := fixtureCommand(t, "runner", mode) + output, err := cmd.CombinedOutput() + want := RunnerFailure + if mode == "file-sink" { + want = SinkFailure + } + if childExit(err) != want || len(output) != 0 { + t.Fatalf("invalid setup: exit=%d output=%q", childExit(err), output) + } + }) + } +} + +func TestBrokenAndFullSinksFailWithinDeadline(t *testing.T) { + for _, mode := range []string{"broken", "full"} { + t.Run(mode, func(t *testing.T) { + cmd := fixtureCommand(t, "runner", mode) + start := time.Now() + output, err := cmd.CombinedOutput() + if childExit(err) != SinkFailure || len(output) != 0 || time.Since(start) > 3*time.Second { + t.Fatalf("sink failure not bounded: exit=%d duration=%s output=%q", childExit(err), time.Since(start), output) + } + }) + } +} + +func TestSignalExitCodeIsPreserved(t *testing.T) { + cmd := fixtureCommand(t, "runner", "signal-exit") + out, err := cmd.Output() + if childExit(err) != 128+int(syscall.SIGKILL) { + t.Fatal("native signal exit changed", err) + } + records(t, out) +} + +func TestPipeFlagsRestored(t *testing.T) { + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + defer r.Close() + defer w.Close() + fd := w.Fd() + before, _, errno := syscall.Syscall(syscall.SYS_FCNTL, fd, syscall.F_GETFL, 0) + if errno != 0 { + t.Fatal(errno) + } + s, err := openSink(int(fd)) + if err != nil { + t.Fatal(err) + } + if err := s.write(aggregate.Snapshot{Schema: 1, Component: "nginx"}, time.Second); err != nil { + t.Fatal(err) + } + if err := s.close(); err != nil { + t.Fatal(err) + } + // Do not call File.Fd here: it may itself restore blocking mode and mask a bug. + after, _, errno := syscall.Syscall(syscall.SYS_FCNTL, fd, syscall.F_GETFL, 0) + if errno != 0 || before != after { + t.Fatalf("shared pipe flags changed: %d -> %d (%v)", before, after, errno) + } +} + +// This fixture is a real subprocess, so core limits and signal handlers never +// alter the parent test process. No production test bypass is exposed by Run. +func TestRunnerFixture(t *testing.T) { + if os.Getenv("PRIVACY_RUNNER_FIXTURE") != "1" { + return + } + args := os.Args + role, mode := args[len(args)-2], args[len(args)-1] + if role == "native" { + nativeFixture(mode) + os.Exit(99) + } + self, err := os.Executable() + if err != nil { + os.Exit(98) + } + opts, ok := parse([]string{"--component", "nginx", "--", self, "-test.run=^TestRunnerFixture$", "--", "native", mode}) + if !ok { + os.Exit(98) + } + opts.interval, opts.sinkWait, opts.stopWait, opts.killWait = 30*time.Millisecond, 150*time.Millisecond, 200*time.Millisecond, 300*time.Millisecond + if strings.HasSuffix(mode, "uwsgi") { + opts.component = aggregate.UWsgi + opts.command = append(opts.command[:len(opts.command)-2], "--die-on-term", "native", mode) + } + var sinkWriter *os.File + if mode == "file-sink" { + file, err := os.CreateTemp("", "counter-file") + if err != nil { + os.Exit(98) + } + opts.sinkFD = int(file.Fd()) + } else if mode == "missing" { + opts.command[0] = "/missing/" + testMarker + } else if mode == "directory" { + opts.command[0] = "/tmp" + } else if mode == "setid" { + file, err := os.CreateTemp("", "counter-setid") + if err != nil || file.Chmod(0755|os.ModeSetuid) != nil { + os.Exit(98) + } + opts.command[0] = file.Name() + } else if mode == "broken" || mode == "full" { + r, w, err := os.Pipe() + if err != nil { + os.Exit(98) + } + sinkWriter = w + opts.sinkFD = int(w.Fd()) + if mode == "broken" { + r.Close() + } else { + if syscall.SetNonblock(opts.sinkFD, true) != nil { + os.Exit(98) + } + for { + _, err := syscall.Write(opts.sinkFD, bytes.Repeat([]byte("x"), 4096)) + if err == syscall.EAGAIN { + break + } + if err != nil { + os.Exit(98) + } + } + } + } + code := run(opts) + if sinkWriter != nil { + sinkWriter.Close() + } + os.Exit(code) +} + +func nativeFixture(mode string) { + if lifecycleFixture(mode) { + return + } + switch mode { + case "output": + fmt.Fprint(os.Stdout, "PRIVACY_REQUEST status=2") + fmt.Fprint(os.Stderr, "PRIVACY_REQUEST status=503 seconds=30.000\n"+testMarker+"\n") + fmt.Fprint(os.Stdout, "00 msecs=99\n"+testMarker+"\x00\n") + fmt.Fprint(os.Stderr, strings.Repeat("x", 8192)+"\n") + fmt.Fprint(os.Stdout, testMarker) + os.Exit(17) + case "signal-exit": + syscall.Kill(os.Getpid(), syscall.SIGKILL) + case "broken", "full": + for { + fmt.Fprintln(os.Stdout, testMarker) + time.Sleep(time.Millisecond) + } + default: + // A forbidden launch would create a detectable non-counter marker. + fmt.Fprint(os.Stdout, testMarker+filepath.Base(os.Args[0])+strconv.Itoa(os.Getpid())) + } +} diff --git a/Docker/privacy-diagnostic/internal/runner/sink_linux.go b/Docker/privacy-diagnostic/internal/runner/sink_linux.go new file mode 100644 index 000000000..34311f2a6 --- /dev/null +++ b/Docker/privacy-diagnostic/internal/runner/sink_linux.go @@ -0,0 +1,76 @@ +package runner + +import ( + "encoding/json" + "errors" + "os" + "syscall" + "time" + + "github.com/jacobyoby/docassemble/privacy-diagnostic/internal/aggregate" +) + +var errSink = errors.New("counter sink unavailable") + +type sink struct { + file *os.File + flags uintptr +} + +// openSink rejects regular files and duplicates only a pipe/socket. Like the +// startup diagnostic, it restores shared descriptor flags when finished. +func openSink(source int) (*sink, error) { + fd, err := syscall.Dup(source) + if err != nil { + return nil, errSink + } + syscall.CloseOnExec(fd) + var st syscall.Stat_t + if syscall.Fstat(fd, &st) != nil || (st.Mode&syscall.S_IFMT != syscall.S_IFIFO && st.Mode&syscall.S_IFMT != syscall.S_IFSOCK) { + syscall.Close(fd) + return nil, errSink + } + flags, _, errno := syscall.Syscall(syscall.SYS_FCNTL, uintptr(fd), syscall.F_GETFL, 0) + if errno != 0 { + syscall.Close(fd) + return nil, errSink + } + _, _, errno = syscall.Syscall(syscall.SYS_FCNTL, uintptr(fd), syscall.F_SETFL, flags|syscall.O_NONBLOCK) + if errno != 0 { + syscall.Close(fd) + return nil, errSink + } + s := &sink{file: os.NewFile(uintptr(fd), "counter-sink"), flags: flags} + if s.file == nil { + // NewFile accepts this nonnegative descriptor; retain a fail-closed guard. + syscall.Syscall(syscall.SYS_FCNTL, uintptr(fd), syscall.F_SETFL, flags) + syscall.Close(fd) + return nil, errSink + } + return s, nil +} + +func (s *sink) write(value aggregate.Snapshot, timeout time.Duration) error { + data, err := json.Marshal(value) + if err != nil || len(data) >= 8191 { + return errSink + } + data = append(data, '\n') + if s.file.SetWriteDeadline(time.Now().Add(timeout)) != nil { + return errSink + } + n, err := s.file.Write(data) + if err != nil || n != len(data) { + return errSink + } + return nil +} + +func (s *sink) close() error { + _, _, errno := syscall.Syscall(syscall.SYS_FCNTL, s.file.Fd(), syscall.F_SETFL, s.flags) + err := s.file.Close() + if errno != 0 || err != nil { + return errSink + } + return nil +} diff --git a/Docker/privacy-diagnostic/main.go b/Docker/privacy-diagnostic/main.go new file mode 100644 index 000000000..b6d849a73 --- /dev/null +++ b/Docker/privacy-diagnostic/main.go @@ -0,0 +1,191 @@ +// privacy-diagnostic emits one fixed failure record to a bounded pipe. +// It accepts no message text, configuration, paths, or exception details. +package main + +import ( + "encoding/json" + "os" + "syscall" + "time" +) + +const ( + usageFailure = 64 + startupFailure = 70 + sinkFailure = 74 + writeTimeout = time.Second +) + +type component uint8 + +const ( + launcher component = iota + nginx + uwsgi + uwsgilog + celery + celerySingle + websockets + mail + cron + initialize + maintenance +) + +type phase uint8 + +const ( + invocation phase = iota + activation + config + configEval + preflight + launch + execution +) + +type failure struct { + component component + phase phase +} + +type wireRecord struct { + Schema int `json:"schema"` + Component string `json:"component"` + Event string `json:"event"` + Phase string `json:"phase"` +} + +func parse(args []string) (failure, int) { + invalid := failure{launcher, invocation} + if len(args) != 2 { + return invalid, usageFailure + } + var f failure + switch args[0] { + case "nginx": + f.component = nginx + case "uwsgi": + f.component = uwsgi + case "uwsgilog": + f.component = uwsgilog + case "celery": + f.component = celery + case "celerysingle": + f.component = celerySingle + case "websockets": + f.component = websockets + case "mail": + f.component = mail + case "cron": + f.component = cron + case "initialize": + f.component = initialize + case "maintenance": + f.component = maintenance + default: + return invalid, usageFailure + } + switch args[1] { + case "activation": + f.phase = activation + case "config": + f.phase = config + case "config_eval": + f.phase = configEval + case "preflight": + f.phase = preflight + case "launch": + f.phase = launch + case "execution": + if f.component != cron { + return invalid, usageFailure + } + f.phase = execution + default: + return invalid, usageFailure + } + return f, startupFailure +} + +func encode(f failure) ([]byte, error) { + // All strings come from these constants, never from CLI values. + components := [...]string{"launcher", "nginx", "uwsgi", "uwsgilog", "celery", "celerysingle", "websockets", "mail", "cron", "initialize", "maintenance"} + phases := [...]string{"invocation", "activation", "config", "config_eval", "preflight", "launch", "execution"} + if int(f.component) >= len(components) || int(f.phase) >= len(phases) || f.phase == execution && f.component != cron { + f = failure{launcher, invocation} + } + event := "startup_failed" + if f.phase == execution { + event = "command_failed" + } + data, err := json.Marshal(wireRecord{1, components[f.component], event, phases[f.phase]}) + return append(data, '\n'), err +} + +func report(args []string, sinkFD int, timeout time.Duration) (code int) { + f, code := parse(args) + data, err := encode(f) + if err != nil || len(data) > 256 || timeout <= 0 { + return sinkFailure + } + + fd, err := syscall.Dup(sinkFD) + if err != nil { + return sinkFailure + } + var output *os.File + var originalFlags uintptr + restoreFlags := false + defer func() { + if restoreFlags { + _, _, errno := syscall.Syscall(syscall.SYS_FCNTL, uintptr(fd), syscall.F_SETFL, originalFlags) + if errno != 0 { + code = sinkFailure + } + } + var closeErr error + if output == nil { + closeErr = syscall.Close(fd) + } else { + closeErr = output.Close() + } + if closeErr != nil { + code = sinkFailure + } + }() + + var stat syscall.Stat_t + if syscall.Fstat(fd, &stat) != nil { + return sinkFailure + } + kind := stat.Mode & syscall.S_IFMT + if kind != syscall.S_IFIFO && kind != syscall.S_IFSOCK { + return sinkFailure + } + flags, _, errno := syscall.Syscall(syscall.SYS_FCNTL, uintptr(fd), syscall.F_GETFL, 0) + if errno != 0 { + return sinkFailure + } + originalFlags = flags + _, _, errno = syscall.Syscall(syscall.SYS_FCNTL, uintptr(fd), syscall.F_SETFL, flags|syscall.O_NONBLOCK) + if errno != 0 { + return sinkFailure + } + restoreFlags = true + // NewFile sees O_NONBLOCK and registers the descriptor with Go's poller. + // The launcher is waiting for this process; restore its shared flags on exit. + output = os.NewFile(uintptr(fd), "privacy-diagnostic") + if output == nil || output.SetWriteDeadline(time.Now().Add(timeout)) != nil { + return sinkFailure + } + written, err := output.Write(data) + if err != nil || written != len(data) { + return sinkFailure + } + return code +} + +func main() { + os.Exit(report(os.Args[1:], 1, writeTimeout)) +} diff --git a/Docker/privacy-diagnostic/main_test.go b/Docker/privacy-diagnostic/main_test.go new file mode 100644 index 000000000..70bfd08a8 --- /dev/null +++ b/Docker/privacy-diagnostic/main_test.go @@ -0,0 +1,236 @@ +package main + +import ( + "bytes" + "encoding/json" + "io" + "os" + "strings" + "syscall" + "testing" + "time" +) + +func pipe(t *testing.T) (*os.File, *os.File) { + t.Helper() + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { r.Close(); w.Close() }) + return r, w +} + +func flags(t *testing.T, fd int) uintptr { + t.Helper() + value, _, errno := syscall.Syscall(syscall.SYS_FCNTL, uintptr(fd), syscall.F_GETFL, 0) + if errno != 0 { + t.Fatal(errno) + } + return value +} + +func capture(t *testing.T, args []string) (int, []byte) { + t.Helper() + r, w := pipe(t) + fd := int(w.Fd()) + // Darwin records that a pipe was written in F_GETFL even for plain write(2). + // Prime and drain it before taking the full flag-preservation baseline. + if _, err := syscall.Write(fd, []byte("x")); err != nil { + t.Fatal(err) + } + if _, err := io.ReadFull(r, make([]byte, 1)); err != nil { + t.Fatal(err) + } + before := flags(t, fd) + code := report(args, fd, 100*time.Millisecond) + if after := flags(t, fd); after != before { + t.Fatalf("descriptor flags changed: %d -> %d", before, after) + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + data, err := io.ReadAll(r) + if err != nil { + t.Fatal(err) + } + return code, data +} + +func TestEveryAllowedComponentAndPhase(t *testing.T) { + for _, comp := range []string{"nginx", "uwsgi", "uwsgilog", "celery", "celerysingle", "websockets", "mail", "cron", "initialize", "maintenance"} { + for _, stage := range []string{"activation", "config", "config_eval", "preflight", "launch"} { + t.Run(comp+"/"+stage, func(t *testing.T) { + code, data := capture(t, []string{comp, stage}) + if code != startupFailure { + t.Fatalf("exit %d", code) + } + if len(data) >= 256 || bytes.Count(data, []byte("\n")) != 1 { + t.Fatalf("invalid record framing: %q", data) + } + var record map[string]any + if err := json.Unmarshal(data, &record); err != nil { + t.Fatal(err) + } + if len(record) != 4 || record["schema"] != float64(1) || record["component"] != comp || record["event"] != "startup_failed" || record["phase"] != stage { + t.Fatalf("wrong record: %#v", record) + } + }) + } + } +} + +func TestInvalidArgumentsCannotEnterOutput(t *testing.T) { + marker := "SYNTHETIC_PRIVATE\n\"config\":true" + cases := [][]string{nil, {"uwsgi"}, {"uwsgi", "config", marker}, {marker, "config"}, {"uwsgi", marker}, {"uwsgi", strings.Repeat(marker, 10000)}} + for _, args := range cases { + code, data := capture(t, args) + if code != usageFailure { + t.Fatalf("exit %d", code) + } + want := "{\"schema\":1,\"component\":\"launcher\",\"event\":\"startup_failed\",\"phase\":\"invocation\"}\n" + if string(data) != want { + t.Fatalf("nonconstant invalid-input record: %q", data) + } + } +} + +func TestCronExecutionFailureUsesAGenericFixedEvent(t *testing.T) { + code, data := capture(t, []string{"cron", "execution"}) + want := "{\"schema\":1,\"component\":\"cron\",\"event\":\"command_failed\",\"phase\":\"execution\"}\n" + if code != startupFailure || string(data) != want { + t.Fatalf("cron execution failure changed: %d %q", code, data) + } + code, _ = capture(t, []string{"mail", "execution"}) + if code != usageFailure { + t.Fatal("cron-only execution phase escaped its component") + } +} + +func TestInvalidEnumBecomesFixedInvocationRecord(t *testing.T) { + data, err := encode(failure{component(255), phase(255)}) + if err != nil || !bytes.Contains(data, []byte(`"phase":"invocation"`)) { + t.Fatalf("invalid enum: %q %v", data, err) + } +} + +func TestClosedAndBrokenPipeReturnSinkFailure(t *testing.T) { + if got := report([]string{"uwsgi", "config"}, -1, time.Second); got != sinkFailure { + t.Fatalf("invalid descriptor: %d", got) + } + r, w := pipe(t) + r.Close() + if got := report([]string{"uwsgi", "config"}, int(w.Fd()), time.Second); got != sinkFailure { + t.Fatalf("broken pipe: %d", got) + } +} + +func TestRegularFileRejectedWithoutWriting(t *testing.T) { + f, err := os.CreateTemp(t.TempDir(), "sink") + if err != nil { + t.Fatal(err) + } + defer f.Close() + if got := report([]string{"uwsgi", "config"}, int(f.Fd()), time.Second); got != sinkFailure { + t.Fatalf("regular file: %d", got) + } + stat, err := f.Stat() + if err != nil || stat.Size() != 0 { + t.Fatalf("unexpected file write: %v", err) + } +} + +func TestUnixSocketCanDeliverFixedRecord(t *testing.T) { + fds, err := syscall.Socketpair(syscall.AF_UNIX, syscall.SOCK_STREAM, 0) + if err != nil { + t.Fatal(err) + } + defer syscall.Close(fds[0]) + defer syscall.Close(fds[1]) + if code := report([]string{"nginx", "preflight"}, fds[0], time.Second); code != startupFailure { + t.Fatalf("socket exit %d", code) + } + buffer := make([]byte, 256) + n, err := syscall.Read(fds[1], buffer) + if err != nil || !json.Valid(bytes.TrimSpace(buffer[:n])) || !bytes.Contains(buffer[:n], []byte(`"phase":"preflight"`)) { + t.Fatalf("socket record %q: %v", buffer[:n], err) + } +} + +func TestFullPipeDeadlineAndFlagRestoration(t *testing.T) { + _, w := pipe(t) + fd := int(w.Fd()) + original := flags(t, fd) + if err := syscall.SetNonblock(fd, true); err != nil { + t.Fatal(err) + } + chunk := bytes.Repeat([]byte("x"), 4096) + filled := false + for total := 0; total < 16*1024*1024; { + n, err := syscall.Write(fd, chunk) + if err == syscall.EAGAIN { + filled = true + break + } + if err != nil { + t.Fatal(err) + } + total += n + } + if !filled { + t.Fatal("did not fill pipe") + } + // Fill any remainder smaller than the atomic chunk before testing timeout. + for { + _, err := syscall.Write(fd, []byte("x")) + if err == syscall.EAGAIN { + break + } + if err != nil { + t.Fatal(err) + } + } + _, _, errno := syscall.Syscall(syscall.SYS_FCNTL, uintptr(fd), syscall.F_SETFL, original) + if errno != 0 { + t.Fatal(errno) + } + // Filling the pipe itself can change kernel-maintained status bits. + original = flags(t, fd) + start := time.Now() + got := report([]string{"nginx", "preflight"}, fd, 40*time.Millisecond) + elapsed := time.Since(start) + if got != sinkFailure || elapsed < 20*time.Millisecond || elapsed > 2*time.Second { + t.Fatalf("deadline: code=%d elapsed=%s", got, elapsed) + } + if flags(t, fd) != original { + t.Fatal("original flags not restored") + } +} + +func TestNonpositiveTimeoutCannotWrite(t *testing.T) { + _, w := pipe(t) + for _, timeout := range []time.Duration{0, -time.Second} { + if code := report([]string{"uwsgi", "config"}, int(w.Fd()), timeout); code != sinkFailure { + t.Fatalf("exit %d", code) + } + } +} + +func FuzzParseHasFixedBoundedOutput(f *testing.F) { + f.Add("uwsgi", "activation") + f.Add("cron", "execution") + f.Add("private\nvalue", "private\nphase") + f.Fuzz(func(t *testing.T, comp, stage string) { + record, code := parse([]string{comp, stage}) + data, err := encode(record) + if err != nil || len(data) > 256 || !json.Valid(bytes.TrimSpace(data)) { + t.Fatal("invalid framing") + } + if code != startupFailure && code != usageFailure { + t.Fatalf("unexpected code %d", code) + } + if code == usageFailure && !bytes.Contains(data, []byte(`"phase":"invocation"`)) { + t.Fatal("invalid input was not replaced") + } + }) +} diff --git a/Docker/privacy/nginx-lifecycle.conf b/Docker/privacy/nginx-lifecycle.conf new file mode 100644 index 000000000..95d815e73 --- /dev/null +++ b/Docker/privacy/nginx-lifecycle.conf @@ -0,0 +1,5 @@ +# REQUIRED companion to the Go privacy-process runner's Linux lifecycle checks. +# Include this file in nginx's MAIN context, outside events/http/server/location. +# File presence does not prove inclusion; verify the effective config and held-request fixture. +# Native graceful shutdown closes remaining held connections after this deadline. +worker_shutdown_timeout 2s; diff --git a/Docker/privacy/nginx-realip.patch b/Docker/privacy/nginx-realip.patch new file mode 100644 index 000000000..469f1bde4 --- /dev/null +++ b/Docker/privacy/nginx-realip.patch @@ -0,0 +1,6 @@ +--- a/Docker/config/nginx-realip ++++ b/Docker/config/nginx-realip +@@ -18,3 +18,2 @@ + # BOTH FILES MUST SHIP TOGETHER -- this line without that file fails nginx -t. +- access_log /var/log/nginx/access.log privacy; + diff --git a/Docker/process-email.sh b/Docker/process-email.sh index 2734d6a50..d30bda536 100755 --- a/Docker/process-email.sh +++ b/Docker/process-email.sh @@ -1,13 +1,25 @@ #!/bin/bash +# The message stays on stdin; retain only fixed diagnostics and Go counters. +exec 3>&1 +exec >/dev/null 2>&1 export DA_ROOT="${DA_ROOT:-/usr/share/docassemble}" export DA_DEFAULT_LOCAL="local3.14" -export DA_ACTIVATE="${DA_PYTHON:-${DA_ROOT}/${DA_DEFAULT_LOCAL}}/bin/activate" -source "${DA_ACTIVATE}" +startup_failure() { + if [ -x "${DA_ROOT}/webapp/privacy-diagnostic" ]; then + "${DA_ROOT}/webapp/privacy-diagnostic" mail "$1" >&3 3>&- + fi + # Exim must retry failed delivery, including absent/broken diagnostics. + exit 75 +} +[ -x "${DA_ROOT}/webapp/privacy-diagnostic" ] || exit 75 +DA_RUNTIME="${DA_PYTHON:-${DA_ROOT}/${DA_DEFAULT_LOCAL}}" +export DA_ACTIVATE="${DA_RUNTIME}/bin/activate" +source "${DA_ACTIVATE}" || startup_failure activation -emailfile="$(mktemp)" - -cat > "$emailfile" -python -m docassemble.webapp.process_email "$emailfile" -rm -f "$emailfile" +shopt -s execfail +exec "${DA_ROOT}/webapp/privacy-process" --component mail -- \ + "${DA_RUNTIME}/bin/python" -m docassemble.webapp.process_email /dev/stdin >&3 3>&- +exec 3>&1 >/dev/null +startup_failure launch diff --git a/Docker/restart-post-logrotate.sh b/Docker/restart-post-logrotate.sh index a478a74c9..56fe7a5c6 100755 --- a/Docker/restart-post-logrotate.sh +++ b/Docker/restart-post-logrotate.sh @@ -1,6 +1,20 @@ #! /bin/bash export DA_ROOT="${DA_ROOT:-/usr/share/docassemble}" +# Internal handoff only; normal invocations always enter native capture. +if [ "${1:-}" != "--privacy-captured" ]; then + exec 3>&1 + exec >/dev/null 2>&1 + [ -x "${DA_ROOT}/webapp/privacy-diagnostic" ] || exit 69 + shopt -s execfail + exec "${DA_ROOT}/webapp/privacy-process" --component maintenance -- \ + /bin/bash "${BASH_SOURCE[0]}" --privacy-captured "$@" >&3 3>&- + exec 3>&1 >/dev/null + "${DA_ROOT}/webapp/privacy-diagnostic" maintenance launch >&3 3>&- + exit $? +fi +shift + export DA_CONFIG_FILE="${DA_CONFIG:-${DA_ROOT}/config/config.yml}" export DA_DEFAULT_LOCAL="local3.14" diff --git a/Docker/run-celery-single.sh b/Docker/run-celery-single.sh index ac3a54912..50129737f 100755 --- a/Docker/run-celery-single.sh +++ b/Docker/run-celery-single.sh @@ -1,19 +1,35 @@ #!/bin/bash +# Keep bootstrap output out of retained logs; report checked failures safely. +exec 3>&1 +exec >/dev/null 2>&1 export DA_ROOT="${DA_ROOT:-/usr/share/docassemble}" export DA_DEFAULT_LOCAL="local3.14" -export DA_ACTIVATE="${DA_PYTHON:-${DA_ROOT}/${DA_DEFAULT_LOCAL}}/bin/activate" - -source "${DA_ACTIVATE}" +startup_failure() { + [ -x "${DA_ROOT}/webapp/privacy-diagnostic" ] || exit 69 + "${DA_ROOT}/webapp/privacy-diagnostic" celerysingle "$1" >&3 3>&- + exit $? +} +[ -x "${DA_ROOT}/webapp/privacy-diagnostic" ] || exit 69 +DA_RUNTIME="${DA_PYTHON:-${DA_ROOT}/${DA_DEFAULT_LOCAL}}" +export DA_ACTIVATE="${DA_RUNTIME}/bin/activate" +source "${DA_ACTIVATE}" || startup_failure activation export DA_CONFIG_FILE="${DA_CONFIG:-${DA_ROOT}/config/config.yml}" -source /dev/stdin < <(source "$DA_ACTIVATE" && python -m docassemble.base.read_config --limited "$DA_CONFIG_FILE") +DA_EXPORTS=$("${DA_RUNTIME}/bin/python" -m docassemble.base.read_config --limited "$DA_CONFIG_FILE") || startup_failure config +source /dev/stdin <<< "$DA_EXPORTS" || startup_failure config_eval +unset DA_EXPORTS set -- $LOCALE export LANG=$1 export HOME=/var/www -exec celery -A docassemble.webapp.worker worker --loglevel=INFO --concurrency=1 -Q single -n worker1@%h +shopt -s execfail +exec "${DA_ROOT}/webapp/privacy-process" --component celerysingle -- \ + "${DA_RUNTIME}/bin/celery" -A docassemble.webapp.worker worker --loglevel=INFO \ + --concurrency=1 -Q single -n worker1@%h >&3 3>&- +exec 3>&1 >/dev/null +startup_failure launch diff --git a/Docker/run-celery.sh b/Docker/run-celery.sh index 8bc29515c..58cbd8dc7 100755 --- a/Docker/run-celery.sh +++ b/Docker/run-celery.sh @@ -1,31 +1,54 @@ #!/bin/bash +# Keep bootstrap output out of retained logs; report checked failures safely. +exec 3>&1 +exec >/dev/null 2>&1 export DA_ROOT="${DA_ROOT:-/usr/share/docassemble}" export DA_DEFAULT_LOCAL="local3.14" -export DA_ACTIVATE="${DA_PYTHON:-${DA_ROOT}/${DA_DEFAULT_LOCAL}}/bin/activate" - -source "${DA_ACTIVATE}" +startup_failure() { + [ -x "${DA_ROOT}/webapp/privacy-diagnostic" ] || exit 69 + "${DA_ROOT}/webapp/privacy-diagnostic" celery "$1" >&3 3>&- + exit $? +} +[ -x "${DA_ROOT}/webapp/privacy-diagnostic" ] || exit 69 +DA_RUNTIME="${DA_PYTHON:-${DA_ROOT}/${DA_DEFAULT_LOCAL}}" +export DA_ACTIVATE="${DA_RUNTIME}/bin/activate" +source "${DA_ACTIVATE}" || startup_failure activation export DA_CONFIG_FILE="${DA_CONFIG:-${DA_ROOT}/config/config.yml}" -source /dev/stdin < <(source "$DA_ACTIVATE" && python -m docassemble.base.read_config --limited "$DA_CONFIG_FILE") +DA_EXPORTS=$("${DA_RUNTIME}/bin/python" -m docassemble.base.read_config --limited "$DA_CONFIG_FILE") || startup_failure config +source /dev/stdin <<< "$DA_EXPORTS" || startup_failure config_eval +unset DA_EXPORTS set -- $LOCALE export LANG=$1 export HOME=/var/www -NUMPROCS=$(nproc --all) +valid_workers() { + [[ "$1" =~ ^[1-9][0-9]{0,6}$ ]] && (( $1 <= 1048576 )) +} +NUMPROCS=$(nproc --all) || startup_failure config_eval +valid_workers "$NUMPROCS" && valid_workers "${DAMAXCELERYWORKERS:-}" || startup_failure config_eval +if [ -n "${DACELERYWORKERS:-}" ]; then + valid_workers "$DACELERYWORKERS" || startup_failure config_eval +fi if ((NUMPROCS > DAMAXCELERYWORKERS)); then NUMPROCS=${DAMAXCELERYWORKERS} fi NUMPROCS=${DACELERYWORKERS:-${NUMPROCS}} -NUMPROCS=$(expr $NUMPROCS - 1) +NUMPROCS=$((NUMPROCS - 1)) if ((NUMPROCS < 1)); then NUMPROCS=1 fi -exec celery -A docassemble.webapp.worker worker --loglevel=INFO --concurrency=$NUMPROCS -Q celery +shopt -s execfail +exec "${DA_ROOT}/webapp/privacy-process" --component celery -- \ + "${DA_RUNTIME}/bin/celery" -A docassemble.webapp.worker worker --loglevel=INFO \ + --concurrency="$NUMPROCS" -Q celery >&3 3>&- +exec 3>&1 >/dev/null +startup_failure launch diff --git a/Docker/run-cron.sh b/Docker/run-cron.sh index ffa734456..1abbf88dd 100755 --- a/Docker/run-cron.sh +++ b/Docker/run-cron.sh @@ -1,21 +1,66 @@ -#! /bin/bash +#!/bin/bash +# Cron keeps its existing output pipe; only fixed diagnostics/counters reach it. +exec 3>&1 +exec >/dev/null 2>&1 export DA_ROOT="${DA_ROOT:-/usr/share/docassemble}" export DA_DEFAULT_LOCAL="local3.14" -export DA_ACTIVATE="${DA_PYTHON:-${DA_ROOT}/${DA_DEFAULT_LOCAL}}/bin/activate" +startup_failure() { + [ -x "${DA_ROOT}/webapp/privacy-diagnostic" ] || exit 69 + "${DA_ROOT}/webapp/privacy-diagnostic" cron "$1" >&3 3>&- + exit $? +} +[ -x "${DA_ROOT}/webapp/privacy-diagnostic" ] || exit 69 -source "${DA_ACTIVATE}" +# Drop privileges before Go establishes its child/process-death boundary. +shopt -s execfail +if [ "$EUID" -eq 0 ]; then + DA_CRON_CHILD='' + DA_CRON_SIGNAL='' + forward_signal() { + DA_CRON_SIGNAL=$1 + DA_CRON_INTERRUPTED=1 + [ -z "$DA_CRON_CHILD" ] || kill -s "$1" "$DA_CRON_CHILD" + } + trap 'forward_signal TERM' TERM + trap 'forward_signal INT' INT + /usr/bin/setpriv --reuid=www-data --regid=www-data --init-groups \ + /bin/bash "${BASH_SOURCE[0]}" "${1:-cron_daily}" >&3 3>&- & + DA_CRON_CHILD=$! + [ -z "$DA_CRON_SIGNAL" ] || kill -s "$DA_CRON_SIGNAL" "$DA_CRON_CHILD" + while :; do + DA_CRON_INTERRUPTED=0 + wait "$DA_CRON_CHILD" + DA_CRON_STATUS=$? + [ "$DA_CRON_INTERRUPTED" -eq 0 ] && break + done + trap '' TERM INT + if [ "$DA_CRON_STATUS" -ne 0 ]; then + "${DA_ROOT}/webapp/privacy-diagnostic" cron execution >&3 3>&- + fi + exit "$DA_CRON_STATUS" +fi +[ "$EUID" -eq "$(/usr/bin/id -u www-data)" ] || startup_failure launch +export USER=www-data LOGNAME=www-data SHELL=/bin/bash HOME=/var/www +DA_RUNTIME="${DA_PYTHON:-${DA_ROOT}/${DA_DEFAULT_LOCAL}}" +export DA_ACTIVATE="${DA_RUNTIME}/bin/activate" +source "${DA_ACTIVATE}" || startup_failure activation -export HOME=/var/www export CRONTYPE=${1:-cron_daily} export DA_CONFIG_FILE="${DA_CONFIG:-${DA_ROOT}/config/config.yml}" -source /dev/stdin < <(su -c "source \"$DA_ACTIVATE\" && python -m docassemble.base.read_config \"$DA_CONFIG_FILE\"" www-data) +DA_EXPORTS=$("${DA_RUNTIME}/bin/python" -m docassemble.base.read_config "$DA_CONFIG_FILE") || startup_failure config +source /dev/stdin <<< "$DA_EXPORTS" || startup_failure config_eval +unset DA_EXPORTS set -- $LOCALE export LANG=$1 export IN_CRON=true -exec nice -n 19 su -c "source \"$DA_ACTIVATE\" && flask --app docassemble.webapp.server cron run $CRONTYPE" www-data +[ -x "${DA_ROOT}/webapp/privacy-process" ] || startup_failure launch +exec /usr/bin/nice -n 19 "${DA_ROOT}/webapp/privacy-process" --component cron -- \ + "${DA_RUNTIME}/bin/flask" --app docassemble.webapp.server cron run "$CRONTYPE" >&3 3>&- +exec 3>&1 >/dev/null +startup_failure launch diff --git a/Docker/run-nginx.sh b/Docker/run-nginx.sh index 37d7f29bb..e1777d4e3 100755 --- a/Docker/run-nginx.sh +++ b/Docker/run-nginx.sh @@ -1,13 +1,25 @@ #!/bin/bash +# Discard raw bootstrap output; report failures with bounded fixed records. +exec 3>&1 +exec >/dev/null 2>&1 +set -o pipefail export CONTAINERROLE=":${CONTAINERROLE:-all}:" export DEBIAN_FRONTEND=noninteractive export DA_ROOT="${DA_ROOT:-/usr/share/docassemble}" export DA_DEFAULT_LOCAL="local3.14" +startup_failure() { + [ -x "${DA_ROOT}/webapp/privacy-diagnostic" ] || exit 69 + "${DA_ROOT}/webapp/privacy-diagnostic" nginx "$1" >&3 3>&- + exit $? +} +[ -x "${DA_ROOT}/webapp/privacy-diagnostic" ] || exit 69 export DA_ACTIVATE="${DA_PYTHON:-${DA_ROOT}/${DA_DEFAULT_LOCAL}}/bin/activate" export DA_CONFIG_FILE="${DA_CONFIG:-${DA_ROOT}/config/config.yml}" -source /dev/stdin < <(su -c "source \"$DA_ACTIVATE\" && python -m docassemble.base.read_config \"$DA_CONFIG_FILE\"" www-data | grep -e '^export LOCALE=' -e '^export DAHOSTNAME=' -e '^export EC2=' -e '^export BEHINDHTTPSLOADBALANCER=' -e '^export USELETSENCRYPT=' -e '^export DALOCATIONREWRITE=' -e '^export WSGIROOT=' -e '^export POSTURLROOT=' -e '^export DAMAXCONTENTLENGTH=' -e '^export DASSLPROTOCOLS=' -e '^export DASSLCIPHERS=' -e '^export DAWEBSOCKETSIP=' -e '^export DAWEBSOCKETSPORT=' -e '^export PORT=' -e '^export USEHTTPS=' -e '^export DAREADONLYFILESYSTEM=') +DA_NGINX_EXPORTS=$(su -c "source \"$DA_ACTIVATE\" && python -m docassemble.base.read_config \"$DA_CONFIG_FILE\"" www-data | grep -e '^export LOCALE=' -e '^export DAHOSTNAME=' -e '^export EC2=' -e '^export BEHINDHTTPSLOADBALANCER=' -e '^export USELETSENCRYPT=' -e '^export DALOCATIONREWRITE=' -e '^export WSGIROOT=' -e '^export POSTURLROOT=' -e '^export DAMAXCONTENTLENGTH=' -e '^export DASSLPROTOCOLS=' -e '^export DASSLCIPHERS=' -e '^export DAWEBSOCKETSIP=' -e '^export DAWEBSOCKETSPORT=' -e '^export PORT=' -e '^export USEHTTPS=' -e '^export DAREADONLYFILESYSTEM=') || startup_failure config +source /dev/stdin <<< "$DA_NGINX_EXPORTS" || startup_failure config_eval +unset DA_NGINX_EXPORTS set -- $LOCALE export LANG=$1 @@ -92,7 +104,6 @@ if [[ $CONTAINERROLE =~ .*:(log):.* ]]; then if [ "${DAREADONLYFILESYSTEM:-false}" == "false" ]; then ln -sf /etc/nginx/sites-available/docassemblelog /etc/nginx/sites-enabled/docassemblelog fi - su -c "source \"$DA_ACTIVATE\" && uwsgi --ini \"${DA_ROOT}/config/docassemblelog.ini\"" www-data & else if [ "${DAREADONLYFILESYSTEM:-false}" == "false" ]; then rm -f /etc/nginx/sites-enabled/docassemblelog @@ -119,31 +130,12 @@ if [ "${DAREADONLYFILESYSTEM:-false}" == "false" ]; then fi fi -function stopfunc { - if [[ $CONTAINERROLE =~ .*:(log):.* ]] && [ -f /var/run/uwsgi/uwsgilog.pid ]; then - UWSGILOG_PID=$(&2 - kill -INT $UWSGILOG_PID - echo "Waiting for uwsgi log to stop" >&2 - wait $UWSGILOG_PID - echo "uwsgi log stopped" >&2 - fi - fi - if [ -f /var/run/nginx.pid ]; then - NGINX_PID=$(&2 - kill -QUIT $NGINX_PID - echo "Waiting for nginx to stop" >&2 - wait $NGINX_PID - echo "nginx stopped" >&2 - fi - fi - exit 0 -} - -trap stopfunc SIGINT SIGTERM - -/usr/sbin/nginx -g "daemon off;" & -wait %1 +# The log-role uWSGI process is separately owned by Supervisor (uwsgilog). +DA_RUNTIME="${DA_PYTHON:-${DA_ROOT}/${DA_DEFAULT_LOCAL}}" +"${DA_ROOT}/webapp/privacy-preflight" nginx || startup_failure preflight +shopt -s execfail +exec "${DA_ROOT}/webapp/privacy-process" \ + --component nginx -- /usr/sbin/nginx -e stderr -g 'daemon off;' >&3 3>&- +# Failed exec keeps its redirections: recover the saved sink from stdout. +exec 3>&1 >/dev/null +startup_failure launch diff --git a/Docker/run-uwsgi.sh b/Docker/run-uwsgi.sh index c17ae5b84..d385f42cb 100755 --- a/Docker/run-uwsgi.sh +++ b/Docker/run-uwsgi.sh @@ -1,36 +1,47 @@ #!/bin/bash +# Discard raw bootstrap output; report failures with bounded fixed records. +exec 3>&1 +exec >/dev/null 2>&1 export DEBIAN_FRONTEND=noninteractive export DA_ROOT="${DA_ROOT:-/usr/share/docassemble}" export DA_CONFIG_FILE="${DA_CONFIG:-${DA_ROOT}/config/config.yml}" export DA_DEFAULT_LOCAL="local3.14" - -export DA_ACTIVATE="${DA_PYTHON:-${DA_ROOT}/${DA_DEFAULT_LOCAL}}/bin/activate" -source "${DA_ACTIVATE}" -source /dev/stdin < <(python -m docassemble.base.read_config --limited "$DA_CONFIG_FILE" ) +startup_failure() { + [ -x "${DA_ROOT}/webapp/privacy-diagnostic" ] || exit 69 + "${DA_ROOT}/webapp/privacy-diagnostic" uwsgi "$1" >&3 3>&- + exit $? +} +[ -x "${DA_ROOT}/webapp/privacy-diagnostic" ] || exit 69 +DA_RUNTIME="${DA_PYTHON:-${DA_ROOT}/${DA_DEFAULT_LOCAL}}" +export DA_ACTIVATE="${DA_RUNTIME}/bin/activate" +source "${DA_ACTIVATE}" || startup_failure activation +DA_EXPORTS=$("${DA_RUNTIME}/bin/python" -m docassemble.base.read_config --limited "$DA_CONFIG_FILE") || startup_failure config +source /dev/stdin <<< "$DA_EXPORTS" || startup_failure config_eval +unset DA_EXPORTS set -- $LOCALE export LANG=$1 export HOME=/var/www -function stopfunc { - UWSGI_PID=$(&2 - kill -INT $UWSGI_PID - echo "Waiting for uwsgi to stop" >&2 - wait $UWSGI_PID - echo "uwsgi stopped" >&2 - exit 0 -} - -trap stopfunc SIGINT SIGTERM +# uWSGI environment options must not add a hidden logger, daemon, or UID change. +for DA_UWSGI_OPTION in ${!UWSGI_@}; do + unset "$DA_UWSGI_OPTION" +done +unset DA_UWSGI_OPTION if [ "${DAWEBSERVER:-nginx}" = "none" ]; then - unset DAWEBSERVER - uwsgi --ini "${DA_ROOT}/config/docassemble-expose-uwsgi.ini" & + DA_INI="${DA_ROOT}/config/docassemble-expose-uwsgi.ini" else - unset DAWEBSERVER - uwsgi --ini "${DA_ROOT}/config/docassemble.ini" & + DA_INI="${DA_ROOT}/config/docassemble.ini" fi -wait %1 +unset DAWEBSERVER +"${DA_ROOT}/webapp/privacy-preflight" uwsgi "$DA_INI" || startup_failure preflight +shopt -s execfail +exec "${DA_ROOT}/webapp/privacy-process" \ + --component uwsgi -- "${DA_RUNTIME}/bin/uwsgi" --ini "$DA_INI" \ + --die-on-term --log-format 'PRIVACY_REQUEST status=%(status) msecs=%(msecs)' >&3 3>&- +# Failed exec keeps its redirections: recover the saved sink from stdout. +exec 3>&1 >/dev/null +startup_failure launch diff --git a/Docker/run-uwsgilog.sh b/Docker/run-uwsgilog.sh new file mode 100644 index 000000000..180af4854 --- /dev/null +++ b/Docker/run-uwsgilog.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Discard raw bootstrap output; report failures with bounded fixed records. +exec 3>&1 +exec >/dev/null 2>&1 + +export DEBIAN_FRONTEND=noninteractive +export DA_ROOT="${DA_ROOT:-/usr/share/docassemble}" +export DA_CONFIG_FILE="${DA_CONFIG:-${DA_ROOT}/config/config.yml}" +export DA_DEFAULT_LOCAL="local3.14" +startup_failure() { + [ -x "${DA_ROOT}/webapp/privacy-diagnostic" ] || exit 69 + "${DA_ROOT}/webapp/privacy-diagnostic" uwsgilog "$1" >&3 3>&- + exit $? +} +[ -x "${DA_ROOT}/webapp/privacy-diagnostic" ] || exit 69 +DA_RUNTIME="${DA_PYTHON:-${DA_ROOT}/${DA_DEFAULT_LOCAL}}" +export DA_ACTIVATE="${DA_RUNTIME}/bin/activate" +source "${DA_ACTIVATE}" || startup_failure activation +DA_EXPORTS=$("${DA_RUNTIME}/bin/python" -m docassemble.base.read_config --limited "$DA_CONFIG_FILE") || startup_failure config +source /dev/stdin <<< "$DA_EXPORTS" || startup_failure config_eval +unset DA_EXPORTS + +set -- $LOCALE +export LANG=$1 +export HOME=/var/www + +# uWSGI environment options must not add a hidden logger, daemon, or UID change. +for DA_UWSGI_OPTION in ${!UWSGI_@}; do + unset "$DA_UWSGI_OPTION" +done +unset DA_UWSGI_OPTION + +DA_INI="${DA_ROOT}/config/docassemblelog.ini" + +unset DAWEBSERVER +"${DA_ROOT}/webapp/privacy-preflight" uwsgi "$DA_INI" || startup_failure preflight +shopt -s execfail +exec "${DA_ROOT}/webapp/privacy-process" \ + --component uwsgi -- "${DA_RUNTIME}/bin/uwsgi" --ini "$DA_INI" \ + --die-on-term --log-format 'PRIVACY_REQUEST status=%(status) msecs=%(msecs)' >&3 3>&- +# Failed exec keeps its redirections: recover the saved sink from stdout. +exec 3>&1 >/dev/null +startup_failure launch diff --git a/Docker/run-websockets.sh b/Docker/run-websockets.sh index 11f6dd95a..e04e73258 100755 --- a/Docker/run-websockets.sh +++ b/Docker/run-websockets.sh @@ -1,27 +1,32 @@ #!/bin/bash +# Keep bootstrap output out of retained logs; report checked failures safely. +exec 3>&1 +exec >/dev/null 2>&1 export DA_ROOT="${DA_ROOT:-/usr/share/docassemble}" export DA_DEFAULT_LOCAL="local3.14" -export DA_ACTIVATE="${DA_PYTHON:-${DA_ROOT}/${DA_DEFAULT_LOCAL}}/bin/activate" -source "${DA_ACTIVATE}" +startup_failure() { + [ -x "${DA_ROOT}/webapp/privacy-diagnostic" ] || exit 69 + "${DA_ROOT}/webapp/privacy-diagnostic" websockets "$1" >&3 3>&- + exit $? +} +[ -x "${DA_ROOT}/webapp/privacy-diagnostic" ] || exit 69 +DA_RUNTIME="${DA_PYTHON:-${DA_ROOT}/${DA_DEFAULT_LOCAL}}" +export DA_ACTIVATE="${DA_RUNTIME}/bin/activate" +source "${DA_ACTIVATE}" || startup_failure activation export DA_CONFIG_FILE="${DA_CONFIG:-${DA_ROOT}/config/config.yml}" -source /dev/stdin < <(source "$DA_ACTIVATE" && python -m docassemble.base.read_config "$DA_CONFIG_FILE") +DA_EXPORTS=$("${DA_RUNTIME}/bin/python" -m docassemble.base.read_config "$DA_CONFIG_FILE") || startup_failure config +source /dev/stdin <<< "$DA_EXPORTS" || startup_failure config_eval +unset DA_EXPORTS set -- $LOCALE export LANG=$1 export HOME=/var/www -python -u -m docassemble.webapp.socketserver & - -WEBSOCKETSPID=%1 - -function stopfunc { - kill -SIGTERM $WEBSOCKETSPID - exit 0 -} - -trap stopfunc SIGINT SIGTERM - -wait $WEBSOCKETSPID +shopt -s execfail +exec "${DA_ROOT}/webapp/privacy-process" --component websockets -- \ + "${DA_RUNTIME}/bin/python" -u -m docassemble.webapp.socketserver >&3 3>&- +exec 3>&1 >/dev/null +startup_failure launch diff --git a/Docker/sync.sh b/Docker/sync.sh index 2640d0c78..1fb1145b4 100755 --- a/Docker/sync.sh +++ b/Docker/sync.sh @@ -1,6 +1,20 @@ #!/bin/bash export DA_ROOT="${DA_ROOT:-/usr/share/docassemble}" +# Internal handoff only; normal invocations always enter native capture. +if [ "${1:-}" != "--privacy-captured" ]; then + exec 3>&1 + exec >/dev/null 2>&1 + [ -x "${DA_ROOT}/webapp/privacy-diagnostic" ] || exit 69 + shopt -s execfail + exec "${DA_ROOT}/webapp/privacy-process" --component maintenance -- \ + /bin/bash "${BASH_SOURCE[0]}" --privacy-captured "$@" >&3 3>&- + exec 3>&1 >/dev/null + "${DA_ROOT}/webapp/privacy-diagnostic" maintenance launch >&3 3>&- + exit $? +fi +shift + export DA_DEFAULT_LOCAL="local3.14" export DA_ACTIVATE="${DA_PYTHON:-${DA_ROOT}/${DA_DEFAULT_LOCAL}}/bin/activate" diff --git a/Docker/syslog-ng.conf b/Docker/syslog-ng.conf index 59e80849a..8c1a5070e 100644 --- a/Docker/syslog-ng.conf +++ b/Docker/syslog-ng.conf @@ -101,7 +101,8 @@ filter f_nginx_access { level(debug) and program("nginx"); }; filter f_nginx_error { level(error) and program("nginx"); }; filter f_docassemble { program("docassemble"); }; filter f_websockets { program("websockets"); }; -filter f_daworker { program("celery"); }; +# Keep the single-queue worker in its own destination. +filter f_daworker { program("^celery$"); }; filter f_daworkersingle { program("celerysingle"); }; filter f_uwsgi { program("uwsgi"); }; filter f_supervisor { program("supervisor"); }; diff --git a/Dockerfile b/Dockerfile index 358e219cd..4000927d1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,37 @@ # syntax=docker/dockerfile:1 +FROM --platform=$BUILDPLATFORM golang:1.27.0 AS privacy-diagnostic-build +WORKDIR /src +COPY Docker/privacy-diagnostic/ ./ +ARG TARGETOS +ARG TARGETARCH +RUN CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH GOTOOLCHAIN=local GOPROXY=off \ + go build -trimpath -buildvcs=false -o /out/privacy-diagnostic . \ + && CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH GOTOOLCHAIN=local GOPROXY=off \ + go build -trimpath -buildvcs=false -o /out/privacy-process ./cmd/privacy-process \ + && CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH GOTOOLCHAIN=local GOPROXY=off \ + go build -trimpath -buildvcs=false -o /out/privacy-preflight ./cmd/privacy-preflight \ + && CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH GOTOOLCHAIN=local GOPROXY=off \ + go build -trimpath -buildvcs=false -o /out/privacy-monitor ./cmd/privacy-monitor + FROM jhpyle/docassemble-os USER root +COPY --from=privacy-diagnostic-build --chown=0:0 --chmod=0755 \ + /out/privacy-diagnostic /usr/share/docassemble/webapp/privacy-diagnostic +COPY --from=privacy-diagnostic-build --chown=0:0 --chmod=0755 \ + /out/privacy-process /usr/share/docassemble/webapp/privacy-process +COPY --from=privacy-diagnostic-build --chown=0:0 --chmod=0755 \ + /out/privacy-preflight /usr/share/docassemble/webapp/privacy-preflight +COPY --from=privacy-diagnostic-build --chown=0:0 --chmod=0755 \ + /out/privacy-monitor /usr/share/docassemble/webapp/privacy-monitor RUN --mount=type=bind,source=.,target=/tmp/docassemble \ DEBIAN_FRONTEND=noninteractive TERM=xterm LC_CTYPE=C.UTF-8 LANG=C.UTF-8 \ bash -c \ "cp /tmp/docassemble/docassemble_webapp/docassemble.wsgi /usr/share/docassemble/webapp/ \ && cp /tmp/docassemble/Docker/*.sh /usr/share/docassemble/webapp/ \ +&& install -d -m 0755 /usr/local/lib/docassemble-privacy \ +&& cp /tmp/docassemble/Docker/privacy/nginx-lifecycle.conf /usr/local/lib/docassemble-privacy/ \ +&& chmod -R go-w /usr/local/lib/docassemble-privacy \ +&& cp /tmp/docassemble/Docker/nginx.conf /etc/nginx/nginx.conf \ && cp /tmp/docassemble/Docker/VERSION /usr/share/docassemble/webapp/ \ && cp /tmp/docassemble/Docker/config/* /usr/share/docassemble/config/ \ && cp /tmp/docassemble/Docker/cgi-bin/index.sh /usr/lib/cgi-bin/ \ diff --git a/docassemble_webapp/docassemble/webapp/log_initialize.py b/docassemble_webapp/docassemble/webapp/log_initialize.py index ebdcce3e8..59c03ca5a 100644 --- a/docassemble_webapp/docassemble/webapp/log_initialize.py +++ b/docassemble_webapp/docassemble/webapp/log_initialize.py @@ -1,86 +1,6 @@ -import time -import sys -import logging -import re -import os -from flask import request -from flask import has_request_context -from flask_login import current_user -from docassemble.base.logger import set_logmessage -from docassemble.base.thread_context import this_thread -from docassemble.webapp.config import ( - LOGSERVER, - LOG_DIRECTORY, - daconfig, - in_celery, - in_cron, -) -from docassemble.webapp.utils.request import get_requester_ip - -sys_logger = None - -def syslog_message(message): - message = re.sub(r'\n', ' ', message) - if has_request_context(): - try: - if current_user and current_user.is_authenticated: - the_user = current_user.email - else: - the_user = "anonymous" - the_current_info = getattr(this_thread, 'current_info', {}) - sys_logger.debug('%s', LOGFORMAT % {'message': message, 'clientip': get_requester_ip(request), 'yamlfile': the_current_info.get('yaml_filename', 'na'), 'user': the_user, 'session': the_current_info.get('session', 'na')}) - except BaseException as err: - sys.stderr.write("Error writing log message " + str(message) + "\n") - try: - sys.stderr.write("Error was " + err.__class__.__name__ + ": " + str(err) + "\n") - except: - pass - else: - try: - sys_logger.debug('%s', LOGFORMAT % {'message': message, 'clientip': 'localhost', 'yamlfile': 'na', 'user': 'na', 'session': 'na'}) - except BaseException as err: - sys.stderr.write("Error writing log message " + str(message) + "\n") - try: - sys.stderr.write("Error was " + err.__class__.__name__ + ": " + str(err) + "\n") - except: - pass - - -def syslog_message_with_timestamp(message): - syslog_message(time.strftime("%Y-%m-%d %H:%M:%S") + " " + message) - -LOGFORMAT = daconfig.get('log format', 'docassemble: ip=%(clientip)s i=%(yamlfile)s uid=%(session)s user=%(user)s %(message)s') - - -class UnsilenceableLogger(logging.Logger): - def isEnabledFor(self, level): - return level >= self.level - - -def add_log_handler(): - tries = 0 - while tries < 5: - try: - docassemble_log_handler = logging.FileHandler(filename=os.path.join(LOG_DIRECTORY, 'docassemble.log')) - except PermissionError: - sys.stderr.write("Unable to open docassemble.log; trying again\n") - time.sleep(1) - tries += 1 - continue - sys_logger.addHandler(docassemble_log_handler) - if os.environ.get('SUPERVISORLOGLEVEL', 'info') == 'debug': - stderr_log_handler = logging.StreamHandler(stream=sys.stderr) - sys_logger.addHandler(stderr_log_handler) - break - -if not (in_celery or in_cron or daconfig.get('log to std', False)): - logging.setLoggerClass(UnsilenceableLogger) - sys_logger = logging.getLogger('docassemble') - logging.setLoggerClass(logging.Logger) - sys_logger.setLevel(logging.DEBUG) - sys_logger.propagate = False - add_log_handler() - if LOGSERVER is None: - set_logmessage(syslog_message_with_timestamp) - else: - set_logmessage(syslog_message) +# Preserve the application import boundary without installing a file logger. +# The existing docassemble.base.logger callback writes to stderr. Supported +# service launchers must route stdout/stderr through privacy-process; that Go +# process owns fixed counter output, delivery deadlines, and failure reporting. +# Do not install this package without the complete privacy service profile. +# See docs/privacy/application-logger-go.md in the maintained source checkout. diff --git a/docassemble_webapp/docassemble/webapp/process_email.py b/docassemble_webapp/docassemble/webapp/process_email.py index ca705ed75..fbcdf79e5 100644 --- a/docassemble_webapp/docassemble/webapp/process_email.py +++ b/docassemble_webapp/docassemble/webapp/process_email.py @@ -22,13 +22,10 @@ def main(): - fp = open("/tmp/mail.log", "a", encoding="utf-8") - # fp.write("The file is " + sys.argv[1] + "\n") try: with open(sys.argv[1], 'r', encoding="utf-8") as email_fp: msg = email.message_from_file(email_fp) - except BaseException as err: - fp.write("Failed to read e-mail message: " + str(err) + "\n") + except BaseException: sys.exit("Failed to read e-mail message") raw_date = msg.get('Date', msg.get('Resent-Date', None)) addr_return_path = msg.get('Return-path', None) @@ -36,9 +33,6 @@ def main(): addr_to = msg.get('Envelope-to', None) addr_from = msg.get('From', msg.get('Sender', None)) subject = msg.get('Subject', None) - fp.write("Message to " + str(addr_to) + "\n") - # fp.write("From was " + str(addr_from) + "\n") - # fp.write("Subject was " + str(subject) + "\n") to_recipients = [] for recipient in getaddresses(msg.get_all('to', []) + msg.get_all('resent-to', [])): to_recipients.append({'name': recipient[0], 'address': recipient[1]}) @@ -50,51 +44,36 @@ def main(): recipients.append({'name': recipient[0], 'address': recipient[1]}) if addr_to is None and len(recipients) > 0: addr_to = recipients[0]['address'] - # fp.write("recipients are " + str(recipients) + "\n") if addr_to is not None: - # fp.write("parsed envelope-to: " + str(parseaddr(addr_to)) + "\n") short_code = re.sub(r'@.*', '', parseaddr(addr_to)[1]) else: short_code = None - # fp.write("short code is " + str(short_code) + "\n") with session_scope() as session: record = session.execute(select(Shortener).filter_by(short=short_code)).scalar() if record is None: - fp.write("short code not found\n") sys.exit("short code not found") - # fp.write("short code found\n") # file_number = get_new_file_number(record.uid, 'email', record.filename) - # #fp.write("file number is " + str(file_number) + "\n") # saved_file_email = SavedFile(file_number, fix=True) if addr_from is not None: - # fp.write("parsed from: " + str(parseaddr(addr_from)[1]) + "\n") addr_from = {'name': parseaddr(addr_from)[0], 'address': parseaddr(addr_from)[1]} else: addr_from = {'empty': True} if addr_return_path is not None: - # fp.write("parsed return_path: " + str(parseaddr(addr_return_path)[1]) + "\n") addr_return_path = {'name': parseaddr(addr_return_path)[0], 'address': parseaddr(addr_return_path)[1]} else: addr_return_path = {'empty': True} - # fp.write("return_path is " + str(addr_return_path) + "\n") if addr_reply_to is not None: - # fp.write("parsed reply-to: " + str(parseaddr(addr_reply_to)[1]) + "\n") addr_reply_to = {'name': parseaddr(addr_reply_to)[0], 'address': parseaddr(addr_reply_to)[1]} - # fp.write("reply-to is " + str(addr_reply_to) + "\n") else: addr_reply_to = {'empty': True} - # fp.write("reply-to is " + str(addr_reply_to) + "\n") msg_current_time = datetime.datetime.now() if raw_date is not None: msg_date = datetime.datetime.fromtimestamp(mktime(parsedate(raw_date))) - # fp.write("msg_date is " + str(msg_date) + "\n") else: msg_date = msg_current_time - # fp.write("msg_date set to current time\n") headers = [] for item in msg.items(): headers.append([item[0], item[1]]) - # fp.write("headers:\n" + json.dumps(headers) + "\n") email_record = Email(short=short_code, to_addr=json.dumps(to_recipients), cc_addr=json.dumps(cc_recipients), from_addr=json.dumps(addr_from), reply_to_addr=json.dumps(addr_reply_to), return_path_addr=json.dumps(addr_return_path), subject=subject, datetime_message=msg_date, datetime_received=msg_current_time) session.add(email_record) @@ -115,14 +94,11 @@ def main(): filename = '%03d-%s' % (counter, safe_filename(filename)) else: filename = '%03d-attachment%s' % (counter, ext) - # fp.write("Filename is " + str(filename) + "\n") - # fp.write("Content type is " + str(part.get_content_type()) + "\n") real_filename = re.sub(r'[0-9][0-9][0-9]-', r'', filename) real_ext = re.sub(r'^\.', r'', ext) save_attachment(session, record.uid, record.filename, real_filename, email_record.id, counter, part.get_content_type(), real_ext, part.get_payload(decode=True)) counter += 1 - fp.close() user = None if record.user_id is not None: user = session.execute(select(UserModel).options(joinedload(UserModel.roles)).filter_by(id=record.user_id)).scalar() diff --git a/docs/privacy/aggregation-go.md b/docs/privacy/aggregation-go.md new file mode 100644 index 000000000..10f043c44 --- /dev/null +++ b/docs/privacy/aggregation-go.md @@ -0,0 +1,99 @@ +# Go native-output aggregation + +Status: integrated into the candidate's [Go native runner](native-runner-go.md). +**Not deployed.** The separate startup diagnostic does not import this library; +cross-compiling only that executable would not verify the native capture path. + +`Docker/privacy-diagnostic/internal/aggregate` uses only the standard library. +It replaces the aggregation algorithm, preserving the reviewed Python draft's +counter semantics. It does not start processes, read descriptors, write logs, +schedule snapshots, or enforce native configuration. The process runner must +provide those boundaries. The candidate launchers now select the Go runner. + +## Interface and retained state + +Construct an `Aggregator` with `New(Nginx)`, `New(UWsgi)`, `New(Celery)`, +`New(CelerySingle)`, `New(Websockets)`, or `New(Mail)`. Call `Feed` with +`Stdout` or `Stderr`, `Finish` at that stream's EOF, and `Snapshot` for a value +copy. The caller must serialize these operations. Invalid components, streams, +nil receivers, and zero values return fixed errors without mutating state. + +Each stream owns one 4,096-byte array. Partial input is copied, so callers may +reuse their read buffer after `Feed` returns. Consumed, overflowing, and finished +owned buffers are cleared. This does not promise erasure of caller buffers, +runtime copies, OS buffers, swap, or core dumps; the runner remains responsible +for its own memory and process protections. + +A line may contain at most 4,096 bytes before its newline, including a trailing +carriage return. Overflow increments `dropped` immediately and only once, then +discards input through the next newline. EOF consumes a valid unterminated tail +once. Repeated EOF is harmless; a later feed starts a fresh line. stdout and +stderr never share partial records. + +## Accepted records + +For nginx/uWSGI request counters, records must contain exactly three space-separated tokens: + +```text +PRIVACY_REQUEST status=200 msecs=99 +PRIVACY_REQUEST status=503 seconds=30.000 +``` + +Status is exactly three ASCII digits in 100–599. Milliseconds are canonical +unsigned decimal, without leading zeroes except `0`, in 0–600000. Only nginx +accepts seconds: one to three canonical whole digits, a decimal point, and +exactly three fractional digits, with the same upper limit. Extra fields, +embedded CR/NUL, malformed UTF-8, and invalid numeric forms cannot become +request counts. A single trailing CR is accepted. Valid text without the exact +first token increments `unclassified`; malformed marked records increment +`rejected`. + +Application profiles count all valid text as `unclassified`, including request +lookalikes. Malformed text and overflow retain the same rejected/dropped rules. +Their status/latency counters always remain zero and serialization rejects +forged nonzero request counters. See [application capture](application-capture-go.md). + +Snapshots contain exactly 14 JSON fields: schema `1`, a fixed component, five +status-family counters, four latency counters, `unclassified`, `rejected`, and +`dropped`. Latency buckets are below 100 ms, 100–999 ms, 1000–29999 ms, and +30000–600000 ms. Every counter is cumulative and saturates at 2^31−1. +Serialization rejects forged components, schema values, and oversized counters +before producing output. Raw input never becomes a snapshot field or error. + +## Verification + +Run from the checkout root: + +```sh +bash tests/verify_privacy.sh +``` + +The command checks all Go packages with formatting, vet, race tests, module +verification, and Linux amd64/arm64 compilation. It runs ten seconds of fuzzing +for each of the startup diagnostic, aggregator, preflight, and monitor, then all 131 native/application-launcher +and 84 frozen application draft tests plus shell syntax and diff whitespace. No production +dependency was added. CI also runs the pinned development scanner +`govulncheck@v1.7.0`; the local aggregator scan found no vulnerabilities. + +The Go tests compare 240 deterministic synthetic scenarios with the frozen +Python algorithm in `tests/privacy_native/reference/log_aggregate.py`, including +both streams, arbitrary fragmentation, intermediate +snapshots, and repeated EOF. The test-only oracle lives in +`tests/privacy_native/aggregate_reference.py`. It verifies the reference SHA-256 +`582582eacbb29d486e1a9b8205577664d5eb239f85493441970197b53f31951f`. +Boundary and malformed-input tests provide expectations independent of that +reference. A mutation control changing the overflow check from `>` to `>=` +fails specifically because a valid 4,096-byte line is dropped. + +Local evidence is in `tests/.aggregate-review/`. The final focused run passed, +including 77,263 aggregate fuzz executions and 974,367 diagnostic executions. +Fuzz debug output attributes long progress pauses to input minimization; no +failure was reported. A separate benchmark processes a 64 KiB oversized line +in one-byte fragments in approximately 0.34 ms on this Apple M4, with zero +allocations. This measures the parser only, not service throughput. Unit tests +also check fixed state size and zero allocations for a 1 MiB oversized chunk. + +Remaining release work is recorded in [the release review](review-2026-09-05.md). +The Go runner now connects this library to bounded pipe capture and snapshot +delivery, with Linux process and launcher tests. Full service installation, +complete application output coverage, monitoring, and rollback remain open. diff --git a/docs/privacy/application-capture-go.md b/docs/privacy/application-capture-go.md new file mode 100644 index 000000000..3e564925d --- /dev/null +++ b/docs/privacy/application-capture-go.md @@ -0,0 +1,114 @@ +# Go capture for Celery and websockets + +Status: integrated into the uncommitted JOS-81 candidate; **not deployed**. +The three application launchers now use the existing Go process runner. This +covers their checked bootstrap steps and captured stdout/stderr, including +output before Python logging initialization. Independently opened files and +other service launchers remain outside this boundary. + +## Launch and output contract + +`run-celery.sh`, `run-celery-single.sh`, and `run-websockets.sh` discard raw +bootstrap output and check virtualenv activation, configuration loading, +export evaluation, and the final exec. Failures use the fixed component/phase +records and exit statuses in [startup diagnostics](startup-diagnostics.md). +A missing diagnostic stops execution before activation. Successful exec closes +the temporary sink descriptor and preserves Supervisor's process identity. +The Go runner then starts the foreground service through its existing Linux +process protections and bounded capture path. + +The selected virtualenv supplies the absolute Python/Celery executable path. +Celery queue names and worker arguments are preserved. Its concurrency remains +the CPU count capped by the configured maximum, replaced by an explicit worker +override when supplied, then reduced by one with a minimum of one. Counts must +be canonical positive decimal integers at most 1,048,576 before shell +arithmetic. Zero, negative, leading-zero, oversized, or arithmetic-expression +values fail configuration evaluation. The single queue always uses one worker. + +The fixed capture components are `celery`, `celerysingle`, and `websockets`. +They use the existing 14-field cumulative snapshot schema. Valid text lines +increment `unclassified`, malformed text increments `rejected`, and oversized +lines increment `dropped`. Application text never becomes request metrics, +even when it resembles `PRIVACY_REQUEST`. All nine status/latency fields stay +zero; serialization also rejects forged nonzero request counters for these +components. Original nginx/uWSGI parsing and the frozen reference remain intact. + +Supervisor retains only the launcher's fixed startup records and the runner's +counter snapshots in the existing worker/websocket log paths. Each application +section now uses one combined stream with 5 MB rotation and seven backups. +The [process monitor](process-monitor-go.md) reports these three services' +unexpected EXITED, BACKOFF, and FATAL states using fixed labels. + +## Shutdown behavior + +| Service | Master grace before group escalation | Supervisor stop wait | +| --- | --- | --- | +| Celery | 60 seconds | 70 seconds | +| Single-queue Celery | 60 seconds | 70 seconds | +| Websockets | 20 seconds | 30 seconds | + +TERM/INT reaches the service master first. Its workers receive the full master +grace window before group signaling and KILL; the original two-second native +profile would interrupt Celery work prematurely. Remaining descendants are +signaled immediately if the master exits. The runner allows one further second +for teardown and retains its five-second output deadline. Supervisor's extra +time accommodates capture/teardown overhead. Linux parent-death protection +sends TERM to the application master if the runner dies. It cannot guarantee +drain completion when the master ignores TERM or changes credentials; the +foreground process and credential restrictions still apply. + +The websocket launcher now execs the runner instead of using a shell trap that +exits without waiting for the child. Native exit codes remain observable. + +## Verification record + +Local verification on September 5, 2026 used synthetic data only: + +- Fail-first application checks reproduced raw bootstrap output and continuation + after startup failures in the old launchers. New profiles also failed against + the previous Go parser before implementation. +- `bash tests/verify_privacy.sh` passed Go formatting, vet, race tests, four + bounded fuzz targets, module verification, four host/Linux amd64/arm64 + executable builds, 126 native/application-launcher Python tests, 84 existing + application logger tests, shell syntax, and diff whitespace. +- All Go packages passed the final Linux race suite and Linux vet. Actual shell + launchers reached Go capture for all three roles and retained only counters + while preserving the synthetic service's exit code. Six lifecycle cases + covered worker completion during normal shutdown and runner death. +- An isolated mutation restoring half-window group signaling failed the Celery + and single-queue drain checks with worker exit 93. Production source and + assertions were unchanged by the control. +- Real Supervisor 4.3.0 ran the production listener and application sections + as `www-data`, invoking the actual launchers with synthetic runtime commands. + All three application logs contained counter records only; the monitor + reported their unexpected exits and ignored expected/unrelated exits. + The fixture uses executable files from a read-only mount and leaves `/tmp` + non-executable. No container permissions were broadened to make it pass. +- Real nginx inspection accepted the safe profile and rejected the unsafe + profile without output. Both profiles first passed native syntax inspection. +- A fresh Linux-targeted `govulncheck@v1.7.0` scan found no vulnerabilities in + the Go candidate. This does not scan the full docassemble dependency set. + +Evidence and before-edit copies are in +`tests/.privacy-build/application-capture-review/`. The final Linux checks used +Go 1.27.0 and the existing development image +`sha256:d318b5a18cc090c52fc2db3a9642f1e3749e6bcd64d71c93cfd36e51ea596a25`. +Containers had no network, a read-only root filesystem, no capabilities, and +no-new-privileges. Runtime state and caches stayed within this worktree. + +## Remaining release work + +The later [application logger replacement](application-logger-go.md) removes +the file-handler override and leaves the existing stderr callback feeding Go +capture. The Python draft and its unconsumed failure counter are now frozen +test references. No `log to std` text check or duplicated cloud configuration +loader is needed for that replacement. +The later [mail capture record](mail-capture-go.md) removes mail's direct +`/tmp/mail.log` and adds a bounded Go command profile. Cron, other independent files, and remaining +service/copy/backup routes need separate coverage. + +The fixture does not run real Flask, Celery tasks, a broker, or websocket +requests. A complete candidate image, persistent-volume installation, retained +output/rotation/restore acceptance, tested rollback, monitor health checks, +and CI for the exact release commit remain required. No commit, push, merge, +or production change was made. See the [release review](review-2026-09-05.md). diff --git a/docs/privacy/application-logger-go.md b/docs/privacy/application-logger-go.md new file mode 100644 index 000000000..d90d2987f --- /dev/null +++ b/docs/privacy/application-logger-go.md @@ -0,0 +1,105 @@ +# Application logging through Go capture + +Status: implemented in the uncommitted JOS-81 candidate; **not deployed**. +The application no longer installs its independent `docassemble.log` handler. +Its existing base logger writes to stderr, which the supported service +launchers feed into the Go capture process. The production change removes the +logging override; it adds no executable Python, runtime dependency, or new +configuration parser. + +## Runtime boundary + +`docassemble.webapp.app_initialize` still imports `log_initialize` at the same +point. That module now contains only explanatory comments. The existing +`docassemble.base.logger.default_logmessage` callback remains unchanged. +First-party source search found no consumers of the removed module's private +functions or `sys_logger` variable. The `docassemble.webapp.utils.logger` +re-export continues to use the base callback. + +This removes the separate file-handler selection and its request-context +formatting. `log to std`, `LOGSERVER`, `log format`, and debug settings no +longer cause this entrypoint to open a log file or change the callback. +The configuration loader, including its cloud overlays and key normalization, +is untouched. This does not disable unrelated application configuration. + +Supported nginx/uWSGI and application launchers must be installed together with +the [Go runner](native-runner-go.md), [startup diagnostic](startup-diagnostics.md), +and [Supervisor monitor](process-monitor-go.md). The Go process captures output +before, during, and after Python logger initialization. It emits cumulative +14-field counter snapshots with fixed component names. Per-message text, +exception details, and the old five-field application event records are not +retained. Application lines contribute to `unclassified`, `rejected`, or +`dropped`; native request metrics retain their existing profile rules. + +The discarded Python handler's local `failure_count` is no longer a production +failure mechanism. Go owns the bounded output sink and process teardown; +delivery failures return 74, and Supervisor failure states provide fixed +reports when the monitor's own sink is available. Existing broken/full-sink, +shutdown, and acknowledgement tests remain applicable. Pipe acceptance is +still not a filesystem durability guarantee. + +## Preserved reference and tests + +The unreleased Python draft moved byte-for-byte into +`tests/privacy_logging/reference/`. Its two SHA-256 values are enforced by the +test loader: + +| Reference | SHA-256 | +| --- | --- | +| `log_initialize.py` | `93f5e7b14ff7162edef330246bf99d30b34d96fadfac1314eaa593103dd89cb1` | +| `privacy_logging.py` | `0cf784e0d0e3997ba334b511c419fc47ed0f37d560777c1ad1b1c0dc39843634` | + +All 84 existing draft tests retain their assertions. They now exercise this +frozen design reference and do not prove current production behavior. The +reference is outside the application package and is not installed by the +Dockerfile. The prior [application logging document](application-logging.md) +describes that historical design. + +Current behavior is checked by `TestApplicationLoggerUsesOnlyCapturedStreams` +and `tests/privacy_native/fixtures/application-logger.py`. The fixture imports +the real base logger and executes the real `app_initialize.init_app` through +the first post-logging boundary. Configuration and downstream dependencies +are synthetic. It emits known private markers before setup, during setup, +after initialization, after standard-library logging reconfiguration, through +the base callback again, and directly to stdout. + +The Linux test runs that fixture through the actual Go runner across five +capture profiles, four application contexts, two log-server settings, and two +debug settings: **80 combinations**, including mail. Each must exit successfully, leave the +independent log directory empty, preserve the base callback across reimport, +and produce exactly six unclassified lines in the final safe snapshot. No +private marker may appear in retained output. A direct fail-first run against +the previous draft stopped because initialization opened an independent file. + +## Verification and remaining work + +Run `bash tests/verify_privacy.sh`. On macOS, also run the Linux-specific Go +tests in the disposable Linux environment. Local September 5 verification +passed the 64-case Linux integration test, the complete Linux race suite, +Linux vet, and the unchanged 84 reference tests. The canonical suite passed +formatting, vet, race tests, four bounded fuzz targets, module verification, +host/Linux builds, 126 native/application-launcher Python tests, and the +reference suite separately. A fresh Linux-targeted Go vulnerability scan found +no vulnerabilities in the Go candidate; it does not cover all application +dependencies. The expanded real Supervisor test also ran the production +application launchers with real logger initialization and the original four +synthetic output controls. Each retained exactly ten unclassified lines and +reported its unexpected exit through the fixed monitor records. +Evidence and before-edit copies are in +`tests/.privacy-build/application-logger-review/`. + +This closes the source-level new-Python and unconsumed-handler-counter issues +for this entrypoint. It does not establish full Flask/Celery/broker startup or +protect independently opened files, custom package handlers, or launchers +that still bypass Go capture. The later [mail capture record](mail-capture-go.md) +removes the direct `/tmp/mail.log` and verifies the expanded 80-case matrix; +actual mail storage and Exim transport acceptance remain open. The synthetic cron +context in the matrix is not an acceptance test of `Docker/run-cron.sh`. +Historical files and their rotation/copy/backup/restore paths remain separate +release work. A package-only installation would leave stderr unprotected in +an older service profile and must not be used. + +The complete image, persistent-volume installation, tested rollback, monitor +health, retained-output acceptance, and CI for the exact release commit remain +required by the [release review](review-2026-09-05.md). No production service, +volume, log, or configuration was changed. diff --git a/docs/privacy/application-logging.md b/docs/privacy/application-logging.md new file mode 100644 index 000000000..b515f895b --- /dev/null +++ b/docs/privacy/application-logging.md @@ -0,0 +1,95 @@ +# Historical Python application logging draft (JOS-81) + +Status: **frozen test reference, superseded in the candidate**. Both draft +modules now live in `tests/privacy_logging/reference/`, with their original +bytes and tests preserved. The production application uses the existing stderr +callback through [Go capture](application-logger-go.md). The behavior and +limitations below describe the historical draft, not the current runtime. + +The draft replaced the `docassemble.webapp.log_initialize` entrypoint +and added its adjacent, dependency-free `privacy_logging` module. Importing the +real module installs the base `docassemble.base.logger` callback in every context. +The staged candidate previously skipped Celery, cron and `log to std`. + +| Context | Safe destination for the base callback | +| --- | --- | +| Web, default | `LOG_DIRECTORY/docassemble.log`, append mode | +| Web, `SUPERVISORLOGLEVEL=debug` | Same file plus stderr; either usable sink suffices | +| Celery or cron | stderr, without opening the application log file | +| `log to std` | stderr, without opening the application log file | + +The stderr choices preserve the original base logger destination in the formerly +skipped branches. Debug does not add a duplicate stderr handler in those modes. +`LOGSERVER` preserves callback selection, but both callbacks now discard legacy +message text without inspecting or stringifying it. The formatter supplies UTC +time; legacy calls produce event `UNKNOWN`, with no equivalent free-text detail. +Direct records sent to the configured `docassemble` logger may supply an exact +allowlisted `event_code` instead. + +Every emitted application record has exactly `ts`, `level`, `svc`, `comp`, and +`event`. Service/component are fixed; timestamp, level and event are validated. +Message, args, exception/stack text, logger name, paths, IPs, identities, sessions, +form names and arbitrary extra attributes are discarded. Safe handlers ignore +attempts to replace their formatter. Existing handlers and filters on this +specific logger are removed and propagation is disabled. + +The safe callback is registered before configuration and sink setup. Startup +probes sinks with an empty write and flush; file opens retry at most five times +with four one-second waits. No usable sink raises the fixed +`RuntimeError("Privacy logging unavailable")`, suppressing both the internal +failure and any active caller exception chain. A NullHandler prevents stdlib's +raw `lastResort` fallback. A missing helper or dependency also fails startup; +there is no fallback to the previous free-text callback once it is available. +If the base logger itself cannot import, startup fails before registration. + +Later write/flush failures emit no fallback text; each safe handler keeps a local +failure count capped at 99. This module does not restart services, recover a +failed sink, bound synchronous stream-write time, or prove continued delivery +when a sink accepts the empty probe and later rejects records. Reinitialization +closes only files opened by this module. Historical contents in an existing log +file are untouched and require separately authorized retention handling. + +## Local verification + +Run from the fork root: + +```sh +python3.14 -m unittest discover -s tests/privacy_logging -v +``` + +84 tests pass on Python 3.14.7. They include all 16 context/LOGSERVER/debug +combinations, hostile messages/fields, exact schema, formatter replacement, +propagation, retry exhaustion, missing/closed/failing stderr, file probe failure, +import/setup failure, write/flush failure, caller exception suppression, +reinitialization and an actual interpreter shutdown subprocess. The latter must +exit zero with empty stderr, so an import failure cannot count as a passing test. + +Tests import the frozen references under their original module names using synthetic +configuration and dependency stubs. They also execute real `server.py`, +`flask_app.py`, `app_initialize.py`, and `worker.py` until the first post-logging +boundary (`secret_key` assignment). The real worker selects `in_celery=True`. +The real `cron.py` launcher runs with a stub subprocess; its `IN_CRON` value then +drives a synthetic config for the same real server import. The Flask CLI and base +configuration loader are not executed. No real application, database, task +broker, request, production configuration or production log is imported/read. + +## Historical acceptance limits + +The tests establish early installation, not complete application startup or +survival through subsequent Celery/Flask logging configuration. In particular: + +- Web configuration imports and `setup.init_app` run before this entrypoint; + Celery loads base configuration first. Output from those earlier operations is + outside this module. Full synthetic startup must verify the outer capture + boundary and later logger configuration before installation. +- Celery root/task loggers, Flask and other third-party loggers, websocket's + separate app, SQL diagnostics, arbitrary logging factories/filters installed + later, native extensions, direct stdout/stderr and independent file writes + remain separate routes. A safe base callback alone does not protect them. +- Packaging must install both adjacent modules into the interpreter used by every + applicable service. Full dependency startup, actual queue/task behavior, + scheduled job execution, log rotation/copy/backup/restore, and process capture + across the complete launcher lifecycle still need acceptance evidence. + +Current replacement behavior, source coverage, and remaining release gates are +recorded in [application logging through Go capture](application-logger-go.md). diff --git a/docs/privacy/cron-capture-go.md b/docs/privacy/cron-capture-go.md new file mode 100644 index 000000000..0b69a0e01 --- /dev/null +++ b/docs/privacy/cron-capture-go.md @@ -0,0 +1,38 @@ +# Interview-cron output capture + +`run-cron.sh` now uses the existing Go capture process for the Flask cron +command. The fixed `cron` profile treats stdout/stderr as application output, +emits one final counter record, and preserves the command's exit status. It +does not forward stdin or use mail's temporary-failure mapping. Existing mail +enum values and behavior remain unchanged. + +Scheduled root invocations drop to `www-data` before starting Go. The launcher +sets the matching identity environment and retains niceness 19. Go directly +starts the absolute virtualenv Flask executable. Checked bootstrap failures +produce fixed startup diagnostics. The waiting root wrapper reports a generic +`command_failed` record when its child fails, including failures in privilege +dropping, while retaining the child's exit code. It does not describe every +command failure as a startup failure. It forwards TERM/INT to its child and +waits for cleanup before exiting. Cron's existing output/mail plumbing and +Exim configuration remain unchanged. + +The full-image rehearsal includes synthetic real queue and cron interviews. +The queue returns a known result after emitting stdout/stderr markers. The +cron interview saves a changed counter while emitting a response and stderr; +it also checks UID/GID, identity environment and niceness. Missing configuration +and a missing privilege-drop executable exercise failure records. A long-running +real interview checks that TERM/INT sent only to the root launcher stop Go, +Flask and a spawned descendant, preserve the child's termination status and +emit only fixed records. Diagnostic +files, gzip logs and container stdout/stderr are scanned for the synthetic +private markers. Separate controls prove that the scan detects raw and +compressed markers across read boundaries. These additions require native CI; +source assertions alone are not an acceptance result. + +Native CI on a32541b4d passed the interview launch, result, failure and shutdown +checks described above. The [maintenance extension](maintenance-capture-go.md) +now wraps parent scheduled scripts and exercises local copies, backup and +restore separately. A stream capture cannot filter bytes copied from files. +Historical logs must not be deleted merely to make a privacy test pass. +Manual Python cron invocation remains outside the supported launch path. +The Go mail replacement remains deferred. diff --git a/docs/privacy/deferred-mail-go.md b/docs/privacy/deferred-mail-go.md new file mode 100644 index 000000000..c6e5a2948 --- /dev/null +++ b/docs/privacy/deferred-mail-go.md @@ -0,0 +1,81 @@ +# Deferred: incoming mail replacement in Go + +User decision on September 5, 2026: implement the incoming-mail replacement in +Go, then **save it for later**. Do not resume this replacement or its proposed +Exim transport changes without a new user request. No reminder or scheduled +resume was created. + +## Saved state + +- Worktree: `/Users/jacobrakai/Projects/docassemble-jos81`; + branch `fix/jos81-private-logging`; HEAD + `63d22c44b7bd008a521f0bcb8ab442b2143ee557`. +- The existing uncommitted [mail capture candidate](mail-capture-go.md) remains + preserved. It routes the legacy processor through Go capture, removes its + diagnostic file and shell spool, and maps command failures to 75. It does + not replace the actual Python mail processor. +- Last candidate verification: 131 native/launcher tests, 84 unchanged frozen + reference tests, full Linux race/vet, Go vulnerability scan, and real + Supervisor acceptance passed. Evidence is in + `tests/.privacy-build/mail-capture-review/`. Those checks are bounded capture + verification, not real incoming-mail delivery acceptance. +- The replacement's parser was discussed but no `internal/mailparse` package + was created before the user deferred work. No production Python repairs, + new dependencies, Exim changes, mail sends, or deployment occurred in this + follow-up. Working agents were interrupted. The task Linux VM remains off. +- Deferral does not resolve the [release gates](review-2026-09-05.md). Other + JOS-81 work may continue without restarting this mail replacement. + +## Findings to carry forward + +1. **The storage caller selects the wrong mode.** `process_email.save_attachment` + passes decoded MIME bytes to `SavedFile.write_content` without its binary + option. The generic writer's default text behavior is correct. The saved + `probe_mail_storage.py` demonstrates that mismatch by invoking the actual + method directly; it must not become a requirement to change that default. + A replacement acceptance test must exercise the real caller/storage contract. +2. **IDs and publication ordering need real transaction tests.** The existing + processor reads `Email.id` before an explicit flush and publishes its task + before `session_scope` commits. Its upload-number allocator uses Flask's + scoped session even though the standalone mail entrypoint establishes no + Flask application context. Use one explicit Go database transaction for + related records and capture scalar task data before commit. Publishing + after commit alone does not provide an outbox or duplicate protection. +3. **Preserve the consumer's storage layout.** `Uploads` stores `key`, `filename`, + `yamlfile`, and default private/persistent flags. Local upload ID 77 maps to + `/000/000/000/04d/file`, with `file.` pointing to the same + content. Cloud object keys use decimal IDs: `files/77/file` and + `files/77/file.`. S3/Azure behavior, MIME metadata, rollback + residues, and existing file collisions require acceptance. +4. **Preserve the existing incoming-mail task.** Its name is + `tasks.background_action`; the action is `incoming_email` with the numeric + email ID. The consumer immediately retrieves committed email/attachment + rows. `tasks/app_object.py` selects the configured `rabbitmq` broker or a + default `pyamqp` URL, with Redis as the result backend. Exact Celery message + protocol, routing, broker variants, confirmations, and retry behavior have + not been mapped or implemented in Go. +5. **Keep Exim transport work explicit.** A proposed dedicated + `docassemble_mail_pipe` would avoid modifying unrelated `address_pipe` + consumers. Candidate settings need real Exim queue tests for 75, execution + failure 127, timeout deferral, and signal freezing. Freezing needs manual + intervention; it is not automatic retry. Native output logging includes + recipient context and cannot stand in for a private fixed-counter sink. + +## Resume boundary + +Start by reading `process_email.py`, `emailserver/models.py`, +`emailserver/helpers.py`, `files/savedfile.py`, `files/file_number.py`, +`main/models.py`, `tasks/app_object.py`, and `config_worker.py` in the webapp +package. Confirm current source and dependencies before implementation. + +The proposed first Go API was `mailparse.Parse(io.Reader, Limits)` returning +typed metadata and MIME parts with fixed, content-free errors. Caller-supplied +positive bounds would cover raw/decoded bytes, part count, and nesting depth. +Preserve ordered headers, addresses, recipient selection, text/PDF bytes, and +consumer-required filenames. Resolve RFC2047, platform MIME extension, and +nested-message differences through tests against the legacy contract. + +That parser would be only one part of a complete Go ingestion path. Do not +activate a parser-only replacement or route production to an incomplete +database/storage/queue implementation. Estimated scope: several hours of +implementation and integration acceptance, beyond a simple email test. diff --git a/docs/privacy/dependency-review-2026-09-05.md b/docs/privacy/dependency-review-2026-09-05.md new file mode 100644 index 000000000..1e0431a56 --- /dev/null +++ b/docs/privacy/dependency-review-2026-09-05.md @@ -0,0 +1,30 @@ +# NLTK advisory review + +GitHub inspection on September 5, 2026 found two open High Dependabot alerts, +[35](https://github.com/jacobyoby/docassemble/security/dependabot/35) and +[36](https://github.com/jacobyoby/docassemble/security/dependabot/36), for one +advisory: [GHSA-8mgp-746c-j5xp](https://github.com/advisories/GHSA-8mgp-746c-j5xp). +Both application manifests pin affected `nltk==3.10.3`. GitHub identifies no +patched release. No dependency version was changed and neither alert was dismissed. + +The advisory concerns model-artifact APIs that bypass NLTK's path restrictions +when untrusted workflows control model import/export paths. Affected APIs named +in the review include `TransitionParser.train/parse`, `AveragedPerceptron.save/load`, +`PerceptronTagger.save_to_json` and `save_maxent_params`. + +Source inspection found no direct calls to those APIs. `docassemble/base/pattern.py` +uses language helpers and fixed corpus downloads; the webapp machine-learning +module uses `docassemble_pattern` KNN/SVM. This does not establish non-reachability: +`docassemble/base/pattern_server.py` dispatches dynamically through a Unix socket, +and installed extensions, transitive implementations and runtime socket access +have not been audited here. No public endpoint exploit was demonstrated. + +Upstream work routes model file operations through `nltk.pathsec.open()` or +guarded helpers. A version bump needs a verified fixed release; if affected +runtime use is established before one exists, evaluate a bounded upstream +backport or a control preventing untrusted model paths from reaching those APIs. +Do not remove the working language features or claim the package is safe based +only on a search with no direct matches. + +The two Dependabot alerts remain the authoritative tracking records. This review +does not close the separate [privacy installation work](https://github.com/jacobyoby/docassemble/issues/22). diff --git a/docs/privacy/install-catalog.json b/docs/privacy/install-catalog.json new file mode 100644 index 000000000..4ca2c32d7 --- /dev/null +++ b/docs/privacy/install-catalog.json @@ -0,0 +1,107 @@ +{ + "schema": 1, + "scope": "non-mail-privacy-overlay", + "owner": "root", + "group": "root", + "files": [ + {"id":"diagnostic","kind":"binary","source":"privacy-diagnostic-linux-{{ARCH}}","target":"{{DA_ROOT}}/webapp/privacy-diagnostic","mode":"0755"}, + {"id":"process","kind":"binary","source":"privacy-process-linux-{{ARCH}}","target":"{{DA_ROOT}}/webapp/privacy-process","mode":"0755"}, + {"id":"preflight","kind":"binary","source":"privacy-preflight-linux-{{ARCH}}","target":"{{DA_ROOT}}/webapp/privacy-preflight","mode":"0755"}, + {"id":"monitor","kind":"binary","source":"privacy-monitor-linux-{{ARCH}}","target":"{{DA_ROOT}}/webapp/privacy-monitor","mode":"0755"}, + {"id":"run-nginx","kind":"source","source":"Docker/run-nginx.sh","target":"{{DA_ROOT}}/webapp/run-nginx.sh","mode":"0755"}, + {"id":"run-uwsgi","kind":"source","source":"Docker/run-uwsgi.sh","target":"{{DA_ROOT}}/webapp/run-uwsgi.sh","mode":"0755"}, + {"id":"run-uwsgilog","kind":"source","source":"Docker/run-uwsgilog.sh","target":"{{DA_ROOT}}/webapp/run-uwsgilog.sh","mode":"0755"}, + {"id":"run-celery","kind":"source","source":"Docker/run-celery.sh","target":"{{DA_ROOT}}/webapp/run-celery.sh","mode":"0755"}, + {"id":"run-celery-single","kind":"source","source":"Docker/run-celery-single.sh","target":"{{DA_ROOT}}/webapp/run-celery-single.sh","mode":"0755"}, + {"id":"run-websockets","kind":"source","source":"Docker/run-websockets.sh","target":"{{DA_ROOT}}/webapp/run-websockets.sh","mode":"0755"}, + {"id":"run-cron","kind":"source","source":"Docker/run-cron.sh","target":"{{DA_ROOT}}/webapp/run-cron.sh","mode":"0755"}, + {"id":"cron-hourly","kind":"source","source":"Docker/cron/docassemble-cron-hourly.sh","target":"/etc/cron.hourly/docassemble","mode":"0755"}, + {"id":"cron-daily","kind":"source","source":"Docker/cron/docassemble-cron-daily.sh","target":"/etc/cron.daily/docassemble","mode":"0755","existing":"apply-privacy-diff"}, + {"id":"cron-weekly","kind":"source","source":"Docker/cron/docassemble-cron-weekly.sh","target":"/etc/cron.weekly/docassemble","mode":"0755"}, + {"id":"cron-monthly","kind":"source","source":"Docker/cron/docassemble-cron-monthly.sh","target":"/etc/cron.monthly/docassemble","mode":"0755"}, + {"id":"sync","kind":"source","source":"Docker/sync.sh","target":"{{DA_ROOT}}/webapp/sync.sh","mode":"0755"}, + {"id":"initialize","kind":"source","source":"Docker/initialize.sh","target":"{{DA_ROOT}}/webapp/initialize.sh","mode":"0755","existing":"apply-privacy-diff"}, + {"id":"app-logger","kind":"source","source":"docassemble_webapp/docassemble/webapp/log_initialize.py","target":"{{SITE_PACKAGES}}/docassemble/webapp/log_initialize.py","mode":"0644"}, + {"id":"uwsgi-template","kind":"source","source":"Docker/config/docassemble.ini.dist","target":"{{DA_ROOT}}/config/docassemble.ini.dist","mode":"0644"}, + {"id":"uwsgilog-template","kind":"source","source":"Docker/config/docassemblelog.ini.dist","target":"{{DA_ROOT}}/config/docassemblelog.ini.dist","mode":"0644"}, + {"id":"uwsgi-exposed","kind":"source","source":"Docker/config/docassemble-expose-uwsgi.ini","target":"{{DA_ROOT}}/config/docassemble-expose-uwsgi.ini","mode":"0644"}, + {"id":"uwsgilog-exposed","kind":"source","source":"Docker/config/docassemblelog-expose-uwsgi.ini","target":"{{DA_ROOT}}/config/docassemblelog-expose-uwsgi.ini","mode":"0644"}, + {"id":"nginx-main","kind":"source","source":"Docker/nginx.conf","target":"/etc/nginx/nginx.conf","mode":"0644"}, + {"id":"nginx-lifecycle","kind":"source","source":"Docker/privacy/nginx-lifecycle.conf","target":"/usr/local/lib/docassemble-privacy/nginx-lifecycle.conf","mode":"0644"}, + {"id":"nginx-realip","kind":"source","source":"Docker/config/nginx-realip","target":"{{DA_ROOT}}/config/nginx-realip","mode":"0644","existing":"apply-privacy-diff","patch":"Docker/privacy/nginx-realip.patch"}, + {"id":"nginx-http","kind":"source","source":"Docker/config/nginx-http.dist","target":"{{DA_ROOT}}/config/nginx-http.dist","mode":"0644","existing":"preserve-and-validate"}, + {"id":"nginx-ssl","kind":"source","source":"Docker/config/nginx-ssl.dist","target":"{{DA_ROOT}}/config/nginx-ssl.dist","mode":"0644","existing":"preserve-and-validate"}, + {"id":"nginx-log","kind":"source","source":"Docker/config/nginx-log.dist","target":"{{DA_ROOT}}/config/nginx-log.dist","mode":"0644","existing":"preserve-and-validate"}, + {"id":"nginx-redirect","kind":"source","source":"Docker/config/nginx-redirect.dist","target":"{{DA_ROOT}}/config/nginx-redirect.dist","mode":"0644","existing":"preserve-and-validate"}, + {"id":"nginx-ssl-redirect","kind":"source","source":"Docker/config/nginx-ssl-redirect.dist","target":"{{DA_ROOT}}/config/nginx-ssl-redirect.dist","mode":"0644","existing":"preserve-and-validate"}, + {"id":"supervisor","kind":"source","source":"Docker/docassemble-supervisor.conf","target":"/etc/supervisor/conf.d/docassemble.conf","mode":"0644"}, + {"id":"syslog-collector","kind":"source","source":"Docker/syslog-ng.conf","target":"{{DA_ROOT}}/webapp/syslog-ng.conf","mode":"0644"}, + {"id":"syslog-forwarder-base","kind":"source","source":"Docker/syslog-ng-docker.conf","target":"{{DA_ROOT}}/webapp/syslog-ng-docker.conf","mode":"0644"}, + {"id":"syslog-forwarder","kind":"source","source":"Docker/docassemble-syslog-ng.conf","target":"{{DA_ROOT}}/webapp/docassemble-syslog-ng.conf","mode":"0644"}, + {"id":"logrotate","kind":"source","source":"Docker/docassemble.logrotate","target":"/etc/logrotate.d/docassemble","mode":"0644"}, + {"id":"logrotate-callback","kind":"source","source":"Docker/restart-post-logrotate.sh","target":"{{DA_ROOT}}/webapp/restart-post-logrotate.sh","mode":"0755"} + ], + "states": [ + {"id":"uwsgi-active","target":"{{DA_ROOT}}/config/docassemble.ini","kind":"rendered-file","sources":["uwsgi-template"],"rollback":"exact-prior-state"}, + {"id":"uwsgilog-active","target":"{{DA_ROOT}}/config/docassemblelog.ini","kind":"rendered-file","sources":["uwsgilog-template"],"rollback":"exact-prior-state"}, + {"id":"nginx-http-active","target":"/etc/nginx/sites-available/docassemblehttp","kind":"rendered-file","sources":["nginx-http"],"rollback":"exact-prior-state"}, + {"id":"nginx-ssl-active","target":"/etc/nginx/sites-available/docassemblessl","kind":"rendered-file","sources":["nginx-ssl"],"rollback":"exact-prior-state"}, + {"id":"nginx-log-active","target":"/etc/nginx/sites-available/docassemblelog","kind":"rendered-file","sources":["nginx-log"],"rollback":"exact-prior-state"}, + {"id":"nginx-redirect-active","target":"/etc/nginx/sites-available/docassembleredirect","kind":"rendered-file","sources":["nginx-redirect"],"rollback":"exact-prior-state"}, + {"id":"nginx-ssl-redirect-active","target":"/etc/nginx/sites-available/docassemblesslredirect","kind":"rendered-file","sources":["nginx-ssl-redirect"],"rollback":"exact-prior-state"}, + {"id":"nginx-http-link","target":"/etc/nginx/sites-enabled/docassemblehttp","kind":"role-selected-link","sources":["nginx-http"],"rollback":"exact-prior-state"}, + {"id":"nginx-ssl-link","target":"/etc/nginx/sites-enabled/docassemblessl","kind":"role-selected-link","sources":["nginx-ssl"],"rollback":"exact-prior-state"}, + {"id":"nginx-log-link","target":"/etc/nginx/sites-enabled/docassemblelog","kind":"role-selected-link","sources":["nginx-log"],"rollback":"exact-prior-state"}, + {"id":"nginx-redirect-link","target":"/etc/nginx/sites-enabled/docassembleredirect","kind":"role-selected-link","sources":["nginx-redirect"],"rollback":"exact-prior-state"}, + {"id":"nginx-ssl-redirect-link","target":"/etc/nginx/sites-enabled/docassemblesslredirect","kind":"role-selected-link","sources":["nginx-ssl-redirect"],"rollback":"exact-prior-state"}, + {"id":"syslog-active","target":"{{DA_ROOT}}/syslogng/syslog-ng.conf","kind":"existing-or-role-selected-file","sources":["syslog-collector","syslog-forwarder-base"],"rollback":"exact-prior-state"}, + {"id":"syslog-include","target":"/etc/syslog-ng/conf.d/docassemble.conf","kind":"role-selected-file","sources":["syslog-forwarder"],"rollback":"exact-prior-state"}, + {"id":"syslog-link","target":"/etc/syslog-ng/syslog-ng.conf","kind":"role-selected-link","sources":["syslog-collector","syslog-forwarder-base"],"rollback":"exact-prior-state"}, + {"id":"certificate-marker","target":"/etc/letsencrypt/da_using_lets_encrypt","kind":"role-selected-file","sources":["run-nginx"],"rollback":"exact-prior-state"}, + {"id":"logger-bytecode","target":"{{SITE_PACKAGES}}/docassemble/webapp/__pycache__","kind":"logger-bytecode-only","sources":["app-logger"],"rollback":"exact-prior-state"}, + {"id":"uwsgi-directory","target":"/var/run/uwsgi","kind":"runtime-directory","sources":[],"rollback":"metadata-only-no-stale-runtime-files"}, + {"id":"helper-directory","target":"/usr/local/lib/docassemble-privacy","kind":"owned-directory","sources":["nginx-lifecycle"],"rollback":"directory-metadata-only"}, + {"id":"celery-log","target":"{{DA_ROOT}}/log/privacy-celery.log","kind":"counter-file","sources":["supervisor"],"rollback":"metadata-only-preserve-log-content"}, + {"id":"celerysingle-log","target":"{{DA_ROOT}}/log/privacy-celerysingle.log","kind":"counter-file","sources":["supervisor"],"rollback":"metadata-only-preserve-log-content"}, + {"id":"uwsgi-log","target":"{{DA_ROOT}}/log/privacy-uwsgi.log","kind":"counter-file","sources":["supervisor"],"rollback":"metadata-only-preserve-log-content"}, + {"id":"uwsgilog-log","target":"{{DA_ROOT}}/log/uwsgilog.log","kind":"counter-file","sources":["supervisor"],"rollback":"metadata-only-preserve-log-content"}, + {"id":"nginx-log-file","target":"{{DA_ROOT}}/log/nginx-safe.log","kind":"counter-file","sources":["supervisor"],"rollback":"metadata-only-preserve-log-content"}, + {"id":"websockets-log","target":"{{DA_ROOT}}/log/privacy-websockets.log","kind":"counter-file","sources":["supervisor"],"rollback":"metadata-only-preserve-log-content"}, + {"id":"monitor-log","target":"{{DA_ROOT}}/log/privacy-monitor.log","kind":"counter-file","sources":["supervisor"],"rollback":"metadata-only-preserve-log-content"} + ], + "protected": [ + {"id":"mail-shell","target":"{{DA_ROOT}}/webapp/process-email.sh","policy":"hash-and-metadata-unchanged"}, + {"id":"mail-processor","target":"{{SITE_PACKAGES}}/docassemble/webapp/process_email.py","policy":"hash-and-metadata-unchanged"}, + {"id":"exim","target":"/etc/exim4","policy":"tree-and-metadata-unchanged"}, + {"id":"user-config","target":"{{DA_ROOT}}/config/config.yml","policy":"hash-and-metadata-unchanged"}, + {"id":"uploads","target":"{{DA_ROOT}}/files","policy":"preserve-contents"}, + {"id":"historical-logs","target":"{{DA_ROOT}}/log","policy":"preserve-contents"}, + {"id":"certificates","target":"/etc/ssl/docassemble","policy":"tree-and-metadata-unchanged"}, + {"id":"letsencrypt-certificates","target":"/etc/letsencrypt/live","policy":"tree-and-metadata-unchanged"}, + {"id":"nginx-includes","target":"/etc/nginx/conf.d","policy":"tree-and-metadata-unchanged"}, + {"id":"nginx-modules","target":"/etc/nginx/modules-enabled","policy":"tree-and-metadata-unchanged"}, + {"id":"forma-site","target":"/etc/nginx/sites-available/formapauperis","policy":"hash-and-metadata-unchanged"}, + {"id":"forma-site-link","target":"/etc/nginx/sites-enabled/formapauperis","policy":"link-and-metadata-unchanged"}, + {"id":"legacy-form-rewrites","target":"/etc/nginx/form_rewrite_rules.conf","policy":"hash-and-metadata-unchanged"}, + {"id":"apache","target":"/etc/apache2","policy":"tree-and-metadata-unchanged"} + ], + "requirements": [ + "verify-target-standard-root-and-python-3.14-layout", + "resolve-active-package-path-and-existing-volume-mounts", + "snapshot-existence-type-content-hash-owner-group-mode-and-symlink-target", + "keep-snapshot-private-and-verify-restore-before-cutover", + "stop-ingress-and-producers-drain-workers-stop-services-monitor-last", + "precreate-all-counter-files-www-data-0600-before-any-service-start", + "never-run-initialize-as-installer-or-use-bulk-package-and-directory-copies", + "render-only-listed-configurations-from-validated-target-inputs", + "preserve-existing-nginx-templates-and-nonprivacy-initializer-lines", + "record-and-preserve-target-service-states-unless-explicitly-required-for-privacy", + "compare-active-syslog-and-includes-before-any-role-selected-replacement", + "invalidate-only-log-initialize-bytecode-and-preserve-other-package-files", + "preserve-mail-sections-inside-shared-configurations", + "validate-effective-nginx-uwsgi-supervisor-and-syslog-configurations", + "start-monitor-first-backends-next-ingress-only-after-acceptance", + "verify-protected-paths-unchanged-and-preserve-log-content-on-rollback", + "require-complete-installation-rollback-and-exact-commit-ci-evidence" + ] +} diff --git a/docs/privacy/installation-plan.md b/docs/privacy/installation-plan.md new file mode 100644 index 000000000..aa4d8280d --- /dev/null +++ b/docs/privacy/installation-plan.md @@ -0,0 +1,153 @@ +# JOS-81 installation and rollback inventory + +GitHub tracking: [JOS-81, issue 22](https://github.com/jacobyoby/docassemble/issues/22). + +Status: the [explicit file catalog](install-catalog.json) is checked by the +existing privacy test suite. It lists 36 required files, 26 rollback/state requirements +and 14 protected paths. It is a file list, not a new release framework or an +installer. [Native CI at c09aba80c](https://github.com/jacobyoby/docassemble/actions/runs/34011607496) +passed the 35-file installation, real queue/cron, maintenance, shutdown/restore, +interrupted-start protection and rollback. The additional one-line migration +of customized `nginx-realip` and native escape parsing need their own candidate +CI. Production has not been changed. The +[release review](review-2026-09-05.md) still blocks deployment. This inventory +does not resume the [deferred Go mail replacement](deferred-mail-go.md). + +Both installed daily cron scripts differ from the fork source. Apply only the +privacy diff to those existing bytes, just as for the customized initializer; +do not replace their unrelated backup code. + +The native amd64 CI job runs `bash tests/privacy_native/test_install_image.sh` +against the exact image digest recorded in the target inventory. It boots a +fresh volume with synthetic data, installs only the non-mail catalog, and checks +custom routes, real application responses, bounded counters, protected files, +and rollback metadata/service states. It does not publish an image or deploy. +The local rehearsal reached a working stock application and custom routes, but +GNU tar failed because both local user-mode emulators lack `openat2`. No candidate +files were installed in that attempt and its partial archive is not rollback +evidence. The first native CI run on PR #23 passed installation, real/custom +routes, counter validation and protected-file checks, but tar rejected restoring +the `/var/run/uwsgi` directory through the image's `/var/run` link. The rehearsal +now restores directory metadata explicitly and archives only files/links. +Queue and interview-cron acceptance subsequently passed on a32541b4d. +The maintenance extension passed whole-container stop/start before rollback +at c09aba80c. The native tests must pass again for each final candidate. + +Existing `tools/deployCore.sh` copies in the NJForms release worktrees export +and install only the base/webapp packages, back up those two package directories +and touch the WSGI entrypoint. They do not deliver the complete native boundary. +Their `.git` directory check also rejects linked worktrees. Those other +worktrees were inspected read-only and are unchanged. + +## Coherent non-mail installation set + +| Source in this worktree | Required installed state | +| --- | --- | +| `Docker/privacy-diagnostic/` | Four architecture-matched binaries in `${DA_ROOT}/webapp`: `privacy-diagnostic`, `privacy-process`, `privacy-preflight`, `privacy-monitor`; root-owned, mode 0755 | +| `Docker/run-nginx.sh`, `run-uwsgi.sh`, `run-uwsgilog.sh`, `run-celery.sh`, `run-celery-single.sh`, `run-websockets.sh`, `run-cron.sh`, `initialize.sh` | Corresponding installed scripts under `${DA_ROOT}/webapp` | +| `Docker/cron/docassemble-cron-*.sh`, `Docker/sync.sh` | Four installed `/etc/cron.{hourly,daily,weekly,monthly}/docassemble` scripts and `${DA_ROOT}/webapp/sync.sh` | +| `docassemble_webapp/docassemble/webapp/log_initialize.py` | Current comments-only module in the actual service virtualenv, installed together with native capture | +| Four `Docker/config/docassemble*.ini*` profiles/templates | Input templates and correctly rendered active profiles under `${DA_ROOT}/config` | +| `Docker/nginx.conf`, `Docker/privacy/nginx-lifecycle.conf`, nginx site templates | `/etc/nginx/nginx.conf`, `/usr/local/lib/docassemble-privacy/nginx-lifecycle.conf`, rendered sites and enabled symlinks | +| `Docker/docassemble-supervisor.conf` | `/etc/supervisor/conf.d/docassemble.conf`, with capture, monitor, stream ownership and shutdown waits | +| `Docker/docassemble-syslog-ng.conf`, `Docker/syslog-ng.conf` | Installed inputs under `${DA_ROOT}/webapp`; preserve existing system syslog configuration or select collector/forwarder only when the verified role requires it; active path `${DA_ROOT}/syslogng/syslog-ng.conf` and forwarding include `/etc/syslog-ng/conf.d/docassemble.conf` | +| `Docker/docassemble.logrotate` and existing rotation callback | Reviewed rotation ownership in `/etc/logrotate.d/docassemble` and matching installed callback | + +The catalog includes the unchanged `syslog-ng-docker.conf` forwarding base as +a required input. It covers collector/forwarder inputs, preservation of existing +system syslog configuration, the main configuration symlink and forwarding-include +absence/presence. It also names nginx enabled +links, the certificate marker, the affected logger bytecode only, runtime-directory +metadata and all seven Supervisor counter files. Runtime sockets and PID files +must be recreated by clean starts; historical logs must retain their contents. + +`DA_ROOT` and `SITE_PACKAGES` in the catalog describe the standard source +layout. The actual service virtualenv, active package path and volume mounts +must be resolved before using it. Current source profiles contain standard +absolute paths, so this catalog does not establish support for a custom root. +The existing test suite checks source existence, protected-path exclusion, +Supervisor command/counter coverage and actual nginx/uWSGI generated paths. +Per-architecture source hashes are review evidence, not target-installation proof. + +The [read-only target inventory](target-inventory-2026-09-05.md) now confirms +the standard root, Python 3.14 virtualenv, active uWSGI package path and persistent +volume mounts on both running containers. It also found target-specific nginx +configuration. The catalog's `existing: preserve-and-validate` entries are +required rendering inputs: retain their existing bytes and metadata, then +validate the resulting effective configuration. Use the source template only +when the target is absent. Do not overwrite a customized template just because +it differs from the source hash. Unsafe retained configuration still blocks +installation; this policy is not permission to bypass preflight. + +For `existing: apply-privacy-diff`, apply only this candidate's privacy changes +to the snapshotted target initializer, daily cron script and `nginx-realip`. Preserve unrelated target lines. Record +the actual resulting hash and inspect the diff before installation; a source +hash alone cannot verify that result. Other entries require reviewed replacement +and the listed ownership/mode. These are review instructions for the existing +copy/patch tools, not an implemented installer. + +The `nginx-realip` entry names an explicit patch because the installed files +contain extensive custom routes absent from the five-line source default. +Require exactly one known legacy `access_log /var/log/nginx/access.log privacy;` +line, apply the patch without fuzz and verify that removing this line is the +entire byte difference. Main HTTP logging then supplies the bounded counter +format. Snapshot and restore the complete original file and metadata; never +install the source default over these customized files. Retain the unused +legacy format definition. Other unsafe directives still fail preflight. + +`initialize.sh` renders the main and log uWSGI profiles; `run-nginx.sh` renders +the site configurations. Read-only mode skips generation. Existing volumes +can mask image-installed binaries, templates and Python packages. Exposed +profiles and Supervisor paths still assume `/usr/share/docassemble`; reconcile +actual runtime paths explicitly before any custom-root installation. + +## Cutover and rollback requirements + +1. Create a manifest and rollback snapshot covering binaries, scripts, package + state, templates, rendered files, symlink targets, ownership and permissions. + Preserve application data, user configuration, certificates and historical + logs outside broad replacement or deletion operations. +2. Quiesce ingress and scheduled producers, stop the websocket/front end, drain + workers and stop uWSGI services. Keep the monitor until affected services + have stopped. Stop NLTK before a `main` group update as well: the existing + initializer removes its socket before starting it, so an already-running + NLTK process would leave startup waiting on an unlinked socket. Rehearse + this order with actual service roles and deadlines. + Precreate and assign ownership to every counter file before restarting any + service. Current initialization starts Celery before its later ownership + block, so that block cannot serve as the installer's pre-start guarantee. + Record actual service states: Forma's `syslogng` is currently stopped, while + demo's is running. Their active file uses system/internal sources and differs + from both shipped templates. Compare its includes and required role before + replacing it. Do not infer the required startup set from `role=all` alone. +3. Install the coherent set while services are stopped. Validate hashes, ELF + architecture, executable ownership, actual virtualenv paths and generated + configurations. Do not invoke `initialize.sh` as a generic installer: it also + performs unrelated initialization, backup and restore operations. +4. Start and verify the monitor, then applicable backends/workers and uWSGI. + Confirm sockets and health before nginx reopens ingress. Exercise complete + synthetic service, failure, output-retention and rotation acceptance. +5. Restore the entire manifest on failure and repeat acceptance. A package-only + rollback can restore raw file logging. Keep ingress closed if the restored + installation does not meet the selected privacy acceptance requirements. + +## Unresolved artifact and retention boundaries + +- The Dockerfile bulk-copies shell scripts and installs the whole webapp package. + This would include the preserved `process-email.sh` and `process_email.py` + capture edits. A non-mail release must explicitly define and verify those + exclusions; the deferral note does not exclude bytes automatically. Shared + Go components can remain intact without activating deferred mail work. +- [Rotation ownership is repaired](rotation-ownership.md): four local + `privacy-*.log` files belong to Supervisor, while the legacy collector files + retain logrotate. Install the matching forwarding input and exact Celery + collector filter with these paths. The precreation/ownership pass must precede + service startup. `uwsgilog` uses Supervisor rotation and needs no logrotate + callback entry. Existing legacy callbacks can still restart application + services; their full installed behavior remains an acceptance requirement. +- Frozen Python references and all test-only fixture dependencies must remain + outside production artifacts. Cron output, independent file writes, log + copies, backup/restore and monitor-sink health still require full coverage. +- A full image build alone cannot prove installation over the existing volume. + Exact candidate commit/CI, complete installation and rollback evidence remain + mandatory release gates. diff --git a/docs/privacy/mail-capture-go.md b/docs/privacy/mail-capture-go.md new file mode 100644 index 000000000..18f9ad552 --- /dev/null +++ b/docs/privacy/mail-capture-go.md @@ -0,0 +1,111 @@ +# Mail command capture through Go + +Status: implemented and locally verified in the uncommitted JOS-81 candidate; +**not deployed**. This protects the mail command's bootstrap and captured +streams. Actual Exim delivery, application storage, and retained Exim logs have +not passed release acceptance. + +The user subsequently deferred the Go replacement of the underlying mail +processor. Its findings and restart boundary are saved in +[the deferred work note](deferred-mail-go.md); this capture candidate remains +preserved and unreleased. + +## Command boundary + +`Docker/process-email.sh` checks activation and replaces itself with +`privacy-process --component mail -- /absolute/python -m +docassemble.webapp.process_email /dev/stdin`. The message stays on inherited +stdin. The shell no longer spools a second raw message through `mktemp`, and +its final cleanup command no longer hides the processor's exit status. + +The existing Python processor no longer opens or writes `/tmp/mail.log`. +Its MIME, database, storage, and task code is otherwise unchanged; no new +executable production Python or dependency was added. Removing diagnostic +writes does not remove the intended operational message and attachment data. + +The Go mail profile inherits the runner's descriptor sealing, core limit, +locked-thread Linux parent-death protection, bounded stream buffers, and +process-group teardown. It alone passes stdin to its child. Mail uses a +20-second master grace window plus one second for bounded teardown. + +Mail emits at most **one final counter record**, below the existing 8,192-byte +record limit. Service profiles retain their periodic reporting. Each mail line +uses the application counter rules; request-looking text cannot become request +metrics. Raw message contents, recipient addresses, exceptions, and paths are +never counter fields. Bootstrap failures emit a single fixed startup record +when the diagnostic helper and sink work. + +The successful command returns zero. Startup, invocation, child, capture, sink, +and teardown failures return 75. A received SIGINT or SIGTERM also returns 75, +even if the child handles shutdown and returns zero. Hard-killing the launcher +or failure to execute it is outside that exit-mapping boundary. + +Exim's [pipe transport contract](https://www.exim.org/exim-html-current/doc/html/spec_html/ch-the_pipe_transport.html) +explains the constraints: stock temporary statuses include 75, combined command +output defaults to a 20 KiB maximum, and `return_output` can reject successful +commands that produce output. Final-only reporting prevents periodic counters +from eventually crossing the output limit. It does not alter Exim's timeout, +transport options, or retained logging policy. + +## Verification + +Run `bash tests/verify_privacy.sh`; on macOS also run the Linux Go suite. +The September 5 mail-cycle results include: + +- Canonical formatting, vet, race tests, four bounded fuzz targets, module + verification, four host/Linux amd64/arm64 executable builds, shell syntax, + 131 native/launcher Python tests, and 84 unchanged frozen-reference tests. +- Full Linux race suite and vet; a fresh Linux-targeted `govulncheck@v1.7.0` + scan found no vulnerabilities in the Go candidate. +- Actual mail shell and Go CLI with 1.25 MiB binary stdin preserved by SHA-256, + successful synthetic processing, native exit 17, read/unknown-recipient, + database, and broker failures; rejected regular-file sinks and missing tools. +- Public CLI child-signal and interrupted-clean-child tests, including SIGINT + and SIGTERM; broken and full pipe sinks return 75 within the deadline. + Three reporting intervals and more than 20 KiB of raw child output produce + one final record with all 3,000 lines counted. This is not a 90-second test. +- The real application logging boundary across 80 profile/context combinations, + plus mail worker-drain and parent-death fixtures. Existing real Supervisor + service acceptance remains separate from mail, which is MTA-invoked. + +The mail processor fixture runs the real module as `__main__` and observes its +configuration-load call. Database, storage, configuration, and broker interfaces +are synthetic. It verifies MIME values and interface calls, **not actual +attachment persistence, transaction ordering, or broker delivery**. + +Fail-first evidence caught the old independent log, swallowed processor exits, +periodic mail records, and interrupted delivery falsely returning success. +Source copies and logs live in `tests/.privacy-build/mail-capture-review/`. +Three independent read-only reviews covered Go lifecycle, processor contracts, +and Exim integration; one writer owns the candidate worktree. + +## Unresolved release gates + +The review exposed existing processor defects that the synthetic interfaces +cannot prove safe: + +1. Decoded MIME payloads are bytes, while `save_attachment` calls + `SavedFile.write_content` without its binary option. The actual method's + default text writer raises `TypeError`. Run + `TMPDIR="$PWD/tests/.privacy-build/tmp" python3.14 -B + tests/privacy_native/probe_mail_storage.py` from the checkout root. This + separate probe intentionally remains nonzero; its textual control passes + before decoded bytes fail. The method is extracted from real source and + uses in-memory handles, with no application import or storage writes. +2. The processor consumes the new email ID before an explicit session flush + and queues work before the surrounding transaction commits. The synthetic + fixture's immediate ID and task ledger do not establish correct linkage or + ordering. Real database/broker tests, retry replay, and duplicate processing + acceptance remain required. +3. The candidate inherits `address_pipe` from its unpinned base image. Inspect + `exim4 -bP transport address_pipe` and + `exim4 -bP log_file_path log_selector` inside the exact image. Verify exit-75, + timeout and missing-launcher behavior, execution as `www-data`, configured + runtime paths, and writable/read-only initialization. The stock transport + discards successful/deferred command output: these counters are not yet a + retained mail-health record. Exim main/reject/panic logs remain separate. + +The complete image, persistent-volume installation, rollback, cron coverage, +retained log/copy/restore acceptance, and exact-commit CI remain required by +the [release review](review-2026-09-05.md). No production service, volume, +configuration, log, or mail queue was changed during this cycle. diff --git a/docs/privacy/maintenance-capture-go.md b/docs/privacy/maintenance-capture-go.md new file mode 100644 index 000000000..1827f9984 --- /dev/null +++ b/docs/privacy/maintenance-capture-go.md @@ -0,0 +1,69 @@ +# Maintenance and initialization capture + +The candidate reuses the Go runner for the four scheduled docassemble scripts, +`sync.sh`, the rotation callback and `initialize.sh`. A small re-exec prelude +enters capture before activation or configuration. Existing commands, role +checks, arguments and working directory are retained. No backup framework or +content filter is added. + +The internal `--privacy-captured` handoff prevents recursion. Supported callers +invoke the scripts normally without it; it is not an authorization boundary. +Missing capture executables fail before activation with a fixed diagnostic. +The finite `maintenance` profile emits one final counter snapshot and preserves +the script exit code. Its 30-second shutdown grace encloses the inner cron +runner's 20-second grace, one-second teardown and five-second sink deadline. +Supervisor allows 40 seconds for sync. + +Nested interview-cron records are counted as application text by the outer +runner. Their detailed phase/event labels are not forwarded. Legacy shell +exit behavior also remains: an early failing command can be followed by a +successful final command. This change does not claim per-command failure +detection or introduce blanket `set -e`. + +The `initialize` profile reports periodic counters. Its 610-second shutdown +grace accommodates PostgreSQL's existing 600-second budget and final cleanup; +Supervisor allows 630 seconds. The shared `main` group still stops initializer, +PostgreSQL and Redis together. The monitor accepts only the exact +`initialize/main` pair and emits fixed unexpected-exit/backoff/fatal records. +Other accepted process/group pairs remain unchanged. + +The native image rehearsal now covers: + +- Real installed hourly/daily/weekly/monthly and sync operations, local and + rolling log backups, and historical sentinel bytes. +- The actual non-mail logrotate stanza and callback in its configured cron + role context, preserved rotated bytes and expected worker PID changes. +- TERM through the outer monthly script into a real cron interview, plus + the existing inner TERM/INT checks. +- A real initializer startup module that checks root identity, working + directory and closed inherited fd3, emits stdout/stderr markers, and + writes a success receipt. A newer receipt is required after restart. +- Whole-container orderly shutdown, absent running/ready markers, a + shutdown-only backup sentinel, valid final initializer counters and + same-volume restart. The host inserts a backup-only file while stopped; + its later appearance in live logs proves the actual restore path ran. +- A one-shot startup hold in that same synthetic module, before the real + initializer installs its shutdown trap. Supervisor interrupts the captured + initializer, and its unfinished-start marker must survive. A new backup-only + file must remain absent from live logs on the next startup, proving the + existing unsafe-restore guard still applies. +- Preserved configuration, upload and database session state, working + queue/cron after restart, stopped-service reconciliation and rollback. + +The container's explicit 1000-second stop budget covers serial Supervisor +group shutdown; it is not just the initializer's timeout. Each main-group +update waits for initialization readiness before checking state or restoring +counter metadata. The old initializer's normal log-directory ownership pass +must finish first. + +[Native CI at c09aba80c](https://github.com/jacobyoby/docassemble/actions/runs/34011607496) +passes these lifecycle additions, including the full install and rollback job. +Historical +diagnostic content is preserved separately from new private-output markers. +Scans cover current, rotated, copied and local/rolling backup diagnostic +destinations; application documents and database dumps are not treated as logs. +Target checks establish local backups and disable S3/Azure. Conditional cloud, +Apache and explicit log-role paths, arbitrary third-party handlers and manual +bypasses are not proven by this fixture. Final candidate CI and target-specific +release checks remain open. Mail stays +deferred and its rotation stanza is excluded. diff --git a/docs/privacy/native-config.md b/docs/privacy/native-config.md new file mode 100644 index 000000000..0231b6271 --- /dev/null +++ b/docs/privacy/native-config.md @@ -0,0 +1,80 @@ +# Native configuration preflight + +Status: integrated into the JOS-81 candidate's three native launchers; **not deployed**. The standard-library Go CLI at `/usr/share/docassemble/webapp/privacy-preflight` never prints configuration, paths, command output, exception details, or a success message. Exit `0` means accepted; every rejection or operational failure returns `70`. + +``` +privacy-preflight uwsgi CONFIG_PATH +privacy-preflight nginx +``` + +The dedicated Linux process disables core dumps and seals inherited descriptors beyond standard input/output/error. Other platforms reject invocation. The uWSGI command opens the path nonblocking, rejects nonregular files before reading, and reads at most 1 MiB plus one overflow byte. Its exact option-name allowlist comes from the four maintained `Docker/config/docassemble*.ini*` templates. It rejects duplicate options/sections, other sections/options, interpolation, unrendered placeholders, arbitrary modules/mount targets, and noncanonical values. `master = true`, `die-on-term = true`, and `log-format = PRIVACY_REQUEST status=%(status) msecs=%(msecs)` are mandatory. This deliberately does not accept the full uWSGI configuration language. The launcher separately clears `UWSGI_*` environment overrides and must use the checked file without an intervening mutation. + +The nginx command executes only `/usr/sbin/nginx -T -e stderr`. Both output streams count toward a combined 1 MiB limit; only stdout is retained for parsing. Execution has a 10-second deadline and at most one additional second to finish readers. A nonzero exit, overflow, timeout, malformed dump, or unsupported syntax rejects startup. Captured output is never replayed. The child runs in a new session with Linux parent-death protection on a locked OS thread; set-id and file-capability executables are rejected. The subprocess group is killed on cancellation and after leader exit, including descendants that retain capture pipes. + +The parser handles quoted strings, comments, braces, semicolons and `${variable}` tokens. It resolves the dump's file-header sections and include graph without reading another file, preserving each include's native context. Repeated includes in separate contexts and identical repeated file sections are supported. Missing exact includes, unreachable sections, conflicting repeated sections, include cycles, and ambiguous/unsupported syntax reject. Backslash escapes and partial quoted/unquoted token concatenation are intentionally unsupported. Include globs cannot cross directory boundaries. A `#` embedded in an unquoted token remains part of that token. The parser limits each file to 100,000 tokens and 63 nested blocks, and include expansion to 100,000 visited directives. Invalid UTF-8, disallowed control bytes, and input over 1 MiB reject before parsing. + +Every explicit `access_log` must be `off` or `/dev/stdout privacy_counts`; no options or conditional logging are accepted. Every explicit `error_log` must be `stderr`, with at most one standard severity. Each context permits at most one of each. The main context requires `error_log stderr` and `worker_shutdown_timeout 2s`. Exactly one HTTP context must define the exact one-string format `PRIVACY_REQUEST status=$status seconds=$request_time` as `privacy_counts`, plus the `/dev/stdout privacy_counts` baseline. Local `access_log off` is permitted. Safe directives inside a server/location cannot supply missing global baselines. + +Explicit `master_process` must be `on`; explicit `daemon` must be `off`, both only in the main context and without duplicates. These may be absent: the maintained launcher supplies `-g 'daemon off;'` and relies on nginx's normal master-process default. The validator does not alter launch arguments, log files, configuration, environment, or retention settings. + +The check covers native access/error routing in the complete tested dump. It does not prove arbitrary third-party module code, Python output, historical logs, or downstream copies safe. No custom include is exempt from inspection; deployment must also prevent config/module changes between preflight and launch. Native `-T` remains a real executable/configuration test, not a side-effect-free parser. Its behavior and the actual launcher remain part of disposable-image acceptance. + +The Dockerfile builds and installs the preflight, process runner, process monitor, and startup +diagnostic as static, root-owned executables with mode 0755. Only the nginx +lifecycle data file is copied from `Docker/privacy`. All four former native +Python helpers are frozen test references under `tests/privacy_native/reference`; +none is installed as a production helper. + +## Verification and limits + +Run the focused suite from the checkout root without installing the runtime or +setting `PYTHONPATH`: + +``` +bash tests/verify_privacy.sh +``` + +This checks all Go packages with formatting, vet, race tests, four bounded fuzz +targets, module verification, and host/Linux amd64/arm64 builds. It also runs 126 +native/application-launcher tests and 84 frozen application draft tests, shell +syntax, and diff checks. On Linux the [application logger tests](application-logger-go.md) +also exercise 64 combinations through the actual Go capture process. +The frozen preflight reference is checked against SHA-256 +`694f6a7bb916882234969544f994ba396efe16d2936631c94d5947762821f9ac`; +the Go decisions and 10,000 deterministic include-glob cases are compared with +that reference. The reference is evidence of preserved behavior, not independent +proof of native configuration correctness. + +Linux tests exercise actual Go preflight and process capture through both uWSGI +launchers, including standard, exposed, and log profiles. Unsafe selected INI +files and missing helpers stop before native execution. Synthetic capture tests +cover successful output, nonzero exits, stream/combined overflow, timeouts, and +orphan cleanup. The fixture launcher tests retain their original exec-failure +expectations after replacing the former Python preflight stub. These tests use +synthetic activation, application configuration, and native programs. + +A separate smoke check runs the compiled Go helper against real nginx `-T`: + +```sh +bash tests/privacy_native/test_nginx_preflight_image.sh EXISTING_IMAGE +``` + +Supply a local Linux amd64/arm64 image containing `/usr/sbin/nginx`. The script +never pulls an image and starts only configuration inspection, with no public +listener. It mounts synthetic configuration and the compiled helper read-only +in a disposable container with no network, no capabilities, no-new-privileges, +and a temporary writable `/tmp`. Both fixtures must first pass native `-T` +inspection, so rejection cannot be attributed to a syntax or installation +failure. The Go helper must then return `0` for the safe include and `70` for an +unsafe access-log destination. Both helper paths must emit zero bytes. +These checks passed with the cached docassemble image's nginx 1.28.3. Full +Linux race tests, Linux vet, and a Linux-targeted `govulncheck@v1.7.0` scan also +passed; that scan found no Go vulnerabilities. The [integration record](preflight-integration-2026-09-05.md) +records evidence and the remaining release gates. + +The validator itself remains silent. Its launcher now reports failed preflight +through the [bounded startup diagnostic](startup-diagnostics.md), with no raw +validator output. These checks do not boot uWSGI, Supervisor, or the complete +application, install into a persistent volume, or prove monitoring and rollback. +See the [release review](review-2026-09-05.md) for those open gates and the +[Go runner](native-runner-go.md) for the subsequent process-capture boundary. diff --git a/docs/privacy/native-runner-go.md b/docs/privacy/native-runner-go.md new file mode 100644 index 000000000..b8ef6b3d3 --- /dev/null +++ b/docs/privacy/native-runner-go.md @@ -0,0 +1,135 @@ +# Go foreground native runner + +Status: integrated into the JOS-81 candidate's nginx, uWSGI, log-role, Celery, +single-queue Celery, websocket, and mail launchers; **not deployed**. Process and launcher tests have run on Linux, +including the race detector. [Real uWSGI acceptance](native-runtime-2026-09-05.md) +now covers all four profiles with synthetic application/configuration modules. +A complete docassemble image, full native/application services, persistent-volume +installation, monitoring, and rollback still need acceptance before release. + +The standard-library Go executable at +`/usr/share/docassemble/webapp/privacy-process` replaces the unreleased Python +process wrapper and parent-death helper. It imports the +[Go aggregation library](aggregation-go.md). The Dockerfile builds all four Go +executables as static binaries and installs them root-owned, mode 0755. It +copies only the nginx lifecycle configuration from `Docker/privacy`. The +[Go preflight](native-config.md) validates native configuration before capture. +All four superseded Python helper files have moved unchanged +to `tests/privacy_native/reference`; they are no longer production runtime +dependencies. The canonical test suite still checks the frozen counter +algorithm against its pinned SHA-256. + +## Invocation and lifecycle + +```text +privacy-process --component nginx -- /usr/sbin/nginx -e stderr -g 'daemon off;' +privacy-process --component uwsgi -- /absolute/uwsgi --ini /absolute/profile --die-on-term +privacy-process --component celery -- /absolute/celery -A docassemble.webapp.worker worker +privacy-process --component websockets -- /absolute/python -u -m docassemble.webapp.socketserver +privacy-process --component mail -- /absolute/python -m docassemble.webapp.process_email /dev/stdin +``` + +The component, separator, absolute executable path, and NUL-free arguments are +validated before launch. uWSGI requires `--die-on-term`. Production invocation +has no bypass for parent-death protection and rejects non-Linux platforms. +Configuration validation remains a separate launcher step; the runner does +not claim arbitrary commands or configurations are safe native profiles. + +The runner permanently disables core dumps, seals descriptors beyond standard +input/output/error with close-on-exec, and starts a new child session. Linux +`PDEATHSIG` sends nginx `SIGQUIT`, or uWSGI/application services `SIGTERM`, if the runner dies. The +creating Go thread stays locked for the child lifecycle so thread retirement +cannot trigger premature parent-death signals. Set-id and file-capability +executables are rejected because those transitions can clear this protection. +Service binaries/configuration must remain immutable through validation and +exec; later credential changes and daemonization remain prohibited by the +maintained native profile. + +Service children receive `/dev/null` on stdin; mail alone inherits the message +stream. All children receive distinct stdout/stderr pipes. Two +readers use fixed 4 KiB buffers and a two-slot event channel. Aggregation occurs +on one goroutine. Captured bytes are never replayed or included in errors; +reader buffers, consumed events, and retained partial lines are cleared on +consumption or teardown. This bounds retained application payloads, not total +Go runtime RSS, kernel buffers, or every possible memory copy. + +Cumulative service snapshots are emitted initially, at most once per second +while running when input changed, and finally after both streams finish. Mail +emits at most one final record to bound total output to the MTA. One writer +may be active; no unbounded output queue exists. Output accepts only a pipe or +socket. Each complete record has a five-second write deadline; collection and +signal handling continue while the sink is blocked. A partial stream write may +leave a truncated final JSON record if the sink fails; it is never reported as +success and is never followed by a raw-output fallback. + +TERM/INT requests graceful native shutdown. HUP and USR1 are forwarded to the +master while running. The runner signals the process group if descendants +remain after master exit. Native profiles send group TERM/QUIT halfway through +a two-second grace window and group KILL at its end. Application profiles +allow the master its full 60-second Celery or 20-second websocket/mail grace window +before group escalation. All profiles allow one further second for teardown. +The [application capture record](application-capture-go.md) explains the service +shutdown and Supervisor margins. Escaped descendants retaining pipes cannot +extend the wait indefinitely; incomplete teardown returns failure. Native +exit codes are preserved; signal exits become 128 plus the signal number. +Mail maps every failure and interrupted delivery to temporary exit 75. + +| Exit | Meaning | +| --- | --- | +| 64 | Invalid invocation | +| 70 | Unsupported platform, setup, capture, or teardown failure | +| 74 | Invalid, broken, timed-out, or unsuccessfully closed output sink | +| 75 | Mail-only temporary failure, including interrupted delivery | + +Runner failures produce fixed exit statuses without error text. The candidate's +[Go process monitor](process-monitor-go.md) reports Supervisor failure states; +its installation and operational health checks remain release gates. A launcher unable to execute the binary reports its existing +bounded startup diagnostic for phase `launch`. + +## Verification and limits + +`bash tests/verify_privacy.sh` runs all applicable Go tests, formatting, vet, +fuzzing, host and Linux builds, 140 native/application-launcher Python tests, 84 frozen application +draft tests, shell syntax, and diff whitespace. On Linux it includes real process +lifecycle and launcher-to-Go capture tests under the race detector. On macOS +those Linux-specific tests must additionally run in a Linux environment. +`GORACE=atexit_sleep_ms=0` removes only the race runtime's artificial exit sleep, +which would interfere with shortened fixture shutdown deadlines; race +diagnostics and failure exits remain enabled. + +Local Linux checks used disposable containers with no network, a read-only +root filesystem, all capabilities dropped, no-new-privileges, and an init +reaper. Source was mounted read-only, with caches confined to this worktree. +The Linux Go 1.27.0 archive was verified against the official SHA-256 +`51798d2c42d0e1c6ed7fd9f48728b4193abac9e8aad6dbac2fe96a81f5909bda`. +The tests exercise real Linux pipes/processes/signals but use synthetic native +programs and bootstrap configuration; they do not boot docassemble services. + +Checks cover final counters and native exit codes, separate streams, malformed +and oversized output, regular-file/broken/full sink rejection, graceful signal +selection, HUP/USR1 forwarding, parent death, stubborn masters, orphan workers, +core limits, inherited descriptors, and shared sink-flag restoration. The +actual uWSGI and log-role shell launchers reach both Go preflight and process +CLIs and preserve the synthetic native exit code. Unsafe selected profiles and +missing preflight helpers stop before native launch. The nginx shell's argument routing remains covered +by the existing synthetic launcher test. An isolated mutation removing +descriptor sealing fails the descriptor-inheritance test with native exit 95. +Linux `go vet` and a Linux-targeted `govulncheck@v1.7.0` scan passed; the scanner +found no vulnerabilities. These checks cover the Go candidate, not the entire +docassemble dependency set or installed service image. + +Evidence is recorded in `tests/.privacy-build/runner-review/`, including test +logs, before-edit copies, the mutation control, security scan, and source +hashes. Earlier aggregator/startup evidence is historical and does not describe +the new source layout. The later [preflight integration record](preflight-integration-2026-09-05.md) +records current launcher checks and real nginx configuration inspection. +The later [application capture record](application-capture-go.md) covers Celery +and websocket integration, full worker grace windows, and real Supervisor +acceptance using synthetic runtime commands. +The [application logger replacement](application-logger-go.md) removes the +independent file handler and verifies the real base logger under Go capture +across 80 profile/context combinations. The [mail capture record](mail-capture-go.md) +documents stdin preservation, bounded total output, retry exits, and unresolved +real mail processing and transport acceptance. +The [release review](review-2026-09-05.md) retains the +remaining implementation, full-installation, monitoring, and release gates. diff --git a/docs/privacy/native-runtime-2026-09-05.md b/docs/privacy/native-runtime-2026-09-05.md new file mode 100644 index 000000000..6a767a5a0 --- /dev/null +++ b/docs/privacy/native-runtime-2026-09-05.md @@ -0,0 +1,83 @@ +# Real uWSGI acceptance — 2026-09-05 + +Status: local runtime acceptance passed for all four maintained uWSGI profiles. +The candidate is uncommitted and **not deployed**. This follow-up changes test +fixtures, CI wiring and documentation; production candidate bytes are unchanged. +The [Go mail replacement and Exim work remain deferred](deferred-mail-go.md). + +## Scope and evidence + +The fixture compiles real uWSGI 2.0.31, using the source archive digest from +[PyPI](https://pypi.org/project/uWSGI/2.0.31/#files): +`e8f8b350ccc106ff93a65247b9136f529c14bf96b936ac5b264c6ff9d0c76257`. +The test image uses Python 3.14.7 and the repository's existing setuptools +83.0.0 build-backend pin. Its local Linux arm64 image ID is +`sha256:7456f402a0bcf0806b17dcafd36535f10a8b0a68e986c1d72d3bb7d1a3cea5ab`. +These are test dependencies; no production dependency was added. + +The test renders the actual `docassemble.ini.dist`, `docassemblelog.ini.dist` +and both exposed-port profiles. Main, exposed and log roles use the actual +shell launchers with freshly built Go preflight, diagnostic and capture +binaries. No current launcher selects log-exposed, so that profile is checked +by direct Go preflight and runner invocation. Application and configuration +modules are synthetic; this is not a complete docassemble application boot. + +Each profile receives four real protocol requests: 200, 503, explicit 500 and +an uncaught WSGI exception. The native exception closes its connection without +response headers while recording status 500; a direct native control confirmed +that behavior. The synthetic application emits distinct private markers on +stdout, stderr and in its exception. Retained wrapper output must contain only +the exact fixed JSON schema, bounded integers, one 2xx and three 5xx requests, +four total latency observations, and captured unclassified output. No marker, +query value or synthetic interview filename may appear in retained output. + +Shutdown checks record the actual master and worker PIDs, their Linux start +times and isolated process group. After TERM, the wrapper must return zero and +the observed native processes must disappear. Cleanup only signals a verified +native group. Appending an unsafe `logto` directive must fail preflight with a +fixed startup record, before the requested file exists. + +An intentionally unsafe runner built from an ignored source copy bypassed the +capture pipes. The original acceptance run rejected it because request counters +remained zero. That control proves missing capture is detected; it did not +demonstrate a retained-marker failure. The production runner was never replaced. + +Evidence under `tests/.privacy-build/native-runtime-review/`: + +- `image-build.log`: successful native image build; the earlier missing-header + build failure is preserved separately. +- `native-control.log`: direct native behavior with synthetic data only. +- `capture-control.log`: intentionally unsafe capture failed acceptance. +- `uwsgi-reviewed.log`: all four profiles, latency/status accounting, native + cleanup and unsafe configuration rejection passed; image and Go binary hashes. +- `verification.log`: canonical repository verification for this follow-up. + +## Reproduction and bounds + +From the worktree root, with an isolated Docker runtime selected: + +```sh +bash tests/verify_privacy.sh +docker build -t privacy-candidate-check \ + -f tests/privacy_native/fixtures/privacy-check.Dockerfile tests/privacy_native/fixtures +docker build -t privacy-uwsgi-check \ + -f tests/privacy_native/fixtures/uwsgi-check.Dockerfile tests/privacy_native/fixtures +bash tests/privacy_native/test_uwsgi_image.sh privacy-uwsgi-check +``` + +The standalone acceptance script rebuilds all three mounted Go executables from +the current checkout; cached executable existence is insufficient. Per-run +artifacts remain in the ignored `tests/.privacy-build/uwsgi-run.*` directory. +Containers have unique names and bounded cleanup, no network, no Docker log +retention, a read-only root and source mounts, no capabilities, no privilege +escalation, 512 MiB memory and 128 PIDs. Temporary filesystems are bounded and +non-executable. The inner deadline is 45 seconds plus 5 seconds to kill; the +host command deadline is 55 seconds, followed by bounded removal verification. +The workflow now runs this fixture after its nginx/Supervisor checks. + +Review added actual process-removal assertions, latency totals, fresh builds and +container resource/deadline controls. Passing these checks does not establish +real Flask/Celery behavior, complete retained-output coverage, rotation, +persistent-volume installation, rollback or exact-commit remote CI. The +[installation inventory](installation-plan.md) and [release gates](review-2026-09-05.md) +remain open. No incoming mail was sent or processed by this fixture. diff --git a/docs/privacy/preflight-integration-2026-09-05.md b/docs/privacy/preflight-integration-2026-09-05.md new file mode 100644 index 000000000..aa1a9e564 --- /dev/null +++ b/docs/privacy/preflight-integration-2026-09-05.md @@ -0,0 +1,83 @@ +# Native preflight integration — 2026-09-05 + +Status: the Go configuration preflight is integrated and locally verified in +`fix/jos81-private-logging`, based on +`63d22c44b7bd008a521f0bcb8ab442b2143ee557`. **The candidate is uncommitted and +not deployed.** This record covers the native preflight and its launcher +integration; it does not close the [full release gates](review-2026-09-05.md). + +## Changes and review findings + +The three native launchers call `privacy-preflight` before `privacy-process`. +The Dockerfile builds all three standard-library Go executables and installs +them root-owned, mode 0755. Its native helper data directory now contains only +the nginx lifecycle configuration. The former Python preflight moved unchanged +to `tests/privacy_native/reference/check_native_config.py`; all four replaced +native Python helpers are now test references. The new application Python +logging draft remains an unresolved implementation requirement. + +The interrupted integration left three launcher exec-failure tests failing: +their injection still lived in the retired Python preflight fixture. Moved that +injection into the Go-helper fixture, retained the same assertions, and checked +its argument routing. The fixture still emits synthetic private markers on both +streams to verify bootstrap suppression. The preflight reference harness now +resolves its test import from its new location without `PYTHONPATH`. + +Linux launcher tests now exercise the actual Go preflight and capture code, +including standard/exposed uWSGI and the log role. Only the selected profile is +safe in each positive case; the alternatives deliberately reject. Unsafe +selected files and missing preflight helpers must stop before the native +program creates its startup marker. A separate real-nginx smoke script checks +the compiled production helper rather than a test binary. + +## Machine feedback + +| Check | Verified result | Boundary | +| --- | --- | --- | +| `bash tests/verify_privacy.sh` | Passed: formatting, vet, race tests, module verification, host and static Linux amd64/arm64 builds, 122 native and 84 application Python tests, shell syntax, diff whitespace | macOS host; Linux-specific tests run separately | +| Three 10-second fuzz targets | Passed: diagnostic 936,101 executions; aggregation 200,873; preflight 126,091 | Bounded fuzzing, not exhaustive proof | +| All Go packages on Linux with `-race -count=1 -timeout=30s` | Passed, including preflight subprocess lifecycle and both uWSGI launcher paths | Real Linux processes/pipes/signals; synthetic native programs | +| Linux `go vet ./...` | Passed | Go candidate only | +| Linux-targeted `govulncheck@v1.7.0 ./...` | No vulnerabilities found | Reachable Go code and dependencies, not the complete service image | +| Real nginx `-T` smoke | Both fixtures pass native inspection; Go helper accepts safe configuration with exit 0 and rejects unsafe access logging with exit 70; both helper paths emit zero bytes | Configuration inspection only, no service boot or requests | +| Frozen native reference hashes | All four unchanged | Preserves prior design/test material | +| Final hygiene and documentation | 81 candidate files checked; no supported credential markers, generated junk, broken local documentation links, or trailing whitespace; workflow YAML parsed; all Go files below 500 lines | Scoped checks, not a full secrets or application security audit | + +The canonical suite gained shell syntax validation for the new smoke script +after its complete passing run; that added check and the revised smoke script +were also run separately. No production code changed after those Go builds and +checks. The smoke test's native success controls ensure its unsafe rejection is +not merely an nginx syntax or installation error. + +The smoke command is reproducible with a caller-selected, already-present image: + +```sh +bash tests/verify_privacy.sh +bash tests/privacy_native/test_nginx_preflight_image.sh EXISTING_IMAGE +``` + +This local run used nginx 1.28.3 in the cached Linux arm64 image +`sha256:c0b0a707a6cd2149d5777ee83af1ea1de544210e597371eac93e67a1503c395c`. +Containers had no network, a read-only root filesystem, no capabilities, +no-new-privileges, and an init reaper. Source/configuration was read-only; +host-side caches and evidence stayed inside this worktree. No production +configuration, services, logs, or persistent volumes were changed. + +Evidence is under `tests/.privacy-build/preflight-review/`: the original fixture +failure, repaired launcher tests, complete canonical output, Linux CLI and full +race logs, Linux vet, vulnerability scan, real-nginx smoke, hygiene results, +before-edit copies, and source hashes. The older `runner-review`, aggregator, +and startup records remain historical and are not current source manifests. + +## Remaining release work + +At this stage, the complete candidate still needed a compliant application logging boundary, +observable sink/process failures, full service and retained-output coverage, +an atomic installer for the existing persistent volume, and tested rollback. +The full candidate image has not been built and booted. The current GitHub +checks are for the unchanged base, not this patch; the candidate workflow has +not run remotely. Complete those requirements before deployment. + +Later [process monitoring](process-monitor-go.md) and [application logger +replacement](application-logger-go.md) address the corresponding source-level +gaps. The [release review](review-2026-09-05.md) tracks current acceptance work. diff --git a/docs/privacy/process-monitor-go.md b/docs/privacy/process-monitor-go.md new file mode 100644 index 000000000..f0e33526d --- /dev/null +++ b/docs/privacy/process-monitor-go.md @@ -0,0 +1,127 @@ +# Go process-failure monitor + +Status: implemented in the JOS-81 candidate and verified with synthetic +processes under real Supervisor 4.3.0; **not deployed**. This covers nginx, +uWSGI, log-role, Celery, single-queue Celery, and websocket process failures. +The later [application logger replacement](application-logger-go.md) removes +the Python handler's unconsumed failure counter. Full installation and monitor +health gates remain open. + +`privacy-monitor` is a standard-library Go executable installed at +`/usr/share/docassemble/webapp/privacy-monitor`. The Dockerfile builds it with +the other three Go helpers, root-owned with mode 0755. The new Supervisor +listener runs as `www-data`, starts before the native services, and restarts +after failure. It receives process EXITED, BACKOFF, FATAL, and TICK_60 events. +Supervisor's [documented event protocol](https://supervisord.org/events.html#event-notification-protocol) +provides the input framing and acknowledgement contract. + +## Output and failure behavior + +Only fixed service/state labels reach the report sink: + +```json +{"schema":1,"component":"uwsgi","event":"process_failed","state":"exited"} +``` + +Components are `nginx`, `uwsgi`, `uwsgilog`, `celery`, `celerysingle`, +`websockets`, `initialize`, and `monitor`. The initializer is accepted only +with Supervisor group `main`; the other process/group pairs must match. +The new initializer pairing is covered by wire tests and a real Supervisor +fixture and awaits native CI with the maintenance extension. Process failures use +states `exited`, `backoff`, or `fatal`; monitor readiness/liveness uses `ready`. +An unexpected exit produces a failure record. A missing executable, early exit, +or exhausted startup retries produces BACKOFF/FATAL records as supplied by +Supervisor. Expected exits and unrelated services produce no failure record. +No PID, process/group name from input, serial, server name, command, path, raw +log output, or exception is retained in these records. These state events do +not contain the numeric native exit code, so the monitor does not invent it. + +Stdout carries only `READY`/`RESULT` protocol messages and is not logged. Fixed +JSON records go to stderr, which Supervisor retains in `privacy-monitor.log` +with 5 MB rotation and seven backups. `redirect_stderr` remains false so records +cannot corrupt the acknowledgement stream. A readiness record precedes the +first READY; each received TICK_60 produces a liveness record. A malformed frame +produces a fixed `monitor`/`protocol_failed`/`invalid` record when deliverable. + +The dedicated Linux process disables core dumps and seals extra inherited +descriptors. It accepts only pipes/sockets for its three standard descriptors. +Header storage is 1 KiB including its newline; payloads are limited to 1 KiB, +with at most 16 fields per token set. Control/non-ASCII bytes, duplicate fields, +invalid framing, and oversized declared payloads reject. Idle waits are allowed; +after a frame begins, five seconds bounds its complete header/payload. Each +protocol or record write has a five-second deadline. Buffers are cleared after +processing; this does not promise erasure of runtime copies or kernel memory. + +| Exit | Meaning | +| --- | --- | +| 0 | Supervisor closed input between events | +| 64 | Unsupported invocation arguments | +| 70 | Unsupported platform, process protection, or event-protocol failure | +| 74 | Unavailable, unsupported, broken, blocked, or unsuccessfully closed pipe | + +The listener acknowledges a reportable event only after the record is accepted +by its output pipe. A failed delivery exits without acknowledgement, allowing +Supervisor to requeue the event. Acknowledgement loss can therefore produce +duplicates; output acceptance is not a filesystem durability guarantee. The +listener pool buffers 256 events. Queue overflow and the monitor's own +unavailability remain visible through Supervisor's activity log/state; they +must be covered by deployment health checks. A missing liveness record is not +proof that services are healthy. No external notification or service-control +API call is performed. + +## Verification + +```sh +bash tests/verify_privacy.sh +docker build -t privacy-candidate-check \ + -f tests/privacy_native/fixtures/privacy-check.Dockerfile tests/privacy_native/fixtures +bash tests/privacy_native/test_nginx_preflight_image.sh privacy-candidate-check +bash tests/privacy_native/test_supervisor_monitor_image.sh privacy-candidate-check +``` + +The disposable test image's Python, gcc, Bash, Supervisor, and nginx are +development tools: Python runs the existing reference suites and fixtures, gcc +supports Linux race tests, and Bash/native services exercise their real +protocols. They add no Go module dependencies. The Supervisor harness runs as +the same `www-data` identity and uses the production listener section, changing +only its log destination to a temporary directory. Native programs are +synthetic: a clean exit, an unexpected exit, a missing executable, and an +unrelated process. The container has no network, a read-only root filesystem, +no capabilities, no-new-privileges, and temporary writable `/tmp`. +The expanded harness also runs the actual application launchers and compiled +Go capture with synthetic runtime executables from the read-only source mount; +it checks their retained counters and unexpected process exits. + +Original monitor verification record on September 5, 2026: + +- The canonical suite passed: formatting, vet, race tests, module verification, + four host/Linux amd64/arm64 executable builds, 122 native-profile and 84 + application Python tests, shell syntax, and diff whitespace. +- All Go packages passed Linux race tests; Linux vet passed. The + Linux-targeted `govulncheck@v1.7.0` scan found no vulnerabilities. +- Four bounded fuzz targets passed; the new monitor target ran 1,094,705 + executions. Parser tests validate fixed output, malformed input, and ignored + events. Linux pipe tests cover truncated/oversized/stalled frames, full and + broken sinks, delivery before acknowledgement, and descriptor-flag restoration. +- The compiled monitor passed real Supervisor 4.3.0 acceptance with the synthetic + processes. An isolated mutation acknowledging before delivery failed the + broken-sink test with an unexpected `RESULT` token. Production source and + assertions were unchanged by that mutation. +- Final checks covered 92 candidate files: no supported credential markers, + generated junk, broken local documentation links, or trailing whitespace; + workflow YAML parsed and all Go files remained below 500 lines. This is scoped + hygiene, not a complete service/dependency security audit. + +The workflow now includes both real-protocol checks, but it has not run on +GitHub for this uncommitted candidate. Evidence, before-edit copies, the +mutation, and manifests are in `tests/.privacy-build/monitor-review/`. The +shared Linux runtime was found stopped; this cycle used a separate runtime +whose files/caches are confined to this worktree. Its test image was +`sha256:d318b5a18cc090c52fc2db3a9642f1e3749e6bcd64d71c93cfd36e51ea596a25` +(Linux arm64, Python 3.14.7, Supervisor 4.3.0, nginx 1.30.4). No production +services, logs, or volumes were changed. All disposable containers finished, +and the worktree's VM was stopped after verification. The application boundary, +complete candidate image, persistent-volume installer, rollback, retained-output +coverage, and exact-commit CI remain required by the [release review](review-2026-09-05.md). +The later [application capture record](application-capture-go.md) records the +expanded six-service coverage and current verification. diff --git a/docs/privacy/review-2026-09-05.md b/docs/privacy/review-2026-09-05.md new file mode 100644 index 000000000..6ae12ed1b --- /dev/null +++ b/docs/privacy/review-2026-09-05.md @@ -0,0 +1,184 @@ +# JOS-81 release review — 2026-09-05 + +**Decision: release verification remains open.** Production is unchanged. +The candidate is in draft [PR 23](https://github.com/jacobyoby/docassemble/pull/23) +under [issue 22](https://github.com/jacobyoby/docassemble/issues/22). + +**Current follow-up:** the non-mail candidate now uses the existing Go +runner, aggregator, preflight and fixed diagnostic/monitor records. The five +superseded Python implementations remain test references, outside production +artifacts. [Native CI at a32541b4d](https://github.com/jacobyoby/docassemble/actions/runs/34009497006) +passes the prior 30-file populated-volume installation, real queue and cron +interviews, root-launcher TERM/INT, retained-output checks and complete rollback. +That run also passes 146 native/launcher tests, 84 frozen references, Go checks, +real nginx/uWSGI/Supervisor/rotation fixtures and vulnerability scanning. +[CodeQL](https://github.com/jacobyoby/docassemble/actions/runs/34009496992) and +its security check pass for the same commit. + +The [maintenance extension](maintenance-capture-go.md) adds seven small re-exec +preludes, two Go profiles and initializer failure monitoring. It keeps the +existing backup operations and preserves customized daily/initializer code by +applying only the privacy diff. The [catalog](install-catalog.json) now lists +36 required files, including a narrow target nginx migration. Native acceptance exercises scheduled maintenance, +local copies, rotation callbacks, orderly shutdown/restore, interrupted-start +restore suppression and queue/cron after restart. These checks pass on +[c09aba80c](https://github.com/jacobyoby/docassemble/actions/runs/34011607496), +along with 147 native/launcher tests, 84 frozen references, Go checks, +vulnerability scanning and [CodeQL](https://github.com/jacobyoby/docassemble/actions/runs/34011607521). +The PR code-scanning query returned no open alerts for that revision. + +Target review subsequently found a legacy file-access-log override in each +customized `nginx-realip`, plus valid quoted/unquoted escapes rejected by the +Go parser. The follow-up preserves every other byte while removing that one +override and handles the native escape rules, with escaped includes still +rejected. Both saved target dumps pass Go policy after this migration; +new candidate native syntax, installation and rollback acceptance remain open. +The old local artifact must be rebuilt for the changed parser and catalog. + +The [target inventory](target-inventory-2026-09-05.md) confirms the installed +runtime, custom nginx inputs, maintenance schedules and local backup settings. +Both targets have no custom syslog includes; retain their system configuration +and differing service states. Final target-specific installation, a private +restorable snapshot, served verification and release provenance remain required. +The [installation plan](installation-plan.md) defines the complete non-mail +set; a package-only install cannot deliver it. + +The user has [deferred the Go mail replacement](deferred-mail-go.md). Existing +mail-capture edits are preserved but excluded from the installation catalog; +the full Dockerfile is not a non-mail release artifact. No mail sends or Exim +changes belong to this release. The original blockers below are historical +review findings; the current follow-up describes which parts are resolved. +The original review evidence below is preserved as the baseline; it is not a +claim that the candidate's production files are still unchanged. + +Scope: `fix/jos81-private-logging` in `docassemble-jos81`, based on +`63d22c44b7bd008a521f0bcb8ab442b2143ee557`. Exact candidate CI remains required. +One writer owns this worktree; three independent read-only reviewers were +authorized for the mail follow-up. This review does +not promote the separate metrics or SSE worktrees. + +## Release blockers + +1. **P1 — The existing delivery path cannot install the complete change.** + `njCourtForms/tools/deployCore.sh` installs only `docassemble_base` and + `docassemble_webapp` and reloads the application. This candidate also needs + the launchers, native helper directory, rendered uWSGI profiles, nginx main + configuration, and Supervisor configuration in the running environment. + Its Dockerfile describes an image build, but no exact candidate image has + been built and accepted with the existing persistent volume. A package-only + promotion would omit the native capture boundary. Prepare and test an atomic + installation and rollback for all affected files before production use. + +2. **P1 — Startup and runtime failures can lose their diagnostic signal.** + The original bootstrap-silence defect is addressed for checked activation, + configuration, evaluation, preflight, and exec failures. The compiled Go + helper now reports fixed component/phase records with a bounded sink deadline; + missing helpers and failed sinks produce distinct nonzero exits. + Selected native/application Supervisor failure states now have a fixed-record Go event listener, + verified with synthetic services and real Supervisor. This does not expose + arbitrary application handler failures that never cause a process-state + transition. Follow-up: the draft `privacy_logging.py:133` local failure + counter is now test-only. The application entrypoint retains its existing + stderr callback and Go owns capture/delivery failures. Monitor health and + retained-sink delivery still require full-install acceptance; no raw + configuration, message, path, or exception may become a fallback record. + +3. **P1 — Full application and retained-output coverage is incomplete.** + The application tests use synthetic configuration and dependency stubs. They + stop at the early logging boundary; they do not prove real Flask/Celery + startup, queue execution, or subsequent logger reconfiguration. Websocket + protocol logging (`socketserver.py:57`) and worker stdout/stderr now have an + outer Go capture boundary tested with synthetic programs. Full application + execution, direct cron response output (`cron_tasks/cli.py:287`), and + independent file writes still require coverage. Mail's real storage probe + exposes an existing bytes/text mismatch; its database ID/commit ordering, + actual Exim transport, timeout/retry behavior, and retained Exim logs also + require acceptance. `Docker/sync.sh` and syslog/backup routes can copy + existing logs. Successful formatter and parser tests do not establish that + the complete installation keeps sensitive content out of retained output. + Use synthetic markers across the complete service lifecycle, including + rotation, copies, backup/restore, startup failure, and rollback. + +4. **P1 — The draft conflicts with the current implementation requirements.** + The prior draft adds five Python production modules: four in `Docker/privacy` + and the adjacent application `privacy_logging.py`. The current goal forbids + new Python production code and untyped production code. An AST inventory + also found functions missing annotations in the new preflight, aggregator, + formatter, and modified application entrypoint. This inventory is not a + type check. Preserve these drafts as design/test material; establish a + compliant implementation before promoting them. Do not rewrite unrelated + stable legacy code for language purity. + Follow-up: the standard-library Go preflight, runner, and aggregator now replace four + of those modules. Their Python sources are retained only under + `tests/privacy_native/reference`. The application draft now lives in + `tests/privacy_logging/reference`; its production entrypoint installs no + override, leaving the stable base logger feeding the Go capture boundary. + No new executable Python remains in this candidate's implementation. + +5. **P1 — Exact-candidate release verification is missing.** + The successful e2e run and CodeQL run are for the unchanged base commit, + not this candidate. The current e2e + workflow copies selected earlier fixes into a stock container; it neither + installs the revised logging entrypoint nor the native privacy profile. + Extend acceptance to install the complete candidate, then require green + checks and a development verification record for the exact release commit. + +## Initial review fix and machine feedback + +The first combined native-suite invocation produced four import errors: +`ModuleNotFoundError: No module named 'log_aggregate'`. Four test modules relied +on an undocumented `PYTHONPATH`. Added `tests/privacy_native/native_support.py` +to load the runtime directly from this checkout, and changed only those four +imports. No assertions, expected values, production files, or security controls +were changed during this review. + +Verified on Python 3.14.7 with `PYTHONPATH` removed for the native run: + +| Check | Result | Limit | +| --- | --- | --- | +| Application logging suite | 84 passed | Synthetic dependencies; early logging boundary | +| Native-profile suite after import fix | 118 passed | Parser, stub launchers, aggregation; no native services | +| Python syntax | 17 candidate files compiled in memory | Not type checking | +| Shell syntax | Four revised launcher/initialize scripts passed `bash -n` | Not service startup | +| Supervisor file | Parsed; three revised sections have bounded rotation settings | Not a native Supervisor boot | +| Diff whitespace | `git diff --check` passed | Tracked diff only | +| Candidate production-file hashes | All 19 unchanged by this review | Original draft remains unverified for release | + +Commands from the checkout root: + +```sh +mkdir -p tests/.review-tmp +env -u PYTHONPATH TMPDIR="$PWD/tests/.review-tmp" PYTHONDONTWRITEBYTECODE=1 \ + python3.14 -B -m unittest discover -s tests/privacy_native -v +TMPDIR="$PWD/tests/.review-tmp" PYTHONDONTWRITEBYTECODE=1 \ + python3.14 -B -m unittest discover -s tests/privacy_logging -v +bash -n Docker/run-nginx.sh +bash -n Docker/run-uwsgi.sh +bash -n Docker/run-uwsgilog.sh +bash -n Docker/initialize.sh +git diff --check +``` + +Evidence is preserved in `tests/.review-output/`: initial failure, passing +outputs, baseline file hashes, original copies of the four edited test files, +and `structural-review.json`. It contains synthetic test evidence, not service +logs. No new production dependencies were added. `ruff`, `mypy`, `pylint`, and +`shellcheck` are unavailable in the current interpreter/PATH, so their checks +were not run. No dependency or full security scan of the complete installed +candidate has passed; no such pass is claimed. + +Final hygiene: a scoped scan found no high-confidence private-key or supported +token markers in candidate files. This is not a full secrets scan. The two +known generated bytecode files were removed, and the empty review scratch +directory was removed. Intentional test evidence and backups remain under +`tests/.review-output/`; `final-review-checks.json` records these checks. All +pre-existing production draft bytes remain unchanged. + +Base-only CI evidence: + +- [e2e at the base commit](https://github.com/jacobyoby/docassemble/actions/runs/33584689870) +- [CodeQL at the base commit](https://github.com/jacobyoby/docassemble/actions/runs/33593328986) + +Next release step: resolve the implementation and diagnostic requirements in +this one worktree, then exercise a complete synthetic installation and rollback. +The present passing unit suites are not authorization to bypass those gates. diff --git a/docs/privacy/rotation-ownership.md b/docs/privacy/rotation-ownership.md new file mode 100644 index 000000000..f75ce5567 --- /dev/null +++ b/docs/privacy/rotation-ownership.md @@ -0,0 +1,99 @@ +# Counter-file rotation and forwarding + +Status: the candidate's overlapping rotation ownership and Celery collector +misrouting are repaired and locally verified. **Not deployed.** Complete +installation, callback behavior, backup/restore and retained-output coverage +remain [release gates](review-2026-09-05.md). The [mail replacement](deferred-mail-go.md) +remains deferred. + +## Source repair + +Supervisor and the syslog collector previously wrote the same four filenames. +Supervisor rotated these files by size while logrotate could independently +rename them. The candidate now gives the local counter streams distinct names: + +| Component | Local file owned by Supervisor | Collector file owned by logrotate | +| --- | --- | --- | +| Celery | `privacy-celery.log` | `worker.log` | +| Single-queue Celery | `privacy-celerysingle.log` | `single_worker.log` | +| uWSGI | `privacy-uwsgi.log` | `uwsgi.log` | +| Websockets | `privacy-websockets.log` | `websockets.log` | + +All paths are under `/usr/share/docassemble/log`. The forwarding file sources +move with the local files and retain their existing program labels. Initialization +precreates the four new files before its ownership pass, alongside the existing +legacy files. This prevents a root Supervisor from creating them after the +`www-data` ownership pass under a restrictive umask. Actual installed modes and +ownership still require installation acceptance. + +The native test also showed `celerysingle` records entering `worker.log` through +the existing unanchored `celery` filter. The collector now matches `^celery$`. +The [syslog-ng filter documentation](https://syslog-ng.github.io/admin-guide/080_Log/030_Filters/005_Filter_functions/009_program.html) +confirms that `program()` accepts a regular expression; the real native test +verifies the resulting separation. + +Existing `uwsgilog.log`, `nginx-safe.log` and `privacy-monitor.log` already have +distinct Supervisor destinations. Supervisor keeps its production 5 MB limit +and seven backups. Existing collector destinations, logrotate rules, Apache +rotation and the postrotate callback remain otherwise unchanged. Legacy log +rotation can still restart application services through that callback; this +repair does not claim restart-free production rotation. + +## Verification + +Five ownership tests cover external writer/rotator exclusion, forwarding and +collector mapping, precreation order, bounded Supervisor rotation and separate +Celery routing. The first run failed on the original ownership/mapping conflict; +a later fail-first test reproduced the collector filter overlap. + +The native fixture uses real Supervisor 4.3.0, logrotate 3.22.0 and syslog-ng +4.11.0 with the freshly built Go runner. All application output is synthetic. +It runs the actual initialization precreation/ownership fragment in a private +directory under umask 0077, then uses the four actual forwarding source lines +and collector filter/destination rules over loopback TCP. Local capture and +collector files share one directory, matching the ownership hazard. + +The fixture reduces Supervisor's rollover threshold to 256 bytes to exercise +seven backups promptly; source checks retain the production 5 MB requirement. +It pauses Supervisor for less than 4.5 seconds while forcing actual logrotate +over the legacy files. All capture-file inodes and hashes must remain unchanged. +The original callback is replaced by an inert test hook. The test explicitly +signals syslog-ng to reopen its collector files, then emits an 8,192-line burst +per component. Each newly created active collector file must receive the new +counter totals with the correct component and without private markers. Final +daemon liveness and the original four capture PID/start-time identities must +remain stable across a settled observation before deliberate shutdown. Final +capture archives are inspected after Supervisor stops, rejecting partial JSON +records and more than seven backups. + +The disposable container has no network beyond loopback, fixed local hostname +resolution, a read-only root/source mount, no capabilities, no privilege +escalation, no Docker log retention, 512 MiB memory, 128 PIDs, and 64 MiB of +non-executable temporary storage. Inner and outer command deadlines and named +container cleanup bound failures. The native packages are test-only additions +to the existing protocol fixture image, not production dependencies. + +Evidence: `tests/.privacy-build/rotation-review/` contains `fail-first.log`, +`filter-fail-first.log`, the native misrouting reproduction `native-routing.log`, +the accepted run `native-final.log`, image build/version evidence, canonical +`verification.log`, source hashes and the final hygiene record. The immutable +local Linux arm64 image is +`sha256:53259b84ca8ed8b8c0affc26a7b5f60d7be7239daf2fe54852ab2e18bf4074d1`. +The initial syslog startup timeout was resolved by providing local hostname +resolution inside the network-isolated fixture; the test's syntax deadline +remained unchanged. + +```sh +bash tests/verify_privacy.sh +docker build -t privacy-candidate-check \ + -f tests/privacy_native/fixtures/privacy-check.Dockerfile tests/privacy_native/fixtures +docker build -t privacy-rotation-check \ + -f tests/privacy_native/fixtures/rotation-check.Dockerfile tests/privacy_native/fixtures +bash tests/privacy_native/test_rotation_image.sh privacy-rotation-check +``` + +These checks establish native ownership, rollover and continued forwarding for +the selected routes. They do not prove lossless forwarding of every snapshot, +production callback behavior, full application operation, historical-log +treatment, remote transport deployment, installation over existing volumes or +rollback. No historical logs were removed and no mail work was resumed. diff --git a/docs/privacy/startup-diagnostics.md b/docs/privacy/startup-diagnostics.md new file mode 100644 index 000000000..132ff7bb2 --- /dev/null +++ b/docs/privacy/startup-diagnostics.md @@ -0,0 +1,107 @@ +# Bounded startup failure diagnostics + +Status: implemented in the JOS-81 candidate; not deployed. This resolves the +tested bootstrap failure-reporting paths only. Full native startup, monitor +health, retained-sink acceptance, and complete logging coverage remain open. + +`Docker/privacy-diagnostic` is a Go 1.27.0 module using only the standard library. +The Dockerfile builds a static binary in a separate `golang:1.27.0` stage and +installs it, root-owned with mode 0755, at +`/usr/share/docassemble/webapp/privacy-diagnostic`. The Go compiler and scanner +are not added to the final service image. Existing persistent volumes still +require explicit installation; image construction alone does not establish that +the running volume contains this binary. + +## Record and process contract + +The uWSGI, log-role uWSGI, nginx, Celery, single-queue Celery, websocket, and mail launchers discard raw bootstrap output and +call the diagnostic with a fixed component and phase when a checked step fails: + +```json +{"schema":1,"component":"uwsgi","event":"startup_failed","phase":"activation"} +``` + +Components: `uwsgi`, `uwsgilog`, `nginx`, `celery`, `celerysingle`, `websockets`, `mail`. +Phases: `activation`, `config`, +`config_eval`, `preflight`, `launch`. Nginx combines activation and configuration +loading in its existing `su` command and reports that failure as `config`. +Unknown arguments produce only a fixed `launcher`/`invocation` record. No input +argument is copied into output, including invalid strings. + +Output is one newline-terminated JSON record, at most 256 bytes. The helper +accepts only a pipe or socket, registers a nonblocking duplicate with Go's I/O +poller, and imposes a one-second write deadline. It restores the inherited +descriptor's flags before closing the duplicate. The launcher is waiting for +that process, so successful service execution is unaffected by these temporary +flags. Kernel-managed status bits caused by any write are not writable flags. + +| Exit | Meaning | +| --- | --- | +| 64 | Invalid diagnostic invocation; fixed invocation record if deliverable | +| 69 | Diagnostic executable missing or not executable; launcher stops before bootstrap | +| 70 | Startup failed; fixed component/phase record delivered | +| 74 | Diagnostic sink unavailable, unsupported, broken, or timed out | + +An OS failure to execute an otherwise executable diagnostic propagates its +nonzero shell status in the service launchers. The [mail launcher](mail-capture-go.md) +maps all its startup failures to 75, including a missing diagnostic or failed +diagnostic delivery. Failure to deliver a record is never success and never +falls back to raw stderr. Supervisor must retain/monitor these process exits; +the missing-helper and failed-sink paths cannot promise a delivered record. +The candidate now includes a [Go process-failure monitor](process-monitor-go.md), +verified with real Supervisor and synthetic services. Installing it and checking +its operational health remain part of full deployment acceptance. + +Successful launch still uses `exec`, preserving Supervisor's process identity. +The extra sink descriptor is closed before native-wrapper startup. Bash keeps +the failed `exec` redirections, so the failure path recovers the sink from its +already-redirected stdout before reporting `launch`. + +## Verification + +```sh +bash tests/verify_privacy.sh +``` + +The runner keeps build/test caches in `tests/.privacy-build/`, checks formatting, +runs `go vet`, race tests, ten seconds of bounded fuzzing for each fuzz target, +module verification, host compilation, Linux amd64/arm64 compilation of all Go +packages, the Python candidate +suites, shell syntax, and tracked diff whitespace. The new GitHub workflow runs +this on Linux and adds `govulncheck@v1.7.0`. That pinned scanner is a development +tool only; it is not a runtime/module dependency. Its downloads and vulnerability +database require network access. No dependency was added to the diagnostic's +`go.mod` beyond the Go language version. + +Fail-first evidence: six activation/config/preflight cases failed against the +previous silent launchers; the successful handoff control remained passing. +Expanded launcher checks execute the compiled diagnostic and cover all checked +phases, absent executables, broken/full output pipes, successful PID handoff, +and closure of the extra sink descriptor. Nginx tests use its read-only profile +with synthetic `su`, configuration, and native-execution boundaries; they do not +write system nginx paths or start nginx. Go tests also cover invalid arguments, +fixed-schema output, regular-file rejection, socket delivery, deadline behavior, +and descriptor-flag restoration. + +These are focused checks, not full service acceptance. The initial image build, +installation into a persistent volume, real native services, complete logging +coverage, exit monitoring, and rollback are not established by them. + +Original diagnostic verification record (2026-09-05): 122 native-profile and 84 application tests +pass. Go formatting, vet, race tests, 1,001,159 fuzz executions, module checks, +host compilation, and static Linux amd64/arm64 cross-builds pass. A source-mode +`govulncheck@v1.7.0` scan found no vulnerabilities in the diagnostic's reachable +Go code. The new workflow is authored but has not run on GitHub; the complete +Docker image has not been built. Evidence and pre-edit backups are retained in +`tests/.startup-review/`. + +The later [Go aggregator](aggregation-go.md), [Go native runner](native-runner-go.md), +and [Go preflight](native-config.md) extend the candidate's capture path. The +launchers now select the Go preflight and runner. Their evidence is separate +from this original diagnostic verification record; see the current +[preflight integration record](preflight-integration-2026-09-05.md). +The later [application capture record](application-capture-go.md) extends the +checked bootstrap paths to Celery and websockets and records the current +126 native/application-launcher tests. The later [application logger replacement](application-logger-go.md) +adds 64 Linux capture combinations; the 84 Python logger tests now cover frozen +references only. diff --git a/docs/privacy/target-inventory-2026-09-05.md b/docs/privacy/target-inventory-2026-09-05.md new file mode 100644 index 000000000..97446bc17 --- /dev/null +++ b/docs/privacy/target-inventory-2026-09-05.md @@ -0,0 +1,177 @@ +# JOS-81 target inventory + +Read-only observations collected on `forms-vps` around 2026-09-06 02:31 UTC +(September 5 Pacific). No candidate files were installed, no services restarted, +and no logs or application data were copied. This resolves target paths and +reveals configuration differences; it does not prove installation or rollback. + +## Confirmed runtime + +| Property | Forma | Retiring demo | +| --- | --- | --- | +| Container | `docassemble-forma` | `docassemble-demo` | +| Architecture | Linux amd64 | Linux amd64 | +| Container role | `all` | `all` | +| Persistent volume at `/usr/share/docassemble` | `da_forma`, read/write | `da_live`, read/write | +| Active uWSGI profile | `config/docassemble.ini` | `config/docassemble.ini` | +| uWSGI application mount | `/nj` | `/` | +| Python | 3.14.4 | 3.14.4 | +| uWSGI / Supervisor | 2.0.31 / 4.3.0 | 2.0.31 / 4.3.0 | +| nginx / syslog-ng / logrotate | 1.28.3 / 4.8.1 / 3.22.0 | 1.28.3 / 4.8.1 / 3.22.0 | + +Both containers use image ID +`sha256:c0b0a707a6cd2149d5777ee83af1ea1de544210e597371eac93e67a1503c395c`. +The running uWSGI executable is +`/usr/share/docassemble/local3.14/bin/uwsgi`; the rendered profile names +`/usr/share/docassemble/local3.14/bin/python` and the same virtualenv. Interpreter +inspection, without importing the application, resolves its package directory +to `/usr/share/docassemble/local3.14/lib/python3.14/site-packages`. + +The webapp directory and root are root-owned mode 0755. The log and uWSGI runtime +directories are owned by `www-data` (uid/gid 33), mode 0755. All four privacy +binaries, the lifecycle include and `run-uwsgilog.sh` are absent. Image changes +alone would be masked beneath the existing volume. + +Both configurations select nginx, an HTTPS load balancer, no local HTTPS or +Let's Encrypt, and omit read-only filesystem mode. Observed container environment +agrees on the load balancer and route prefix. Configuration rendering and +startup behavior must still be exercised against the candidate. + +## Preserve target differences + +- Forma has an additional enabled `formapauperis` nginx site. Its config and + symlink are now explicit protected catalog entries. Demo instead customizes + `nginx-http.dist` with additional public hostnames and + `/etc/nginx/form_rewrite_rules.conf`; that include is also protected. Preserve + all existing nginx templates and validate their rendered outputs. Do not + replace demo's HTTP template with the unmodified repository template. +- Both installed initializers lack the fork base's unrelated Apache HTTPS-port + changes. Apply only the privacy diff to the installed initializer; copying + the entire candidate would also change that unrelated behavior. +- Both targets already have `sharedscripts` in the application logrotate stanza; + Forma differs only in its placement. Keep exactly one directive and preserve + the separate mail stanza. The installed rotation callback matches the source. +- The ordinary launchers, application logger, uWSGI templates/profiles, + Supervisor configuration and syslog input templates match the repository base. + The nginx main configuration is a distro-style file rather than the base's + empty placeholder; its effective directives still need comparison when the + candidate is rehearsed. + +Both targets use `/etc/syslog-ng/syslog-ng.conf` as a symlink into the volume; +the forwarding include is absent. The active files share hash +`b267e9e9a1b9e75536205511b49744e8189a6e031fc720c29ab2cce9e77db18a`, which +matches neither shipped syslog template. A follow-up source comparison finds +`system()` and `internal()` inputs, ordinary system-log destinations and a +`/etc/syslog-ng/conf.d/*.conf` include; the main file has none of the collector +template's application TCP routes. Its effective includes and required routing +still need review before any replacement. Preserve the existing system logging +configuration unless a reviewed privacy requirement calls for a specific change. + +Supervisor reports `syslogng` stopped on Forma and running on demo. nginx, uWSGI, +both Celery workers, websockets, cron +and Exim are running on both. The privacy monitor and `uwsgilog` are not configured. +Demo reports Apache `FATAL` while Forma reports it stopped; nginx is the configured +web server. These are baseline observations, not changes caused by this candidate. +Do not start every role-compatible service merely to make the two targets alike. + +## Evidence and remaining work + +Private local evidence is in `tests/.privacy-build/target-review/`: selected +Docker identity/mount metadata, catalog path hashes and ownership, process path +observations, safe configuration selections, native versions, and selected source +diffs. Root could not inspect some other-user `/proc` entries; a second probe as +`www-data` verified the active uWSGI executable and profile without expanding +container capabilities. `supervisorctl --version` was unsupported; the successful +`supervisord --version` check reported 4.3.0 on both targets. + +The path inventory covers all 29 required files and 26 state entries. Protected +regular-file hashes and selected link metadata were read; protected directory +trees, uploads and historical logs were not recursively read or snapshotted. +Counter-file contents were not read. Free space was approximately 112.5 GB on +the volume filesystem, which does not establish snapshot size or restore safety. + +Still required: a private restorable snapshot, complete protected-tree verification, +target-specific rendered configuration checks, installation and rollback rehearsal, +full application/output coverage and exact candidate commit/CI. Earlier native +fixtures used nginx 1.30.4 and syslog-ng 4.11.0, so their passes alone do not prove +the installed versions above. Go mail replacement and Exim changes remain +[deferred](deferred-mail-go.md). + +## Cron and backup follow-up + +A subsequent read-only probe covers the added thirtieth catalog file, +`webapp/run-cron.sh`: both installed copies match the repository base hash +`0dff2d464606dd708864cb96e88a64410ad57a4fc9f698b50dab8f4533115b8c`. +Both targets provide `setpriv` and the virtualenv Flask entrypoint. Flask is +mode 0755, owned by uid/gid 33; ordinary ownership is compatible with the +native runner and needs no broad permission change. + +Both targets have S3 and Azure backup disabled, 14 rolling-backup days, a +writable filesystem, the default application log directory, and existing local +`backup/log` and `backup/nginxlogs` directories. The probe reported only these +settings and directory existence; it did not read retained log contents or +print credentials, bucket names, or application data. Local maintenance, +copy, backup and restore are therefore the next acceptance scope. Cloud, +Apache and explicit log-role paths are conditional rather than active-target +requirements established by this probe. Effective crontab contents were not +inspected. Evidence: `tests/.privacy-build/queue-cron-review/target-copy-settings.txt` +and `target-cron-runtime.txt`. Production remains unchanged. + +The same bounded probe finds `allow updates` disabled on both targets, although +`update on start` is true; the initializer requires both switches before its +package-update branch. Neither target configures error-notification email or +its interview-variable attachments. These settings distinguish conditional +package-update temporary diagnostics and error-mail attachments from active +ordinary service logging. They do not cover manually invoked updates or +third-party package handlers. The evidence records only booleans in +`target-diagnostic-settings.txt`. + +The maintenance follow-up confirms that all four installed interval scripts +and `sync.sh` are root-owned mode 0755. Four script hashes match the pre-change +source on both targets. The daily script differs on both, so its catalog entry +applies only the privacy diff, as the initializer already does. Its existing +backup implementation must be retained. The actual crontab contains all four +run-parts schedules and the colon-delimited all-role setting used by the rotation +callback. The probe emitted only hashes, permissions and schedule booleans, +not crontab contents. Evidence is in +`tests/.privacy-build/maintenance-review/target-maintenance-paths.txt`. + +The final read-only syslog include check finds the same active hash on both +targets, only the standard `scl.conf` and `/etc/syslog-ng/conf.d/*.conf` +includes, and zero matching local include files. The uncommented main file +contains system/internal sources, no network source and no application-log +path. Preserve that existing system logging configuration and each target's +service state; no custom application forwarding include needs migration. +This is configuration evidence, not a scan of historical system-log contents. +Evidence: `tests/.privacy-build/maintenance-review/target-syslog-includes.json`. + +Both live `config/config.yml` files are uid/gid 33:33, mode 0644. The populated +fixture now starts its protected configuration with that same ownership. +The unchanged initializer's local restore branch unconditionally assigns +33:33; a fresh image's root-owned configuration is a different starting state. +Contents and all ownership/permission comparisons remain exact throughout the +fixture. Evidence: `tests/.privacy-build/maintenance-review/target-config-metadata.json`. + + +## Customized nginx follow-up + +Successful read-only `nginx -T -e stderr` checks expose nine file sections on +both targets. Their `config/nginx-realip` files contain 519 lines (Forma) and +598 lines (Demo), including routes and page customizations. Both override the +HTTP logger with exactly one `access_log /var/log/nginx/access.log privacy;` +line. The five-line repository default must never replace these files. + +The explicit catalog patch removes only that line; local copies from both +targets accept it without fuzz and preserve all other bytes. Both synthesized +candidate dumps pass Go policy after this migration, retaining all observed +include sections. This is not a native check of the candidate or an install. +Current production syntax was checked; candidate native/served checks remain. + +The parser now follows Nginx 1.28.3 escape decoding for quoted payloads and +quoted/unquoted regexes, preserving unknown escapes. It does not expand support +for escaped include globs. Primary reference: [ngx_conf_read_token, pinned +Nginx source](https://github.com/nginx/nginx/blob/9b958b000776c88036cd800c66e7e4ad39e6fd41/src/core/ngx_conf_file.c#L627). +Two exact frozen-reference inputs intentionally change from rejection to +acceptance; the frozen source itself remains unchanged. Synthetic native +fixtures cover escaped quotes, a quoted fake dump header, regex escapes and +an unsafe file destination containing an escaped delimiter. diff --git a/tests/privacy_logging/module_support.py b/tests/privacy_logging/module_support.py new file mode 100644 index 000000000..973a3a6e6 --- /dev/null +++ b/tests/privacy_logging/module_support.py @@ -0,0 +1,88 @@ +"""Exercise the frozen Python draft, not the production Go capture boundary.""" +import contextlib +import hashlib +import importlib.util +import io +import logging +import os +from pathlib import Path +import sys +import tempfile +import types +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[2] +WEBAPP = ROOT / "docassemble_webapp/docassemble/webapp" +REFERENCE = ROOT / "tests/privacy_logging/reference" +MODULE = "docassemble.webapp.log_initialize" + +# Keep this historical design executable without allowing it to masquerade as +# the current production logger. New capture tests use the actual source module. +for filename, expected in ( + ("log_initialize.py", "93f5e7b14ff7162edef330246bf99d30b34d96fadfac1314eaa593103dd89cb1"), + ("privacy_logging.py", "0cf784e0d0e3997ba334b511c419fc47ed0f37d560777c1ad1b1c0dc39843634"), +): + if hashlib.sha256((REFERENCE / filename).read_bytes()).hexdigest() != expected: + raise RuntimeError("frozen application logger reference changed") + + +def package(name, path=None): + module = types.ModuleType(name) + module.__path__ = [] if path is None else [str(path)] + return module + + +def load(name, path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +# No installed docassemble package or service configuration is imported. +for name, path in (("docassemble", None), ("docassemble.base", None), + ("docassemble.webapp", WEBAPP)): + sys.modules[name] = package(name, path) +sys.modules["docassemble.webapp"].__path__.insert(0, str(REFERENCE)) +privacy = load("docassemble.webapp.privacy_logging", REFERENCE / "privacy_logging.py") + + +@contextlib.contextmanager +def fixture(context="web", logserver=None, debug=False, stderr=None): + with tempfile.TemporaryDirectory() as directory: + config = types.ModuleType("docassemble.webapp.config") + config.LOGSERVER = logserver + config.LOG_DIRECTORY = directory + config.daconfig = {"log to std": context == "std"} + config.in_celery = context == "celery" + config.in_cron = context == "cron" + base = types.ModuleType("docassemble.base.logger") + base.callback = lambda message: sys.stderr.write(message) + base.set_logmessage = lambda cb: setattr(base, "callback", cb) + base.logmessage = lambda message: base.callback(message) + capture = io.StringIO() if stderr is None else stderr + previous = logging.Logger.manager.loggerDict.pop("docassemble", None) + original_class = logging.getLoggerClass() + state = types.SimpleNamespace(config=config, base=base, stderr=capture, + directory=Path(directory), module=None) + modules = {"docassemble.webapp.config": config, + "docassemble.base.logger": base} + with patch.dict(sys.modules, modules), patch.object(sys, "stderr", capture), \ + patch.dict(os.environ, {"SUPERVISORLOGLEVEL": "debug" if debug else "info"}): + try: + yield state + finally: + module = sys.modules.pop(MODULE, None) + if module is not None: + module._silence_logger() + module._close_owned_files() + logging.Logger.manager.loggerDict.pop("docassemble", None) + if previous is not None: + logging.Logger.manager.loggerDict["docassemble"] = previous + logging.setLoggerClass(original_class) + + +def start(state): + state.module = load(MODULE, REFERENCE / "log_initialize.py") + return state.module diff --git a/tests/privacy_logging/reference/log_initialize.py b/tests/privacy_logging/reference/log_initialize.py new file mode 100644 index 000000000..f7c5c594b --- /dev/null +++ b/tests/privacy_logging/reference/log_initialize.py @@ -0,0 +1,144 @@ +"""Install the allowlisted application logger in every application context.""" + +import logging +import os +import sys +import time + +sys_logger = None +_configured = False +_log_files = [] + + +class UnsilenceableLogger(logging.Logger): + def isEnabledFor(self, level): + return level >= self.level + + +def syslog_message(message): + """Discard legacy free text without inspecting or stringifying it.""" + try: + if sys_logger is not None: + record = logging.LogRecord("docassemble", logging.DEBUG, "", 0, + "*", (), None) + record.event_code = "UNKNOWN" + sys_logger.handle(record) + except Exception: + # Neither the message nor the logging failure is safe to print. + pass + + +def syslog_message_with_timestamp(message): + # The safe formatter supplies UTC time; do not concatenate the message. + syslog_message(message) + + +def _close_owned_files(): + for stream in _log_files: + try: + stream.close() + except Exception: + pass + _log_files.clear() + + +def _silence_logger(): + if sys_logger is not None: + sys_logger.handlers.clear() + sys_logger.filters.clear() + sys_logger.propagate = False + # Prevent logging.lastResort from becoming a raw stderr fallback. + sys_logger.addHandler(logging.NullHandler()) + + +def _attach_stream(stream): + from .privacy_logging import PrivacyStreamHandler + try: + if stream is None: + return False + # Detect already closed or failing sinks without logging user data. + stream.write("") + stream.flush() + sys_logger.addHandler(PrivacyStreamHandler(stream=stream)) + return True + except Exception: + return False + + +def add_log_handler(log_directory=None, *, to_std=False): + """Return whether at least one usable safe sink was installed.""" + if to_std: + return _attach_stream(sys.stderr) + if log_directory is None: + from docassemble.webapp.config import LOG_DIRECTORY + log_directory = LOG_DIRECTORY + added = False + for attempt in range(5): + try: + stream = open(os.path.join(log_directory, "docassemble.log"), + "a", encoding="utf-8") + _log_files.append(stream) + added = _attach_stream(stream) + if not added: + _close_owned_files() + break + except OSError: + if attempt < 4: + time.sleep(1) + except Exception: + break + if os.environ.get("SUPERVISORLOGLEVEL", "info") == "debug": + added = _attach_stream(sys.stderr) or added + return added + + +def initialize(): + """Fail startup with a fixed error if no safe application sink exists. + + Celery, cron and log-to-std retain their stderr destination. All contexts + replace the base callback, including the failure path once it is available. + """ + global sys_logger, _configured + _configured = False + previous_class = None + failed = False + try: + previous_class = logging.getLoggerClass() + logging.setLoggerClass(UnsilenceableLogger) + sys_logger = logging.getLogger("docassemble") + _silence_logger() + _close_owned_files() + sys_logger.disabled = False + sys_logger.setLevel(logging.DEBUG) + from docassemble.base.logger import set_logmessage + # Install before config/sink setup so failure cannot restore raw output. + set_logmessage(syslog_message) + from docassemble.webapp.config import ( + LOGSERVER, LOG_DIRECTORY, daconfig, in_celery, in_cron, + ) + to_std = in_celery or in_cron or daconfig.get("log to std", False) + if not add_log_handler(LOG_DIRECTORY, to_std=to_std): + raise RuntimeError("Privacy logging unavailable") + if LOGSERVER is None: + set_logmessage(syslog_message_with_timestamp) + _configured = True + except Exception: + failed = True + try: + _silence_logger() + except Exception: + pass + _close_owned_files() + finally: + if previous_class is not None: + try: + logging.setLoggerClass(previous_class) + except Exception: + failed = True + if failed: + _configured = False + # Drop our raw error context and suppress any active caller exception. + raise RuntimeError("Privacy logging unavailable") from None + + +initialize() diff --git a/tests/privacy_logging/reference/privacy_logging.py b/tests/privacy_logging/reference/privacy_logging.py new file mode 100644 index 000000000..feeed0c2d --- /dev/null +++ b/tests/privacy_logging/reference/privacy_logging.py @@ -0,0 +1,160 @@ +import logging +import json +import datetime + + +SAFE_EVENT_CODES = frozenset({ + "FORM_START", + "FORM_SUBMIT", + "FORM_ERROR", + "SESSION_START", + "SESSION_END", + "AUTH_CHECK", + "PDF_GENERATE", +}) + +EVENT_UNKNOWN = "UNKNOWN" + +_LEVEL_MAP = { + logging.DEBUG: "DEBUG", + logging.INFO: "INFO", + logging.WARNING: "WARNING", + logging.ERROR: "ERROR", + logging.CRITICAL: "CRITICAL", +} + +SVC = "docassemble" +COMP = "app" + +_TS_EPOCH = 0.0 +_TS_UPPER = 4102444800.0 + +_MISSING = "___" + +_FALLBACK = json.dumps({ + "ts": "1970-01-01T00:00:00Z", + "level": "UNKNOWN", + "svc": SVC, + "comp": COMP, + "event": EVENT_UNKNOWN, +}, sort_keys=True) + + +class PrivacyFormatter(logging.Formatter): + """Strict-allowlist logging formatter. + + Output contains ONLY: + - ts: validated finite numeric UTC timestamp from record.created + - level: canonical string from exact type(levelno) is int mapping + - svc: fixed service label + - comp: fixed component label + - event: membership-checked code from SAFE_EVENT_CODES + + ALL other LogRecord fields are ignored: name, msg, args, pathname, + lineno, safe_diag, exc_info, stack_info, and any arbitrary attributes + (clientip, yamlfile, session, user, etc). + + Untrusted objects are never stringified. Exact type() checks prevent + hostile subclasses with custom __str__/__repr__/__eq__. + + On any error, returns a static JSON fallback string. + """ + + @classmethod + def _safe_str(cls, x): + if type(x) is str and x is not _MISSING: + return x + return _MISSING + + def format(self, record): + try: + if not isinstance(record, logging.LogRecord): + return _FALLBACK + + lvl = record.levelno + if type(lvl) is int and lvl in _LEVEL_MAP: + level = _LEVEL_MAP[lvl] + else: + level = "UNKNOWN" + + event = self._safe_str(getattr(record, "event_code", None)) + if event not in SAFE_EVENT_CODES: + event = EVENT_UNKNOWN + + ts_raw = record.created + if type(ts_raw) not in (int, float): + ts_out = "1970-01-01T00:00:00Z" + elif ts_raw != ts_raw or ts_raw == float("inf") or ts_raw == float("-inf"): + ts_out = "1970-01-01T00:00:00Z" + elif not (_TS_EPOCH <= ts_raw <= _TS_UPPER): + ts_out = "1970-01-01T00:00:00Z" + else: + ts_out = datetime.datetime.fromtimestamp( + float(ts_raw), tz=datetime.timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") + + out = { + "ts": ts_out, + "level": level, + "svc": SVC, + "comp": COMP, + "event": event, + } + return json.dumps(out, sort_keys=True) + except Exception: + return _FALLBACK + + +class PrivacyStreamHandler(logging.StreamHandler): + """Stream handler that never emits raw record data. + + Always uses PrivacyFormatter regardless of setFormatter() calls. + Overrides emit() to bypass the standard formatter dispatch, so that + default construction, setFormatter(None), or replacement with a + generic logging.Formatter cannot cause raw record output. + + Overrides handleError to prevent the standard behavior of writing + raw record.msg/args to stderr when a write or flush fails. + Maintains a bounded local failure counter. + + Does not open files, call networking, or invoke subprocess. + """ + + _FAILURE_MAX = 99 + + def __init__(self, stream=None): + super().__init__(stream) + self._failure_count = 0 + self._safe_fmt = PrivacyFormatter() + + def setFormatter(self, fmt=None): + pass + + def emit(self, record): + try: + msg = self._safe_fmt.format(record) + self.stream.write(msg + self.terminator) + self.flush() + except Exception: + self.handleError(record) + + def flush(self): + try: + self.stream.flush() + except Exception: + if self._failure_count < self._FAILURE_MAX: + self._failure_count += 1 + + def handleError(self, record): + if self._failure_count < self._FAILURE_MAX: + self._failure_count += 1 + + @property + def failure_count(self): + return self._failure_count + + +def safe_format_record(record): + """Format a LogRecord into safe JSON. Returns _FALLBACK for non-records.""" + fmt = PrivacyFormatter() + return fmt.format(record) diff --git a/tests/privacy_logging/test_entrypoints.py b/tests/privacy_logging/test_entrypoints.py new file mode 100644 index 000000000..61dcb092f --- /dev/null +++ b/tests/privacy_logging/test_entrypoints.py @@ -0,0 +1,255 @@ +import builtins +import io +import itertools +import json +import logging +import sys +import traceback +import unittest +from unittest.mock import patch + +from module_support import fixture, privacy, start + +MARKER = "SYNTHETIC_PRIVATE_MARKER" +KEYS = {"ts", "level", "svc", "comp", "event"} + + +class Hostile: + def __str__(self): + raise AssertionError("message must not be stringified") + + def __repr__(self): + raise AssertionError("message must not be represented") + + +class BrokenStream(io.StringIO): + fail_write = False + fail_flush = False + + def write(self, value): + if self.fail_write: + raise OSError(MARKER) + return super().write(value) + + def flush(self): + if self.fail_flush: + raise OSError(MARKER) + return super().flush() + + +class TestRealModuleEntrypoint(unittest.TestCase): + def assert_safe(self, data, count=1, event="UNKNOWN"): + self.assertNotIn(MARKER, data) + rows = [json.loads(line) for line in data.splitlines()] + self.assertEqual(len(rows), count) + for row in rows: + self.assertEqual(set(row), KEYS) + self.assertEqual(row["svc"], "docassemble") + self.assertEqual(row["comp"], "app") + self.assertEqual(row["event"], event) + + def test_every_context_sink_logserver_and_debug_combination(self): + for context, server, debug in itertools.product( + ("web", "celery", "cron", "std"), (None, "synthetic"), (False, True)): + with self.subTest(context=context, server=server, debug=debug): + with fixture(context, server, debug) as state: + module = start(state) + self.assertTrue(module._configured) + self.assertFalse(module.sys_logger.propagate) + callback = (module.syslog_message_with_timestamp if server is None + else module.syslog_message) + self.assertIs(state.base.callback, callback) + state.base.logmessage(MARKER) + state.base.logmessage(Hostile()) + path = state.directory / "docassemble.log" + if context == "web": + self.assert_safe(path.read_text(), count=2) + self.assert_safe(state.stderr.getvalue(), count=2 if debug else 0) + else: + self.assertFalse(path.exists()) + self.assert_safe(state.stderr.getvalue(), count=2) + + def test_replaces_existing_raw_handlers_filters_and_disables_propagation(self): + with fixture() as state: + raw = io.StringIO() + old = logging.getLogger("docassemble") + old.addHandler(logging.StreamHandler(raw)) + old.addFilter(lambda record: (_ for _ in ()).throw(RuntimeError(MARKER))) + root = logging.getLogger() + handler = logging.StreamHandler(raw) + root.addHandler(handler) + try: + module = start(state) + module.sys_logger.error(MARKER, exc_info=(ValueError, ValueError(MARKER), None), + extra={"event_code": "FORM_ERROR", "user": MARKER}) + self.assert_safe((state.directory / "docassemble.log").read_text(), + event="FORM_ERROR") + self.assertEqual(raw.getvalue(), "") + finally: + root.removeHandler(handler) + + def test_reinitialization_closes_owned_file_without_duplicate_output(self): + with fixture() as state: + module = start(state) + previous = module._log_files[0] + module.initialize() + self.assertTrue(previous.closed) + self.assertEqual(len(module._log_files), 1) + state.base.logmessage(MARKER) + self.assert_safe((state.directory / "docassemble.log").read_text()) + + def test_no_file_sink_fails_closed_after_five_attempts(self): + with fixture() as state, patch.object(builtins, "open", side_effect=OSError(MARKER)) as opened, \ + patch("time.sleep") as sleep: + try: + start(state) + except RuntimeError as error: + self.assertEqual(str(error), "Privacy logging unavailable") + self.assertNotIn(MARKER, traceback.format_exc()) + else: + self.fail("missing sink did not stop initialization") + self.assertEqual(opened.call_count, 5) + self.assertEqual(sleep.call_count, 4) + state.base.logmessage(MARKER) + logging.getLogger("docassemble").error(MARKER) + self.assertEqual(state.stderr.getvalue(), "") + + def test_debug_stderr_remains_safe_when_file_cannot_open(self): + with fixture(debug=True) as state, patch.object(builtins, "open", side_effect=OSError(MARKER)), \ + patch("time.sleep"): + self.assertTrue(start(state)._configured) + state.base.logmessage(MARKER) + self.assert_safe(state.stderr.getvalue()) + + def test_each_stderr_context_rejects_missing_closed_and_failed_sink(self): + for context, failure in itertools.product(("celery", "cron", "std"), + ("missing", "closed", "write", "flush")): + with self.subTest(context=context, failure=failure): + stream = BrokenStream() + if failure == "closed": + stream.close() + stream.fail_write = failure == "write" + stream.fail_flush = failure == "flush" + with fixture(context, stderr=stream) as state: + with patch("sys.stderr", None if failure == "missing" else stream): + with self.assertRaisesRegex(RuntimeError, "^Privacy logging unavailable$"): + start(state) + state.base.logmessage(MARKER) + logging.getLogger("docassemble").error(MARKER) + if failure != "closed": + self.assertEqual(stream.getvalue(), "") + + def test_later_sink_failures_and_shutdown_do_not_print_record_or_error(self): + for context, mode in itertools.product(("web", "celery", "cron", "std"), + ("write", "flush")): + with self.subTest(context=context, mode=mode): + stream = BrokenStream() + with fixture(context, stderr=stream) as state: + module = start(state) + if context == "web": + safe = next(h for h in module.sys_logger.handlers + if isinstance(h, privacy.PrivacyStreamHandler)) + safe.stream = stream + stream.fail_write = mode == "write" + stream.fail_flush = mode == "flush" + state.base.logmessage(MARKER) + module.sys_logger.error(MARKER) + for handler in module.sys_logger.handlers: + handler.flush() + self.assert_safe(stream.getvalue(), count=0 if mode == "write" else 2) + + def test_configuration_exception_has_fixed_error_and_safe_registered_callback(self): + class BrokenConfig: + def get(self, *args): + raise ValueError(MARKER) + with fixture() as state: + state.config.daconfig = BrokenConfig() + try: + start(state) + except RuntimeError as error: + self.assertEqual(str(error), "Privacy logging unavailable") + self.assertNotIn(MARKER, traceback.format_exc()) + else: + self.fail("configuration failure was ignored") + state.base.logmessage(MARKER) + self.assertEqual(state.stderr.getvalue(), "") + + def test_callback_logging_exception_does_not_fall_back_to_raw_stderr(self): + with fixture() as state: + module = start(state) + with patch.object(module.sys_logger, "handle", side_effect=RuntimeError(MARKER)): + state.base.logmessage(MARKER) + module.syslog_message_with_timestamp(Hostile()) + self.assertEqual(state.stderr.getvalue(), "") + + def test_dependency_import_errors_are_fixed_and_have_no_raw_context(self): + original_import = builtins.__import__ + for dependency in ("docassemble.base.logger", "docassemble.webapp.config", "privacy_logging"): + with self.subTest(dependency=dependency), fixture() as state: + def importing(name, *args, **kwargs): + if name == dependency: + raise ImportError(MARKER) + return original_import(name, *args, **kwargs) + with patch.object(builtins, "__import__", side_effect=importing): + with self.assertRaisesRegex(RuntimeError, "^Privacy logging unavailable$") as caught: + start(state) + self.assertIsNone(caught.exception.__context__) + self.assertIsNone(caught.exception.__cause__) + self.assertEqual(state.stderr.getvalue(), "") + self.assertFalse(sys.modules["docassemble.webapp.log_initialize"]._log_files) + if dependency != "docassemble.base.logger": + state.base.logmessage(MARKER) + self.assertEqual(state.stderr.getvalue(), "") + + def test_logger_setup_and_cleanup_errors_do_not_escape_raw(self): + for method in ("getLoggerClass", "getLogger"): + with self.subTest(method=method), fixture() as state: + with patch.object(logging, method, side_effect=RuntimeError(MARKER)): + with self.assertRaisesRegex(RuntimeError, "^Privacy logging unavailable$") as caught: + start(state) + self.assertIsNone(caught.exception.__context__) + self.assertEqual(state.stderr.getvalue(), "") + with fixture() as state: + logger = logging.getLogger("docassemble") + with patch.object(logger, "addHandler", side_effect=RuntimeError(MARKER)): + with self.assertRaisesRegex(RuntimeError, "^Privacy logging unavailable$") as caught: + start(state) + self.assertIsNone(caught.exception.__context__) + self.assertEqual(state.stderr.getvalue(), "") + + def test_bad_file_probe_closes_stream_and_debug_file_survives_bad_stderr(self): + stream = BrokenStream() + stream.fail_flush = True + with fixture() as state, patch.object(builtins, "open", return_value=stream): + with self.assertRaisesRegex(RuntimeError, "^Privacy logging unavailable$"): + start(state) + self.assertTrue(stream.closed) + self.assertEqual(state.stderr.getvalue(), "") + stream = BrokenStream() + stream.fail_write = True + with fixture(debug=True, stderr=stream) as state: + self.assertTrue(start(state)._configured) + state.base.logmessage(MARKER) + self.assert_safe((state.directory / "docassemble.log").read_text()) + self.assertEqual(stream.getvalue(), "") + + def test_fixed_initialization_failure_suppresses_active_caller_exception(self): + with fixture() as state, patch.object(builtins, "open", side_effect=OSError(MARKER)), \ + patch("time.sleep"): + try: + raise ValueError("CALLER_PRIVATE_MARKER") + except ValueError: + try: + start(state) + except RuntimeError as error: + self.assertEqual(str(error), "Privacy logging unavailable") + rendered = traceback.format_exc() + self.assertNotIn("CALLER_PRIVATE_MARKER", rendered) + self.assertNotIn(MARKER, rendered) + self.assertTrue(error.__suppress_context__) + else: + self.fail("missing sink did not fail initialization") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/privacy_logging/test_privacy_handlers.py b/tests/privacy_logging/test_privacy_handlers.py new file mode 100644 index 000000000..61e010a31 --- /dev/null +++ b/tests/privacy_logging/test_privacy_handlers.py @@ -0,0 +1,306 @@ +import unittest +import logging +import json +import io +import sys + +from module_support import privacy, ROOT + +from docassemble.webapp.privacy_logging import ( + PrivacyFormatter, + PrivacyStreamHandler, + SAFE_EVENT_CODES, + EVENT_UNKNOWN, + SVC, + COMP, + _FALLBACK, + safe_format_record, +) + + +def _make_record(**overrides): + defaults = dict( + name="test.logger", + level=logging.INFO, + pathname="test.py", + lineno=1, + msg="ignored message", + args=(), + exc_info=None, + ) + defaults.update(overrides) + return logging.LogRecord(**defaults) + + +class TestPrivacyStreamHandlerSafeError(unittest.TestCase): + + def test_write_failure_no_stderr_output(self): + class FailStream(io.StringIO): + def write(self, s): + raise OSError("disk full") + + buf = io.StringIO() + handler = PrivacyStreamHandler(stream=FailStream()) + handler.setFormatter(PrivacyFormatter()) + + old_stderr = sys.stderr + sys.stderr = buf + try: + r = _make_record(msg="192.0.2.1 secret data") + r.event_code = "FORM_START" + handler.emit(r) + finally: + sys.stderr = old_stderr + + self.assertEqual(buf.getvalue(), "") + self.assertNotIn("192.0.2.1", buf.getvalue()) + self.assertNotIn("secret data", buf.getvalue()) + + def test_flush_failure_no_stderr_output(self): + class FlushFailStream(io.StringIO): + def flush(self): + raise OSError("flush broken") + + buf = io.StringIO() + handler = PrivacyStreamHandler(stream=FlushFailStream()) + handler.setFormatter(PrivacyFormatter()) + + old_stderr = sys.stderr + sys.stderr = buf + try: + r = _make_record(msg="198.51.100.1 private info") + handler.emit(r) + finally: + sys.stderr = old_stderr + + self.assertEqual(buf.getvalue(), "") + + def test_failure_counter_increments(self): + class FailStream(io.StringIO): + def write(self, s): + raise OSError("fail") + + handler = PrivacyStreamHandler(stream=FailStream()) + handler.setFormatter(PrivacyFormatter()) + self.assertEqual(handler.failure_count, 0) + + r = _make_record() + handler.emit(r) + self.assertEqual(handler.failure_count, 1) + handler.emit(r) + self.assertEqual(handler.failure_count, 2) + + def test_failure_counter_bounded(self): + class FailStream(io.StringIO): + def write(self, s): + raise OSError("fail") + + handler = PrivacyStreamHandler(stream=FailStream()) + handler.setFormatter(PrivacyFormatter()) + r = _make_record() + for _ in range(200): + handler.emit(r) + self.assertLessEqual( + handler.failure_count, PrivacyStreamHandler._FAILURE_MAX + ) + + def test_successful_write_to_stream(self): + stream = io.StringIO() + handler = PrivacyStreamHandler(stream=stream) + handler.setFormatter(PrivacyFormatter()) + + r = _make_record() + r.event_code = "FORM_START" + handler.emit(r) + + output = stream.getvalue() + self.assertTrue(len(output) > 0) + parsed = json.loads(output.strip()) + self.assertEqual(parsed["event"], "FORM_START") + self.assertEqual(parsed["svc"], SVC) + self.assertEqual(parsed["comp"], COMP) + + +class TestHandlerFormatterLockdown(unittest.TestCase): + """Handler must always use PrivacyFormatter; no raw fallback path.""" + + def test_default_construction_no_formatter_set(self): + stream = io.StringIO() + handler = PrivacyStreamHandler(stream=stream) + r = _make_record(msg="SYNTHETIC_PRIVATE_MARKER") + handler.emit(r) + output = stream.getvalue() + self.assertNotIn("SYNTHETIC_PRIVATE_MARKER", output) + parsed = json.loads(output.strip()) + self.assertEqual(parsed["svc"], SVC) + + def test_setFormatter_none_still_safe(self): + stream = io.StringIO() + handler = PrivacyStreamHandler(stream=stream) + handler.setFormatter(None) + r = _make_record(msg="SYNTHETIC_PRIVATE_MARKER") + handler.emit(r) + output = stream.getvalue() + self.assertNotIn("SYNTHETIC_PRIVATE_MARKER", output) + parsed = json.loads(output.strip()) + self.assertEqual(parsed["event"], EVENT_UNKNOWN) + + def test_setFormatter_generic_formatter_rejected(self): + stream = io.StringIO() + handler = PrivacyStreamHandler(stream=stream) + handler.setFormatter(logging.Formatter("%(message)s")) + r = _make_record(msg="SYNTHETIC_PRIVATE_MARKER") + handler.emit(r) + output = stream.getvalue() + self.assertNotIn("SYNTHETIC_PRIVATE_MARKER", output) + parsed = json.loads(output.strip()) + self.assertEqual(parsed["svc"], SVC) + + def test_setFormatter_raw_formatter_rejected(self): + stream = io.StringIO() + handler = PrivacyStreamHandler(stream=stream) + handler.setFormatter(logging.Formatter( + "%(name)s %(levelname)s %(message)s %(safe_diag)s" + )) + r = _make_record( + name="private_client_192.0.2.1", + msg="raw message content", + ) + r.safe_diag = "192.0.2.1 /interview?session=secret" + handler.emit(r) + output = stream.getvalue() + self.assertNotIn("private_client", output) + self.assertNotIn("192.0.2.1", output) + self.assertNotIn("raw message content", output) + self.assertNotIn("session=secret", output) + parsed = json.loads(output.strip()) + self.assertEqual(set(parsed.keys()), {"ts", "level", "svc", "comp", "event"}) + + def test_handle_via_handler_handle_default_construction(self): + stream = io.StringIO() + handler = PrivacyStreamHandler(stream=stream) + r = _make_record(msg="SYNTHETIC_PRIVATE_MARKER") + handler.handle(r) + output = stream.getvalue() + self.assertNotIn("SYNTHETIC_PRIVATE_MARKER", output) + + def test_write_failure_after_formatter_override_no_stderr(self): + class FailStream(io.StringIO): + def write(self, s): + raise OSError("disk full") + + buf = io.StringIO() + handler = PrivacyStreamHandler(stream=FailStream()) + handler.setFormatter(logging.Formatter("%(message)s")) + + old_stderr = sys.stderr + sys.stderr = buf + try: + r = _make_record(msg="SYNTHETIC_PRIVATE_MARKER") + handler.emit(r) + finally: + sys.stderr = old_stderr + + self.assertEqual(buf.getvalue(), "") + self.assertNotIn("SYNTHETIC_PRIVATE_MARKER", buf.getvalue()) + + +class TestHandlerFlushSafety(unittest.TestCase): + """flush() must not propagate exceptions or leak to stderr.""" + + def test_direct_flush_failure_does_not_raise(self): + class FlushFailStream(io.StringIO): + def flush(self): + raise RuntimeError("SYNTHETIC_PRIVATE_MARKER") + + handler = PrivacyStreamHandler(stream=FlushFailStream()) + handler.flush() + + def test_direct_flush_failure_no_stderr(self): + class FlushFailStream(io.StringIO): + def flush(self): + raise RuntimeError("SYNTHETIC_PRIVATE_MARKER") + + handler = PrivacyStreamHandler(stream=FlushFailStream()) + buf = io.StringIO() + old_stderr = sys.stderr + sys.stderr = buf + try: + handler.flush() + finally: + sys.stderr = old_stderr + self.assertEqual(buf.getvalue(), "") + self.assertNotIn("SYNTHETIC_PRIVATE_MARKER", buf.getvalue()) + + def test_flush_failure_counts_once(self): + class FlushFailStream(io.StringIO): + def flush(self): + raise OSError("broken") + + handler = PrivacyStreamHandler(stream=FlushFailStream()) + self.assertEqual(handler.failure_count, 0) + handler.flush() + self.assertEqual(handler.failure_count, 1) + handler.flush() + self.assertEqual(handler.failure_count, 2) + + def test_close_with_flush_failure_does_not_raise(self): + class FlushFailStream(io.StringIO): + def flush(self): + raise RuntimeError("SYNTHETIC_PRIVATE_MARKER") + + handler = PrivacyStreamHandler(stream=FlushFailStream()) + handler.close() + + def test_shutdown_simulation_no_stderr_leak(self): + class FlushFailStream(io.StringIO): + def flush(self): + raise RuntimeError("SYNTHETIC_PRIVATE_MARKER") + + handler = PrivacyStreamHandler(stream=FlushFailStream()) + buf = io.StringIO() + old_stderr = sys.stderr + sys.stderr = buf + try: + handler.flush() + handler.close() + finally: + sys.stderr = old_stderr + self.assertEqual(buf.getvalue(), "") + self.assertNotIn("SYNTHETIC_PRIVATE_MARKER", buf.getvalue()) + + def test_subprocess_shutdown_no_leak(self): + script = ( + "import logging, sys, io\n" + "from module_support import privacy\n" + "from docassemble.webapp.privacy_logging import PrivacyStreamHandler\n" + "class F(io.StringIO):\n" + " def flush(self):\n" + " raise RuntimeError('SYNTHETIC_PRIVATE_MARKER')\n" + "h = PrivacyStreamHandler(stream=F())\n" + "logging.getLogger('x').addHandler(h)\n" + ) + old_stderr = sys.stderr + capture = io.StringIO() + sys.stderr = capture + try: + import subprocess + result = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + timeout=10, + cwd=ROOT / "tests/privacy_logging", + ) + finally: + sys.stderr = old_stderr + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stderr, "") + self.assertNotIn("SYNTHETIC_PRIVATE_MARKER", result.stderr) + self.assertNotIn("SYNTHETIC_PRIVATE_MARKER", result.stdout) + self.assertNotIn("SYNTHETIC_PRIVATE_MARKER", capture.getvalue()) + + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/privacy_logging/test_privacy_logging.py b/tests/privacy_logging/test_privacy_logging.py new file mode 100644 index 000000000..bf5aec4f5 --- /dev/null +++ b/tests/privacy_logging/test_privacy_logging.py @@ -0,0 +1,352 @@ +import unittest +import logging +import json +import io +import sys + +from module_support import privacy + +from docassemble.webapp.privacy_logging import ( + PrivacyFormatter, + PrivacyStreamHandler, + SAFE_EVENT_CODES, + EVENT_UNKNOWN, + SVC, + COMP, + _FALLBACK, + safe_format_record, +) + + +def _make_record(**overrides): + defaults = dict( + name="test.logger", + level=logging.INFO, + pathname="test.py", + lineno=1, + msg="ignored message", + args=(), + exc_info=None, + ) + defaults.update(overrides) + return logging.LogRecord(**defaults) + + +class TestPrivacyFormatterNoLeaks(unittest.TestCase): + """No untrusted field content may appear in formatted output.""" + + def setUp(self): + self.fmt = PrivacyFormatter() + + def out(self, record): + return self.fmt.format(record) + + def test_ipv4_in_msg_not_in_output(self): + r = _make_record(msg="client 192.0.2.1 submitted form") + self.assertNotIn("192.0.2.1", self.out(r)) + + def test_ipv6_in_msg_not_in_output(self): + r = _make_record(msg="client 2001:db8::1 submitted form") + self.assertNotIn("2001:db8::1", self.out(r)) + + def test_ipv4_in_logger_name_not_in_output(self): + r = _make_record(name="private_client_192.0.2.1") + self.assertNotIn("192.0.2.1", self.out(r)) + self.assertNotIn("private_client", self.out(r)) + + def test_ipv4_in_safe_diag_not_in_output(self): + r = _make_record() + r.safe_diag = "192.0.2.1 /interview?i=synthetic.yml session=synthetic-secret" + self.assertNotIn("192.0.2.1", self.out(r)) + self.assertNotIn("synthetic.yml", self.out(r)) + self.assertNotIn("synthetic-secret", self.out(r)) + + def test_encoded_query_in_msg_not_in_output(self): + r = _make_record(msg="GET /interview?q=%3Ftoken%3Dsecret%26user%3Dadmin") + self.assertNotIn("token", self.out(r)) + self.assertNotIn("secret", self.out(r)) + self.assertNotIn("%3F", self.out(r)) + + def test_multiline_msg_not_in_output(self): + r = _make_record(msg="line1\n192.0.2.1\nline3") + out = self.out(r) + self.assertNotIn("line1", out) + self.assertNotIn("192.0.2.1", out) + self.assertNotIn("\n", out) + + def test_logger_name_injection_not_in_output(self): + r = _make_record(name="ERROR\x1b[31mINJECTED") + out = self.out(r) + self.assertNotIn("INJECTED", out) + self.assertNotIn("\x1b", out) + + def test_levelname_injection_ignored(self): + r = _make_record() + r.levelname = "SUPERCRITICAL" + out = self.out(r) + self.assertNotIn("SUPERCRITICAL", out) + parsed = json.loads(out) + self.assertEqual(parsed["level"], "INFO") + + def test_lineno_not_in_output(self): + r = _make_record() + r.lineno = 31337 + self.assertNotIn("31337", self.out(r)) + + def test_exception_text_not_in_output(self): + try: + raise ValueError("secret error detail 203.0.113.5") + except ValueError: + ei = sys.exc_info() + r = _make_record(msg="fail", exc_info=ei) + out = self.out(r) + self.assertNotIn("secret error detail", out) + self.assertNotIn("203.0.113.5", out) + self.assertNotIn("Traceback", out) + + def test_clientip_attr_not_in_output(self): + r = _make_record() + r.clientip = "198.51.100.7" + self.assertNotIn("198.51.100.7", self.out(r)) + + def test_yamlfile_attr_not_in_output(self): + r = _make_record() + r.yamlfile = "/interviews/private_form.yml" + self.assertNotIn("private_form", self.out(r)) + + def test_session_attr_not_in_output(self): + r = _make_record() + r.session = "sess_abc123secret" + self.assertNotIn("sess_abc123secret", self.out(r)) + + def test_user_attr_not_in_output(self): + r = _make_record() + r.user = "filer_jane_doe" + self.assertNotIn("filer_jane_doe", self.out(r)) + + def test_pathname_not_in_output(self): + r = _make_record(pathname="/srv/app/secret_module.py") + self.assertNotIn("secret_module", self.out(r)) + + +class TestHostileStringConversion(unittest.TestCase): + """Formatter must never call __str__, __repr__, or __eq__ on msg.""" + + def setUp(self): + self.fmt = PrivacyFormatter() + + def test_hostile_str_not_called(self): + class HostileStr: + called = False + def __str__(self): + HostileStr.called = True + return "192.0.2.1 leaked via __str__" + r = _make_record(msg=HostileStr()) + self.fmt.format(r) + self.assertFalse(HostileStr.called) + + def test_hostile_repr_not_called(self): + class HostileRepr: + called = False + def __repr__(self): + HostileRepr.called = True + return "192.0.2.1 leaked via __repr__" + r = _make_record(msg=HostileRepr()) + self.fmt.format(r) + self.assertFalse(HostileRepr.called) + + def test_hostile_eq_not_called(self): + class HostileEq: + called = False + def __eq__(self, other): + HostileEq.called = True + return True + r = _make_record(msg=HostileEq()) + self.fmt.format(r) + self.assertFalse(HostileEq.called) + + def test_hostile_str_in_args_not_called(self): + class HostileArg: + called = False + def __str__(self): + HostileArg.called = True + return "leaked" + r = _make_record(msg="msg %s", args=(HostileArg(),)) + self.fmt.format(r) + self.assertFalse(HostileArg.called) + + +class TestInvalidTimestampTypes(unittest.TestCase): + + def setUp(self): + self.fmt = PrivacyFormatter() + + def _ts_out(self, created): + r = _make_record() + r.created = created + return json.loads(self.fmt.format(r))["ts"] + + def test_string_timestamp_rejected(self): + self.assertEqual(self._ts_out("2026-01-01"), "1970-01-01T00:00:00Z") + + def test_none_timestamp_rejected(self): + self.assertEqual(self._ts_out(None), "1970-01-01T00:00:00Z") + + def test_nan_timestamp_rejected(self): + self.assertEqual(self._ts_out(float("nan")), "1970-01-01T00:00:00Z") + + def test_inf_timestamp_rejected(self): + self.assertEqual(self._ts_out(float("inf")), "1970-01-01T00:00:00Z") + + def test_negative_inf_timestamp_rejected(self): + self.assertEqual(self._ts_out(float("-inf")), "1970-01-01T00:00:00Z") + + def test_future_out_of_range_rejected(self): + self.assertEqual(self._ts_out(99999999999.0), "1970-01-01T00:00:00Z") + + def test_negative_timestamp_rejected(self): + self.assertEqual(self._ts_out(-1.0), "1970-01-01T00:00:00Z") + + def test_valid_float_accepted(self): + self.assertIn("2024", self._ts_out(1725500000.0)) + + def test_valid_int_accepted(self): + self.assertIn("2024", self._ts_out(1725500000)) + + def test_bool_timestamp_rejected(self): + self.assertEqual(self._ts_out(True), "1970-01-01T00:00:00Z") + + def test_list_timestamp_rejected(self): + self.assertEqual(self._ts_out([1, 2]), "1970-01-01T00:00:00Z") + + +class TestInvalidLevelTypes(unittest.TestCase): + + def setUp(self): + self.fmt = PrivacyFormatter() + + def _level_out(self, levelno): + r = _make_record() + r.levelno = levelno + return json.loads(self.fmt.format(r))["level"] + + def test_string_level_rejected(self): + self.assertEqual(self._level_out("INFO"), "UNKNOWN") + + def test_bool_level_rejected(self): + self.assertEqual(self._level_out(True), "UNKNOWN") + + def test_float_level_rejected(self): + self.assertEqual(self._level_out(20.0), "UNKNOWN") + + def test_none_level_rejected(self): + self.assertEqual(self._level_out(None), "UNKNOWN") + + def test_unknown_int_level(self): + self.assertEqual(self._level_out(999), "UNKNOWN") + + def test_valid_int_levels_accepted(self): + for lvl_int, expected in [ + (logging.DEBUG, "DEBUG"), + (logging.INFO, "INFO"), + (logging.WARNING, "WARNING"), + (logging.ERROR, "ERROR"), + (logging.CRITICAL, "CRITICAL"), + ]: + self.assertEqual(self._level_out(lvl_int), expected) + + +class TestUnknownEventCodes(unittest.TestCase): + + def setUp(self): + self.fmt = PrivacyFormatter() + + def _event_out(self, event_code): + r = _make_record() + if event_code is not None: + r.event_code = event_code + return json.loads(self.fmt.format(r))["event"] + + def test_missing_event_code(self): + self.assertEqual(self._event_out(None), EVENT_UNKNOWN) + + def test_unknown_string_event(self): + self.assertEqual(self._event_out("DROP_TABLE"), EVENT_UNKNOWN) + + def test_empty_string_event(self): + self.assertEqual(self._event_out(""), EVENT_UNKNOWN) + + def test_int_event_rejected(self): + self.assertEqual(self._event_out(42), EVENT_UNKNOWN) + + def test_bool_event_rejected(self): + self.assertEqual(self._event_out(True), EVENT_UNKNOWN) + + def test_str_subclass_event_rejected(self): + class EvilStr(str): + pass + self.assertEqual(self._event_out(EvilStr("FORM_START")), EVENT_UNKNOWN) + + +class TestSafeEventCodeAcceptance(unittest.TestCase): + + def setUp(self): + self.fmt = PrivacyFormatter() + + def test_all_safe_event_codes_accepted(self): + for code in SAFE_EVENT_CODES: + r = _make_record() + r.event_code = code + parsed = json.loads(self.fmt.format(r)) + self.assertEqual(parsed["event"], code) + + +class TestOutputStructure(unittest.TestCase): + + def setUp(self): + self.fmt = PrivacyFormatter() + + def test_output_is_valid_json(self): + r = _make_record() + parsed = json.loads(self.fmt.format(r)) + self.assertIsInstance(parsed, dict) + + def test_output_has_exactly_five_keys(self): + r = _make_record() + parsed = json.loads(self.fmt.format(r)) + self.assertEqual( + set(parsed.keys()), {"ts", "level", "svc", "comp", "event"} + ) + + def test_fixed_service_and_component(self): + r1 = _make_record(name="completely.different.logger") + r2 = _make_record(name="192.0.2.1.evil.logger", msg="10.0.0.1") + for r in (r1, r2): + parsed = json.loads(self.fmt.format(r)) + self.assertEqual(parsed["svc"], SVC) + self.assertEqual(parsed["comp"], COMP) + + def test_fallback_is_valid_json(self): + parsed = json.loads(_FALLBACK) + self.assertEqual( + set(parsed.keys()), {"ts", "level", "svc", "comp", "event"} + ) + + def test_non_logrecord_returns_fallback(self): + self.assertEqual(self.fmt.format("not a record"), _FALLBACK) + self.assertEqual(self.fmt.format(None), _FALLBACK) + self.assertEqual(self.fmt.format(42), _FALLBACK) + + def test_safe_format_record_helper(self): + r = _make_record() + r.event_code = "FORM_SUBMIT" + parsed = json.loads(safe_format_record(r)) + self.assertEqual(parsed["event"], "FORM_SUBMIT") + + def test_safe_format_record_non_record(self): + self.assertEqual(safe_format_record(None), _FALLBACK) + self.assertEqual(safe_format_record("string"), _FALLBACK) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/privacy_logging/test_startup_paths.py b/tests/privacy_logging/test_startup_paths.py new file mode 100644 index 000000000..253a02840 --- /dev/null +++ b/tests/privacy_logging/test_startup_paths.py @@ -0,0 +1,126 @@ +"""Execute real startup modules up to the first post-logger dependency boundary.""" +import contextlib +import importlib +import io +import json +import runpy +import sys +import types +import unittest +from unittest.mock import patch + +from module_support import MODULE, WEBAPP, fixture, package + + +class AfterLogging(Exception): + """Stop the synthetic application before database or service initialization.""" + + +class SyntheticApp: + @property + def secret_key(self): + return None + + @secret_key.setter + def secret_key(self, value): + raise AfterLogging() + + +def stub(name, **attrs): + result = types.ModuleType(name) + result.__dict__.update(attrs) + return result + + +@contextlib.contextmanager +def startup_dependencies(state): + base_config = stub("docassemble.base.config", loaded=False) + def base_load(**kwargs): + state.config.in_celery = kwargs.get("in_celery", False) + base_config.loaded = True + base_config.load = base_load + modules = { + "werkzeug": package("werkzeug"), + "werkzeug.middleware": package("werkzeug.middleware"), + "werkzeug.middleware.proxy_fix": stub("werkzeug.middleware.proxy_fix", ProxyFix=object), + "docassemble.base.plugin_manager": stub("docassemble.base.plugin_manager", pm=object()), + "docassemble.webapp.extensions": stub("docassemble.webapp.extensions", **{ + name: object() for name in ("csrf", "cors", "babel", "lm", "the_user_manager", "kv_session", "db")}), + "docassemble.webapp.setup": stub("docassemble.webapp.setup", init_app=lambda app: None), + "docassemble.webapp.app_object": stub("docassemble.webapp.app_object", flaskapp=SyntheticApp()), + "docassemble.base.config": base_config, + "celery": stub("celery", Celery=object, chord=object), + } + root = sys.modules["docassemble"] + base = sys.modules["docassemble.base"] + webapp = sys.modules["docassemble.webapp"] + with patch.dict(sys.modules, modules), \ + patch.object(root, "base", base, create=True), \ + patch.object(root, "webapp", webapp, create=True), \ + patch.object(base, "config", base_config, create=True): + for name in ("app_initialize", "flask_app", "worker", "server"): + sys.modules.pop("docassemble.webapp." + name, None) + yield base_config + + +class TestActualStartupPaths(unittest.TestCase): + def test_web_and_log_to_std_reach_import_time_protection(self): + for context in ("web", "std"): + with self.subTest(context=context), fixture(context) as state, startup_dependencies(state): + with self.assertRaises(AfterLogging): + importlib.import_module("docassemble.webapp.server") + module = sys.modules[MODULE] + self.assertTrue(module._configured) + state.base.logmessage("SYNTHETIC_PRIVATE_MARKER") + output = ((state.directory / "docassemble.log").read_text() + if context == "web" else state.stderr.getvalue()) + self.assertEqual(json.loads(output)["event"], "UNKNOWN") + self.assertNotIn("SYNTHETIC_PRIVATE_MARKER", output) + + def test_actual_worker_sets_celery_then_installs_safe_callback(self): + # Begin in default web context: the real worker must select Celery. + with fixture() as state, startup_dependencies(state) as base_config: + with self.assertRaises(AfterLogging): + importlib.import_module("docassemble.webapp.worker") + self.assertTrue(base_config.loaded) + self.assertTrue(state.config.in_celery) + self.assertTrue(sys.modules[MODULE]._configured) + state.base.logmessage("SYNTHETIC_PRIVATE_MARKER") + self.assertEqual(json.loads(state.stderr.getvalue())["event"], "UNKNOWN") + self.assertFalse((state.directory / "docassemble.log").exists()) + + def test_actual_cron_launcher_selects_cron_for_server_startup(self): + calls = [] + def launch(argv, **kwargs): + calls.append((argv, kwargs)) + return types.SimpleNamespace(returncode=0) + with patch("subprocess.run", side_effect=launch), \ + patch.object(sys, "argv", ["cron.py", "-type", "cron_daily"]): + with self.assertRaises(SystemExit) as stopped: + runpy.run_path(str(WEBAPP / "cron.py"), run_name="__main__") + self.assertEqual(stopped.exception.code, 0) + argv, options = calls[0] + self.assertEqual(argv[:3], ["flask", "--app", "docassemble.webapp.server"]) + self.assertEqual(options["env"]["IN_CRON"], "true") + # The Flask CLI and config loader are synthetic, not live dependencies. + with fixture() as state, startup_dependencies(state): + state.config.in_cron = options["env"]["IN_CRON"] == "true" + with self.assertRaises(AfterLogging): + importlib.import_module("docassemble.webapp.server") + self.assertTrue(sys.modules[MODULE]._configured) + state.base.logmessage("SYNTHETIC_PRIVATE_MARKER") + self.assertEqual(json.loads(state.stderr.getvalue())["event"], "UNKNOWN") + self.assertFalse((state.directory / "docassemble.log").exists()) + + def test_worker_no_sink_fails_before_post_logging_startup(self): + stream = io.StringIO() + stream.close() + with fixture(stderr=stream) as state, startup_dependencies(state): + with self.assertRaisesRegex(RuntimeError, "^Privacy logging unavailable$"): + importlib.import_module("docassemble.webapp.worker") + self.assertTrue(state.config.in_celery) + state.base.logmessage("SYNTHETIC_PRIVATE_MARKER") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/privacy_native/aggregate_reference.py b/tests/privacy_native/aggregate_reference.py new file mode 100644 index 000000000..aff36d9ef --- /dev/null +++ b/tests/privacy_native/aggregate_reference.py @@ -0,0 +1,33 @@ +"""Synthetic differential-test oracle; never imported by production services.""" + +import base64 +import hashlib +import json +from pathlib import Path +import sys + +from native_support import LogAggregator + + +def main() -> None: + root = Path(__file__).resolve().parents[2] + source = root / "tests/privacy_native/reference/log_aggregate.py" + cases = json.loads(sys.stdin.buffer.read(8 * 1024 * 1024)) + result = [] + for case in cases: + aggregate = LogAggregator(case["component"]) + steps = [aggregate.snapshot()] + for operation in case["operations"]: + if operation["finish"]: + aggregate.finish(operation["stream"]) + else: + aggregate.feed(operation["stream"], base64.b64decode(operation["data"] or "", validate=True)) + steps.append(aggregate.snapshot()) + result.append(steps) + json.dump({"source_sha256": hashlib.sha256(source.read_bytes()).hexdigest(), + "snapshots": result}, sys.stdout) + sys.stdout.write("\n") + + +if __name__ == "__main__": + main() diff --git a/tests/privacy_native/check_cron_launcher.py b/tests/privacy_native/check_cron_launcher.py new file mode 100644 index 000000000..cc943db1b --- /dev/null +++ b/tests/privacy_native/check_cron_launcher.py @@ -0,0 +1,115 @@ +"""Terminate actual installed cron launchers in the disposable synthetic image.""" +from http.cookies import SimpleCookie +import importlib.util +import json +import os +from pathlib import Path +import signal +import subprocess +import tempfile +import time + +REPO = Path(__file__).resolve().parents[2] +spec = importlib.util.spec_from_file_location('install_check', Path(__file__).with_name('check_install_image.py')) +install = importlib.util.module_from_spec(spec) +spec.loader.exec_module(install) + + +def active(pid): + try: + value = Path(f'/proc/{pid}/stat').read_text() + except FileNotFoundError: + return False + return value.rsplit(')', 1)[1].split()[0] != 'Z' + + +def check_stop(sig, task, pid_file, outer=False): + pid_file.write_text('') + env = dict(os.environ, JOS81_CRON_PID_FILE=str(pid_file)) + args = ['bash', '/etc/cron.monthly/docassemble'] if outer else ['bash', install.ROOT + '/webapp/run-cron.sh', task] + process = subprocess.Popen(args, + env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + start_new_session=True) + children = [] + try: + deadline = time.monotonic() + 45 + while time.monotonic() < deadline: + assert process.poll() is None, 'cron launcher exited before the interview' + try: + children = json.loads(pid_file.read_text()) + except json.JSONDecodeError: + time.sleep(0.1) + continue + break + assert len(children) == 3 and all(type(pid) is int and pid > 1 for pid in children) + assert len(set(children + [process.pid])) == 4 + ancestor = children[0] + for unused in range(8): + parent = int(Path(f'/proc/{ancestor}/stat').read_text().rsplit(')', 1)[1].split()[1]) + if parent == process.pid: + break + assert parent > 1 and parent not in children + children.append(parent) + ancestor = parent + else: + raise AssertionError('cron fixture did not descend from the tested launcher') + assert all(active(pid) for pid in children), 'expected Go, Flask and descendant processes' + # Signal only the entrypoint PID, not its process group or descendants. + process.send_signal(sig) + stdout, stderr = process.communicate(timeout=40 if outer else 30) + assert process.returncode == 143 and not stderr, 'launcher did not preserve child termination status' + lines = stdout.splitlines() + if outer: + assert len(lines) == 1 and install.records(lines[0], 'maintenance')[0]['unclassified'] >= 1 + else: + assert len(lines) == 2, 'unexpected termination records' + assert install.records(lines[0], 'cron')[0]['unclassified'] >= 2 + assert json.loads(lines[1]) == dict(schema=1, component='cron', event='command_failed', phase='execution') + assert all(not active(pid) for pid in children), 'cron descendant survived launcher termination' + finally: + # Use the launcher cleanup even if readiness failed before PID publication. + # The enclosing disposable-container trap is the final containment bound. + if process.poll() is None: + process.terminate() + try: + process.communicate(timeout=40 if outer else 25) + except subprocess.TimeoutExpired: + process.kill() + finally: + for pid in reversed(children): + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.stdout.close() + process.stderr.close() + process.wait(timeout=5) + + +def main(): + source = (REPO / 'tests/privacy_native/fixtures/cron-stop.yml').read_text() + assert source.count('JOS81_CRON_STOP_EVENT') == 1 + with tempfile.TemporaryDirectory(prefix='jos81-cron-stop-') as directory: + base = Path(directory) + pid_file = base / 'processes.json' + pid_file.touch(mode=0o600) + os.chown(base, 33, 33) + os.chown(pid_file, 33, 33) + for sig, task in ((signal.SIGTERM, 'cron_stop_term'), (signal.SIGINT, 'cron_stop_int'), + (signal.SIGTERM, 'cron_monthly')): + # Distinct interviews avoid waiting for the killed session's lock TTL. + target = Path(install.SITE + f'/docassemble/privacyfixture/data/questions/{task}.yml') + target.write_text(source.replace('JOS81_CRON_STOP_EVENT', task)) + cookies = SimpleCookie() + path = f'/nj/?i=docassemble.privacyfixture:data/questions/{task}.yml' + assert b'JOS81_CRON_STOP_READY' in install.fetch(path, cookies=cookies) + try: + check_stop(sig, task, pid_file, outer=task == 'cron_monthly') + finally: + target.unlink() # Prevent subsequent scheduled jobs entering this fixture. + install.private_output_absent((Path(install.ROOT + '/log'), Path('/var/log'))) + print('inner TERM/INT and outer monthly TERM stop Go, Flask and descendants; only safe output remains', flush=True) + + +if __name__ == '__main__': + main() diff --git a/tests/privacy_native/check_install_image.py b/tests/privacy_native/check_install_image.py new file mode 100644 index 000000000..6c4e546a2 --- /dev/null +++ b/tests/privacy_native/check_install_image.py @@ -0,0 +1,498 @@ +"""Full-image installation/rollback test using only synthetic local data.""" +import hashlib +import gzip +from http.cookies import SimpleCookie +from http.client import HTTPConnection, HTTPException +import json +import os +from pathlib import Path +import shutil +import socket +import stat +import subprocess +import sys +import time +from urllib.parse import urlsplit +import xmlrpc.client + +REPO = Path(__file__).resolve().parents[2] +WORK = Path('/tmp/jos81-rehearsal') +ROOT = '/usr/share/docassemble' +SITE = ROOT + '/local3.14/lib/python3.14/site-packages' +CATALOG = json.loads((REPO / 'docs/privacy/install-catalog.json').read_text()) +RPC = xmlrpc.client.ServerProxy('http://localhost:9001/RPC2') +COMPONENTS = ('celery', 'celerysingle', 'websockets', 'uwsgi', 'nginx') +COUNTERS = {'privacy-celery.log': 'celery', 'privacy-celerysingle.log': 'celerysingle', + 'privacy-websockets.log': 'websockets', 'privacy-uwsgi.log': 'uwsgi', + 'nginx-safe.log': 'nginx'} + + +def records(data, component): + def unique(pairs): + assert len({key for key, unused in pairs}) == len(pairs), 'duplicate counter fields' + return dict(pairs) + values = [json.loads(line, object_pairs_hook=unique) for line in data.splitlines()] + assert values, 'missing counters for ' + component + keys = {'schema', 'component', 'status_1xx', 'status_2xx', 'status_3xx', 'status_4xx', + 'status_5xx', 'latency_fast', 'latency_medium', 'latency_slow', 'latency_timeout', + 'unclassified', 'rejected', 'dropped'} + for value in values: + assert isinstance(value, dict) and set(value) == keys, 'unexpected counter fields' + assert value['schema'] == 1 and value['component'] == component, 'counter identity' + assert all(type(count) is int and 0 <= count <= 2147483647 + for key, count in value.items() if key != 'component'), 'counter bounds' + if component in ('celery', 'celerysingle', 'websockets', 'cron', 'initialize', 'maintenance'): + assert all(count == 0 for key, count in value.items() + if key.startswith(('status_', 'latency_'))), 'application request counts' + return values + + +def read_counters(): + return {component: records(Path(ROOT + '/log/' + name).read_text(), component) + for name, component in COUNTERS.items()} + + +def run(*args, timeout=100, **kwargs): + result = subprocess.run(args, capture_output=True, timeout=timeout, **kwargs) + if result.returncode: + raise RuntimeError(str(args[:3]) + '\n' + result.stdout.decode(errors='replace')[-2500:] + result.stderr.decode(errors='replace')[-2500:]) + return result.stdout + + +def resolve(value): + return Path(value.replace('{{DA_ROOT}}', ROOT).replace('{{SITE_PACKAGES}}', SITE)) + + +def state(path, contents=True): + try: + value = path.lstat() + except FileNotFoundError: + return {'kind': 'absent'} + result = {'mode': stat.S_IMODE(value.st_mode), 'uid': value.st_uid, 'gid': value.st_gid} + if path.is_symlink(): + result.update(kind='link', target=os.readlink(path)) + elif path.is_dir(): + result['kind'] = 'directory' + elif path.is_file(): + result['kind'] = 'file' + if contents: + result['sha256'] = hashlib.sha256(path.read_bytes()).hexdigest() + else: + result['kind'] = 'special' + return result + + +def protected(): + result = {} + for item in CATALOG['protected']: + base = resolve(item['target']) + # Active logs and uploads keep a fixed synthetic sentinel, not a changing tree. + if item['id'] in ('uploads', 'historical-logs'): + base /= 'jos81-preserve-sentinel' + result[str(base)] = state(base) + if base.is_dir(): + for path in sorted(base.rglob('*')): + result[str(path)] = state(path) + return result + + +def check_protected(): + expected = json.loads((WORK / 'protected.json').read_text()) + current = protected() + changed = {path: {'before': expected.get(path), 'after': current.get(path)} + for path in expected.keys() | current.keys() if expected.get(path) != current.get(path)} + assert not changed, 'protected paths changed: ' + json.dumps(changed, sort_keys=True) + + +def statuses(): + return {item['name']: item['statename'] for item in RPC.supervisor.getAllProcessInfo()} + + +def stop_services(): + for name in ('nginx', 'cron', 'exim4', 'websockets', 'celery', 'celerysingle', 'uwsgi', 'nltk', 'privacy-monitor'): + if statuses().get(name) == 'RUNNING': + RPC.supervisor.stopProcess(name, True) + check = subprocess.run(['pgrep', '-x', 'uwsgi'], capture_output=True) + assert check.returncode == 1, 'uWSGI process remains after stop' + sockpath = Path('/var/run/uwsgi/docassemble.sock') + if sockpath.exists(): + assert stat.S_ISSOCK(sockpath.lstat().st_mode) + with socket.socket(socket.AF_UNIX) as probe: + try: + probe.connect(str(sockpath)) + except ConnectionRefusedError: + pass + else: + raise AssertionError('application socket still accepts connections') + sockpath.unlink() + Path('/var/run/uwsgi/uwsgi.pid').unlink(missing_ok=True) + + +def start_services(candidate): + run('supervisorctl', '-s', 'http://localhost:9001', 'reread') + run('supervisorctl', '-s', 'http://localhost:9001', 'update') + # A main-group configuration change restarts initialization and its database + # siblings together. RUNNING alone does not mean startup/chown has finished. + deadline = time.monotonic() + 240 + while not Path('/var/run/docassemble/ready').exists(): + assert statuses().get('initialize') in ('STARTING', 'RUNNING'), 'initializer failed' + assert time.monotonic() < deadline, ( + 'initializer readiness deadline exceeded; states=' + str(statuses()) + + '; nltk_socket=' + str(Path('/var/run/nltk/da_nltk.sock').exists()) + + '; startup_receipt=' + str((WORK / 'startup-receipt').exists())) + time.sleep(0.5) + original = json.loads((WORK / 'services.json').read_text()) + for name, value in original.items(): + if value == 'STOPPED' and statuses().get(name) == 'RUNNING': + assert RPC.supervisor.stopProcess(name, True) + if candidate: + for unused in range(30): + if statuses().get('privacy-monitor') == 'RUNNING': + break + time.sleep(0.2) + assert statuses()['privacy-monitor'] == 'RUNNING' + for name in COMPONENTS: + if statuses().get(name) != 'RUNNING': + assert RPC.supervisor.startProcess(name, True) + assert all(statuses()[name] == 'RUNNING' for name in COMPONENTS) + + +def fetch(path, port=80, cookies=None): + for unused in range(6): + headers = {'Host': 'privacy-install.test'} + if cookies: + headers['Cookie'] = '; '.join(value.OutputString(attrs=[]) for value in cookies.values()) + connection = HTTPConnection('127.0.0.1', port, timeout=15) + try: + connection.request('GET', path, headers=headers) + response = connection.getresponse() + body = response.read(2_000_001) + assert len(body) <= 2_000_000, 'oversized synthetic response' + if cookies is not None: + for key, value in response.getheaders(): + if key.lower() == 'set-cookie': + cookies.load(value) + location = response.getheader('Location') + finally: + connection.close() + if response.status in (301, 302, 303, 307, 308): + assert location, 'redirect has no destination' + target = urlsplit(location) + assert target.netloc in ('', 'privacy-install.test', '127.0.0.1'), 'non-fixture redirect' + path = (target.path or '/') + ('?' + target.query if target.query else '') + continue + assert response.status == 200 + return body + raise AssertionError('too many fixture redirects') + + +def ready(path, marker, port=80): + deadline = time.monotonic() + 30 + while True: + try: + body = fetch(path, port) + assert marker in body and b'is starting' not in body + return + except (OSError, HTTPException, AssertionError): + if time.monotonic() >= deadline: + raise + time.sleep(0.5) + + +def routes(): + ready('/kept-realip.pdf', b'JOS81_KEEP_REALIP\n') + ready('/kept-rewrite', b'JOS81_KEEP_REWRITE') + ready('/', b'JOS81_KEEP_SITE', 8088) + ready('/nj/', b'JOS81_APPLICATION_OK') + + +def private_output_absent(roots): + marker = b'JOS81_PRIVATE_' + total = 0 + for root in roots: + assert root.is_dir(), 'missing retained-output directory' + for path in root.rglob('*'): + if path.is_symlink() or not path.is_file(): + continue + opener = gzip.open if path.suffix == '.gz' else open + tail = b'' + with opener(path, 'rb') as stream: + while chunk := stream.read(65536): + total += len(chunk) + assert total <= 64 * 1024 * 1024, 'synthetic log scan exceeded bound' + assert marker not in tail + chunk, 'private output retained in ' + str(path) + tail = chunk[-len(marker):] + + +def queued_job(): + before = read_counters()['celery'][-1]['unclassified'] + # Explicit cookies are confined to this fixed loopback fixture. Secure cookies + # otherwise would be dropped by an HTTP CookieJar behind the HTTPS proxy fixture. + cookies = SimpleCookie() + path = '/nj/?i=docassemble.privacyfixture:data/questions/queue.yml' + deadline = time.monotonic() + 45 + while time.monotonic() < deadline: + body = fetch(path, cookies=cookies) + if b'JOS81_QUEUE_OK' in body: + break + assert b'JOS81_QUEUE_WAIT' in body, 'unexpected queue interview response' + time.sleep(0.5) + else: + raise AssertionError('queued job did not return its expected result') + assert cookies, 'queue check did not preserve an interview session' + for unused in range(50): + if read_counters()['celery'][-1]['unclassified'] >= before + 2: + break + time.sleep(0.2) + else: + raise AssertionError('queued stdout/stderr did not advance Celery counters') + private_output_absent((Path(ROOT + '/log'), Path('/var/log'))) + print('real queued job returned its result; stdout/stderr counted; private markers absent from logs', flush=True) + + +def cron_interview(): + cookies = SimpleCookie() + path = '/nj/?i=docassemble.privacyfixture:data/questions/cron.yml' + assert b'JOS81_CRON_COUNT_0' in fetch(path, cookies=cookies) + output = run('bash', ROOT + '/webapp/run-cron.sh', 'cron_hourly') + values = records(output, 'cron') + assert len(values) == 1 and values[0]['unclassified'] >= 2, 'cron output was not counted once' + assert b'JOS81_CRON_COUNT_1' in fetch(path, cookies=cookies), 'cron did not save its session changes' + (WORK / 'cron-session.json').write_text(json.dumps({key: value.value for key, value in cookies.items()})) + failure = subprocess.run(['env', 'DA_CONFIG=/nonexistent/JOS81_PRIVATE_CONFIG', + 'bash', ROOT + '/webapp/run-cron.sh', 'cron_hourly'], + capture_output=True, timeout=30) + assert failure.returncode == 70 and not failure.stderr + expected = [dict(schema=1, component='cron', event='startup_failed', phase='config'), + dict(schema=1, component='cron', event='command_failed', phase='execution')] + assert [json.loads(line) for line in failure.stdout.splitlines()] == expected + # Inject a missing privilege-drop executable into an otherwise exact launcher copy. + source = Path(ROOT + '/webapp/run-cron.sh').read_text() + assert source.count('/usr/bin/setpriv') == 1 + broken = WORK / 'missing-privilege-drop.sh' + broken.write_text(source.replace('/usr/bin/setpriv', '/nonexistent/JOS81_PRIVATE_EXEC')) + failure = subprocess.run(['bash', str(broken)], capture_output=True, timeout=15) + assert failure.returncode == 127 and not failure.stderr + assert [json.loads(line) for line in failure.stdout.splitlines()] == expected[-1:] + private_output_absent((Path(ROOT + '/log'), Path('/var/log'))) + print('real cron interview saved its result; response/stderr reduced to one safe snapshot', flush=True) + + +def configure(): + import yaml # Existing application dependency, not a new test/service dependency. + package = Path(SITE + '/docassemble/privacyfixture') + (package / 'data/questions').mkdir(parents=True) + (package / '__init__.py').write_text('') + (package / 'data/questions/install.yml').write_text( + 'metadata:\n title: Privacy installation check\n---\nmandatory: True\n' + 'question: Privacy installation check\nsubquestion: JOS81_APPLICATION_OK\n') + shutil.copyfile(REPO / 'tests/privacy_native/fixtures/queue.yml', package / 'data/questions/queue.yml') + shutil.copyfile(REPO / 'tests/privacy_native/fixtures/cron.yml', package / 'data/questions/cron.yml') + path = Path(ROOT + '/config/config.yml') + config = yaml.safe_load(path.read_text()) + assert config['allow demo'] is False + assert config['enable playground'] is False + assert config['allow log viewing'] is False + assert config['allow configuration editing'] is False + assert config['behind https load balancer'] is True + config['default interview'] = 'docassemble.privacyfixture:data/questions/install.yml' + path.write_text(yaml.safe_dump(config, sort_keys=False)) + # Both populated targets already use this ownership; clean restore retains it. + os.chown(path, 33, 33) + assert RPC.supervisor.stopProcess('uwsgi', True) + assert RPC.supervisor.startProcess('uwsgi', True) + ready('/nj/', b'JOS81_APPLICATION_OK') + print('real application serves synthetic interview with demo access disabled', flush=True) + + +def seed(): + assert Path('/var/run/docassemble/ready').exists() + assert all(statuses()[name] == 'RUNNING' for name in COMPONENTS) + WORK.mkdir(mode=0o700) + run(sys.executable, '-I', '-B', str(REPO / 'tests/privacy_native/check_maintenance_image.py'), 'baseline') + template = Path(ROOT + '/config/nginx-http.dist') + text = template.read_text() + assert 'form_rewrite_rules' not in text + template.write_text(text.replace(' location {{DAWSGIROOT}}', ' include /etc/nginx/form_rewrite_rules.conf;\n location {{DAWSGIROOT}}', 1)) + Path('/etc/nginx/form_rewrite_rules.conf').write_text('location = /kept-rewrite { return 200 "JOS81_KEEP_REWRITE"; }\n') + site = Path('/etc/nginx/sites-available/formapauperis') + site.write_text('server { listen 127.0.0.1:8088; server_name privacy-install.test; location / { return 200 "JOS81_KEEP_SITE"; } }\n') + Path('/etc/nginx/sites-enabled/formapauperis').symlink_to(site) + for dirname in ('files', 'log'): + sentinel = Path(ROOT + '/' + dirname + '/jos81-preserve-sentinel') + sentinel.write_text('JOS81_PRESERVE_' + dirname) + os.chown(sentinel, 33, 33) + # Exercise metadata restoration for existing counter files, without historical data. + for item in CATALOG['states']: + if item['kind'] == 'counter-file': + path = resolve(item['target']) + if item['id'] == 'monitor-log': + assert not path.exists() + continue + assert not path.exists() or path.stat().st_size == 0 + path.touch() + os.chmod(path, 0o400) + os.chown(path, 0, 0) + # A stopped producer must remain stopped after rollback. + if statuses().get('cron') == 'RUNNING': + assert RPC.supervisor.stopProcess('cron', True) + assert RPC.supervisor.stopProcess('nginx', True) + assert RPC.supervisor.startProcess('nginx', True) + routes() + snapshot() + + +def snapshot(): + assert all(statuses()[name] == 'RUNNING' for name in COMPONENTS) + routes() + original = statuses() + assert not set(original.values()) & {'STARTING', 'STOPPING', 'BACKOFF'}, 'unstable baseline' + (WORK / 'services.json').write_text(json.dumps(original, indent=2)) + stop_services() + metadata = {str(resolve(item['target'])): state(resolve(item['target']), contents=False) + for item in CATALOG['states'] if item['kind'] == 'counter-file'} + (WORK / 'counter-metadata.json').write_text(json.dumps(metadata, indent=2)) + paths = [resolve(item['target']) for item in CATALOG['files']] + for item in CATALOG['states']: + if item['kind'] == 'counter-file': + continue + path = resolve(item['target']) + if item['kind'] == 'logger-bytecode-only': + paths.extend(path.glob('log_initialize.*.pyc')) + else: + paths.append(path) + paths = sorted(set(paths)) + baseline = {str(path): state(path) for path in paths} + (WORK / 'baseline.json').write_text(json.dumps(baseline, indent=2)) + (WORK / 'protected.json').write_text(json.dumps(protected(), indent=2)) + # Directory metadata is restored explicitly; archiving /var/run/uwsgi would + # make tar traverse the image's /var/run -> /run link during extraction. + existing = [str(path).lstrip('/') for path in paths + if baseline[str(path)]['kind'] not in ('absent', 'directory')] + (WORK / 'restore-list.txt').write_text('\n'.join(existing) + '\n') + run('tar', '--numeric-owner', '--no-recursion', '-cpf', str(WORK / 'rollback.tar'), '-C', '/', '-T', str(WORK / 'restore-list.txt')) + print('baseline routes pass; file snapshot, counter metadata and service states saved', flush=True) + + +def install(): + expected = json.loads((WORK / 'baseline.json').read_text()) + assert all(state(Path(path)) == value for path, value in expected.items()) + patched = WORK / 'patched' + for item in CATALOG['files']: + if item.get('existing') == 'apply-privacy-diff': + destination = patched / item['source'] + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(resolve(item['target']), destination) + run('patch', '--batch', '--fuzz=0', '-p1', '-d', str(patched), '-i', str(REPO / 'tests/.privacy-build/overlay-privacy.patch')) + installed = {} + for item in CATALOG['files']: + target = resolve(item['target']) + if item.get('existing') == 'preserve-and-validate' and target.exists(): + installed[str(target)] = state(target) + continue + if item['kind'] == 'binary': + source = REPO / 'tests/.privacy-build' / item['source'].replace('{{ARCH}}', 'amd64') + raw = source.read_bytes() + assert raw[:4] == b'\x7fELF' and int.from_bytes(raw[18:20], 'little') == 62 + elif item.get('existing') == 'apply-privacy-diff': + source = patched / item['source'] + else: + source = REPO / item['source'] + target.parent.mkdir(parents=True, exist_ok=True) + run('install', '-o', 'root', '-g', 'root', '-m', item['mode'], str(source), str(target)) + installed[str(target)] = state(target) + assert installed[str(target)]['sha256'] == hashlib.sha256(source.read_bytes()).hexdigest() + for source, destination in (('docassemble.ini.dist', 'docassemble.ini'), ('docassemblelog.ini.dist', 'docassemblelog.ini')): + text = Path(ROOT + '/config/' + source).read_text() + text = text.replace('{{DA_PYTHON}}', ROOT + '/local3.14').replace('{{DA_ROOT}}', ROOT).replace('{{DAWSGIROOT}}', '/nj') + assert '{{' not in text + Path(ROOT + '/config/' + destination).write_text(text) + for path in Path(SITE + '/docassemble/webapp/__pycache__').glob('log_initialize.*.pyc'): + path.unlink() + for item in CATALOG['states']: + if item['kind'] == 'counter-file': + path = resolve(item['target']) + path.touch(exist_ok=True) + os.chmod(path, 0o600) + os.chown(path, 33, 33) + (WORK / 'installed.json').write_text(json.dumps(installed, indent=2)) + check_protected() + run(sys.executable, '-I', '-B', str(REPO / 'tests/privacy_native/check_maintenance_image.py'), 'seed') + start_services(True) + time.sleep(2) + before = read_counters() + routes() + check_protected() + for unused in range(50): + time.sleep(0.2) + counters = read_counters() + if all(counters[name][-1]['status_2xx'] > before[name][-1]['status_2xx'] + for name in ('nginx', 'uwsgi')): + break + else: + raise AssertionError('real requests did not advance nginx/uWSGI counters') + (WORK / 'counters.json').write_text(json.dumps(counters, indent=2)) + print('candidate services and custom/application routes pass; protected paths unchanged', flush=True) + + +def rollback(): + stop_services() + metadata = json.loads((WORK / 'counter-metadata.json').read_text()) + counter_hashes = {path: state(Path(path))['sha256'] for path in metadata} + expected = json.loads((WORK / 'baseline.json').read_text()) + for path, value in expected.items(): + item = Path(path) + if value['kind'] == 'absent' and (item.exists() or item.is_symlink()): + if item.is_dir(): + continue + item.unlink() + run('tar', '--numeric-owner', '-xpf', str(WORK / 'rollback.tar'), '-C', '/') + for path, value in sorted(expected.items(), key=lambda item: len(item[0]), reverse=True): + item = Path(path) + if value['kind'] == 'directory': + assert item.is_dir() and not item.is_symlink() + os.chown(item, value['uid'], value['gid']) + os.chmod(item, value['mode']) + if value['kind'] == 'absent' and item.is_dir(): + item.rmdir() + assert state(item) == value, path + ' did not restore exactly' + check_protected() + start_services(False) + routes() + assert 'privacy-monitor' not in statuses() + original = json.loads((WORK / 'services.json').read_text()) + for name, value in original.items(): + if value == 'RUNNING' and statuses().get(name) != 'RUNNING': + assert RPC.supervisor.startProcess(name, True) + elif value == 'STOPPED' and statuses().get(name) == 'RUNNING': + assert RPC.supervisor.stopProcess(name, True) + assert statuses() == original, 'service states differ from baseline' + check_protected() + # A main-group config update restarts the old initializer, which chowns log + # files. Restore saved counter metadata after that normal startup completes. + for path, value in metadata.items(): + item = Path(path) + if value['kind'] != 'absent': + assert value['kind'] == 'file' + os.chown(item, value['uid'], value['gid']) + os.chmod(item, value['mode']) + assert state(item, contents=False) == value, 'counter metadata changed' + assert state(item)['sha256'] == counter_hashes[path], 'counter contents changed' + print('rollback files/metadata, preserved counters, original service states and routes pass', flush=True) + + +if __name__ == '__main__': + if sys.argv[1:] not in (['prepare'], ['resume']): + raise SystemExit('use prepare or resume inside the disposable test container') + socket.setdefaulttimeout(90) + if sys.argv[1] == 'prepare': + configure() + seed() + install() + queued_job() + cron_interview() + print(run(sys.executable, '-I', '-B', str(REPO / 'tests/privacy_native/check_cron_launcher.py'), timeout=160).decode(), end='', flush=True) + print(run(sys.executable, '-I', '-B', str(REPO / 'tests/privacy_native/check_maintenance_image.py'), sys.argv[1], timeout=240).decode(), end='', flush=True) + if sys.argv[1] == 'resume': + rollback() + private_output_absent((Path(ROOT + '/log'), Path('/var/log'))) diff --git a/tests/privacy_native/check_install_lifecycle.py b/tests/privacy_native/check_install_lifecycle.py new file mode 100644 index 000000000..7e0369e22 --- /dev/null +++ b/tests/privacy_native/check_install_lifecycle.py @@ -0,0 +1,135 @@ +"""Host-side orderly stop/start of the already-owned native installation fixture.""" +import importlib.util +import io +import json +from pathlib import Path +import subprocess +import sys +import tarfile +import time + +spec = importlib.util.spec_from_file_location('install_check', Path(__file__).with_name('check_install_image.py')) +install = importlib.util.module_from_spec(spec) +spec.loader.exec_module(install) + + +def docker(*args, timeout=30, check=True): + return subprocess.run(['docker', *args], capture_output=True, timeout=timeout, check=check) + + +def stopped_file(container, path, missing=False): + result = docker('cp', container + ':' + path, '-', check=False) + if result.returncode: + assert missing and b'Could not find' in result.stderr, 'cannot inspect stopped-container marker' + return None + assert len(result.stdout) <= 2_000_000, 'oversized synthetic archive' + with tarfile.open(fileobj=io.BytesIO(result.stdout), mode='r:') as archive: + members = archive.getmembers() + assert len(members) == 1 and members[0].isfile() and members[0].size < 1_000_000 + return archive.extractfile(members[0]).read() + + +def interrupted_start(container, directory): + flag = directory / 'hold-startup' + flag.write_text('hold') + docker('cp', str(flag), container + ':/tmp/jos81-rehearsal/hold-startup') + docker('start', container) + probe = """from pathlib import Path +assert Path('/tmp/jos81-rehearsal/startup-held').read_text() == 'held' +assert Path('/var/run/docassemble/da_running').exists() +assert not Path('/var/run/docassemble/ready').exists() +assert Path('/usr/share/docassemble/log/jos81-restored').read_bytes() == b'JOS81_BACKUP_ONLY_RESTORE' +""" + deadline = time.monotonic() + 240 + while time.monotonic() < deadline: + result = docker('exec', container, 'python3.14', '-I', '-B', '-c', probe, check=False) + if result.returncode == 0: + break + time.sleep(1) + else: + raise AssertionError('initializer did not reach the one-shot startup hold') + stop = """from pathlib import Path +import json +import time +import xmlrpc.client +rpc = xmlrpc.client.ServerProxy('http://localhost:9001/RPC2') +assert rpc.supervisor.stopProcess('main:initialize', False) +assert rpc.supervisor.getProcessInfo('main:initialize')['statename'] in ('STOPPING', 'STOPPED') +Path('/tmp/jos81-rehearsal/release-startup').touch() +deadline = time.monotonic() + 40 +while rpc.supervisor.getProcessInfo('main:initialize')['statename'] != 'STOPPED': + assert time.monotonic() < deadline, 'captured initializer did not stop after its child was released' + time.sleep(0.2) +assert Path('/var/run/docassemble/da_running').exists() +assert not Path('/var/run/docassemble/ready').exists() +info = rpc.supervisor.getProcessInfo('main:initialize') +assert info['exitstatus'] == 143, 'initializer interruption exit changed' +print(json.dumps({'initialize_log': info['stdout_logfile']})) +""" + output = docker('exec', container, 'python3.14', '-I', '-B', '-c', stop, timeout=50) + logpath = json.loads(output.stdout)['initialize_log'] + docker('stop', '--time', '1000', container, timeout=1020) + state = json.loads(docker('inspect', '--format', '{{json .State}}', container).stdout) + assert state['Running'] is False and state['OOMKilled'] is False and state['ExitCode'] == 0 + assert stopped_file(container, '/var/run/docassemble/da_running') is not None + assert stopped_file(container, '/var/run/docassemble/ready', missing=True) is None + install.records(stopped_file(container, logpath), 'initialize') + marker = directory / 'jos81-unsafe-restore' + marker.write_bytes(b'JOS81_MUST_NOT_RESTORE') + docker('cp', str(marker), container + ':/usr/share/docassemble/backup/log/jos81-unsafe-restore') + print('actual initializer interruption preserved unsafe-start state before a new backup-only marker was inserted', flush=True) + + +def main(container, directory): + # Names and labels must identify the caller's unique disposable fixture. + metadata = json.loads(docker('inspect', container).stdout)[0] + assert metadata['Config']['Labels'].get('task') == 'JOS-81-install-review' + assert metadata['Name'].startswith('/jos81-install-run.') + assert metadata['State']['Running'] + assert metadata['HostConfig']['NetworkMode'] == 'none' + before = json.loads(docker('exec', container, 'cat', '/tmp/jos81-rehearsal/lifecycle.json').stdout) + # Supervisor stops its groups serially. Cover all enabled group budgets, + # including the concurrent initialize/PostgreSQL/Redis group, with margin. + docker('stop', '--time', '1000', container, timeout=1020) + state = json.loads(docker('inspect', '--format', '{{json .State}}', container).stdout) + assert state['Running'] is False and state['OOMKilled'] is False and state['ExitCode'] == 0 + for name in ('ready', 'da_running', 'status-postgres-running', 'status-redis-running', + 'status-rabbitmq-running'): + assert stopped_file(container, '/var/run/docassemble/' + name, missing=True) is None, 'unclean shutdown marker remains' + assert stopped_file(container, '/usr/share/docassemble/backup/log/jos81-shutdown') == b'JOS81_SHUTDOWN_BACKUP' + install.records(stopped_file(container, before['initialize_log']), 'initialize') + logs = docker('logs', container) + assert b'JOS81_PRIVATE_' not in logs.stdout + logs.stderr + # Insert a new synthetic file into only the stopped container's backup tree. + # Its later appearance in live logs must come from the actual startup restore. + marker = directory / 'jos81-restored' + marker.write_bytes(b'JOS81_BACKUP_ONLY_RESTORE') + docker('cp', str(marker), container + ':/usr/share/docassemble/backup/log/jos81-restored') + interrupted_start(container, directory) + docker('start', container) + probe = """from pathlib import Path +import json +import xmlrpc.client +assert Path('/var/run/docassemble/ready').exists() +rpc = xmlrpc.client.ServerProxy('http://localhost:9001/RPC2') +states = {p['name']: p['statename'] for p in rpc.supervisor.getAllProcessInfo()} +assert all(states.get(n) == 'RUNNING' for n in ('initialize', 'nginx', 'uwsgi', 'celery', 'celerysingle', 'websockets')) +original = json.loads(Path('/tmp/jos81-rehearsal/services.json').read_text()) +for name, value in original.items(): + if value == 'STOPPED' and states.get(name) == 'RUNNING': + assert rpc.supervisor.stopProcess(name, True) +""" + deadline = time.monotonic() + 240 + while time.monotonic() < deadline: + result = docker('exec', container, 'python3.14', '-I', '-B', '-c', probe, check=False, timeout=20) + if result.returncode == 0: + break + time.sleep(2) + else: + raise AssertionError('candidate did not become ready after orderly restart') + print('whole-container shutdown was clean, produced log backup and retained safe initializer output; same-volume restart is ready', flush=True) + + +if __name__ == '__main__': + assert len(sys.argv) == 3 + main(sys.argv[1], Path(sys.argv[2])) diff --git a/tests/privacy_native/check_maintenance_image.py b/tests/privacy_native/check_maintenance_image.py new file mode 100644 index 000000000..f928b54fd --- /dev/null +++ b/tests/privacy_native/check_maintenance_image.py @@ -0,0 +1,193 @@ +"""Real local maintenance/copy/restart acceptance in the owned synthetic image.""" +from http.cookies import SimpleCookie +import importlib.util +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys + +spec = importlib.util.spec_from_file_location('install_check', Path(__file__).with_name('check_install_image.py')) +install = importlib.util.module_from_spec(spec) +spec.loader.exec_module(install) +ROOT = Path(install.ROOT) +WORK = install.WORK +HOOK = Path(install.SITE + '/docassemble/webapp/starthook.py') +HISTORY = b'JOS81_HISTORICAL_NGINX' +SHUTDOWN = b'JOS81_SHUTDOWN_BACKUP' +RESTORE = b'JOS81_BACKUP_ONLY_RESTORE' +LEGACY_ACCESS = b' access_log /var/log/nginx/access.log privacy;\n' + + +def baseline(): + realip = ROOT / 'config/nginx-realip' + # Model the legacy override and preserved regex/payload syntax observed on + # both targets, while keeping all fixture routes and contents synthetic. + realip.write_bytes(realip.read_bytes() + b''' + # context, /etc/nginx/conf.d/) because log_format is invalid in a server block. + # BOTH FILES MUST SHIP TOGETHER -- this line without that file fails nginx -t. + access_log /var/log/nginx/access.log privacy; + + location ~ ^/kept-realip\\.pdf$ { return 200 'JOS81_KEEP_REALIP\\n'; } +''') + Path('/etc/nginx/conf.d/privacy-log.conf').write_text( + "log_format privacy '[$time_local] $request_method $uri $status';\n") + (WORK / 'realip-original').write_bytes(realip.read_bytes()) + + +def nginx_customization(): + original = (WORK / 'realip-original').read_bytes() + assert original.count(LEGACY_ACCESS) == 1 + assert (ROOT / 'config/nginx-realip').read_bytes() == original.replace(LEGACY_ACCESS, b'', 1) + install.ready('/kept-realip.pdf', b'JOS81_KEEP_REALIP\n') + + +def scan(): + install.private_output_absent((ROOT / 'log', Path('/var/log'), + ROOT / 'backup/log', ROOT / 'backup/nginxlogs')) + day = (WORK / 'maintenance-day').read_text() + install.private_output_absent((ROOT / 'backup' / day / 'log',)) + + +def initializer(): + info = install.RPC.supervisor.getProcessInfo('main:initialize') + assert info['statename'] == 'RUNNING' + assert Path(f"/proc/{info['pid']}/exe").resolve().name == 'privacy-process' + children = [] + for task in Path(f"/proc/{info['pid']}/task").glob('*/children'): + children.extend(int(pid) for pid in task.read_text().split()) + assert len(children) == 1, 'initializer must have one captured Bash child' + argv = Path(f'/proc/{children[0]}/cmdline').read_bytes().split(b'\0')[:3] + assert argv == [b'/bin/bash', str(ROOT / 'webapp/initialize.sh').encode(), b'--privacy-captured'] + values = install.records(install.RPC.supervisor.readProcessStdoutLog('main:initialize', 0, 1_000_000), + 'initialize') + assert values[-1]['unclassified'] >= 2, 'startup hook output did not advance initializer counters' + assert (WORK / 'startup-receipt').read_text().count('emitted\n') > 0, 'startup hook did not execute its positive control' + return info + + +def seed(): + original = HOOK.read_bytes() + (WORK / 'starthook-original.py').write_bytes(original) + # A real initializer startup module emits both markers before normal config + # loading. Other services do not execute this module. + prefix = b"""import os +assert os.getuid() == 0 and os.getcwd() == '/tmp' +try: + os.fstat(3) +except OSError: + pass +else: + raise AssertionError('launcher fd3 reached the startup module') +os.write(1, b'JOS81_PRIVATE_INITIALIZER_STDOUT\\n') +os.write(2, b'JOS81_PRIVATE_INITIALIZER_STDERR\\n') +with open('/tmp/jos81-rehearsal/startup-receipt', 'a') as receipt: + receipt.write('emitted\\n') +from pathlib import Path +hold = Path('/tmp/jos81-rehearsal/hold-startup') +if hold.exists(): + import sys + import time + hold.unlink() + hold.with_name('startup-held').write_text('held') + deadline = time.monotonic() + 60 + while not hold.with_name('release-startup').exists(): + assert time.monotonic() < deadline, 'startup hold expired' + time.sleep(0.1) + sys.exit(0) +""" + HOOK.write_bytes(prefix + original) + + +def command(path, expected=0, **env): + result = subprocess.run(['bash', str(path)], env=dict(os.environ, **env), + capture_output=True, timeout=120) + assert result.returncode == expected and not result.stderr, 'maintenance exit/stderr changed' + values = install.records(result.stdout, 'maintenance') + assert len(values) == 1, 'finite maintenance command emitted periodic/raw output' + return values[0] + + +def prepare(): + nginx_customization() + initializer() + history = Path('/var/log/nginx/jos81-history') + history.write_bytes(HISTORY) + os.chmod(history, 0o600) + # These are the installed scripts and real local operations, not copied command fragments. + command(ROOT / 'webapp/sync.sh') + command('/etc/cron.hourly/docassemble') + command('/etc/cron.weekly/docassemble') + command('/etc/cron.monthly/docassemble') + command('/etc/cron.daily/docassemble') + day = install.run('date', '+%m-%d').decode().strip() + (WORK / 'maintenance-day').write_text(day) + for path in (ROOT / 'log/jos81-history', ROOT / 'backup/log/jos81-history', + ROOT / 'backup/nginxlogs/jos81-history', ROOT / 'backup' / day / 'log/jos81-history'): + assert path.read_bytes() == HISTORY, 'local maintenance did not preserve/copy historical bytes' + assert (ROOT / 'backup/log/jos81-preserve-sentinel').read_text() == 'JOS81_PRESERVE_log' + assert (ROOT / 'backup/files/jos81-preserve-sentinel').read_text() == 'JOS81_PRESERVE_files' + assert (ROOT / 'backup/config.yml').read_bytes() == (ROOT / 'config/config.yml').read_bytes() + assert any((ROOT / 'backup/postgres').iterdir()), 'database backup missing' + # Force the actual non-mail logrotate stanza and callback after local copying. + rotation = Path('/etc/logrotate.d/docassemble').read_text() + assert rotation.count('/var/mail/mail\n') == 1 + nonmail = WORK / 'non-mail-logrotate' + nonmail.write_text(rotation.split('/var/mail/mail\n', 1)[0]) + legacy = ROOT / 'log/worker.log' + prior = legacy.read_bytes() + b'JOS81_HISTORICAL_ROTATION\n' + legacy.write_bytes(prior) + pids = {name: install.RPC.supervisor.getProcessInfo(name)['pid'] + for name in ('celery', 'celerysingle', 'websockets', 'uwsgi')} + output = install.run('logrotate', '-f', '-s', str(WORK / 'logrotate-state'), str(nonmail), + env=dict(os.environ, CONTAINERROLE=':all:'), timeout=180) + assert len(install.records(output, 'maintenance')) == 1, 'rotation callback did not emit safe output' + assert (ROOT / 'log/worker.log.1').read_bytes() == prior, 'rotation changed historical bytes' + for name, pid in pids.items(): + current = install.RPC.supervisor.getProcessInfo(name) + assert current['statename'] == 'RUNNING' + assert (current['pid'] == pid) == (name == 'uwsgi'), 'rotation callback service behavior changed' + install.routes() + install.read_counters() + install.check_protected() + scan() + # This marker appears only after daily backup, proving the shutdown path copies it. + assert not (ROOT / 'backup/log/jos81-shutdown').exists() + (ROOT / 'log/jos81-shutdown').write_bytes(SHUTDOWN) + os.chown(ROOT / 'log/jos81-shutdown', 33, 33) + assert not (ROOT / 'log/jos81-restored').exists() + (WORK / 'lifecycle.json').write_text(json.dumps({ + 'initialize_log': initializer()['stdout_logfile'], + 'startup_receipts': (WORK / 'startup-receipt').read_text().count('emitted\n')})) + print('installed hourly/daily/weekly/monthly/sync and rotation callback pass; historical copies preserved', flush=True) + + +def resume(): + nginx_customization() + install.routes() + initializer() + before = json.loads((WORK / 'lifecycle.json').read_text()) + assert (WORK / 'startup-receipt').read_text().count('emitted\n') >= before['startup_receipts'] + 2, 'both restart attempts must execute the startup marker control' + assert (ROOT / 'log/jos81-restored').read_bytes() == RESTORE, 'backup-only sentinel was not restored' + assert (ROOT / 'backup/log/jos81-unsafe-restore').read_bytes() == b'JOS81_MUST_NOT_RESTORE' + assert not (ROOT / 'log/jos81-unsafe-restore').exists(), 'interrupted-start guard restored backup logs' + for path in (ROOT / 'log/jos81-shutdown', ROOT / 'backup/log/jos81-shutdown'): + assert path.read_bytes() == SHUTDOWN + assert (ROOT / 'files/jos81-preserve-sentinel').read_text() == 'JOS81_PRESERVE_files' + install.check_protected() + cookies = SimpleCookie() + cookies.load(json.loads((WORK / 'cron-session.json').read_text())) + path = '/nj/?i=docassemble.privacyfixture:data/questions/cron.yml' + assert b'JOS81_CRON_COUNT_2' in install.fetch(path, cookies=cookies), 'database session did not survive restart' + install.records(install.run('bash', str(ROOT / 'webapp/run-cron.sh'), 'cron_hourly'), 'cron') + assert b'JOS81_CRON_COUNT_3' in install.fetch(path, cookies=cookies) + install.queued_job() + scan() + shutil.copyfile(WORK / 'starthook-original.py', HOOK) + print('orderly same-volume restart restored backup-only logs and preserved configuration, upload and database session; queue/cron still work', flush=True) + + +if __name__ == '__main__': + assert sys.argv[1:] in (['baseline'], ['seed'], ['prepare'], ['resume']) + {'baseline': baseline, 'seed': seed, 'prepare': prepare, 'resume': resume}[sys.argv[1]]() diff --git a/tests/privacy_native/check_rotation_image.py b/tests/privacy_native/check_rotation_image.py new file mode 100644 index 000000000..a28998739 --- /dev/null +++ b/tests/privacy_native/check_rotation_image.py @@ -0,0 +1,240 @@ +"""Real Supervisor, Go capture, syslog TCP forwarding and native logrotate.""" +import configparser +import hashlib +import json +import os +from pathlib import Path +import re +import signal +import socket +import subprocess +import tempfile +import time + +from test_rotation_ownership import FORWARDED, ROOT, supervisor + + +def stop(process): + if process.poll() is None: + os.killpg(process.pid, signal.SIGTERM) + try: + process.wait(timeout=8) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=2) + raise AssertionError('native rotation fixture did not stop') + + +def rows(path, component, syslog=False, live=True): + data = path.read_text() if path.exists() else '' + assert 'SYNTHETIC_PRIVATE' not in data and 'synthetic.yml' not in data, path + result = [] + if not live: + assert not data or data.endswith('\n'), ('truncated retained counter', path) + expected = {'schema', 'component', 'status_1xx', 'status_2xx', 'status_3xx', 'status_4xx', + 'status_5xx', 'latency_fast', 'latency_medium', 'latency_slow', 'latency_timeout', + 'unclassified', 'rejected', 'dropped'} + # A file can be observed between writes; only inspect complete records. + for line in data.splitlines(keepends=True): + if not line.endswith('\n'): + continue + if syslog: + line = line[line.index('{'):] + row = json.loads(line) + assert set(row) == expected and row['schema'] == 1 and row['component'] == component, (path, row) + assert all(type(value) is int and 0 <= value <= 2147483647 + for key, value in row.items() if key != 'component') + result.append(row) + return result + + +def await_condition(condition, processes, message, timeout=20): + deadline = time.monotonic() + timeout + while True: + assert all(process.poll() is None for process in processes), 'native fixture exited early' + if condition(): + return + assert time.monotonic() < deadline, message + time.sleep(0.05) + + +def capture_identity(pid): + try: + fields = Path(f'/proc/{pid}/stat').read_text().rsplit(')', 1)[1].split() + except FileNotFoundError: + return None + if fields[0] in ('Z', 'X'): + return None + return int(fields[1]), int(fields[19]) # Parent PID and Linux start time. + + +def syslog_config(directory, name, text): + config = directory / (name + '.conf') + config.write_text('@version: 4.3\n@include "scl.conf"\n' + 'options { flush_lines(0); stats(freq(0)); };\n' + text) + command = ['syslog-ng', '--foreground', '--no-caps', '--cfgfile', str(config), + '--persist-file', str(directory / (name + '.persist')), + '--pidfile', str(directory / (name + '.pid')), + '--control', str(directory / (name + '.ctl'))] + try: + subprocess.run([*command, '--syntax-only'], check=True, timeout=5, capture_output=True) + except subprocess.SubprocessError as error: + print(getattr(error, 'stderr', b'') or b'') + raise + return command + + +def main(): + os.umask(0o077) + with tempfile.TemporaryDirectory(prefix='privacy-rotation-') as temporary: + directory = Path(temporary) + local = remote = directory / 'logs' + local.mkdir() + # Execute the exact production precreation/ownership fragment, with only + # its absolute log directory mapped into this private fixture directory. + initialization = (ROOT / 'Docker/initialize.sh').read_text() + start = initialization.index('touch /usr/share/docassemble/log/') + end = initialization.index('chown -R www-data:www-data /usr/share/docassemble/log', start) + end += len('chown -R www-data:www-data /usr/share/docassemble/log') + subprocess.run(['bash', '-c', initialization[start:end].replace('/usr/share/docassemble/log', str(local))], + check=True, capture_output=True, timeout=5) + original = supervisor() + capture = {name: local / Path(original['program:' + name]['stdout_logfile']).name for name in FORWARDED} + for path in capture.values(): + assert path.stat().st_uid == os.getuid() and path.stat().st_mode & 0o777 == 0o600 + # Keep actual source lines and program labels. Only fixture directories + # and the loopback TCP port differ from the maintained syslog route. + shipper_source = (ROOT / 'Docker/docassemble-syslog-ng.conf').read_text() + file_lines = [line for line in shipper_source.splitlines() + if any(f'program-override("{name}")' in line for name in FORWARDED)] + assert len(file_lines) == 4 + shipper = ('source s_docassemble {\n' + '\n'.join(file_lines) + '\n};\n' + 'destination d_network { syslog("127.0.0.1" transport("tcp") port(10514)); };\n' + 'log { source(s_docassemble); destination(d_network); };\n') + shipper = shipper.replace('/usr/share/docassemble/log', str(local)) + receiver_source = (ROOT / 'Docker/syslog-ng.conf').read_text() + keys = ('daworker', 'daworkersingle', 'uwsgi', 'websockets') + receiver_lines = [line for line in receiver_source.splitlines() + if any(line.startswith(prefix + key + ' ') for key in keys + for prefix in ('destination d_', 'filter f_')) + or any('filter(f_' + key + '); destination(d_' + key + ');' in line for key in keys)] + assert len(receiver_lines) == 12 + receiver = 'source s_network { syslog(transport("tcp") port(10514)); };\n' + '\n'.join(receiver_lines) + receiver = receiver.replace('/usr/share/docassemble/log', str(remote)) + receiver_command = syslog_config(directory, 'receiver', receiver) + shipper_command = syslog_config(directory, 'shipper', shipper) + config = configparser.ConfigParser(interpolation=None) + config['supervisord'] = {'nodaemon': 'true', 'logfile': str(directory / 'supervisor.log'), + 'pidfile': str(directory / 'supervisor.pid'), 'childlogdir': temporary, + 'loglevel': 'info'} + for name, path in capture.items(): + section = dict(original['program:' + name]) + section.update(command='/usr/share/docassemble/webapp/privacy-process --component ' + name + + ' -- /bin/sh /review/tests/privacy_native/fixtures/rotation-command.sh ' + + str(directory / 'after-rotation') + ' --die-on-term', + stdout_logfile=str(path), stdout_logfile_maxbytes='256', + autostart='true', autorestart='false') + assert section['stdout_logfile_backups'] == '7' + config['program:' + name] = section + path = directory / 'supervisor.conf' + with path.open('w') as stream: + config.write(stream) + processes = [] + with (directory / 'console').open('wb') as console: + try: + collector = subprocess.Popen(receiver_command, stdout=console, stderr=subprocess.STDOUT, + start_new_session=True) + processes.append(collector) + def collector_ready(): + try: + with socket.create_connection(('127.0.0.1', 10514), timeout=0.2): + return True + except OSError: + return False + await_condition(collector_ready, processes, 'collector did not bind its test port', timeout=5) + forwarder = subprocess.Popen(shipper_command, stdout=console, stderr=subprocess.STDOUT, + start_new_session=True) + processes.append(forwarder) + manager = subprocess.Popen(['supervisord', '-c', str(path)], stdout=console, + stderr=subprocess.STDOUT, start_new_session=True) + processes.append(manager) + await_condition(lambda: all(Path(str(path) + '.7').exists() for path in capture.values()), + processes, 'Supervisor did not retain seven rotations') + await_condition(lambda: all(rows(remote / legacy, name, True) for name, legacy in FORWARDED.items()), + processes, 'native forwarding did not deliver all four program labels') + before = {name: rows(remote / legacy, name, True)[-1]['unclassified'] for name, legacy in FORWARDED.items()} + child_file = Path(f'/proc/{manager.pid}/task/{manager.pid}/children') + children = child_file.read_text().split() + assert len(children) == 4 + identities = {pid: capture_identity(pid) for pid in children} + assert all(identity and identity[0] == manager.pid for identity in identities.values()) + # Pause only Supervisor so its own rotation cannot race the + # independent logrotate inode/hash comparison. Its pipes buffer + # the tiny synthetic input until it is resumed below. + os.kill(manager.pid, signal.SIGSTOP) + paused = time.monotonic() + try: + await_condition(lambda: Path(f'/proc/{manager.pid}/status').read_text().split('State:', 1)[1].lstrip().startswith('T'), + processes, 'Supervisor did not pause', timeout=2) + def fingerprints(): + return {str(path): (path.stat().st_ino, hashlib.sha256(path.read_bytes()).hexdigest()) + for base in capture.values() for path in local.glob(base.name + '*')} + previous = fingerprints() + # Collector and local capture files share the same directory, + # as on a combined service/log-role installation. The actual + # callback's unrelated service restarts remain out of scope. + rotation = (ROOT / 'Docker/docassemble.logrotate').read_text() + rotation = rotation.replace('/usr/share/docassemble/log', str(local)) + rotation = rotation.replace('/var/mail/mail', str(directory / 'absent-mail')) + rotation = rotation.replace('/usr/share/docassemble/webapp/restart-post-logrotate.sh', '/bin/true') + rotation_path = directory / 'logrotate.conf' + rotation_path.write_text(rotation) + subprocess.run(['logrotate', '--force', '--state', str(directory / 'logrotate.state'), str(rotation_path)], + check=True, capture_output=True, timeout=3) + assert fingerprints() == previous, 'logrotate touched Supervisor-owned files' + assert all((local / (legacy + '.1')).stat().st_size for legacy in FORWARDED.values()) + assert child_file.read_text().split() == children, 'rotation replaced a capture process' + finally: + os.kill(manager.pid, signal.SIGCONT) + assert time.monotonic() - paused < 4.5, 'fixture paused capture too long' + # Explicitly reopen collector destinations. This exercises native + # syslog reopening, not the unmodified production callback. + os.kill(collector.pid, signal.SIGHUP) + (directory / 'after-rotation').touch() + def delivered_after_rotation(): + for name, legacy in FORWARDED.items(): + observed = rows(remote / legacy, name, True) + if not observed or observed[-1]['unclassified'] < before[name] + 8192: + return False + return True + await_condition(delivered_after_rotation, + processes, 'forwarding stopped after independent log rotation', timeout=10) + for observation in range(2): + assert all(process.poll() is None for process in processes), 'native daemon exited after forwarding' + assert {pid: capture_identity(pid) for pid in children} == identities, 'capture exited or was replaced after rotation' + if observation == 0: + time.sleep(0.2) + stop(manager) + assert manager.returncode == 0, 'Supervisor did not shut down cleanly' + for name, base in capture.items(): + files = list(local.glob(base.name + '*')) + assert len(files) == 8 and not Path(str(base) + '.8').exists() + observed = [row for path in files for row in rows(path, name, live=False)] + assert observed and max(row['unclassified'] for row in observed) >= before[name] + 8192 + print('Rotation acceptance: four Go streams retained seven Supervisor backups; legacy logrotate left capture files unchanged; all four syslog TCP routes continued with fixed counters', flush=True) + except Exception: + print((directory / 'console').read_text()) + raise + finally: + failures = [] + for process in reversed(processes): + try: + stop(process) + except Exception as error: + failures.append(error) + if failures: + raise ExceptionGroup('native fixture cleanup failed', failures) + + +if __name__ == '__main__': + main() diff --git a/tests/privacy_native/check_supervisor_monitor_image.py b/tests/privacy_native/check_supervisor_monitor_image.py new file mode 100644 index 000000000..dd48412dd --- /dev/null +++ b/tests/privacy_native/check_supervisor_monitor_image.py @@ -0,0 +1,138 @@ +"""Run only in a disposable Linux test container; all processes/data are synthetic.""" +import configparser +import json +import os +from pathlib import Path +import pwd +import shutil +import signal +import subprocess +import sys +import tempfile +import time + + +def main(): + root = Path('/review') + source = configparser.ConfigParser(interpolation=None) + source.read(root / 'Docker/docassemble-supervisor.conf') + section = 'eventlistener:privacy-monitor' + assert source[section]['redirect_stderr'] == 'false' + assert source[section]['stdout_logfile'] == 'NONE' + assert source[section]['user'] == 'www-data' + assert os.getuid() == pwd.getpwnam('www-data').pw_uid + with tempfile.TemporaryDirectory(prefix='synthetic-monitor-') as temporary: + directory = Path(temporary) + report = directory / 'monitor.log' + config = configparser.ConfigParser(interpolation=None) + config['supervisord'] = { + 'nodaemon': 'true', 'logfile': str(directory / 'supervisor.log'), + 'pidfile': str(directory / 'supervisor.pid'), + 'childlogdir': temporary, 'loglevel': 'info', + } + config[section] = dict(source[section]) + # Keep the production command, identity and protocol policy. Only the + # retained destination changes to this fixture's temporary directory. + config[section]['stderr_logfile'] = str(report) + commands = { + 'nginx': '/bin/sh -c "sleep 2; exit 0"', + 'uwsgi': '/bin/sh -c "sleep 2; exit 74"', + 'uwsgilog': '/missing-synthetic-privacy-helper', + 'initialize': '/bin/sh -c "sleep 2; exit 74"', + 'SYNTHETIC_PRIVATE_192.0.2.1': '/bin/false', + } + for name, command in commands.items(): + config['program:' + name] = { + 'command': command, 'autostart': 'true', 'autorestart': 'false', + 'startsecs': '1', 'startretries': '0', 'priority': '500', + 'stdout_logfile': 'NONE', 'stderr_logfile': 'NONE', + 'stopasgroup': 'true', 'killasgroup': 'true', + } + config['group:main'] = {'programs': 'initialize'} + application = directory / 'application' + (application / 'runtime/bin').mkdir(parents=True) + (application / 'webapp').mkdir() + (application / 'webapp/privacy-process').symlink_to('/usr/share/docassemble/webapp/privacy-process') + (application / 'webapp/privacy-diagnostic').symlink_to('/usr/share/docassemble/webapp/privacy-diagnostic') + (application / 'runtime/bin/activate').write_text("printf 'SYNTHETIC_PRIVATE_BOOTSTRAP\\n' >&2\n") + # Keep /tmp noexec. Synthetic executables live in the read-only source + # mount, just as installed service executables live outside writable logs. + command = root / 'tests/privacy_native/fixtures/application-command.sh' + for name in ('python', 'celery'): + (application / 'runtime/bin' / name).symlink_to(command) + profiles = {'celery': 'run-celery.sh', 'celerysingle': 'run-celery-single.sh', + 'websockets': 'run-websockets.sh'} + for name, script in profiles.items(): + program = 'program:' + name + config[program] = dict(source[program]) + config[program]['command'] = 'bash /review/Docker/' + script + config[program]['environment'] = f'DA_ROOT="{application}",DA_PYTHON="{application}/runtime"' + config[program]['autostart'] = 'true' + config[program]['autorestart'] = 'false' + config[program]['stdout_logfile'] = str(directory / (name + '.log')) + assert config[program]['redirect_stderr'] == 'true' + path = directory / 'supervisor.conf' + with path.open('w') as stream: + config.write(stream) + executable = shutil.which('supervisord') + assert executable is not None, 'Supervisor is missing from test image' + with (directory / 'console').open('wb') as console: + process = subprocess.Popen([executable, '-c', str(path)], stdout=console, + stderr=subprocess.STDOUT, start_new_session=True) + try: + wanted = {('monitor', 'monitor_ready', 'ready'), + ('uwsgi', 'process_failed', 'exited'), + ('uwsgilog', 'process_failed', 'backoff'), + ('uwsgilog', 'process_failed', 'fatal')} + wanted.add(('initialize', 'process_failed', 'exited')) + wanted.update((name, 'process_failed', 'exited') for name in profiles) + deadline = time.monotonic() + 15 + while time.monotonic() < deadline: + assert process.poll() is None, 'Supervisor exited before verification' + data = report.read_text() if report.exists() else '' + assert 'SYNTHETIC_PRIVATE' not in data + rows = [json.loads(line) for line in data.splitlines()] + for row in rows: + assert set(row) == {'schema', 'component', 'event', 'state'} + assert row['schema'] == 1 + observed = {(row['component'], row['event'], row['state']) for row in rows} + assert not any(row['component'] == 'nginx' for row in rows), 'expected exit reported as failure' + if wanted <= observed: + break + time.sleep(0.05) + else: + raise AssertionError('real Supervisor did not deliver required fixed records') + activity = (directory / 'supervisor.log').read_text() + assert "spawned: 'privacy-monitor'" in activity + assert 'unknown state' not in activity.lower() + assert 'buffer overflow' not in activity.lower() + for name in profiles: + data = (directory / (name + '.log')).read_text() + assert 'SYNTHETIC_PRIVATE' not in data + records = [json.loads(line) for line in data.splitlines()] + assert records and all(len(row) == 14 and row['component'] == name for row in records) + assert records[-1]['unclassified'] == 10 + assert all(row[key] == 0 for row in records for key in row if key.startswith(('status_', 'latency_'))) + print('Supervisor integration: native/application failures reported; real application launchers and logger initialization emitted counters only; expected/unrelated exits ignored') + except Exception: + # This directory contains only configuration and output from + # this synthetic fixture. Preserve failure evidence in CI output. + for name in ('console', 'supervisor.log', 'monitor.log', + *(profile + '.log' for profile in profiles)): + artifact = directory / name + if artifact.exists(): + print(name + ':\n' + artifact.read_text(), file=sys.stderr) + raise + finally: + if process.poll() is None: + os.killpg(process.pid, signal.SIGTERM) + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=2) + raise AssertionError('Supervisor did not stop its fixture processes') + + +if __name__ == '__main__': + main() diff --git a/tests/privacy_native/check_uwsgi_image.py b/tests/privacy_native/check_uwsgi_image.py new file mode 100644 index 000000000..631c12d25 --- /dev/null +++ b/tests/privacy_native/check_uwsgi_image.py @@ -0,0 +1,206 @@ +"""Real uWSGI + Go capture acceptance with synthetic configuration/application.""" +import json +import os +from pathlib import Path +import signal +import socket +import struct +import subprocess +import tempfile +import time + +ROOT = Path('/review') +DA_ROOT = Path('/usr/share/docassemble') +RUNTIME = DA_ROOT / 'local3.14' +MARKER = b'SYNTHETIC_PRIVATE' +APP = '''import sys +def application(environ, start_response): + print('SYNTHETIC_PRIVATE stdout ' + environ.get('QUERY_STRING', ''), flush=True) + print('SYNTHETIC_PRIVATE stderr ' + environ.get('REMOTE_ADDR', ''), file=sys.stderr, flush=True) + if environ['PATH_INFO'] == '/failure': + raise RuntimeError('SYNTHETIC_PRIVATE exception synthetic.yml') + status = {'/unavailable': '503 Service Unavailable', '/error-response': '500 Internal Server Error'}.get(environ['PATH_INFO'], '200 OK') + start_response(status, [('Content-Type', 'text/plain')]) + return [b'synthetic-response'] +app = application +''' + + +def request(endpoint, http, path, address): + query = 'form=synthetic.yml&session=SYNTHETIC_PRIVATE' + if http: + data = (f'GET {path}?{query} HTTP/1.0\r\nHost: synthetic.invalid\r\n\r\n').encode() + else: + fields = {'REQUEST_METHOD': 'GET', 'SCRIPT_NAME': '', 'PATH_INFO': path, + 'QUERY_STRING': query, 'REQUEST_URI': path + '?' + query, + 'SERVER_NAME': 'synthetic.invalid', 'SERVER_PORT': '80', + 'SERVER_PROTOCOL': 'HTTP/1.0', 'REMOTE_ADDR': address} + payload = bytearray() + for key, value in fields.items(): + key, value = key.encode(), value.encode() + payload.extend(struct.pack('= deadline: + raise AssertionError('real uWSGI did not open its configured socket') + time.sleep(0.02) + with client: + client.sendall(data) + response = bytearray() + while chunk := client.recv(16384): + response.extend(chunk) + assert len(response) < 65536, 'unbounded synthetic response' + return bytes(response) + + +def records(data): + assert MARKER not in data and b'synthetic.yml' not in data, 'request/application content retained' + result = [json.loads(line) for line in data.splitlines()] + assert result, 'missing native counters' + expected = {'schema', 'component', 'status_1xx', 'status_2xx', 'status_3xx', 'status_4xx', + 'status_5xx', 'latency_fast', 'latency_medium', 'latency_slow', 'latency_timeout', + 'unclassified', 'rejected', 'dropped'} + for item in result: + assert set(item) == expected, item + assert item['schema'] == 1 and item['component'] == 'uwsgi', item + for key, value in item.items(): + if key != 'component': + assert type(value) is int and 0 <= value <= 2147483647, key + return result[-1] + + +def process_state(pid): + try: + fields = Path(f'/proc/{pid}/stat').read_text().rsplit(')', 1)[1].split() + except FileNotFoundError: + return None + return {'parent': int(fields[1]), 'group': int(fields[2]), + 'session': int(fields[3]), 'start': int(fields[19])} + + +def native_processes(process, log_role): + pidfile = Path('/var/run/uwsgi/uwsgilog.pid' if log_role else '/var/run/uwsgi/uwsgi.pid') + master = int(pidfile.read_text()) + state = process_state(master) + assert state and state['parent'] == process.pid, 'native master is not the wrapper child' + assert state['group'] == state['session'] == master, 'native session is not isolated' + members = {} + for path in Path('/proc').iterdir(): + if path.name.isdecimal(): + pid = int(path.name) + state = process_state(pid) + if state and state['group'] == master and state['session'] == master: + members[pid] = state['start'] + assert master in members and len(members) >= 2, 'native master/worker pair not observed' + return master, members + + +def remaining_processes(members): + return [pid for pid, start in members.items() + if (state := process_state(pid)) and state['start'] == start] + + +def await_native_exit(members): + deadline = time.monotonic() + 2 + while remaining := remaining_processes(members): + assert time.monotonic() < deadline, ('native processes survived wrapper exit', remaining) + time.sleep(0.02) + + +def main(): + with tempfile.TemporaryDirectory(prefix='uwsgi-runtime-') as directory: + modules = Path(directory) / 'modules' + for package in ('docassemble', 'docassemble/base', 'docassemble/webapp'): + target = modules / package + target.mkdir(parents=True, exist_ok=True) + (target / '__init__.py').write_text('') + (modules / 'docassemble/base/read_config.py').write_text( + 'import sys\nprint("SYNTHETIC_PRIVATE bootstrap", file=sys.stderr)\n' + 'print(\'export LOCALE="C.UTF-8 UTF-8"\')\n') + for module in ('run', 'listlog'): + (modules / ('docassemble/webapp/' + module + '.py')).write_text(APP) + config = DA_ROOT / 'config' + for name in ('docassemble.ini.dist', 'docassemblelog.ini.dist', + 'docassemble-expose-uwsgi.ini', 'docassemblelog-expose-uwsgi.ini'): + text = (ROOT / 'Docker/config' / name).read_text() + for key, value in {'DA_PYTHON': str(RUNTIME), 'DA_ROOT': str(DA_ROOT), 'DAWSGIROOT': '/'}.items(): + text = text.replace('{{' + key + '}}', value) + (config / name.removesuffix('.dist')).write_text(text) + for name, http, log_role in (('main', False, False), ('exposed', True, False), + ('log', False, True), ('log-exposed', True, True)): + env = dict(os.environ, PYTHONPATH=str(modules), PYTHONDONTWRITEBYTECODE='1', + DA_ROOT=str(DA_ROOT), DA_PYTHON=str(RUNTIME), DAWEBSERVER='none' if http else 'nginx') + if name == 'log-exposed': + ini = config / 'docassemblelog-expose-uwsgi.ini' + preflight = subprocess.run([str(DA_ROOT / 'webapp/privacy-preflight'), 'uwsgi', str(ini)], + capture_output=True, timeout=5) + assert preflight.returncode == 0 and not preflight.stdout and not preflight.stderr + command = [str(DA_ROOT / 'webapp/privacy-process'), '--component', 'uwsgi', '--', + str(RUNTIME / 'bin/uwsgi'), '--ini', str(ini), '--die-on-term'] + else: + launcher = 'run-uwsgilog.sh' if log_role else 'run-uwsgi.sh' + command = ['bash', str(ROOT / 'Docker' / launcher)] + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) + endpoint = ('127.0.0.1', 80) if http else '/var/run/uwsgi/docassemble' + ('log' if log_role else '') + '.sock' + master, members = None, {} + try: + for path, address, expected in (('/', '203.0.113.7', b'200'), + ('/unavailable', '2001:db8::7', b'503'), + ('/error-response', '203.0.113.8', b'500'), + ('/failure', '203.0.113.9', None)): + response = request(endpoint, http, path, address) + if expected is None: + # Native control confirms uncaught WSGI exceptions close + # the connection without headers but increment status 500. + assert response == b'', (name, response) + else: + assert response.split(b'\r\n', 1)[0].split()[1] == expected, (name, response) + master, members = native_processes(process, log_role) + process.send_signal(signal.SIGTERM) + out, err = process.communicate(timeout=8) + assert process.returncode == 0 and err == b'', (name, process.returncode, out, err) + await_native_exit(members) + final = records(out) + assert final['status_2xx'] == 1 and final['status_5xx'] == 3, (name, final) + assert final['status_1xx'] == final['status_3xx'] == final['status_4xx'] == 0, (name, final) + assert sum(final[key] for key in ('latency_fast', 'latency_medium', + 'latency_slow', 'latency_timeout')) == 4, (name, final) + assert final['unclassified'] >= 8, (name, final) + print(f'uWSGI {name}: responses 200/503/500 and uncaught exception passed; safe status/latency counters; native master/workers exited', flush=True) + finally: + # Only signal the isolated group if one of the observed process + # identities still belongs to it. The outer container deadline + # also handles failures before a native PID could be observed. + for pid in remaining_processes(members): + state = process_state(pid) + if state and state['start'] == members[pid] and state['group'] == master: + try: + os.killpg(master, signal.SIGKILL) + except ProcessLookupError: + pass + break + if process.poll() is None: + process.kill() + process.communicate(timeout=8) + # The actual launcher must reject a retained file logger before native startup. + with (config / 'docassemble.ini').open('a') as stream: + stream.write('\nlogto = /tmp/SYNTHETIC_PRIVATE.log\n') + env['DAWEBSERVER'] = 'nginx' + rejected = subprocess.run(['bash', str(ROOT / 'Docker/run-uwsgi.sh')], env=env, + capture_output=True, timeout=5) + assert rejected.returncode == 70 and rejected.stderr == b'' + assert json.loads(rejected.stdout) == {'schema': 1, 'component': 'uwsgi', 'event': 'startup_failed', 'phase': 'preflight'} + assert not Path('/tmp/SYNTHETIC_PRIVATE.log').exists() + print('uWSGI unsafe logger: rejected before native startup', flush=True) + + +if __name__ == '__main__': + main() diff --git a/tests/privacy_native/fixtures/application-command.sh b/tests/privacy_native/fixtures/application-command.sh new file mode 100755 index 000000000..9a4d13b19 --- /dev/null +++ b/tests/privacy_native/fixtures/application-command.sh @@ -0,0 +1,17 @@ +#!/bin/sh +# Synthetic service/configuration command for the disposable Supervisor test. +if [ "$1" = '-m' ]; then + printf '%s\n' 'export LOCALE="C.UTF-8 UTF-8"' 'DAMAXCELERYWORKERS=4' 'DACELERYWORKERS=3' + printf 'SYNTHETIC_PRIVATE_BOOTSTRAP\n' >&2 + exit 0 +fi +# Run real base logging/application initialization through the production +# launcher and Go binary before the synthetic native-output controls. +application_logs=$(mktemp -d) || exit 98 +python3.14 -B /review/tests/privacy_native/fixtures/application-logger.py \ + /review "$application_logs" web none info || exit 98 +rmdir "$application_logs" || exit 98 +printf '%s\n' 'SYNTHETIC_PRIVATE_NATIVE' 'PRIVACY_REQUEST status=200 msecs=0' +printf '%s\n' 'SYNTHETIC_PRIVATE_NATIVE' 'PRIVACY_REQUEST status=500 msecs=0' >&2 +sleep 6 +exit 74 diff --git a/tests/privacy_native/fixtures/application-logger.py b/tests/privacy_native/fixtures/application-logger.py new file mode 100644 index 000000000..b9c55cbdf --- /dev/null +++ b/tests/privacy_native/fixtures/application-logger.py @@ -0,0 +1,81 @@ +"""Synthetic dependencies around real application startup and base logging.""" +import importlib +import logging +import os +from pathlib import Path +import sys +import types + + +def module(name, **attributes): + value = types.ModuleType(name) + value.__dict__.update(attributes) + sys.modules[name] = value + return value + + +def package(name, path=None): + return module(name, __path__=[] if path is None else [str(path)]) + + +def main(): + root = Path(sys.argv[1]) + directory = Path(sys.argv[2]) + context, logserver, debug = sys.argv[3:6] + package('docassemble') + package('docassemble.base', root / 'docassemble_base/docassemble/base') + webapp = package('docassemble.webapp', root / 'docassemble_webapp/docassemble/webapp') + from docassemble.base import logger + original_callback = logger.the_logmessage + config = module('docassemble.webapp.config', LOG_DIRECTORY=str(directory), + LOGSERVER=None if logserver == 'none' else 'synthetic', + daconfig={'log to std': context == 'std'}, + in_celery=context == 'celery', in_cron=context == 'cron') + os.environ['SUPERVISORLOGLEVEL'] = debug + package('werkzeug') + package('werkzeug.middleware') + module('werkzeug.middleware.proxy_fix', ProxyFix=object) + module('docassemble.base.plugin_manager', pm=object()) + module('docassemble.webapp.extensions', **{name: object() for name in + ('csrf', 'cors', 'babel', 'lm', 'the_user_manager', 'kv_session', 'db')}) + module('docassemble.webapp.setup', init_app=lambda app: logger.logmessage('SYNTHETIC_PRIVATE_SETUP')) + + class AfterLogging(Exception): + pass + + class SyntheticApp: + @property + def secret_key(self): + return None + + @secret_key.setter + def secret_key(self, value): + logger.logmessage('SYNTHETIC_PRIVATE_INITIALIZED') + raise AfterLogging() + + logger.logmessage('SYNTHETIC_PRIVATE_BEFORE') + startup = importlib.import_module('docassemble.webapp.app_initialize') + try: + startup.init_app(SyntheticApp()) + except AfterLogging: + pass + else: + raise AssertionError('actual startup did not reach the post-logging boundary') + assert not list(directory.iterdir()), 'application initialization opened an independent log file' + assert logger.the_logmessage is original_callback, 'base stderr callback was replaced' + entrypoint = sys.modules['docassemble.webapp.log_initialize'] + importlib.reload(entrypoint) + assert logger.the_logmessage is original_callback + assert not list(directory.iterdir()), 'reimport opened an independent log file' + # Exercise third-party reconfiguration after initialization as well as the + # real base callback. Go owns retained output regardless of these formatters. + logging.basicConfig(level=logging.DEBUG, force=True) + logging.getLogger('SYNTHETIC_PRIVATE_LOGGER').warning('SYNTHETIC_PRIVATE_AFTER') + logger.logmessage('SYNTHETIC_PRIVATE_BASE_AFTER') + print('SYNTHETIC_PRIVATE_STDOUT', flush=True) + assert config.daconfig == {'log to std': context == 'std'} + assert webapp.__path__ == [str(root / 'docassemble_webapp/docassemble/webapp')] + + +if __name__ == '__main__': + main() diff --git a/tests/privacy_native/fixtures/cron-stop.yml b/tests/privacy_native/fixtures/cron-stop.yml new file mode 100644 index 000000000..58e730c13 --- /dev/null +++ b/tests/privacy_native/fixtures/cron-stop.yml @@ -0,0 +1,24 @@ +metadata: + title: Synthetic cron termination check +--- +mandatory: True +code: | + allow_cron = True + multi_user = True +--- +event: JOS81_CRON_STOP_EVENT +code: | + import json + import os + import signal + import subprocess + worker = subprocess.Popen(['/bin/sleep', '120']) + os.write(1, b'JOS81_PRIVATE_CRON_STOP_STDOUT\n') + os.write(2, b'JOS81_PRIVATE_CRON_STOP_STDERR\n') + with open(os.environ['JOS81_CRON_PID_FILE'], 'w') as stream: + json.dump([os.getppid(), os.getpid(), worker.pid], stream) + while True: + signal.pause() +--- +mandatory: True +question: JOS81_CRON_STOP_READY diff --git a/tests/privacy_native/fixtures/cron.yml b/tests/privacy_native/fixtures/cron.yml new file mode 100644 index 000000000..c58066851 --- /dev/null +++ b/tests/privacy_native/fixtures/cron.yml @@ -0,0 +1,22 @@ +metadata: + title: Synthetic scheduled-interview check +--- +mandatory: True +code: | + allow_cron = True + multi_user = True + cron_count = 0 +--- +event: cron_hourly +code: | + import os + assert os.getuid() == os.getgid() == 33 + assert os.environ['USER'] == os.environ['LOGNAME'] == 'www-data' + assert os.environ['HOME'] == '/var/www' + assert os.nice(0) == 19 + cron_count += 1 + os.write(2, b'JOS81_PRIVATE_CRON_STDERR\n') + response('JOS81_PRIVATE_CRON_RESPONSE') +--- +mandatory: True +question: JOS81_CRON_COUNT_${ cron_count } diff --git a/tests/privacy_native/fixtures/mail-runtime.py b/tests/privacy_native/fixtures/mail-runtime.py new file mode 100755 index 000000000..e6364e095 --- /dev/null +++ b/tests/privacy_native/fixtures/mail-runtime.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3.14 +"""Synthetic runtime command for exercising the actual mail shell/Go boundary.""" +import hashlib +import os +from pathlib import Path +import sys + +assert sys.argv[1:] == ['-m', 'docassemble.webapp.process_email', '/dev/stdin'] +with open('/dev/stdin', 'rb') as stream: + message = stream.read() +assert hashlib.sha256(message).hexdigest() == os.environ['SYNTHETIC_MAIL_SHA256'] +mode = os.environ.get('SYNTHETIC_MAIL_MODE', 'input') +if mode != 'input': + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + from mail_support import run_processor + state = run_processor(os.environ['SYNTHETIC_MAIL_DIRECTORY'], + None if mode == 'success' else mode, message) + assert not any(path == '/tmp/mail.log' or access != 'r' for path, access in state.opens) + if state.error is not None: + raise state.error + if state.exit is not None: + sys.exit(state.exit) + assert state.completed and len(state.emails) == 1 and len(state.attachments) == 3 + assert len(state.tasks) == 1 +print('SYNTHETIC_PRIVATE_MAIL_STDOUT') +print('PRIVACY_REQUEST status=200 msecs=0') +print('SYNTHETIC_PRIVATE_MAIL_STDERR', file=sys.stderr) +sys.exit(int(os.environ.get('SYNTHETIC_MAIL_EXIT', '0'))) diff --git a/tests/privacy_native/fixtures/nginx-preflight-safe.conf b/tests/privacy_native/fixtures/nginx-preflight-safe.conf new file mode 100644 index 000000000..b11866be0 --- /dev/null +++ b/tests/privacy_native/fixtures/nginx-preflight-safe.conf @@ -0,0 +1,10 @@ +server { + listen 127.0.0.1:18081; + set $quoted "quote\" and \' and \\ and \n"; + set $header "escaped\" +# configuration file /fake: +end"; + if ($request_uri ~ "\?id=\b[0-9]+\b") { set $quoted 'matched'; } + location ~ \.pdf$ { return 200 'synthetic\n'; } + location / { return 200 'synthetic'; } +} diff --git a/tests/privacy_native/fixtures/nginx-preflight-unsafe.conf b/tests/privacy_native/fixtures/nginx-preflight-unsafe.conf new file mode 100644 index 000000000..46e55e19e --- /dev/null +++ b/tests/privacy_native/fixtures/nginx-preflight-unsafe.conf @@ -0,0 +1,5 @@ +server { + listen 127.0.0.1:18081; + access_log /tmp/SYNTHETIC_PRIVATE_CONFIG\;escaped privacy_counts; + location / { return 200 'synthetic'; } +} diff --git a/tests/privacy_native/fixtures/nginx-preflight.conf b/tests/privacy_native/fixtures/nginx-preflight.conf new file mode 100644 index 000000000..4b710460b --- /dev/null +++ b/tests/privacy_native/fixtures/nginx-preflight.conf @@ -0,0 +1,15 @@ +# Synthetic native -T acceptance fixture. No public listener is started. +pid /tmp/nginx.pid; +error_log stderr; +worker_shutdown_timeout 2s; +events { worker_connections 32; } +http { + client_body_temp_path /tmp/nginx-client; + proxy_temp_path /tmp/nginx-proxy; + fastcgi_temp_path /tmp/nginx-fastcgi; + uwsgi_temp_path /tmp/nginx-uwsgi; + scgi_temp_path /tmp/nginx-scgi; + log_format privacy_counts 'PRIVACY_REQUEST status=$status seconds=$request_time'; + access_log /dev/stdout privacy_counts; + include /etc/nginx/preflight-override.conf; +} diff --git a/tests/privacy_native/fixtures/privacy-check.Dockerfile b/tests/privacy_native/fixtures/privacy-check.Dockerfile new file mode 100644 index 000000000..85d0ffdfd --- /dev/null +++ b/tests/privacy_native/fixtures/privacy-check.Dockerfile @@ -0,0 +1,5 @@ +FROM python:3.14-alpine +# Development only: gcc supports Go race tests, bash runs the real launchers, +# and Supervisor/nginx exercise their real protocols with synthetic fixtures. +RUN apk add --no-cache build-base bash supervisor nginx +RUN adduser -S -D -H -u 82 -G www-data www-data diff --git a/tests/privacy_native/fixtures/queue.yml b/tests/privacy_native/fixtures/queue.yml new file mode 100644 index 000000000..eeb0f415a --- /dev/null +++ b/tests/privacy_native/fixtures/queue.yml @@ -0,0 +1,28 @@ +metadata: + title: Synthetic queued-job check +--- +mandatory: True +code: | + queue_job + if queue_job.ready(): + assert queue_job.get() == 'JOS81_QUEUE_RESULT' + queue_done + else: + queue_wait +--- +code: | + queue_job = background_action('queue_work') +--- +event: queue_work +code: | + import os + os.write(1, b'JOS81_PRIVATE_QUEUE_STDOUT\n') + os.write(2, b'JOS81_PRIVATE_QUEUE_STDERR\n') + background_response('JOS81_QUEUE_RESULT') +--- +event: queue_wait +question: JOS81_QUEUE_WAIT +reload: True +--- +event: queue_done +question: JOS81_QUEUE_OK diff --git a/tests/privacy_native/fixtures/rotation-check.Dockerfile b/tests/privacy_native/fixtures/rotation-check.Dockerfile new file mode 100644 index 000000000..61891c357 --- /dev/null +++ b/tests/privacy_native/fixtures/rotation-check.Dockerfile @@ -0,0 +1,4 @@ +ARG NATIVE_BASE=privacy-candidate-check +FROM ${NATIVE_BASE} +# Test-only native file rotation and forwarding acceptance; no service image changes. +RUN apk add --no-cache logrotate syslog-ng diff --git a/tests/privacy_native/fixtures/rotation-command.sh b/tests/privacy_native/fixtures/rotation-command.sh new file mode 100644 index 000000000..e2be573eb --- /dev/null +++ b/tests/privacy_native/fixtures/rotation-command.sh @@ -0,0 +1,15 @@ +#!/bin/sh +# Synthetic stream only; executable stays in the read-only source mount. +trap 'exit 0' TERM INT QUIT +burst=0 +while :; do + if [ "$burst" -eq 0 ] && [ -e "$1" ]; then + while [ "$burst" -lt 8192 ]; do + printf '%s\n' 'SYNTHETIC_PRIVATE after rotation' + burst=$((burst + 1)) + done + fi + printf '%s\n' 'SYNTHETIC_PRIVATE stdout query=synthetic.yml' 'PRIVACY_REQUEST status=200 msecs=1' + printf '%s\n' 'SYNTHETIC_PRIVATE stderr 203.0.113.7' >&2 + sleep 0.05 +done diff --git a/tests/privacy_native/fixtures/uwsgi-check.Dockerfile b/tests/privacy_native/fixtures/uwsgi-check.Dockerfile new file mode 100644 index 000000000..4f32fc909 --- /dev/null +++ b/tests/privacy_native/fixtures/uwsgi-check.Dockerfile @@ -0,0 +1,12 @@ +ARG NATIVE_BASE=privacy-candidate-check +FROM ${NATIVE_BASE} +# Test-only compiler support comes from the existing protocol fixture image. +# Reuse the application's pinned build backend; no service image is changed. +COPY uwsgi-requirements.txt /tmp/uwsgi-requirements.txt +RUN apk add --no-cache linux-headers \ + && python3.14 -m pip install --no-cache-dir --no-deps setuptools==83.0.0 \ + && UWSGI_BUILD_CORES=2 python3.14 -m pip install --no-cache-dir --no-deps \ + --no-build-isolation --require-hashes -r /tmp/uwsgi-requirements.txt \ + && rm /tmp/uwsgi-requirements.txt \ + && python3.14 -m venv --without-pip /usr/share/docassemble/local3.14 \ + && ln -s /usr/local/bin/uwsgi /usr/share/docassemble/local3.14/bin/uwsgi diff --git a/tests/privacy_native/fixtures/uwsgi-requirements.txt b/tests/privacy_native/fixtures/uwsgi-requirements.txt new file mode 100644 index 000000000..353bd9941 --- /dev/null +++ b/tests/privacy_native/fixtures/uwsgi-requirements.txt @@ -0,0 +1,2 @@ +# Test-only native runtime; source digest published by PyPI for uWSGI 2.0.31. +uWSGI==2.0.31 --hash=sha256:e8f8b350ccc106ff93a65247b9136f529c14bf96b936ac5b264c6ff9d0c76257 diff --git a/tests/privacy_native/mail_support.py b/tests/privacy_native/mail_support.py new file mode 100644 index 000000000..ceb3628f9 --- /dev/null +++ b/tests/privacy_native/mail_support.py @@ -0,0 +1,112 @@ +"""Run the real mail processor with synthetic database, storage, and task sinks.""" +import builtins +import contextlib +import io +from pathlib import Path +import runpy +import sys +import types +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[2] +MESSAGE = (b'From: Synthetic Sender \n' + b'To: testcode@example.invalid\nEnvelope-to: testcode@example.invalid\n' + b'Subject: SYNTHETIC_PRIVATE_SUBJECT\nMIME-Version: 1.0\n' + b'Content-Type: multipart/mixed; boundary="synthetic-boundary"\n\n' + b'--synthetic-boundary\nContent-Type: text/plain\n\nSYNTHETIC_PRIVATE_BODY\n' + b'--synthetic-boundary\nContent-Type: application/pdf\n' + b'Content-Disposition: attachment; filename="synthetic.pdf"\n' + b'Content-Transfer-Encoding: base64\n\nAP9QREY=\n' + b'--synthetic-boundary--\n') + + +def run_processor(directory, failure=None, message=MESSAGE): + state = types.SimpleNamespace(opens=[], emails=[], attachments=[], saved=[], tasks=[], config=[], + completed=False, exit=None, error=None) + modules = {} + + def module(name, **attrs): + value = types.ModuleType(name) + value.__dict__.update(attrs) + modules[name] = value + parent, _, child = name.rpartition('.') + if parent in modules: + setattr(modules[parent], child, value) + return value + + for name in ('docassemble', 'docassemble.base', 'docassemble.webapp', + 'docassemble.webapp.emailserver', 'docassemble.webapp.files', + 'docassemble.webapp.users', 'docassemble.webapp.tasks', 'sqlalchemy'): + module(name, __path__=[]) + module('docassemble.base.config', load=lambda **kwargs: state.config.append(kwargs)) + + class Query: + def filter_by(self, **kwargs): + return self + + class Session: + def execute(self, query): + if failure == 'database': + raise OSError('SYNTHETIC_PRIVATE_DATABASE') + row = None if failure == 'unknown' else types.SimpleNamespace( + uid='synthetic-session', filename='synthetic.yml', user_id=None, temp_user_id=7) + return types.SimpleNamespace(scalar=lambda: row) + + def add(self, record): + pass + + @contextlib.contextmanager + def session_scope(): + yield Session() + + def email_record(**kwargs): + state.emails.append(kwargs) + return types.SimpleNamespace(id=42, **kwargs) + + def attachment(**kwargs): + state.attachments.append(kwargs) + return types.SimpleNamespace(**kwargs) + + def saved_file(*args, **kwargs): + return types.SimpleNamespace(write_content=state.saved.append, finalize=lambda: None) + + def signature(*args, **kwargs): + state.tasks.append((args, kwargs)) + def delay(): + if failure == 'broker': + raise OSError('SYNTHETIC_PRIVATE_BROKER') + return types.SimpleNamespace(delay=delay) + + modules['sqlalchemy'].select = lambda *args: Query() + module('sqlalchemy.orm', joinedload=lambda *args: None) + module('docassemble.webapp.db', session_scope=session_scope) + module('docassemble.webapp.emailserver.models', Shortener=object, Email=email_record, + EmailAttachment=attachment) + module('docassemble.webapp.files.file_number', get_new_file_number=lambda *args: 77) + module('docassemble.webapp.files.savedfile', SavedFile=saved_file) + module('docassemble.webapp.users.models', UserModel=object) + module('docassemble.webapp.tasks.app', celery_app=types.SimpleNamespace(signature=signature)) + original_open = builtins.open + message_path = str(Path(directory) / 'message') + + def opening(path, mode='r', *args, **kwargs): + state.opens.append((str(path), mode)) + if str(path) == '/tmp/mail.log': + # Never touch a real log even when executing the fail-first source. + return io.StringIO() + if str(path) == message_path: + if failure == 'read': + raise OSError('SYNTHETIC_PRIVATE_READ') + return io.StringIO(message.decode('utf-8')) + return original_open(path, mode, *args, **kwargs) + + with patch.dict(sys.modules, modules), patch.object(sys, 'argv', ['process_email.py', message_path]), \ + patch.object(builtins, 'open', opening): + try: + runpy.run_path(str(ROOT / 'docassemble_webapp/docassemble/webapp/process_email.py'), run_name='__main__') + state.completed = True + except SystemExit as exc: + state.exit = exc.code + except Exception as exc: + state.error = exc + return state diff --git a/tests/privacy_native/native_support.py b/tests/privacy_native/native_support.py new file mode 100644 index 000000000..ef0c5d454 --- /dev/null +++ b/tests/privacy_native/native_support.py @@ -0,0 +1,17 @@ +"""Load the exact checkout runtime without installation or PYTHONPATH setup.""" + +import importlib.util +from pathlib import Path + + +RUNTIME = Path(__file__).resolve().parent / "reference/log_aggregate.py" +SPEC = importlib.util.spec_from_file_location("reviewed_log_aggregate", RUNTIME) +if SPEC is None or SPEC.loader is None: + raise RuntimeError("Cannot load the checkout's privacy aggregator") +AGGREGATE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(AGGREGATE) + +LogAggregator = AGGREGATE.LogAggregator +_MAX_LINE = AGGREGATE._MAX_LINE +_COUNTER_MAX = AGGREGATE._COUNTER_MAX +_SCHEMA_VERSION = AGGREGATE._SCHEMA_VERSION diff --git a/tests/privacy_native/probe_mail_storage.py b/tests/privacy_native/probe_mail_storage.py new file mode 100644 index 000000000..91a782a94 --- /dev/null +++ b/tests/privacy_native/probe_mail_storage.py @@ -0,0 +1,26 @@ +"""Independent release-gate probe, intentionally nonzero while legacy storage fails. + +Run directly; this is not a green synthetic mail acceptance test. The actual +SavedFile method runs against in-memory text/binary handles, without importing +the application or touching storage. Its default must accept the MIME payload +type passed by process_email.save_attachment before deployment is accepted. +""" +import ast +import io +import os +from pathlib import Path +import types + +ROOT = Path(__file__).resolve().parents[2] +source = ROOT / 'docassemble_webapp/docassemble/webapp/files/savedfile.py' +tree = ast.parse(source.read_text(), filename=str(source)) +saved_file = next(node for node in tree.body if isinstance(node, ast.ClassDef) and node.name == 'SavedFile') +method = next(node for node in saved_file.body if isinstance(node, ast.FunctionDef) and node.name == 'write_content') +namespace = {'os': os, 'directory_for': lambda *_: '/synthetic', + 'open': lambda *args, **kwargs: io.BytesIO() if args[1] == 'wb' else io.StringIO()} +exec(compile(ast.Module(body=[method], type_ignores=[]), str(source), 'exec'), namespace) +instance = types.SimpleNamespace(filename='attachment', fix=lambda: None, save=lambda: None) +namespace['write_content'](instance, '{"synthetic": true}') +print('Text header control passed; probing decoded MIME bytes.', flush=True) +namespace['write_content'](instance, b'SYNTHETIC_PRIVATE_BODY') +print('Decoded MIME byte storage contract passed.', flush=True) diff --git a/tests/privacy_native/reference/check_native_config.py b/tests/privacy_native/reference/check_native_config.py new file mode 100644 index 000000000..611b8017e --- /dev/null +++ b/tests/privacy_native/reference/check_native_config.py @@ -0,0 +1,339 @@ +#!/usr/bin/env python3 +"""Silent, bounded preflight for the maintained native privacy profile.""" + +import fnmatch +import os +import posixpath +import re +import selectors +import signal +import stat +import subprocess +import sys +import time + + +MAX_BYTES = 1024 * 1024 +TIMEOUT_SECONDS = 10 +INVALID = 70 +UWSGI_FORMAT = "PRIVACY_REQUEST status=%(status) msecs=%(msecs)" +NGINX_FORMAT = "PRIVACY_REQUEST status=$status seconds=$request_time" +NGINX_COMMAND = ("/usr/sbin/nginx", "-T", "-e", "stderr") +_BOOL_OPTIONS = {"master", "enable-threads", "vhost", "manage-script-name", "die-on-term"} +_PATH_OPTIONS = {"socket", "venv", "pidfile", "touch-reload", "py-executable"} +_INT_OPTIONS = {"processes", "threads", "buffer-size", "max-fd"} +_OPTIONS = _BOOL_OPTIONS | _PATH_OPTIONS | _INT_OPTIONS | { + "mount", "module", "callable", "http-socket", "log-format", +} +_LEVELS = {"debug", "info", "notice", "warn", "error", "crit", "alert", "emerg"} +_HEADER = re.compile(r"# configuration file (/[^\r\n]+):\r?\n?\Z") + + +class InvalidConfig(Exception): + """Deliberately carries no input or exception detail.""" + + +def _require(condition): + if not condition: + raise InvalidConfig() + + +def _text(data): + _require(isinstance(data, bytes) and len(data) <= MAX_BYTES) + text = data.decode("utf-8") + _require(all(ord(char) >= 32 or char in "\n\r\t" for char in text)) + return text + + +def _absolute(value): + return bool(re.fullmatch(r"/[A-Za-z0-9_./-]+", value)) and ".." not in value.split("/") + + +def validate_uwsgi(data): + """Validate bytes; raise on rejection, return None on acceptance.""" + options = {} + section = False + text = _text(data) + _require("{{" not in text and "}}" not in text) + for raw in text.splitlines(): + line = raw.strip() + if not line or line.startswith(("#", ";")): + continue + if line.startswith("["): + _require(not section and line == "[uwsgi]") + section = True + continue + _require(section and "=" in line) + key, value = (part.strip() for part in line.split("=", 1)) + _require(key in _OPTIONS and key not in options and bool(value)) + if key in _BOOL_OPTIONS: + _require(value == "true") + elif key in _PATH_OPTIONS: + _require(_absolute(value)) + elif key in _INT_OPTIONS: + _require(bool(re.fullmatch(r"[1-9][0-9]{0,6}", value)) and int(value) <= 1048576) + elif key == "mount": + prefix, separator, target = value.partition("=") + _require(separator and (prefix == "/" or _absolute(prefix))) + _require(target == "docassemble.webapp.run:application") + elif key == "module": + _require(value == "docassemble.webapp.listlog") + elif key == "callable": + _require(value == "app") + elif key == "http-socket": + _require(value == ":80") + elif key == "log-format": + _require(value == UWSGI_FORMAT) + options[key] = value + _require(section and options.get("master") == "true") + _require(options.get("die-on-term") == "true" and options.get("log-format") == UWSGI_FORMAT) + + +def _dump_files(text): + """Split nginx -T headers only outside quoted strings and comments.""" + files = {} + first = None + current = None + body = [] + quote = None + active = False + + def finish(): + if current is not None: + content = "".join(body) + _require(current not in files or files[current] == content) + files[current] = content + + for line in text.splitlines(keepends=True): + header = _HEADER.fullmatch(line) if quote is None else None + if header: + finish() + current = header.group(1) + _require(posixpath.normpath(current) == current) + if first is None: + first = current + body = [] + continue + _require(current is not None or not line.strip()) + body.append(line) + for char in line: + if quote: + _require(char != "\\") + if char == quote: + quote = None + elif char == "#" and not active: + break + elif char in "\"'": + quote = char + active = True + elif char == "\\": + raise InvalidConfig() + elif char.isspace() or char in ";{}": + active = False + else: + active = True + if quote is None: + active = False + _require(quote is None and first is not None) + finish() + return first, files + + +def _tokens(text): + """Tokenize native delimiters, respecting quotes/comments and ${variables}.""" + result = [] + token = [] + active = False + quote = None + after_quote = False + index = 0 + while index < len(text): + char = text[index] + if after_quote: + _require(char.isspace() or char in ";{})") + if char == ")": + result.append(("word", "".join(token))) + token, active = [], False + after_quote = False + if quote: + _require(char != "\\") + if char == quote: + quote = None + after_quote = True + else: + token.append(char) + elif char == "#" and not active: + index = text.find("\n", index) + if index < 0: + break + elif char in "\"'": + _require(not active) + quote, active = char, True + elif char == "\\": + raise InvalidConfig() + elif char == "$" and text[index:index + 2] == "${": + end = text.find("}", index + 2) + _require(end >= 0 and bool(re.fullmatch(r"[A-Za-z0-9_]+", text[index + 2:end]))) + token.append(text[index:end + 1]) + active, index = True, end + elif char.isspace() or char in ";{}": + if active: + result.append(("word", "".join(token))) + token, active = [], False + if char in ";{}": + result.append((char, char)) + else: + token.append(char) + active = True + index += 1 + _require(quote is None) + if active: + result.append(("word", "".join(token))) + _require(len(result) <= 100000) + return result + + +def _parse(text): + root = [] + stack = [root] + words = [] + for kind, value in _tokens(text): + if kind == "word": + words.append(value) + elif kind in (";", "{"): + _require(bool(words) and bool(words[0])) + children = [] if kind == "{" else None + stack[-1].append((tuple(words), children)) + words = [] + if children is not None: + stack.append(children) + _require(len(stack) <= 64) + else: + _require(not words and len(stack) > 1) + stack.pop() + _require(not words and len(stack) == 1) + return root + + +def validate_nginx_dump(data): + """Validate the complete successful -T stdout, including include contexts.""" + first, files = _dump_files(_text(data)) + trees = {name: _parse(body) for name, body in files.items()} + seen = set() + budget = 100000 + main = {} + http_scopes = [] + + def walk(nodes, context, scope, chain): + nonlocal budget + for words, children in nodes: + budget -= 1 + _require(budget >= 0) + name, args = words[0], words[1:] + if name == "include": + _require(children is None and len(args) == 1 and "$" not in args[0]) + pattern = args[0] + if not pattern.startswith("/"): + pattern = posixpath.join(posixpath.dirname(first), pattern) + pattern = posixpath.normpath(pattern) + parts = pattern.split("/") + matches = sorted(path for path in trees + if len(path.split("/")) == len(parts) + and all(fnmatch.fnmatchcase(part, glob) + for part, glob in zip(path.split("/"), parts))) + _require(matches or any(char in pattern for char in "*?[")) + for path in matches: + _require(path not in chain) + seen.add(path) + walk(trees[path], context, scope, chain + (path,)) + continue + if name in {"access_log", "error_log", "worker_shutdown_timeout", "master_process", "daemon"}: + _require(children is None and name not in scope) + scope[name] = args + if name == "access_log": + _require(args in (("off",), ("/dev/stdout", "privacy_counts"))) + _require(bool(context) and context[0] == "http") + elif name == "error_log": + _require(args == ("stderr",) or (len(args) == 2 and args[0] == "stderr" and args[1] in _LEVELS)) + elif name == "worker_shutdown_timeout": + _require(not context and args == ("2s",)) + elif name == "master_process": + _require(not context and args == ("on",)) + elif name == "daemon": + _require(not context and args == ("off",)) + elif name == "log_format": + _require(children is None and len(args) >= 2 and context == ("http",)) + key = ("log_format", args[0]) + _require(key not in scope) + scope[key] = args[1:] + if args[0] == "privacy_counts": + _require(args == ("privacy_counts", NGINX_FORMAT)) + if children is not None: + child_scope = {} + if name == "http": + _require(not context and not args) + http_scopes.append(child_scope) + walk(children, context + (name,), child_scope, chain) + + seen.add(first) + walk(trees[first], (), main, (first,)) + _require(seen == set(trees)) + _require("error_log" in main and main.get("worker_shutdown_timeout") == ("2s",)) + _require(len(http_scopes) == 1) + _require(http_scopes[0].get("access_log") == ("/dev/stdout", "privacy_counts")) + _require(http_scopes[0].get(("log_format", "privacy_counts")) == (NGINX_FORMAT,)) + + +def _nginx_dump(): + process = subprocess.Popen(NGINX_COMMAND, stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + start_new_session=True, close_fds=True) + total = 0 + output = bytearray() + deadline = time.monotonic() + TIMEOUT_SECONDS + try: + with selectors.DefaultSelector() as selector: + selector.register(process.stdout, selectors.EVENT_READ, True) + selector.register(process.stderr, selectors.EVENT_READ, False) + while selector.get_map(): + remaining = deadline - time.monotonic() + _require(remaining > 0) + for key, _ in selector.select(min(remaining, 0.1)): + chunk = os.read(key.fd, 8192) + if not chunk: + selector.unregister(key.fileobj) + continue + total += len(chunk) + _require(total <= MAX_BYTES) + if key.data: + output.extend(chunk) + remaining = deadline - time.monotonic() + _require(remaining > 0 and process.wait(timeout=remaining) == 0) + return bytes(output) + finally: + if process.poll() is None: + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=1) + process.stdout.close() + process.stderr.close() + + +def main(argv=None): + try: + args = sys.argv[1:] if argv is None else argv + if len(args) == 2 and args[0] == "uwsgi": + descriptor = os.open(args[1], os.O_RDONLY | os.O_NONBLOCK) + with os.fdopen(descriptor, "rb") as source: + _require(stat.S_ISREG(os.fstat(source.fileno()).st_mode)) + validate_uwsgi(source.read(MAX_BYTES + 1)) + elif args == ["nginx"]: + validate_nginx_dump(_nginx_dump()) + else: + return INVALID + return 0 + except BaseException: + return INVALID + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/privacy_native/reference/log_aggregate.py b/tests/privacy_native/reference/log_aggregate.py new file mode 100644 index 000000000..148090ace --- /dev/null +++ b/tests/privacy_native/reference/log_aggregate.py @@ -0,0 +1,186 @@ +import re + +_MAX_LINE = 4096 +_COUNTER_MAX = 2**31 - 1 +_SCHEMA_VERSION = 1 + +VALID_COMPONENTS = frozenset({"nginx", "uwsgi"}) +VALID_STREAMS = frozenset({"stdout", "stderr"}) + +_STATUS_FAMILIES = { + "1xx": (100, 199), + "2xx": (200, 299), + "3xx": (300, 399), + "4xx": (400, 499), + "5xx": (500, 599), +} + +_STATUS_RE = re.compile(r"^[1-5][0-9]{2}$") +_MSECS_RE = re.compile(r"^(?:0|[1-9][0-9]*)$") +_SECONDS_RE = re.compile(r"(0|[1-9][0-9]{0,2})\.([0-9]{3})") + + +class LogAggregator: + + def __init__(self, component): + if type(component) is not str or component not in VALID_COMPONENTS: + raise ValueError( + "component must be exactly 'nginx' or 'uwsgi'" + ) + self._component = component + self._buf = {"stdout": b"", "stderr": b""} + self._dropping = {"stdout": False, "stderr": False} + self._counters = { + "schema": _SCHEMA_VERSION, + "component": component, + "status_1xx": 0, + "status_2xx": 0, + "status_3xx": 0, + "status_4xx": 0, + "status_5xx": 0, + "latency_fast": 0, + "latency_medium": 0, + "latency_slow": 0, + "latency_timeout": 0, + "unclassified": 0, + "rejected": 0, + "dropped": 0, + } + + def _inc(self, key): + if self._counters[key] < _COUNTER_MAX: + self._counters[key] += 1 + + def _validate_stream(self, stream): + if type(stream) is not str: + raise TypeError("stream must be str") + if stream not in VALID_STREAMS: + raise ValueError("stream must be 'stdout' or 'stderr'") + + def feed(self, stream, chunk): + self._validate_stream(stream) + if type(chunk) is not bytes: + raise TypeError("chunk must be bytes") + + buf = self._buf[stream] + chunk_len = len(chunk) + cpos = 0 + + while cpos < chunk_len: + if self._dropping[stream]: + nl = chunk.find(b"\n", cpos) + if nl == -1: + return + self._dropping[stream] = False + cpos = nl + 1 + continue + + nl = chunk.find(b"\n", cpos) + if nl == -1: + remaining = chunk_len - cpos + if len(buf) + remaining > _MAX_LINE: + self._dropping[stream] = True + self._inc("dropped") + self._buf[stream] = b"" + return + self._buf[stream] = buf + chunk[cpos:] + return + + line_len = len(buf) + (nl - cpos) + if line_len > _MAX_LINE: + self._inc("dropped") + cpos = nl + 1 + buf = b"" + continue + + line = buf + chunk[cpos:nl] + cpos = nl + 1 + buf = b"" + self._process_line(line) + + self._buf[stream] = buf + + def _process_line(self, line): + if line.endswith(b"\r"): + line = line[:-1] + + try: + text = line.decode("utf-8") + except UnicodeDecodeError: + self._inc("rejected") + return + + if b"\x00" in line or b"\r" in line: + self._inc("rejected") + return + + parts = text.split(" ") + if not parts or parts[0] != "PRIVACY_REQUEST": + self._inc("unclassified") + return + + if len(parts) != 3: + self._inc("rejected") + return + + if not parts[1].startswith("status="): + self._inc("rejected") + return + status_str = parts[1][7:] + if not _STATUS_RE.fullmatch(status_str): + self._inc("rejected") + return + status = int(status_str) + + if parts[2].startswith("msecs="): + msecs_str = parts[2][6:] + if len(msecs_str) > 6 or not _MSECS_RE.fullmatch(msecs_str): + self._inc("rejected") + return + msecs = int(msecs_str) + elif self._component == "nginx" and parts[2].startswith("seconds="): + match = _SECONDS_RE.fullmatch(parts[2][8:]) + if match is None: + self._inc("rejected") + return + msecs = int(match[1]) * 1000 + int(match[2]) + else: + self._inc("rejected") + return + + if not (0 <= msecs <= 600000): + self._inc("rejected") + return + + for fam, (lo, hi) in _STATUS_FAMILIES.items(): + if lo <= status <= hi: + self._inc("status_" + fam) + break + + if msecs < 100: + self._inc("latency_fast") + elif msecs < 1000: + self._inc("latency_medium") + elif msecs < 30000: + self._inc("latency_slow") + else: + self._inc("latency_timeout") + + def snapshot(self): + return dict(self._counters) + + def finish(self, stream): + self._validate_stream(stream) + was_dropping = self._dropping[stream] + self._dropping[stream] = False + buf = self._buf[stream] + self._buf[stream] = b"" + if was_dropping: + if buf: + self._inc("dropped") + return + if buf: + if len(buf) <= _MAX_LINE: + self._process_line(buf) + else: + self._inc("dropped") diff --git a/tests/privacy_native/reference/pdeath_exec.py b/tests/privacy_native/reference/pdeath_exec.py new file mode 100644 index 000000000..75283293b --- /dev/null +++ b/tests/privacy_native/reference/pdeath_exec.py @@ -0,0 +1,74 @@ +"""Linux foreground exec helper. No fallback or raw diagnostics on failure.""" +from __future__ import annotations + +import errno +import os +import signal +import stat +import sys +from collections.abc import Sequence + +FAILURE = 70 + + +def _arm_parent_death(value: int) -> bool: + import ctypes + libc = ctypes.CDLL(None, use_errno=True) + libc.prctl.argtypes = [ctypes.c_int, ctypes.c_ulong, ctypes.c_ulong, + ctypes.c_ulong, ctypes.c_ulong] + libc.prctl.restype = ctypes.c_int + return libc.prctl(1, value, 0, 0, 0) == 0 + + +def _ordinary_executable(path: str) -> bool: + mode = os.stat(path).st_mode + if not stat.S_ISREG(mode) or mode & (stat.S_ISUID | stat.S_ISGID): + return False + try: + return not os.getxattr(path, "security.capability") + except OSError as error: + if error.errno in {errno.ENODATA, errno.ENOTSUP}: + return True + raise + + +def exec_guarded(parent: int, component: str, command: Sequence[str]) -> int: + """Arm a native graceful signal, close the parent race, then replace this PID. + + Later credential changes inside the service can clear the kernel setting; + launch profiles must prohibit those unless separately verified. + """ + try: + import resource + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + if ( + sys.platform != "linux" or type(parent) is not int or not 1 < parent < 2**31 + or type(component) is not str or component not in {"nginx", "uwsgi"} + or isinstance(command, (str, bytes)) or not isinstance(command, Sequence) + or not command or any(type(arg) is not str or "\0" in arg for arg in command) + or not os.path.isabs(command[0]) + or (component == "uwsgi" and "--die-on-term" not in command[1:]) + or not _ordinary_executable(command[0]) + ): + return FAILURE + graceful = signal.SIGQUIT if component == "nginx" else signal.SIGTERM + if not _arm_parent_death(graceful) or os.getppid() != parent: + return FAILURE + os.execv(command[0], list(command)) + except BaseException: + return FAILURE + return FAILURE # A successful exec never returns. + + +def main(argv: Sequence[str] | None = None) -> int: + try: + args = list(sys.argv[1:] if argv is None else argv) + if len(args) < 4 or args[2] != "--": + return FAILURE + return exec_guarded(int(args[0]), args[1], args[3:]) + except BaseException: + return FAILURE + + +if __name__ == "__main__": + os._exit(main()) diff --git a/tests/privacy_native/reference/preflight_reference.py b/tests/privacy_native/reference/preflight_reference.py new file mode 100644 index 000000000..1848f9325 --- /dev/null +++ b/tests/privacy_native/reference/preflight_reference.py @@ -0,0 +1,51 @@ +"""Run unchanged legacy assertions and export synthetic parser decisions only.""" + +import base64 +import fnmatch +import hashlib +import io +import json +from pathlib import Path +import random +import sys +import unittest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import test_native_config as legacy + + +def main() -> None: + cases = [] + + def record(kind, original): + def checked(data): + item = {"kind": kind, "data": base64.b64encode(data).decode(), "accepted": False} + cases.append(item) + result = original(data) + item["accepted"] = True + return result + return checked + + legacy.CHECK.validate_uwsgi = record("uwsgi", legacy.CHECK.validate_uwsgi) + legacy.CHECK.validate_nginx_dump = record("nginx", legacy.CHECK.validate_nginx_dump) + output = io.StringIO() + result = unittest.TextTestRunner(stream=output).run(unittest.defaultTestLoader.loadTestsFromModule(legacy)) + if not result.wasSuccessful(): + raise RuntimeError(output.getvalue()) + + rng = random.Random(81) + globs = [] + alphabet = "abz-!]^[]Ω☃" + for i in range(10000): + pattern = "[" + "".join(rng.choice(alphabet) for _ in range(rng.randrange(1, 8))) + "]" + if i % 3 == 0: + pattern = rng.choice(("*", "?", "x")) + pattern + rng.choice(("*", "?", "")) + name = "".join(rng.choice(alphabet + ".x\n") for _ in range(rng.randrange(0, 4))) + globs.append({"pattern": pattern, "name": name, "accepted": fnmatch.fnmatchcase(name, pattern)}) + + print(json.dumps({"source_sha256": hashlib.sha256(legacy.SCRIPT.read_bytes()).hexdigest(), + "legacy_tests": result.testsRun, "cases": cases, "globs": globs})) + + +if __name__ == "__main__": + main() diff --git a/tests/privacy_native/reference/privacy_process.py b/tests/privacy_native/reference/privacy_process.py new file mode 100644 index 000000000..5ed2e3ab0 --- /dev/null +++ b/tests/privacy_native/reference/privacy_process.py @@ -0,0 +1,380 @@ +"""Counters-only foreground process wrapper. POSIX, Python 3.14, no dependencies.""" + +from __future__ import annotations + +import fcntl +import json +import math +import os +import resource +import selectors +import signal +import stat +import subprocess +import sys +import time +from collections.abc import Callable, Sequence +from typing import Protocol + +CHUNK_BYTES = 4096 +MAX_SNAPSHOT_BYTES = 8192 +USAGE_FAILURE = 64 +RUNNER_FAILURE = 70 +SINK_FAILURE = 74 +COUNTERS = ( + "status_1xx", "status_2xx", "status_3xx", "status_4xx", "status_5xx", + "latency_fast", "latency_medium", "latency_slow", "latency_timeout", + "unclassified", "rejected", "dropped", +) + + +class Aggregator(Protocol): + def feed(self, stream: str, data: bytes) -> None: ... + def finish(self, stream: str) -> None: ... + def snapshot(self) -> dict[str, int | str]: ... + + +def _snapshot(aggregator: Aggregator, component: str) -> bytes: + value = aggregator.snapshot() + if type(value) is not dict or set(value) != {"schema", "component", *COUNTERS}: + raise ValueError("invalid snapshot") + if type(value["schema"]) is not int or value["schema"] != 1: + raise ValueError("invalid snapshot") + if type(value["component"]) is not str or value["component"] != component: + raise ValueError("invalid snapshot") + for key in COUNTERS: + if type(value[key]) is not int or not 0 <= value[key] <= 2**31 - 1: + raise ValueError("invalid snapshot") + clean = {"schema": 1, "component": component} + clean.update({key: value[key] for key in COUNTERS}) + encoded = json.dumps(clean, ensure_ascii=True, separators=(",", ":")).encode() + b"\n" + if len(encoded) >= MAX_SNAPSHOT_BYTES: + raise ValueError("invalid snapshot") + return encoded + + +def _signal(pid: int, value: int, *, group: bool = False) -> None: + try: + (os.killpg if group else os.kill)(pid, value) + except ProcessLookupError: + pass + + +def _group_alive(pid: int) -> bool: + try: + os.killpg(pid, 0) + return True + except ProcessLookupError: + return False + + +def _run( + command: Sequence[str], + component: str, + sink_fd: int = 1, + aggregator_class: Callable[[str], Aggregator] | None = None, + *, + require_parent_death: bool = True, + snapshot_interval: float = 1.0, + sink_timeout: float = 5.0, + stop_grace: float = 2.0, + kill_grace: float = 1.0, +) -> int: + """Run one foreground master; return its exit code or a fixed wrapper failure. + + Call from the main thread of a dedicated process. This permanently disables + core dumps for that process. Timing overrides exist for synthetic tests. + Raw bytes are held only in a fixed read chunk and the injected aggregator. + """ + if ( + type(component) is not str + or component not in {"nginx", "uwsgi"} + or isinstance(command, (str, bytes)) + or not isinstance(command, Sequence) + or not command + or any(type(arg) is not str or "\0" in arg for arg in command) + or not os.path.isabs(command[0]) + or (component == "uwsgi" and "--die-on-term" not in command[1:]) + or type(sink_fd) is not int + or sink_fd < 0 + or type(require_parent_death) is not bool + ): + return USAGE_FAILURE + timings = (snapshot_interval, sink_timeout, stop_grace, kill_grace) + if any(type(v) not in (int, float) or not math.isfinite(v) or v <= 0 for v in timings): + return USAGE_FAILURE + + child: subprocess.Popen[bytes] | None = None + sink: int | None = None + sink_flags: int | None = None + old_handlers: dict[int, object] = {} + selector: selectors.BaseSelector | None = None + signals = {"stop": 0, "forward": 0} + failure = 0 + pipes: dict[int, str] = {} + stop_started: float | None = None + group_stopped = False + killed_at: float | None = None + pending: bytes | None = None + pending_offset = 0 + pending_started = 0.0 + pending_version = 0 + version = 1 + written_version = 0 + next_snapshot = 0.0 + final = False + aggregator_ok = True + + def handle(signum: int, _frame: object) -> None: + if signum in (signal.SIGINT, signal.SIGTERM): + signals["stop"] = signum + elif signum == signal.SIGHUP: + signals["forward"] |= 1 + elif signum == signal.SIGUSR1: + signals["forward"] |= 2 + + def stop(now: float) -> None: + nonlocal stop_started + if child is not None and stop_started is None: + stop_started = now + graceful = signal.SIGQUIT if component == "nginx" else signal.SIGTERM + _signal(child.pid, graceful) + # A master may already have exited while descendants retain pipes. + if child.poll() is not None: + _signal(child.pid, graceful, group=True) + + try: + selector = selectors.DefaultSelector() + resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) + child_command = list(command) + if require_parent_death: + if sys.platform != "linux": + return RUNNER_FAILURE + helper = os.path.join(os.path.dirname(os.path.abspath(__file__)), "pdeath_exec.py") + if not stat.S_ISREG(os.stat(helper).st_mode): + return RUNNER_FAILURE + child_command = [sys.executable, "-I", "-B", helper, str(os.getpid()), + component, "--", *command] + if aggregator_class is None: + from log_aggregate import LogAggregator + aggregator_class = LogAggregator + aggregator = aggregator_class(component) + _snapshot(aggregator, component) # Validate before starting the service. + try: + sink = os.dup(sink_fd) + mode = os.fstat(sink).st_mode + if not (stat.S_ISFIFO(mode) or stat.S_ISSOCK(mode)): + return SINK_FAILURE + sink_flags = fcntl.fcntl(sink, fcntl.F_GETFL) + fcntl.fcntl(sink, fcntl.F_SETFL, sink_flags | os.O_NONBLOCK) + except OSError: + return SINK_FAILURE + for sig in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP, signal.SIGUSR1): + old_handlers[sig] = signal.signal(sig, handle) + old_handlers[signal.SIGPIPE] = signal.signal(signal.SIGPIPE, signal.SIG_IGN) + child = subprocess.Popen( + child_command, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, close_fds=True, start_new_session=True, + bufsize=0, + ) + for pipe, name in ((child.stdout, "stdout"), (child.stderr, "stderr")): + assert pipe is not None + os.set_blocking(pipe.fileno(), False) + pipes[pipe.fileno()] = name + selector.register(pipe, selectors.EVENT_READ) + + while True: + now = time.monotonic() + if signals["stop"] or failure: + stop(now) + forward = signals["forward"] + signals["forward"] = 0 + if child.poll() is None and stop_started is None: + if forward & 1: + _signal(child.pid, signal.SIGHUP) + if forward & 2: + _signal(child.pid, signal.SIGUSR1) + exit_code = child.poll() + if exit_code is not None and (pipes or _group_alive(child.pid)): + stop(now) + + if stop_started is not None and not group_stopped and ( + exit_code is not None or now - stop_started >= stop_grace / 2 + ): + graceful = signal.SIGQUIT if component == "nginx" else signal.SIGTERM + _signal(child.pid, graceful, group=True) + group_stopped = True + if stop_started is not None and now - stop_started >= stop_grace and killed_at is None: + _signal(child.pid, signal.SIGKILL, group=True) + killed_at = now + if killed_at is not None and now - killed_at >= kill_grace: + # Escaped descendants or unreaped groups cannot extend our wait. + if pipes or child.poll() is None or _group_alive(child.pid): + failure = failure or RUNNER_FAILURE + for key in list(selector.get_map().values()): + if aggregator_ok: + try: + aggregator.finish(pipes[key.fd]) + version = (version + 1) & ((1 << 63) - 1) + except Exception: + aggregator_ok = False + selector.unregister(key.fileobj) + key.fileobj.close() + pipes.clear() + final = True + elif exit_code is not None and not pipes and not _group_alive(child.pid): + final = True + + if not failure and aggregator_ok: + try: + if (pending is None or pending_offset == 0) and (final or now >= next_snapshot): + if version != written_version and (pending is None or pending_version != version): + encoded = _snapshot(aggregator, component) + if pending is None: + pending_started = now + pending = encoded + pending_offset = 0 + pending_version = version + next_snapshot = now + snapshot_interval + except Exception: + failure = RUNNER_FAILURE + aggregator_ok = False + stop(now) + + if pending is not None and not failure: + try: + assert sink is not None + count = os.write(sink, memoryview(pending)[pending_offset:]) + if count <= 0: + raise OSError("sink failed") + pending_offset += count + if pending_offset == len(pending): + written_version = pending_version + pending = None + pending_offset = 0 + except (BlockingIOError, InterruptedError): + pass + except OSError: + failure = SINK_FAILURE + stop(now) + if pending is not None and now - pending_started >= sink_timeout: + failure = SINK_FAILURE + stop(now) + + if failure: + pending = None + if final and (failure or (pending is None and written_version == version)): + code = child.poll() + if code is None: + return RUNNER_FAILURE + return failure or (code if code >= 0 else 128 - code) + + for key, _events in selector.select(timeout=0.02): + fd = key.fd + try: + data = os.read(fd, CHUNK_BYTES) + except (BlockingIOError, InterruptedError): + continue + except OSError: + failure = failure or RUNNER_FAILURE + data = b"" + stream = pipes[fd] + if not data: + selector.unregister(key.fileobj) + key.fileobj.close() + del pipes[fd] + if aggregator_ok: + try: + if data: + aggregator.feed(stream, data) + else: + aggregator.finish(stream) + version = (version + 1) & ((1 << 63) - 1) + except Exception: + aggregator_ok = False + failure = failure or RUNNER_FAILURE + except BaseException: + # Never print exception text, argv, paths, or a raw-output fallback. + if child is not None: + try: + graceful = signal.SIGQUIT if component == "nginx" else signal.SIGTERM + _signal(child.pid, graceful, group=True) + deadline = time.monotonic() + stop_grace + while time.monotonic() < deadline and _group_alive(child.pid): + child.poll() + time.sleep(0.02) + except OSError: + pass + return RUNNER_FAILURE + finally: + cleanup_failed = False + if child is not None: + try: + if child.poll() is None or _group_alive(child.pid): + _signal(child.pid, signal.SIGKILL, group=True) + child.wait(timeout=kill_grace) + except (OSError, subprocess.TimeoutExpired): + pass + for pipe in (child.stdout, child.stderr): + if pipe is not None: + try: + pipe.close() + except BaseException: + cleanup_failed = True + if selector is not None: + try: + selector.close() + except BaseException: + cleanup_failed = True + for sig, handler in old_handlers.items(): + try: + signal.signal(sig, handler) + except BaseException: + cleanup_failed = True + if sink is not None: + if sink_flags is not None: + try: + fcntl.fcntl(sink, fcntl.F_SETFL, sink_flags) + except OSError: + cleanup_failed = True + try: + os.close(sink) + except OSError: + cleanup_failed = True + if cleanup_failed: + raise RuntimeError("runner cleanup failed") + + +def run( + command: Sequence[str], component: str, sink_fd: int = 1, + aggregator_class: Callable[[str], Aggregator] | None = None, *, + require_parent_death: bool = True, + snapshot_interval: float = 1.0, sink_timeout: float = 5.0, + stop_grace: float = 2.0, kill_grace: float = 1.0, +) -> int: + """Run a foreground child, requiring Linux parent-death protection by default. + + Portable synthetic callers can explicitly opt out with require_parent_death=False. + Nginx additionally requires the reviewed main-context shutdown companion. + """ + try: + return _run(command, component, sink_fd, aggregator_class, + require_parent_death=require_parent_death, + snapshot_interval=snapshot_interval, sink_timeout=sink_timeout, + stop_grace=stop_grace, kill_grace=kill_grace) + except BaseException: + return RUNNER_FAILURE + + +def main(argv: Sequence[str] | None = None) -> int: + try: + args = list(sys.argv[1:] if argv is None else argv) + if len(args) < 4 or args[0] != "--component" or args[2] != "--": + return USAGE_FAILURE + return run(args[3:], args[1], require_parent_death=True) + except BaseException: + return RUNNER_FAILURE + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/privacy_native/test_application_launchers.py b/tests/privacy_native/test_application_launchers.py new file mode 100644 index 000000000..178ef7e96 --- /dev/null +++ b/tests/privacy_native/test_application_launchers.py @@ -0,0 +1,103 @@ +"""Real shell handoffs and compiled diagnostics, with synthetic bootstrap/native tools.""" +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +import test_native_launchers as native + +PRIVATE, ROOT = native.PRIVATE, native.ROOT + +PROFILES = (('run-celery.sh', 'celery'), ('run-celery-single.sh', 'celerysingle'), + ('run-websockets.sh', 'websockets')) + + +class ApplicationLauncherTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + native.NativeLauncherTests.setUpClass() + + def fixture(self, directory, failure=None): + root = native.NativeLauncherTests().fixture(directory, activation_failure=failure == 'activation') + interpreter = root / 'runtime/bin/python' + interpreter.write_text('#!' + sys.executable + '\n' + r'''import os, sys +print('SYNTHETIC_PRIVATE_BOOTSTRAP', file=sys.stderr) +if sys.argv[1] == '-m': + print('export LOCALE="' if os.environ.get('SYNTHETIC_CONFIG_EVAL_FAIL') else 'export LOCALE="C.UTF-8 UTF-8"') + print('DAMAXCELERYWORKERS=4') + print('DACELERYWORKERS=' + os.environ.get('SYNTHETIC_WORKERS', '3')) + if os.environ.get('SYNTHETIC_LAUNCH_FAIL'): + os.unlink(os.environ['DA_ROOT'] + '/webapp/privacy-process') + raise SystemExit(1 if os.environ.get('SYNTHETIC_CONFIG_FAIL') else 0) +print('SYNTHETIC_PRIVATE_NATIVE') +raise SystemExit(17) +''') + for name, text in {'nproc': '#!/bin/sh\nprintf "8\\n"\n', + 'celery': '#!/bin/sh\nprintf "SYNTHETIC_PRIVATE_NATIVE\\n"\nexit 17\n'}.items(): + executable = root / 'runtime/bin' / name + executable.write_text(text) + executable.chmod(0o755) + return root + + def invoke(self, script, root, **extra): + env = dict(os.environ, DA_ROOT=str(root), DA_PYTHON=str(root / 'runtime'), **extra) + env['PATH'] = str(root / 'runtime/bin') + os.pathsep + os.environ['PATH'] + child = subprocess.Popen(['bash', str(ROOT / 'Docker' / script)], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) + stdout, stderr = child.communicate(timeout=5) + return child, stdout, stderr + + def test_every_application_uses_protected_foreground_capture(self): + for script, component in PROFILES: + with self.subTest(script=script), tempfile.TemporaryDirectory() as directory: + root = self.fixture(directory) + child, stdout, stderr = self.invoke(script, root) + self.assertNotIn(PRIVATE.encode(), stdout + stderr) + self.assertNotIn(b'SYNTHETIC_PRIVATE_NATIVE', stdout + stderr) + self.assertEqual(child.returncode, 0) + self.assertEqual(stderr, b'') + record = json.loads(stdout) + self.assertEqual(record['pid'], child.pid) + self.assertTrue(record['extra_sink_closed']) + self.assertEqual(record['argv'][:3], ['--component', component, '--']) + if component == 'websockets': + self.assertEqual(record['argv'][3:], [str(root / 'runtime/bin/python'), '-u', '-m', 'docassemble.webapp.socketserver']) + else: + args = record['argv'][3:] + self.assertEqual(args[0], str(root / 'runtime/bin/celery')) + self.assertEqual(args[1:5], ['-A', 'docassemble.webapp.worker', 'worker', '--loglevel=INFO']) + self.assertIn('--concurrency=' + ('2' if component == 'celery' else '1'), args) + self.assertEqual(args[args.index('-Q') + 1], 'celery' if component == 'celery' else 'single') + + def test_bootstrap_and_exec_failures_are_fixed_and_bounded(self): + for script, component in PROFILES: + for failure in ('activation', 'config', 'config_eval', 'launch'): + with self.subTest(script=script, failure=failure), tempfile.TemporaryDirectory() as directory: + root = self.fixture(directory, failure) + extra = {} if failure == 'activation' else {'SYNTHETIC_' + failure.upper() + '_FAIL': '1'} + child, stdout, stderr = self.invoke(script, root, **extra) + self.assertEqual(child.returncode, 70) + self.assertEqual(stderr, b'') + self.assertEqual(json.loads(stdout), {'schema': 1, 'component': component, + 'event': 'startup_failed', 'phase': failure}) + + def test_missing_diagnostic_stops_before_activation(self): + for script, component in PROFILES: + with self.subTest(script=script), tempfile.TemporaryDirectory() as directory: + root = self.fixture(directory) + (root / 'webapp/privacy-diagnostic').unlink() + child, stdout, stderr = self.invoke(script, root) + self.assertEqual(child.returncode, 69) + self.assertEqual(stdout + stderr, b'') + + def test_worker_count_cannot_be_shell_arithmetic(self): + for value in ('1+2', '-1', '0', '99999999999999999999', 'name[0]'): + with self.subTest(value=value), tempfile.TemporaryDirectory() as directory: + root = self.fixture(directory) + child, stdout, stderr = self.invoke('run-celery.sh', root, SYNTHETIC_WORKERS=value) + self.assertEqual(child.returncode, 70) + self.assertEqual(stderr, b'') + self.assertEqual(json.loads(stdout)['phase'], 'config_eval') diff --git a/tests/privacy_native/test_install_catalog.py b/tests/privacy_native/test_install_catalog.py new file mode 100644 index 000000000..48c1ba104 --- /dev/null +++ b/tests/privacy_native/test_install_catalog.py @@ -0,0 +1,113 @@ +"""Check the documented overlay against real source paths and runtime bindings.""" +import configparser +import json +from pathlib import Path +import re +import subprocess +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[2] + + +def catalog(): + return json.loads((ROOT / 'docs/privacy/install-catalog.json').read_text()) + + +def target(value): + return value.replace('{{DA_ROOT}}', '/usr/share/docassemble').replace( + '{{SITE_PACKAGES}}', '/usr/share/docassemble/local3.14/lib/python3.14/site-packages') + + +class InstallCatalogTests(unittest.TestCase): + def test_nginx_patch_preserves_custom_content_at_different_offsets(self): + legacy = b' access_log /var/log/nginx/access.log privacy;\n' + for offset in (0, 5, 100): + with self.subTest(offset=offset), tempfile.TemporaryDirectory() as folder: + path = Path(folder) / 'Docker/config/nginx-realip' + path.parent.mkdir(parents=True) + original = b'# preserved prefix\n' * offset + ( + b' # BOTH FILES MUST SHIP TOGETHER -- this line without that file fails nginx -t.\n' + + legacy + b'\n # preserved suffix and custom routing\n') + path.write_bytes(original) + result = subprocess.run(['patch', '--batch', '--fuzz=0', '-p1', '-d', folder, + '-i', str(ROOT / 'Docker/privacy/nginx-realip.patch')], + capture_output=True, timeout=10) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(path.read_bytes(), original.replace(legacy, b'', 1)) + + def test_sources_exist_and_exclude_deferred_mail_and_test_material(self): + data = catalog() + self.assertEqual(data['scope'], 'non-mail-privacy-overlay') + self.assertEqual((data['owner'], data['group']), ('root', 'root')) + for item in data['files']: + with self.subTest(item=item['id']): + self.assertIn(item['mode'], ('0644', '0755')) + self.assertIn(item.get('existing'), (None, 'preserve-and-validate', 'apply-privacy-diff')) + if item['kind'] == 'binary': + self.assertRegex(item['source'], r'^privacy-(diagnostic|process|preflight|monitor)-linux-\{\{ARCH\}\}$') + continue + self.assertEqual(item['kind'], 'source') + path = ROOT / item['source'] + self.assertTrue(path.is_file()) + self.assertTrue(path.resolve().is_relative_to(ROOT)) + self.assertFalse(item['source'].startswith('tests/')) + self.assertNotIn('exim', item['source']) + self.assertNotIn(path.name, ('process-email.sh', 'process_email.py')) + if 'patch' in item: + self.assertEqual(item['existing'], 'apply-privacy-diff') + patch = ROOT / item['patch'] + self.assertTrue(patch.resolve().is_relative_to(ROOT / 'Docker/privacy')) + self.assertTrue(patch.read_text().startswith('--- a/' + item['source'] + '\n+++ b/' + item['source'] + '\n')) + if path.suffix == '.py': + self.assertEqual(path.name, 'log_initialize.py') + self.assertTrue(all(not line.strip() or line.lstrip().startswith('#') for line in path.read_text().splitlines())) + + def test_unique_bindings_and_generated_source_references(self): + data = catalog() + entries = data['files'] + data['states'] + self.assertEqual(len(entries), len({item['id'] for item in entries})) + self.assertEqual(len(entries), len({target(item['target']) for item in entries})) + source_ids = {item['id'] for item in data['files']} + for item in data['states']: + self.assertTrue(set(item['sources']) <= source_ids) + self.assertTrue(item['rollback']) + for item in entries + data['protected']: + resolved = target(item['target']) + self.assertTrue(resolved.startswith('/')) + self.assertNotIn('..', Path(resolved).parts) + self.assertNotIn('{{', resolved) + for item in data['files']: + for protected in data['protected']: + self.assertFalse(Path(target(item['target'])).is_relative_to(target(protected['target']))) + + def test_supervisor_launchers_and_counter_files_are_covered(self): + data = catalog() + files = {target(item['target']) for item in data['files']} + counters = {target(item['target']) for item in data['states'] if item['kind'] == 'counter-file'} + config = configparser.ConfigParser(interpolation=None) + config.read(ROOT / 'Docker/docassemble-supervisor.conf') + for name in ('nginx', 'uwsgi', 'uwsgilog', 'celery', 'celerysingle', 'websockets'): + section = config['program:' + name] + self.assertIn(section['command'].removeprefix('bash '), files) + self.assertIn(section['stdout_logfile'], counters) + monitor = config['eventlistener:privacy-monitor'] + self.assertIn(monitor['command'], files) + self.assertIn(monitor['stderr_logfile'], counters) + + def test_actual_generated_nginx_and_uwsgi_paths_have_rollback_entries(self): + states = {target(item['target']) for item in catalog()['states']} + script = (ROOT / 'Docker/run-nginx.sh').read_text() + generated = set(re.findall(r'"(/etc/nginx/sites-available/[^"\s]+)"', script)) + links = set(re.findall(r'/etc/nginx/sites-enabled/[a-z]+', script)) + self.assertEqual(len(generated), 5) + self.assertEqual(len(links), 5) + self.assertTrue(generated | links <= states) + initializer = (ROOT / 'Docker/initialize.sh').read_text() + generated_uwsgi = re.findall(r'> "\$\{DA_ROOT\}(/config/docassemble(?:log)?\.ini)"', initializer) + self.assertEqual(len(generated_uwsgi), 2) + self.assertTrue({'/usr/share/docassemble' + path for path in generated_uwsgi} <= states) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/privacy_native/test_install_image.sh b/tests/privacy_native/test_install_image.sh new file mode 100644 index 000000000..1e9c110c7 --- /dev/null +++ b/tests/privacy_native/test_install_image.sh @@ -0,0 +1,99 @@ +#!/bin/bash +# Disposable populated-volume rehearsal, not a production installation command. +set -euo pipefail +test "$#" -eq 0 +if [ "$(uname -s)/$(uname -m)" != Linux/x86_64 ]; then + printf '%s\n' 'This test requires native Linux amd64; user-mode emulation lacks required tar syscalls.' >&2 + exit 64 +fi +case "$(docker info --format '{{.Architecture}}')" in x86_64|amd64) ;; *) exit 64 ;; esac +REVIEW_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +REVIEW_BUILD="$REVIEW_ROOT/tests/.privacy-build" +mkdir -p "$REVIEW_BUILD/go-cache" "$REVIEW_BUILD/go-mod" "$REVIEW_BUILD/go-tmp" "$REVIEW_BUILD/tmp" +export GOCACHE="$REVIEW_BUILD/go-cache" GOMODCACHE="$REVIEW_BUILD/go-mod" +export GOTMPDIR="$REVIEW_BUILD/go-tmp" TMPDIR="$REVIEW_BUILD/tmp" +export TEST_TELEMETRY_DIR="$REVIEW_BUILD/telemetry" GOTOOLCHAIN=local GOPROXY=off +export PYTHONDONTWRITEBYTECODE=1 +REVIEW_RUN=$(mktemp -d "$REVIEW_BUILD/install-run.XXXXXXXX") +REVIEW_NAME="jos81-$(basename "$REVIEW_RUN")" +REVIEW_CONTAINER='' +REVIEW_VOLUME='' +cleanup() { + REVIEW_STATUS=$? + trap - EXIT + if [ -n "$REVIEW_CONTAINER" ]; then + if ! timeout -k 5 30 docker rm --force "$REVIEW_CONTAINER" >/dev/null; then REVIEW_STATUS=125; fi + fi + if [ -n "$REVIEW_VOLUME" ]; then + if ! timeout -k 5 15 docker volume rm "$REVIEW_VOLUME" >/dev/null; then REVIEW_STATUS=125; fi + fi + exit "$REVIEW_STATUS" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM +# Generate the source privacy delta from the reviewed base; do not duplicate it in a fixture. +git -C "$REVIEW_ROOT" diff 63d22c44b7bd008a521f0bcb8ab442b2143ee557 -- Docker/initialize.sh Docker/cron/docassemble-cron-daily.sh > "$REVIEW_BUILD/overlay-privacy.patch" +cat "$REVIEW_ROOT/Docker/privacy/nginx-realip.patch" >> "$REVIEW_BUILD/overlay-privacy.patch" +test -s "$REVIEW_BUILD/overlay-privacy.patch" +for REVIEW_COMPONENT in diagnostic process preflight monitor; do + REVIEW_PACKAGE="./cmd/privacy-$REVIEW_COMPONENT" + if [ "$REVIEW_COMPONENT" = diagnostic ]; then REVIEW_PACKAGE=.; fi + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go -C "$REVIEW_ROOT/Docker/privacy-diagnostic" build \ + -trimpath -buildvcs=false -o "$REVIEW_BUILD/privacy-$REVIEW_COMPONENT-linux-amd64" "$REVIEW_PACKAGE" +done +REVIEW_IMAGE='jhpyle/docassemble@sha256:c0b0a707a6cd2149d5777ee83af1ea1de544210e597371eac93e67a1503c395c' +timeout -k 5 600 docker pull --platform linux/amd64 "$REVIEW_IMAGE" +test "$(docker image inspect --format '{{.Architecture}}' "$REVIEW_IMAGE")" = amd64 +printf 'Installation test image: %s\n' "$REVIEW_IMAGE" +go version +sha256sum "$REVIEW_BUILD"/privacy-*-linux-amd64 +# Names are unique and cleanup acts only on resources created by this invocation. +if docker volume inspect "$REVIEW_NAME" >/dev/null 2>&1; then exit 73; fi +REVIEW_VOLUME=$(docker volume create --label task=JOS-81-install-review "$REVIEW_NAME") +REVIEW_CONTAINER=$(docker create --name "$REVIEW_NAME" --platform linux/amd64 --network none \ + --hostname privacy-install-test --add-host privacy-install-test:127.0.0.1 \ + --add-host privacy-install.test:127.0.0.1 --security-opt no-new-privileges \ + --memory 4g --memory-swap 4g --cpus 4 --pids-limit 1024 \ + --log-driver json-file --log-opt max-size=5m --log-opt max-file=2 \ + --label task=JOS-81-install-review \ + --mount "type=volume,src=$REVIEW_VOLUME,dst=/usr/share/docassemble" \ + --mount "type=bind,src=$REVIEW_ROOT,dst=/review,readonly" \ + -e DAHOSTNAME=privacy-install.test -e USEHTTPS=false -e BEHINDHTTPSLOADBALANCER=true \ + -e POSTURLROOT=/nj/ -e WSGIROOT=/nj -e DAALLOWUPDATES=false -e DAUPDATEONSTART=false \ + -e DAENABLEPLAYGROUND=false -e DAALLOWCONFIGURATIONEDITING=false -e DAALLOWLOGVIEWING=false \ + -e DAROOTOWNED=true -e DADEBUG=false -e PYTHONDONTWRITEBYTECODE=1 "$REVIEW_IMAGE") +docker start "$REVIEW_CONTAINER" >/dev/null +python3.14 -B - "$REVIEW_CONTAINER" <<'PY' +import subprocess +import sys +import time +probe = """from pathlib import Path +import xmlrpc.client +assert Path('/var/run/docassemble/ready').exists() +rpc = xmlrpc.client.ServerProxy('http://localhost:9001/RPC2') +states = {p['name']: p['statename'] for p in rpc.supervisor.getAllProcessInfo()} +assert all(states.get(n) == 'RUNNING' for n in ('nginx', 'uwsgi', 'celery', 'celerysingle', 'websockets')) +""" +deadline = time.monotonic() + 240 +while time.monotonic() < deadline: + result = subprocess.run(['docker', 'exec', sys.argv[1], 'python3.14', '-I', '-B', '-c', probe], + capture_output=True, timeout=20) + if result.returncode == 0: + break + time.sleep(2) +else: + raise SystemExit('stock application did not become ready within 240 seconds') +PY +timeout -k 5 600 docker exec "$REVIEW_CONTAINER" /usr/share/docassemble/local3.14/bin/python \ + -I -B /review/tests/privacy_native/check_install_image.py prepare +python3.14 -B "$REVIEW_ROOT/tests/privacy_native/check_install_lifecycle.py" "$REVIEW_CONTAINER" "$REVIEW_RUN" +timeout -k 5 600 docker exec "$REVIEW_CONTAINER" /usr/share/docassemble/local3.14/bin/python \ + -I -B /review/tests/privacy_native/check_install_image.py resume +python3.14 -B - "$REVIEW_CONTAINER" <<'PY' +import subprocess +import sys +result = subprocess.run(['docker', 'logs', sys.argv[1]], capture_output=True, timeout=15, check=True) +assert b'JOS81_PRIVATE_' not in result.stdout + result.stderr, 'private output in container logs' +print('private queue/cron markers absent from container stdout/stderr logs') +PY diff --git a/tests/privacy_native/test_install_records.py b/tests/privacy_native/test_install_records.py new file mode 100644 index 000000000..1908e4c22 --- /dev/null +++ b/tests/privacy_native/test_install_records.py @@ -0,0 +1,56 @@ +"""The integration check must reject retained raw data, even when valid JSON.""" +import json +import gzip +from pathlib import Path +import tempfile +import unittest + +from check_install_image import private_output_absent, records + + +class InstallRecordChecks(unittest.TestCase): + def setUp(self): + self.record = dict(schema=1, component='uwsgi', status_1xx=0, status_2xx=1, + status_3xx=0, status_4xx=0, status_5xx=0, latency_fast=1, + latency_medium=0, latency_slow=0, latency_timeout=0, + unclassified=0, rejected=0, dropped=0) + + def test_accepts_safe_request_record(self): + self.assertEqual(records(json.dumps(self.record), 'uwsgi'), [self.record]) + + def test_rejects_raw_or_ambiguous_json(self): + for data in ('{"message":"synthetic private content"}', + json.dumps(self.record | {'message': 'synthetic private content'}), + '{"status_2xx":"synthetic private content",' + json.dumps(self.record)[1:], + '[]', ''): + with self.subTest(data=data), self.assertRaises(AssertionError): + records(data, 'uwsgi') + + def test_rejects_wrong_identity_and_invalid_numbers(self): + for replacement in ({'schema': 2}, {'component': 'nginx'}, {'status_2xx': True}, + {'status_2xx': -1}, {'status_2xx': 2147483648}, {'status_2xx': 1.5}): + with self.subTest(replacement=replacement), self.assertRaises(AssertionError): + records(json.dumps(self.record | replacement), 'uwsgi') + + def test_rejects_request_buckets_for_application_streams(self): + for component in ('celery', 'celerysingle', 'websockets', 'cron'): + with self.subTest(component=component), self.assertRaises(AssertionError): + records(json.dumps(self.record | {'component': component}), component) + + +class RetainedOutputChecks(unittest.TestCase): + def test_raw_and_compressed_markers_are_detected_across_read_boundaries(self): + for suffix in ('.log', '.log.gz'): + with self.subTest(suffix=suffix), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + opener = gzip.open if suffix.endswith('.gz') else open + with opener(root / ('synthetic' + suffix), 'wb') as stream: + stream.write(b'x' * 65530 + b'JOS81_PRIVATE_SENTINEL') + with self.assertRaises(AssertionError): + private_output_absent((root,)) + + def test_safe_counters_pass_the_retained_output_scan(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'safe.log').write_text('{"schema":1,"component":"cron"}\n') + private_output_absent((root,)) diff --git a/tests/privacy_native/test_log_aggregate.py b/tests/privacy_native/test_log_aggregate.py new file mode 100644 index 000000000..5d1454bde --- /dev/null +++ b/tests/privacy_native/test_log_aggregate.py @@ -0,0 +1,413 @@ +import unittest +import json + +from native_support import LogAggregator, _MAX_LINE, _COUNTER_MAX, _SCHEMA_VERSION + + +SYNTH_IPv4 = "192.0.2.1" +SYNTH_IPv6 = "2001:db8::1" +SYNTH_FORM = "private_form.yml" +SYNTH_SESSION = "sess_abc123secret" +SYNTH_TOKENS = [SYNTH_IPv4, SYNTH_IPv6, SYNTH_FORM, SYNTH_SESSION] + +EXPECTED_KEYS = { + "schema", "component", + "status_1xx", "status_2xx", "status_3xx", + "status_4xx", "status_5xx", + "latency_fast", "latency_medium", + "latency_slow", "latency_timeout", + "unclassified", "rejected", "dropped", +} + + +def _snapshot_str(agg): + return json.dumps(agg.snapshot()) + + +class TestBasicValidLines(unittest.TestCase): + + def test_single_200_fast(self): + agg = LogAggregator("nginx") + agg.feed("stdout", b"PRIVACY_REQUEST status=200 msecs=50\n") + s = agg.snapshot() + self.assertEqual(s["status_2xx"], 1) + self.assertEqual(s["latency_fast"], 1) + + def test_multiple_lines(self): + agg = LogAggregator("uwsgi") + agg.feed("stdout", + b"PRIVACY_REQUEST status=200 msecs=50\n" + b"PRIVACY_REQUEST status=404 msecs=500\n" + b"PRIVACY_REQUEST status=500 msecs=5000\n") + s = agg.snapshot() + self.assertEqual(s["status_2xx"], 1) + self.assertEqual(s["status_4xx"], 1) + self.assertEqual(s["status_5xx"], 1) + self.assertEqual(s["latency_fast"], 1) + self.assertEqual(s["latency_medium"], 1) + self.assertEqual(s["latency_slow"], 1) + + def test_component_in_snapshot(self): + agg = LogAggregator("nginx") + self.assertEqual(agg.snapshot()["component"], "nginx") + agg2 = LogAggregator("uwsgi") + self.assertEqual(agg2.snapshot()["component"], "uwsgi") + + +class TestFragmentedLines(unittest.TestCase): + + def test_line_split_across_two_chunks(self): + agg = LogAggregator("nginx") + agg.feed("stdout", b"PRIVACY_REQUEST status=200 ") + self.assertEqual(agg.snapshot()["status_2xx"], 0) + agg.feed("stdout", b"msecs=50\n") + s = agg.snapshot() + self.assertEqual(s["status_2xx"], 1) + self.assertEqual(s["latency_fast"], 1) + + def test_line_split_byte_by_byte(self): + agg = LogAggregator("nginx") + line = b"PRIVACY_REQUEST status=301 msecs=99\n" + for byte in line: + agg.feed("stdout", bytes([byte])) + s = agg.snapshot() + self.assertEqual(s["status_3xx"], 1) + self.assertEqual(s["latency_fast"], 1) + + def test_multiple_lines_in_one_chunk(self): + agg = LogAggregator("nginx") + agg.feed("stdout", + b"PRIVACY_REQUEST status=200 msecs=10\n" + b"PRIVACY_REQUEST status=200 msecs=20\n") + self.assertEqual(agg.snapshot()["status_2xx"], 2) + + +class TestInterleavedStreams(unittest.TestCase): + + def test_stdout_stderr_independent_buffers(self): + agg = LogAggregator("nginx") + agg.feed("stdout", b"PRIVACY_REQUEST status=200 ") + agg.feed("stderr", b"PRIVACY_REQUEST status=500 ") + s = agg.snapshot() + self.assertEqual(s["status_2xx"], 0) + self.assertEqual(s["status_5xx"], 0) + + agg.feed("stdout", b"msecs=10\n") + s = agg.snapshot() + self.assertEqual(s["status_2xx"], 1) + self.assertEqual(s["status_5xx"], 0) + + agg.feed("stderr", b"msecs=999\n") + s = agg.snapshot() + self.assertEqual(s["status_2xx"], 1) + self.assertEqual(s["status_5xx"], 1) + + def test_stderr_only(self): + agg = LogAggregator("uwsgi") + agg.feed("stderr", b"PRIVACY_REQUEST status=200 msecs=50\n") + self.assertEqual(agg.snapshot()["status_2xx"], 1) + + +class TestBinaryAndInvalidInput(unittest.TestCase): + + def test_invalid_utf8_rejected(self): + agg = LogAggregator("nginx") + agg.feed("stdout", b"\xff\xfe\xfd\n") + s = agg.snapshot() + self.assertEqual(s["rejected"], 1) + + def test_nul_byte_rejected(self): + agg = LogAggregator("nginx") + agg.feed("stdout", b"PRIVACY_REQUEST\x00 status=200 msecs=50\n") + self.assertEqual(agg.snapshot()["rejected"], 1) + + def test_crlf_normalized_trailing_cr_stripped(self): + agg = LogAggregator("nginx") + agg.feed("stdout", + b"PRIVACY_REQUEST status=200 msecs=50\r\n") + s = agg.snapshot() + self.assertEqual(s["status_2xx"], 1) + self.assertEqual(s["rejected"], 0) + + def test_embedded_cr_rejected(self): + agg = LogAggregator("nginx") + agg.feed("stdout", + b"PRIVACY_REQUEST status=200\r msecs=50\n") + self.assertEqual(agg.snapshot()["rejected"], 1) + + def test_all_zero_bytes_rejected(self): + agg = LogAggregator("nginx") + agg.feed("stdout", b"\x00\x00\x00\n") + self.assertEqual(agg.snapshot()["rejected"], 1) + + def test_mixed_valid_and_invalid(self): + agg = LogAggregator("nginx") + agg.feed("stdout", + b"PRIVACY_REQUEST status=200 msecs=50\n" + b"\xff\xfe\n" + b"PRIVACY_REQUEST status=404 msecs=100\n") + s = agg.snapshot() + self.assertEqual(s["status_2xx"], 1) + self.assertEqual(s["status_4xx"], 1) + self.assertEqual(s["rejected"], 1) + + +class TestOversizedLines(unittest.TestCase): + + def test_oversized_single_chunk_dropped(self): + agg = LogAggregator("nginx") + agg.feed("stdout", b"X" * (_MAX_LINE + 1) + b"\n") + s = agg.snapshot() + self.assertEqual(s["dropped"], 1) + self.assertEqual(s["rejected"], 0) + + def test_oversized_delimited_no_dropping_state(self): + agg = LogAggregator("nginx") + agg.feed("stdout", + b"X" * (_MAX_LINE + 1) + b"\n" + b"PRIVACY_REQUEST status=200 msecs=50\n") + s = agg.snapshot() + self.assertEqual(s["dropped"], 1) + self.assertEqual(s["status_2xx"], 1) + self.assertFalse(agg._dropping["stdout"]) + + def test_oversized_split_across_chunks(self): + agg = LogAggregator("nginx") + agg.feed("stdout", b"X" * (_MAX_LINE + 10)) + self.assertEqual(agg.snapshot()["dropped"], 1) + agg.feed("stdout", b"more_garbage\n") + s = agg.snapshot() + self.assertEqual(s["dropped"], 1) + + def test_recovery_after_drop(self): + agg = LogAggregator("nginx") + agg.feed("stdout", b"X" * (_MAX_LINE + 1) + b"\n") + agg.feed("stdout", b"PRIVACY_REQUEST status=200 msecs=50\n") + s = agg.snapshot() + self.assertEqual(s["dropped"], 1) + self.assertEqual(s["status_2xx"], 1) + + def test_multiple_oversized_in_one_chunk(self): + agg = LogAggregator("nginx") + chunk = ( + b"X" * (_MAX_LINE + 1) + b"\n" + + b"Y" * (_MAX_LINE + 1) + b"\n" + + b"PRIVACY_REQUEST status=200 msecs=10\n" + ) + agg.feed("stdout", chunk) + s = agg.snapshot() + self.assertEqual(s["dropped"], 2) + self.assertEqual(s["status_2xx"], 1) + + def test_exactly_max_line_accepted(self): + agg = LogAggregator("nginx") + line = b"X" * _MAX_LINE + b"\n" + self.assertEqual(len(line) - 1, _MAX_LINE) + agg.feed("stdout", line) + s = agg.snapshot() + self.assertEqual(s["unclassified"], 1) + self.assertEqual(s["status_2xx"], 0) + self.assertEqual(s["dropped"], 0) + + +class TestHugeChunkMemoryBound(unittest.TestCase): + + def test_1mib_chunk_no_newline_bounded(self): + agg = LogAggregator("nginx") + agg.feed("stdout", b"x" * 1048576) + self.assertLessEqual(len(agg._buf["stdout"]), _MAX_LINE) + self.assertEqual(agg.snapshot()["dropped"], 1) + + def test_1mib_chunk_then_valid_line(self): + agg = LogAggregator("nginx") + agg.feed("stdout", b"x" * 1048576 + b"\n") + agg.feed("stdout", b"PRIVACY_REQUEST status=200 msecs=50\n") + s = agg.snapshot() + self.assertEqual(s["dropped"], 1) + self.assertEqual(s["status_2xx"], 1) + self.assertLessEqual(len(agg._buf["stdout"]), _MAX_LINE) + + def test_many_small_chunks_no_newline_bounded(self): + agg = LogAggregator("nginx") + for _ in range(10000): + agg.feed("stdout", b"x") + self.assertLessEqual(len(agg._buf["stdout"]), _MAX_LINE) + + +class TestUnterminatedFinish(unittest.TestCase): + + def test_finish_processes_tail(self): + agg = LogAggregator("nginx") + agg.feed("stdout", b"PRIVACY_REQUEST status=200 msecs=50") + self.assertEqual(agg.snapshot()["status_2xx"], 0) + agg.finish("stdout") + self.assertEqual(agg.snapshot()["status_2xx"], 1) + + def test_finish_empty_buffer_resets_drop(self): + agg = LogAggregator("nginx") + agg._dropping["stdout"] = True + agg.finish("stdout") + self.assertFalse(agg._dropping["stdout"]) + self.assertEqual(agg.snapshot()["dropped"], 0) + + def test_finish_empty_buffer_no_drop_noop(self): + agg = LogAggregator("nginx") + agg.finish("stdout") + s = agg.snapshot() + for k, v in s.items(): + if k in ("schema", "component"): + continue + self.assertEqual(v, 0) + + def test_finish_oversized_tail_dropped(self): + agg = LogAggregator("nginx") + agg.feed("stdout", b"X" * (_MAX_LINE + 1)) + agg.finish("stdout") + self.assertEqual(agg.snapshot()["dropped"], 1) + + +class TestStrictInt(unittest.TestCase): + + def test_plus_sign_rejected(self): + agg = LogAggregator("nginx") + agg.feed("stdout", + b"PRIVACY_REQUEST status=+200 msecs=50\n") + self.assertEqual(agg.snapshot()["rejected"], 1) + self.assertEqual(agg.snapshot()["status_2xx"], 0) + + def test_underscore_rejected(self): + agg = LogAggregator("nginx") + agg.feed("stdout", + b"PRIVACY_REQUEST status=200 msecs=1_0\n") + self.assertEqual(agg.snapshot()["rejected"], 1) + + def test_hex_prefix_rejected(self): + agg = LogAggregator("nginx") + agg.feed("stdout", + b"PRIVACY_REQUEST status=0x200 msecs=50\n") + self.assertEqual(agg.snapshot()["rejected"], 1) + + def test_leading_zero_rejected(self): + agg = LogAggregator("nginx") + agg.feed("stdout", + b"PRIVACY_REQUEST status=200 msecs=050\n") + self.assertEqual(agg.snapshot()["rejected"], 1) + self.assertEqual(agg.snapshot()["status_2xx"], 0) + + def test_empty_status_rejected(self): + agg = LogAggregator("nginx") + agg.feed("stdout", + b"PRIVACY_REQUEST status= msecs=50\n") + self.assertEqual(agg.snapshot()["rejected"], 1) + + def test_empty_msecs_rejected(self): + agg = LogAggregator("nginx") + agg.feed("stdout", + b"PRIVACY_REQUEST status=200 msecs=\n") + self.assertEqual(agg.snapshot()["rejected"], 1) + + def test_prefix_runon_rejected(self): + agg = LogAggregator("nginx") + agg.feed("stdout", + b"PRIVACY_REQUESTX status=200 msecs=1\n") + s = agg.snapshot() + self.assertEqual(s["rejected"], 0) + self.assertEqual(s["unclassified"], 1) + + def test_fullwidth_unicode_digits_rejected(self): + agg = LogAggregator("nginx") + line = "PRIVACY_REQUEST status=\uff12\uff10\uff10 msecs=50\n" + agg.feed("stdout", line.encode("utf-8")) + s = agg.snapshot() + self.assertEqual(s["rejected"], 1) + self.assertEqual(s["status_2xx"], 0) + + def test_status_leading_zeros_rejected(self): + agg = LogAggregator("nginx") + agg.feed("stdout", + b"PRIVACY_REQUEST status=0200 msecs=50\n") + s = agg.snapshot() + self.assertEqual(s["rejected"], 1) + self.assertEqual(s["status_2xx"], 0) + + def test_msecs_leading_zeros_rejected(self): + agg = LogAggregator("nginx") + agg.feed("stdout", + b"PRIVACY_REQUEST status=200 msecs=01\n") + s = agg.snapshot() + self.assertEqual(s["rejected"], 1) + self.assertEqual(s["latency_fast"], 0) + + +class TestUnclassifiedVsRejected(unittest.TestCase): + + def test_ordinary_log_line_unclassified(self): + agg = LogAggregator("nginx") + agg.feed("stdout", b"2026-01-01 INFO starting up\n") + s = agg.snapshot() + self.assertEqual(s["unclassified"], 1) + self.assertEqual(s["rejected"], 0) + + def test_wrong_prefix_unclassified(self): + agg = LogAggregator("nginx") + agg.feed("stdout", + b"ACCESS_REQUEST status=200 msecs=50\n") + s = agg.snapshot() + self.assertEqual(s["unclassified"], 1) + self.assertEqual(s["rejected"], 0) + + def test_malformed_privacy_request_rejected(self): + agg = LogAggregator("nginx") + agg.feed("stdout", + b"PRIVACY_REQUEST status=abc msecs=50\n") + s = agg.snapshot() + self.assertEqual(s["rejected"], 1) + self.assertEqual(s["unclassified"], 0) + + def test_extra_fields_rejected(self): + agg = LogAggregator("nginx") + agg.feed("stdout", + b"PRIVACY_REQUEST status=200 msecs=50 extra=1\n") + self.assertEqual(agg.snapshot()["rejected"], 1) + + def test_missing_msecs_rejected(self): + agg = LogAggregator("nginx") + agg.feed("stdout", b"PRIVACY_REQUEST status=200\n") + self.assertEqual(agg.snapshot()["rejected"], 1) + + def test_empty_line_unclassified(self): + agg = LogAggregator("nginx") + agg.feed("stdout", b"\n") + s = agg.snapshot() + self.assertEqual(s["unclassified"], 1) + self.assertEqual(s["rejected"], 0) + + +class TestStatusBoundaries(unittest.TestCase): + + def _feed_status(self, status): + agg = LogAggregator("nginx") + line = ("PRIVACY_REQUEST status=%d msecs=50\n" % status).encode() + agg.feed("stdout", line) + return agg.snapshot() + + def test_status_99_rejected(self): + self.assertEqual(self._feed_status(99)["rejected"], 1) + + def test_status_100_counted(self): + self.assertEqual(self._feed_status(100)["status_1xx"], 1) + + def test_status_199_counted(self): + self.assertEqual(self._feed_status(199)["status_1xx"], 1) + + def test_status_200_counted(self): + self.assertEqual(self._feed_status(200)["status_2xx"], 1) + + def test_status_599_counted(self): + self.assertEqual(self._feed_status(599)["status_5xx"], 1) + + def test_status_600_rejected(self): + self.assertEqual(self._feed_status(600)["rejected"], 1) + + def test_status_0_rejected(self): + self.assertEqual(self._feed_status(0)["rejected"], 1) diff --git a/tests/privacy_native/test_log_aggregate_2.py b/tests/privacy_native/test_log_aggregate_2.py new file mode 100644 index 000000000..63f0037e7 --- /dev/null +++ b/tests/privacy_native/test_log_aggregate_2.py @@ -0,0 +1,227 @@ +import unittest +import json + +from native_support import LogAggregator, _MAX_LINE, _COUNTER_MAX, _SCHEMA_VERSION + + +SYNTH_IPv4 = "192.0.2.1" +SYNTH_IPv6 = "2001:db8::1" +SYNTH_FORM = "private_form.yml" +SYNTH_SESSION = "sess_abc123secret" +SYNTH_TOKENS = [SYNTH_IPv4, SYNTH_IPv6, SYNTH_FORM, SYNTH_SESSION] + +EXPECTED_KEYS = { + "schema", "component", + "status_1xx", "status_2xx", "status_3xx", + "status_4xx", "status_5xx", + "latency_fast", "latency_medium", + "latency_slow", "latency_timeout", + "unclassified", "rejected", "dropped", +} + + +def _snapshot_str(agg): + return json.dumps(agg.snapshot()) + + +class TestLatencyBoundaries(unittest.TestCase): + + def _feed_msecs(self, msecs): + agg = LogAggregator("nginx") + line = ("PRIVACY_REQUEST status=200 msecs=%d\n" % msecs).encode() + agg.feed("stdout", line) + return agg.snapshot() + + def test_msecs_0_fast(self): + self.assertEqual(self._feed_msecs(0)["latency_fast"], 1) + + def test_msecs_99_fast(self): + self.assertEqual(self._feed_msecs(99)["latency_fast"], 1) + + def test_msecs_100_medium(self): + self.assertEqual(self._feed_msecs(100)["latency_medium"], 1) + + def test_msecs_999_medium(self): + self.assertEqual(self._feed_msecs(999)["latency_medium"], 1) + + def test_msecs_1000_slow(self): + self.assertEqual(self._feed_msecs(1000)["latency_slow"], 1) + + def test_msecs_29999_slow(self): + self.assertEqual(self._feed_msecs(29999)["latency_slow"], 1) + + def test_msecs_30000_timeout(self): + self.assertEqual(self._feed_msecs(30000)["latency_timeout"], 1) + + def test_msecs_600000_timeout(self): + self.assertEqual(self._feed_msecs(600000)["latency_timeout"], 1) + + def test_msecs_600001_rejected(self): + self.assertEqual(self._feed_msecs(600001)["rejected"], 1) + + def test_msecs_negative_rejected(self): + self.assertEqual(self._feed_msecs(-1)["rejected"], 1) + + +class TestCounterSaturation(unittest.TestCase): + + def test_counter_saturates_at_max(self): + agg = LogAggregator("nginx") + agg._counters["status_2xx"] = _COUNTER_MAX + agg.feed("stdout", b"PRIVACY_REQUEST status=200 msecs=50\n") + self.assertEqual(agg.snapshot()["status_2xx"], _COUNTER_MAX) + + def test_rejected_counter_saturates(self): + agg = LogAggregator("nginx") + agg._counters["rejected"] = _COUNTER_MAX + agg.feed("stdout", b"PRIVACY_REQUEST bad\n") + self.assertEqual(agg.snapshot()["rejected"], _COUNTER_MAX) + + +class TestInvalidTypes(unittest.TestCase): + + def test_chunk_string_rejected(self): + agg = LogAggregator("nginx") + with self.assertRaises(TypeError): + agg.feed("stdout", "not bytes") + + def test_chunk_int_rejected(self): + agg = LogAggregator("nginx") + with self.assertRaises(TypeError): + agg.feed("stdout", 42) + + def test_chunk_none_rejected(self): + agg = LogAggregator("nginx") + with self.assertRaises(TypeError): + agg.feed("stdout", None) + + def test_chunk_bytearray_rejected(self): + agg = LogAggregator("nginx") + with self.assertRaises(TypeError): + agg.feed("stdout", bytearray(b"data")) + + def test_bytes_subclass_rejected(self): + class EvilBytes(bytes): + pass + agg = LogAggregator("nginx") + with self.assertRaises(TypeError): + agg.feed("stdout", EvilBytes(b"data")) + + def test_stream_int_rejected(self): + agg = LogAggregator("nginx") + with self.assertRaises(TypeError): + agg.feed(123, b"data") + + def test_stream_str_subclass_rejected(self): + class EvilStr(str): + pass + agg = LogAggregator("nginx") + with self.assertRaises(TypeError): + agg.feed(EvilStr("stdout"), b"data\n") + + def test_stream_invalid_value(self): + agg = LogAggregator("nginx") + with self.assertRaises(ValueError): + agg.feed("stdin", b"data") + + def test_component_invalid(self): + with self.assertRaises(ValueError): + LogAggregator("apache") + + def test_component_int(self): + with self.assertRaises(ValueError): + LogAggregator(42) + + def test_component_str_subclass(self): + class EvilStr(str): + pass + with self.assertRaises(ValueError): + LogAggregator(EvilStr("nginx")) + + +class TestNoSyntheticTokensInOutput(unittest.TestCase): + + def test_no_tokens_after_ipv4_input(self): + agg = LogAggregator("nginx") + agg.feed("stdout", + ("192.0.2.1 PRIVACY_REQUEST status=200 msecs=50\n" + ).encode()) + out = _snapshot_str(agg) + for token in SYNTH_TOKENS: + self.assertNotIn(token, out) + + def test_no_tokens_after_ipv6_input(self): + agg = LogAggregator("nginx") + agg.feed("stdout", + ("2001:db8::1 PRIVACY_REQUEST status=200 msecs=50\n" + ).encode()) + out = _snapshot_str(agg) + for token in SYNTH_TOKENS: + self.assertNotIn(token, out) + + def test_no_tokens_after_form_input(self): + agg = LogAggregator("nginx") + agg.feed("stdout", + ("PRIVACY_REQUEST status=200 msecs=50 private_form.yml\n" + ).encode()) + out = _snapshot_str(agg) + for token in SYNTH_TOKENS: + self.assertNotIn(token, out) + + def test_no_tokens_after_garbage_with_embedded_ips(self): + agg = LogAggregator("nginx") + agg.feed("stdout", + b"192.0.2.1 2001:db8::1 sess_abc123secret " + b"private_form.yml\n") + out = _snapshot_str(agg) + for token in SYNTH_TOKENS: + self.assertNotIn(token, out) + + def test_no_tokens_after_oversized_drop(self): + agg = LogAggregator("nginx") + garbage = SYNTH_IPv4.encode() * 500 + agg.feed("stdout", garbage + b"\n") + out = _snapshot_str(agg) + for token in SYNTH_TOKENS: + self.assertNotIn(token, out) + + +class TestSnapshotSchema(unittest.TestCase): + + def test_snapshot_has_exact_keys(self): + agg = LogAggregator("nginx") + s = agg.snapshot() + self.assertEqual(set(s.keys()), EXPECTED_KEYS) + + def test_schema_field_is_one(self): + agg = LogAggregator("nginx") + self.assertEqual(agg.snapshot()["schema"], _SCHEMA_VERSION) + self.assertEqual(agg.snapshot()["schema"], 1) + + def test_all_values_are_int_except_schema_and_component(self): + agg = LogAggregator("nginx") + agg.feed("stdout", b"PRIVACY_REQUEST status=200 msecs=50\n") + s = agg.snapshot() + self.assertIsInstance(s["schema"], int) + self.assertIsInstance(s["component"], str) + for k, v in s.items(): + if k == "component": + self.assertIsInstance(v, str) + else: + self.assertIsInstance(v, int) + + def test_snapshot_is_copy(self): + agg = LogAggregator("nginx") + s1 = agg.snapshot() + agg.feed("stdout", b"PRIVACY_REQUEST status=200 msecs=50\n") + s2 = agg.snapshot() + self.assertEqual(s1["status_2xx"], 0) + self.assertEqual(s2["status_2xx"], 1) + + def test_snapshot_is_json_serializable(self): + agg = LogAggregator("nginx") + agg.feed("stdout", b"PRIVACY_REQUEST status=200 msecs=50\n") + serialized = json.dumps(agg.snapshot()) + parsed = json.loads(serialized) + self.assertEqual(parsed["status_2xx"], 1) + self.assertEqual(parsed["schema"], 1) diff --git a/tests/privacy_native/test_log_aggregate_regressions.py b/tests/privacy_native/test_log_aggregate_regressions.py new file mode 100644 index 000000000..079797c45 --- /dev/null +++ b/tests/privacy_native/test_log_aggregate_regressions.py @@ -0,0 +1,71 @@ +"""Primary-review regressions, using synthetic bytes only.""" +import random +import sys +import tracemalloc +import unittest + +from native_support import LogAggregator + + +class ReviewedContractTests(unittest.TestCase): + def test_unterminated_overflow_counted_once_and_eof_recovers(self): + aggregator = LogAggregator("nginx") + for stream in ("stdout", "stderr"): + aggregator.feed(stream, b"x" * 4097) + aggregator.feed(stream, b"more" * 4097) + aggregator.finish(stream) + aggregator.finish(stream) + aggregator.feed(stream, b"PRIVACY_REQUEST status=200 msecs=0\n") + snapshot = aggregator.snapshot() + self.assertEqual(snapshot["dropped"], 2) + self.assertEqual(snapshot["status_2xx"], 2) + + def test_arbitrary_fragmentation_preserves_counts(self): + payload = ( + b"ordinary synthetic message\r\n" + + b"x" * 4097 + b"\n" + b"PRIVACY_REQUEST status=200 msecs=99\n" + b"PRIVACY_REQUEST status=503 msecs=30000\n" + b"PRIVACY_REQUEST status=200 msecs=01\n" + b"\xff\n" + b"PRIVACY_REQUEST status=404 msecs=1000" + ) + for seed in range(20): + rng = random.Random(seed) + aggregator = LogAggregator("uwsgi") + position = 0 + while position < len(payload): + count = rng.randint(1, 3000) + aggregator.feed("stdout", payload[position:position + count]) + position += count + aggregator.finish("stdout") + snapshot = aggregator.snapshot() + for key, value in { + "unclassified": 1, "dropped": 1, "rejected": 2, + "status_2xx": 1, "status_4xx": 1, "status_5xx": 1, + "latency_fast": 1, "latency_slow": 1, "latency_timeout": 1, + }.items(): + self.assertEqual(snapshot[key], value, (seed, key)) + + def test_large_input_does_not_allocate_a_proportional_copy(self): + payload = b"SYNTHETIC_PRIVATE_MESSAGE" * 350000 + aggregator = LogAggregator("nginx") + tracemalloc.start() + try: + aggregator.feed("stdout", payload) + _current, peak = tracemalloc.get_traced_memory() + finally: + tracemalloc.stop() + self.assertLess(peak, 256 * 1024) + self.assertEqual(aggregator.snapshot()["dropped"], 1) + + def test_large_numeric_field_rejected_before_integer_conversion(self): + previous = sys.get_int_max_str_digits() + try: + sys.set_int_max_str_digits(640) + aggregator = LogAggregator("uwsgi") + aggregator.feed("stdout", b"PRIVACY_REQUEST status=200 msecs=" + b"9" * 1000 + b"\n") + finally: + sys.set_int_max_str_digits(previous) + self.assertEqual(aggregator.snapshot()["rejected"], 1) + self.assertEqual(aggregator.snapshot()["status_2xx"], 0) diff --git a/tests/privacy_native/test_mail_launcher.py b/tests/privacy_native/test_mail_launcher.py new file mode 100644 index 000000000..2433ed6a9 --- /dev/null +++ b/tests/privacy_native/test_mail_launcher.py @@ -0,0 +1,61 @@ +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +import test_native_launchers as native + + +class MailLauncherTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + native.NativeLauncherTests.setUpClass() + + def fixture(self, directory, failure=None): + root = native.NativeLauncherTests().fixture(directory, activation_failure=failure == 'activation') + (root / 'runtime/bin/activate').write_text( + 'printf "SYNTHETIC_PRIVATE_BOOTSTRAP\\n" >&2\n' + + ('return 1\n' if failure == 'activation' else 'true\n')) + (root / 'runtime/bin/python').write_text('#!' + sys.executable + '\n' + 'import sys\nprint("SYNTHETIC_PRIVATE_MESSAGE")\nraise SystemExit(17)\n') + if failure == 'missing': + (root / 'webapp/privacy-diagnostic').unlink() + if failure == 'launch': + (root / 'webapp/privacy-process').unlink() + return root + + def invoke(self, root): + env = dict(os.environ, DA_ROOT=str(root), DA_PYTHON=str(root / 'runtime')) + env['PATH'] = str(root / 'runtime/bin') + os.pathsep + os.environ['PATH'] + process = subprocess.Popen(['bash', str(native.ROOT / 'Docker/process-email.sh')], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) + out, err = process.communicate(b'SYNTHETIC_PRIVATE_MESSAGE', timeout=5) + return process, out, err + + def test_successful_handoff_keeps_identity_and_stream_input(self): + with tempfile.TemporaryDirectory() as directory: + root = self.fixture(directory) + process, out, err = self.invoke(root) + self.assertEqual(process.returncode, 0) + self.assertEqual(err, b'') + record = json.loads(out) + self.assertEqual(record['pid'], process.pid) + self.assertTrue(record['extra_sink_closed']) + self.assertEqual(record['argv'], ['--component', 'mail', '--', str(root / 'runtime/bin/python'), + '-m', 'docassemble.webapp.process_email', '/dev/stdin']) + + def test_failed_bootstrap_and_exec_defer_delivery(self): + for failure in ('activation', 'missing', 'launch'): + with self.subTest(failure=failure), tempfile.TemporaryDirectory() as directory: + root = self.fixture(directory, failure) + process, out, err = self.invoke(root) + self.assertEqual(process.returncode, 75) + self.assertEqual(err, b'') + if failure == 'missing': + self.assertEqual(out, b'') + else: + self.assertEqual(json.loads(out), {'schema': 1, 'component': 'mail', + 'event': 'startup_failed', 'phase': failure}) diff --git a/tests/privacy_native/test_mail_processor.py b/tests/privacy_native/test_mail_processor.py new file mode 100644 index 000000000..3ca5f2407 --- /dev/null +++ b/tests/privacy_native/test_mail_processor.py @@ -0,0 +1,41 @@ +import json +import tempfile +import unittest + +from mail_support import run_processor + + +class MailProcessorTests(unittest.TestCase): + def test_message_and_attachments_reach_expected_synthetic_interfaces(self): + with tempfile.TemporaryDirectory() as directory: + state = run_processor(directory) + self.assertTrue(state.completed, repr(state.error or state.exit)) + self.assertEqual(state.config, [{'arguments': ['process_email.py', directory + '/message']}]) + self.assertFalse(any(path == '/tmp/mail.log' for path, _ in state.opens)) + self.assertFalse(any(mode != 'r' for _, mode in state.opens)) + self.assertEqual(len(state.emails), 1) + self.assertEqual(state.emails[0]['subject'], 'SYNTHETIC_PRIVATE_SUBJECT') + self.assertEqual(json.loads(state.emails[0]['to_addr']), [{'name': '', 'address': 'testcode@example.invalid'}]) + self.assertEqual(len(state.attachments), 3) + self.assertEqual(state.saved[1:], [b'SYNTHETIC_PRIVATE_BODY', b'\x00\xffPDF']) + self.assertEqual(len(state.tasks), 1) + args, kwargs = state.tasks[0] + self.assertEqual(args, ('tasks.background_action',)) + self.assertEqual(kwargs['args'][-1], {'action': 'incoming_email', 'arguments': {'id': 42}}) + + def test_read_and_unknown_recipient_fail_without_diagnostic_file(self): + for failure, message in (('read', 'Failed to read e-mail message'), ('unknown', 'short code not found')): + with self.subTest(failure=failure), tempfile.TemporaryDirectory() as directory: + state = run_processor(directory, failure) + self.assertEqual(state.exit, message) + self.assertFalse(state.completed) + self.assertFalse(state.tasks) + self.assertFalse(any(path == '/tmp/mail.log' for path, _ in state.opens)) + + def test_database_and_broker_failures_propagate(self): + for failure in ('database', 'broker'): + with self.subTest(failure=failure), tempfile.TemporaryDirectory() as directory: + state = run_processor(directory, failure) + self.assertIsInstance(state.error, OSError) + self.assertFalse(state.completed) + self.assertFalse(any(path == '/tmp/mail.log' for path, _ in state.opens)) diff --git a/tests/privacy_native/test_maintenance_launchers.py b/tests/privacy_native/test_maintenance_launchers.py new file mode 100644 index 000000000..d39728784 --- /dev/null +++ b/tests/privacy_native/test_maintenance_launchers.py @@ -0,0 +1,40 @@ +"""Real script bootstrap must fail closed before any maintenance runs.""" +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + +ROOT = Path(__file__).resolve().parents[2] +SCRIPTS = [('Docker/cron/docassemble-cron-' + period + '.sh', 'maintenance') + for period in ('hourly', 'daily', 'weekly', 'monthly')] +SCRIPTS += [('Docker/sync.sh', 'maintenance'), ('Docker/restart-post-logrotate.sh', 'maintenance'), + ('Docker/initialize.sh', 'initialize')] + + +class MaintenanceLaunchers(unittest.TestCase): + def test_missing_capture_emits_only_a_fixed_failure_before_activation(self): + diagnostic = ROOT / 'tests/.privacy-build/privacy-diagnostic' + self.assertTrue(diagnostic.is_file(), 'run tests/verify_privacy.sh') + for script, component in SCRIPTS: + with self.subTest(script=script), tempfile.TemporaryDirectory() as directory: + base = Path(directory) + (base / 'webapp').mkdir() + (base / 'runtime/bin').mkdir(parents=True) + shutil.copy2(diagnostic, base / 'webapp/privacy-diagnostic') + (base / 'runtime/bin/activate').write_text( + 'printf JOS81_PRIVATE_MAINTENANCE >&2\ntouch "$DA_ROOT/activation-ran"\n') + env = dict(os.environ, DA_ROOT=directory, DA_PYTHON=str(base / 'runtime')) + result = subprocess.run(['bash', str(ROOT / script)], env=env, + capture_output=True, timeout=5) + self.assertEqual(result.returncode, 70) + self.assertEqual(result.stderr, b'') + self.assertEqual(json.loads(result.stdout), + dict(schema=1, component=component, event='startup_failed', phase='launch')) + self.assertFalse((base / 'activation-ran').exists()) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/privacy_native/test_native_config.py b/tests/privacy_native/test_native_config.py new file mode 100644 index 000000000..8702166a6 --- /dev/null +++ b/tests/privacy_native/test_native_config.py @@ -0,0 +1,244 @@ +import contextlib +import importlib.util +import io +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest import mock + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "tests/privacy_native/reference/check_native_config.py" +SPEC = importlib.util.spec_from_file_location("check_native_config", SCRIPT) +CHECK = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(CHECK) +UWSGI = ("[uwsgi]\nmaster = true\ndie-on-term = true\nlog-format = " + + CHECK.UWSGI_FORMAT + "\n") +HTTP = ("log_format privacy_counts '" + CHECK.NGINX_FORMAT + "';\n" + "access_log /dev/stdout privacy_counts;\n") +MAIN = "error_log stderr;\nworker_shutdown_timeout 2s;\nhttp {\n" + HTTP + "}\n" + + +def dump(*files): + return "".join("# configuration file " + name + ":\n" + body.rstrip() + "\n" + for name, body in files).encode() + + +def single(body=MAIN): + return dump(("/etc/nginx/nginx.conf", body)) + + +class UwsgiValidation(unittest.TestCase): + def reject(self, text): + with self.assertRaises((CHECK.InvalidConfig, UnicodeError)): + CHECK.validate_uwsgi(text.encode() if isinstance(text, str) else text) + + def test_required_positive_control(self): + CHECK.validate_uwsgi(UWSGI.encode()) + + def test_all_four_owned_templates_after_rendering(self): + filenames = ("docassemble.ini.dist", "docassemble-expose-uwsgi.ini", + "docassemblelog.ini.dist", "docassemblelog-expose-uwsgi.ini") + for filename in filenames: + with self.subTest(template=filename): + source = (ROOT / "Docker/config" / filename).read_text() + for key, value in (("DA_ROOT", "/usr/share/docassemble"), + ("DA_PYTHON", "/usr/share/docassemble/local3.14"), + ("DAWSGIROOT", "/nj")): + source = source.replace("{{" + key + "}}", value) + CHECK.validate_uwsgi(source.encode()) + + def test_unrendered_templates_are_rejected(self): + self.reject(UWSGI + "venv = {{DA_PYTHON}}\n") + + def test_unknown_override_and_privilege_options(self): + for key in ("uid", "gid", "daemonize", "daemonize2", "logto", "logto2", "logger", + "req-logger", "include", "ini", "inherit", "exec-asap", "chdir"): + with self.subTest(option=key): + self.reject(UWSGI + key + " = SYNTHETIC_PRIVATE\n") + + def test_duplicates_sections_and_missing_required(self): + for extra in ("master=true\n", "[uwsgi]\n", "[other]\nkey=value\n"): + self.reject(UWSGI + extra) + for line in UWSGI.splitlines(): + self.reject(UWSGI.replace(line + "\n", "")) + + def test_unsafe_values_and_interpolation(self): + cases = ("master=false", "die-on-term=false", "processes=0", "threads=-1", + "threads=99999999", "venv=@(exec://echo secret)", "pidfile=/tmp/%(foo)", + "py-executable=$(BAD)", "venv=/tmp/../secret", "http-socket=:9000", + "module=arbitrary.application", "mount=/=arbitrary:app", "callable=other", + "log-format=" + CHECK.UWSGI_FORMAT + " %(uri)") + for item in cases: + with self.subTest(value=item): + key = item.split("=", 1)[0] + base = "\n".join(line for line in UWSGI.splitlines() + if line.split("=", 1)[0].strip() != key) + "\n" + self.reject(base + item + "\n") + + def test_comments_cannot_supply_required_values(self): + self.reject(UWSGI.replace("master = true", "# master = true")) + self.reject(UWSGI.replace("die-on-term = true", "; die-on-term = true")) + self.reject(UWSGI.replace(CHECK.UWSGI_FORMAT, '"' + CHECK.UWSGI_FORMAT + '"')) + + def test_invalid_encoding_controls_and_limit(self): + for data in (b"\xff", UWSGI.encode() + b"\x00", b"#" * (CHECK.MAX_BYTES + 1)): + self.reject(data) + + +class NginxValidation(unittest.TestCase): + def reject(self, data): + with self.assertRaises((CHECK.InvalidConfig, UnicodeError)): + CHECK.validate_nginx_dump(data) + + def test_positive_controls_levels_and_foreground(self): + CHECK.validate_nginx_dump(single()) + for level in sorted(CHECK._LEVELS): + CHECK.validate_nginx_dump(single(MAIN.replace("error_log stderr;", "error_log stderr " + level + ";"))) + CHECK.validate_nginx_dump(single("daemon off; master_process on;\n" + MAIN)) + + def test_nested_off_and_quoted_data_not_directives(self): + extra = ('server { access_log off; location / { error_log "stderr" warn; ' + 'return 200 "error_log /tmp/private; # } { access_log /tmp/private;"; }}') + CHECK.validate_nginx_dump(single(MAIN.replace(HTTP, HTTP + extra))) + CHECK.validate_nginx_dump(single(MAIN.replace(HTTP, HTTP + "server { if ($host != 'example.test') { return 301 /; } }"))) + + def test_header_comment_inside_multiline_quote_is_data(self): + extra = 'server { set $payload "first\n# configuration file /fake:\nlast"; }' + CHECK.validate_nginx_dump(single(MAIN.replace(HTTP, HTTP + extra))) + + def test_complete_includes_relative_glob_and_repeated_contents(self): + root = ("error_log stderr; include lifecycle.conf;\nhttp {\n" + HTTP + + "include /etc/nginx/empty/*.conf;\n" + "server { include shared.conf; } server { include shared.conf; } }") + files = (("/etc/nginx/nginx.conf", root), + ("/etc/nginx/lifecycle.conf", "worker_shutdown_timeout 2s;"), + ("/etc/nginx/shared.conf", "access_log off; error_log stderr;")) + CHECK.validate_nginx_dump(dump(*files, files[-1])) + + def test_unsafe_nested_quoted_and_conditional_overrides(self): + overrides = ('access_log "/tmp/private" combined;', 'error_log "/tmp/private";', + 'error_log syslog:server=127.0.0.1;', 'access_log /dev/stdout combined;', + 'access_log /dev/stdout privacy_counts if=$foo;', 'access_log off extra;', + 'error_log stderr debug extra;', 'error_log stderr unknown;') + for override in overrides: + with self.subTest(override=override): + nested = "server { location /secret { " + override + " } }" + self.reject(single(MAIN.replace(HTTP, HTTP + nested))) + + def test_comments_cannot_hide_bad_directives_or_supply_good_ones(self): + self.reject(single(MAIN + "# access_log off;\nerror_log /tmp/private;\n")) + self.reject(single(MAIN.replace("error_log stderr;", "# error_log stderr;"))) + self.reject(single(MAIN.replace("worker_shutdown_timeout 2s;", "# worker_shutdown_timeout 2s;"))) + # Inside an unquoted token, # belongs to the native filename. + for name, safe in (("error_log", "stderr"), ("access_log", "off")): + nested = "server { " + name + " " + safe + "#private\n; }" + self.reject(single(MAIN.replace(HTTP, HTTP + nested))) + + def test_missing_global_baselines_cannot_be_satisfied_in_location(self): + missing_access = MAIN.replace("access_log /dev/stdout privacy_counts;", "") + missing_access = missing_access.replace(HTTP.splitlines()[0], HTTP.splitlines()[0] + + "server { access_log /dev/stdout privacy_counts; }") + self.reject(single(missing_access)) + self.reject(single(MAIN.replace("error_log stderr;", "").replace(HTTP, HTTP + "server { error_log stderr; }"))) + self.reject(single(MAIN.replace(HTTP.splitlines()[0], ""))) + self.reject(single(MAIN.replace("worker_shutdown_timeout 2s;", ""))) + + def test_duplicate_directives_and_bad_lifecycle(self): + for extra in ("error_log stderr;", "worker_shutdown_timeout 2s;", "daemon on;", "master_process off;"): + self.reject(single(extra + MAIN)) + self.reject(single("daemon off; daemon off;" + MAIN)) + self.reject(single("master_process on; master_process on;" + MAIN)) + self.reject(single(MAIN.replace(HTTP, HTTP + HTTP))) + self.reject(single(MAIN.replace("2s;", "2000ms;"))) + self.reject(single(MAIN.replace(HTTP, HTTP + "server { access_log off; access_log off; }"))) + + def test_format_requires_exact_one_string(self): + for value in (CHECK.NGINX_FORMAT + " $request_uri", "PRIVACY_REQUEST status=$status", + CHECK.NGINX_FORMAT.replace(" ", " ")): + self.reject(single(MAIN.replace(CHECK.NGINX_FORMAT, value))) + self.reject(single(MAIN.replace("'" + CHECK.NGINX_FORMAT + "'", "escape=json '" + CHECK.NGINX_FORMAT + "'"))) + + def test_malformed_and_unsupported_lexical_syntax(self): + for suffix in ('"unterminated', "{", "}", "unknown", ";", "foo \\\n bar;", 'foo "escaped\\n";'): + self.reject(single(MAIN + suffix)) + self.reject(single(MAIN).replace(b"stderr", b"std\x00err")) + self.reject(b"\xff") + self.reject(single(MAIN) + b" " * CHECK.MAX_BYTES) + self.reject(MAIN.encode()) + + def test_partial_quotes_cannot_turn_filenames_into_safe_tokens(self): + for override in ('error_log std"err";', 'access_log o\'ff\';', + 'error_log "stderr"private;', 'error_log "std""err";'): + self.reject(single(MAIN.replace(HTTP, HTTP + "server {" + override + "}"))) + + def test_include_completeness_cycles_and_conflicting_repeats(self): + self.reject(single("include missing.conf;" + MAIN)) + self.reject(single("include $unknown;" + MAIN)) + self.reject(single("include nginx.conf;" + MAIN)) + self.reject(dump(("/etc/nginx/nginx.conf", MAIN), ("/etc/nginx/extra.conf", "error_log /tmp/private;"))) + self.reject(dump(("/etc/nginx/nginx.conf", MAIN), ("/etc/nginx/nginx.conf", MAIN + "error_log /tmp/private;"))) + + def test_globs_do_not_cross_directory_boundaries(self): + root = ("error_log stderr; worker_shutdown_timeout 2s; http {" + "include /etc/nginx/conf.d/*.conf; server { include /etc/nginx/conf.d/nested/safe.conf; }}") + # A wildcard must not promote this server-only baseline into HTTP scope. + self.reject(dump(("/etc/nginx/nginx.conf", root), + ("/etc/nginx/conf.d/nested/safe.conf", HTTP))) + + +class SilentCli(unittest.TestCase): + def test_uwsgi_exit_codes_and_absolute_silence(self): + with tempfile.TemporaryDirectory() as directory: + target = Path(directory) / "SYNTHETIC_PRIVATE.ini" + for content, expected in ((UWSGI, 0), (UWSGI + "logto=/tmp/private\n", 70)): + target.write_text(content) + result = subprocess.run([sys.executable, str(SCRIPT), "uwsgi", str(target)], + capture_output=True, timeout=5) + self.assertEqual((result.returncode, result.stdout, result.stderr), (expected, b"", b"")) + target.unlink() + result = subprocess.run([sys.executable, str(SCRIPT), "uwsgi", str(target)], capture_output=True, timeout=5) + self.assertEqual((result.returncode, result.stdout, result.stderr), (70, b"", b"")) + + def test_special_file_is_rejected_without_blocking(self): + import os + with tempfile.TemporaryDirectory() as directory: + target = Path(directory) / "fifo" + os.mkfifo(target) + result = subprocess.run([sys.executable, str(SCRIPT), "uwsgi", str(target)], capture_output=True, timeout=5) + self.assertEqual((result.returncode, result.stdout, result.stderr), (70, b"", b"")) + + def test_nginx_results_and_exceptions_remain_silent(self): + output, error = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(output), contextlib.redirect_stderr(error): + with mock.patch.object(CHECK, "_nginx_dump", return_value=single()): + self.assertEqual(CHECK.main(["nginx"]), 0) + for failure in (RuntimeError("SYNTHETIC_PRIVATE"), TimeoutError("SYNTHETIC_PRIVATE")): + with mock.patch.object(CHECK, "_nginx_dump", side_effect=failure): + self.assertEqual(CHECK.main(["nginx"]), 70) + for args in ([], ["nginx", "extra"], ["uwsgi"], ["unknown"]): + self.assertEqual(CHECK.main(args), 70) + self.assertEqual((output.getvalue(), error.getvalue()), ("", "")) + + def test_fixed_native_command_and_capture_success(self): + self.assertEqual(CHECK.NGINX_COMMAND, ("/usr/sbin/nginx", "-T", "-e", "stderr")) + command = (sys.executable, "-c", "import sys;sys.stdout.buffer.write(" + repr(single()) + + ");sys.stderr.write('SYNTHETIC_PRIVATE')") + with mock.patch.object(CHECK, "NGINX_COMMAND", command): + self.assertEqual(CHECK.main(["nginx"]), 0) + + def test_native_failure_timeout_and_combined_output_limit(self): + snippets = ("import sys;sys.stderr.write('SYNTHETIC_PRIVATE');sys.exit(1)", + "import time;time.sleep(10)", + "import sys;sys.stderr.buffer.write(b'x' * 1048577)", + "import sys;sys.stdout.buffer.write(b'x' * 700000);sys.stderr.buffer.write(b'x' * 400000)") + for snippet in snippets: + with self.subTest(snippet=snippet), mock.patch.object(CHECK, "TIMEOUT_SECONDS", 0.2), \ + mock.patch.object(CHECK, "NGINX_COMMAND", (sys.executable, "-c", snippet)): + self.assertEqual(CHECK.main(["nginx"]), 70) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/privacy_native/test_native_launchers.py b/tests/privacy_native/test_native_launchers.py new file mode 100644 index 000000000..6db318788 --- /dev/null +++ b/tests/privacy_native/test_native_launchers.py @@ -0,0 +1,202 @@ +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import time +import unittest + +ROOT = Path(__file__).resolve().parents[2] +PRIVATE = "SYNTHETIC_PRIVATE_BOOTSTRAP" + + +class NativeLauncherTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.diagnostic = ROOT / "tests/.privacy-build/privacy-diagnostic" + if not cls.diagnostic.is_file(): + raise RuntimeError("Build the diagnostic first: bash tests/verify_privacy.sh") + + def fixture(self, directory, *, activation_failure=False): + root = Path(directory) + binary = root / "runtime/bin" + binary.mkdir(parents=True) + (root / "config").mkdir() + (root / "webapp").mkdir() + shutil.copy2(self.diagnostic, root / "webapp/privacy-diagnostic") + (binary / "activate").write_text( + "echo SYNTHETIC_PRIVATE_BOOTSTRAP >&2\n" + + ("echo SYNTHETIC_PRIVATE_BOOTSTRAP\nreturn 1\n" if activation_failure else "true\n")) + interpreter = binary / "python" + interpreter.write_text("#!" + sys.executable + "\n" + r'''import json, os, sys +args = sys.argv[1:] +if args[0] == "-m": + print('export LOCALE="' if os.environ.get("SYNTHETIC_CONFIG_EVAL_FAIL") else 'export LOCALE="C.UTF-8 UTF-8"') + print('export DAHOSTNAME="synthetic.invalid"') + print('export DAREADONLYFILESYSTEM="true"') + print('export CONFIG_PRIVATE="SYNTHETIC_PRIVATE_BOOTSTRAP"') + print('SYNTHETIC_PRIVATE_BOOTSTRAP', file=sys.stderr) + raise SystemExit(1 if os.environ.get("SYNTHETIC_CONFIG_FAIL") else 0) +raise SystemExit(99) +''') + interpreter.chmod(0o755) + (root / "config/docassemble.ini").write_text("[synthetic]\n") + (root / "config/docassemble-expose-uwsgi.ini").write_text("[synthetic]\n") + (root / "config/docassemblelog.ini").write_text("[synthetic]\n") + preflight = root / "webapp/privacy-preflight" + preflight.write_text("#!" + sys.executable + "\n" + r'''import os, sys +args = sys.argv[1:] +if not (args == ["nginx"] or len(args) == 2 and args[0] == "uwsgi" and os.path.isfile(args[1])): + raise SystemExit(98) +print('SYNTHETIC_PRIVATE_BOOTSTRAP') +print('SYNTHETIC_PRIVATE_BOOTSTRAP', file=sys.stderr) +if os.environ.get("SYNTHETIC_PREFLIGHT_FAIL"): + raise SystemExit(70) +if os.environ.get("SYNTHETIC_LAUNCH_FAIL"): + os.unlink(os.path.join(os.environ["DA_ROOT"], "webapp/privacy-process")) +raise SystemExit(0) +''') + preflight.chmod(0o755) + runner = root / "webapp/privacy-process" + runner.write_text("#!" + sys.executable + "\n" + r'''import json, os, sys +try: + os.fstat(3) + extra_sink_closed = False +except OSError: + extra_sink_closed = True +print(json.dumps({"argv":sys.argv[1:], "pid":os.getpid(), + "extra_sink_closed":extra_sink_closed, + "uwsgi_options_cleared":not any(key.startswith("UWSGI_") for key in os.environ), + "temporary_exports_cleared":"DA_EXPORTS" not in os.environ})) +''') + runner.chmod(0o755) + (binary / "su").write_text('#!/bin/bash\nexec "$DA_PYTHON/bin/python" -m synthetic.read_config\n') + (binary / "su").chmod(0o755) + return root + + def run_launcher(self, name, root, **extra): + env = dict(os.environ, DA_ROOT=str(root), DA_PYTHON=str(root / "runtime"), + UWSGI_LOGTO="SYNTHETIC_PRIVATE_BOOTSTRAP", UWSGI_DAEMONIZE="synthetic", **extra) + env["PATH"] = str(root / "runtime/bin") + os.pathsep + os.environ["PATH"] + child = subprocess.Popen(["bash", str(ROOT / "Docker" / name)], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) + stdout, stderr = child.communicate(timeout=5) + return child, stdout, stderr + + def test_both_ini_choices_and_log_role_exec_protected_foreground(self): + for name, mode, ini in (("run-uwsgi.sh", "nginx", "docassemble.ini"), + ("run-uwsgi.sh", "none", "docassemble-expose-uwsgi.ini"), + ("run-uwsgilog.sh", "nginx", "docassemblelog.ini")): + with self.subTest(name=name, mode=mode), tempfile.TemporaryDirectory() as directory: + root = self.fixture(directory) + child, stdout, stderr = self.run_launcher(name, root, DAWEBSERVER=mode) + self.assertEqual(child.returncode, 0) + self.assertEqual(stderr, b"") + self.assertNotIn(PRIVATE.encode(), stdout) + record = json.loads(stdout) + self.assertEqual(record["pid"], child.pid) + self.assertTrue(record["uwsgi_options_cleared"]) + self.assertTrue(record["temporary_exports_cleared"]) + self.assertTrue(record["extra_sink_closed"]) + args = record["argv"] + self.assertEqual(args[:3], ["--component", "uwsgi", "--"]) + self.assertEqual(args[3], str(root / "runtime/bin/uwsgi")) + self.assertEqual(args[args.index("--ini")+1], str(root / "config" / ini)) + self.assertIn("--die-on-term", args) + self.assertEqual(args[-1], "PRIVACY_REQUEST status=%(status) msecs=%(msecs)") + + def test_bootstrap_failures_report_only_fixed_stage_and_do_not_launch(self): + for name in ("run-uwsgi.sh", "run-uwsgilog.sh"): + for failure in ("activation", "config", "config_eval", "preflight", "launch"): + with self.subTest(name=name, failure=failure), tempfile.TemporaryDirectory() as directory: + root = self.fixture(directory, activation_failure=failure=="activation") + extra = {} if failure == "activation" else {"SYNTHETIC_" + failure.upper() + "_FAIL": "1"} + child, stdout, stderr = self.run_launcher(name, root, **extra) + self.assertEqual(child.returncode, 70) + self.assertEqual(stderr, b"") + self.assertNotIn(PRIVATE.encode(), stdout) + self.assertLess(len(stdout), 256) + self.assertEqual(json.loads(stdout), { + "schema": 1, + "component": "uwsgilog" if name == "run-uwsgilog.sh" else "uwsgi", + "event": "startup_failed", + "phase": failure, + }) + + def test_missing_diagnostic_is_distinct_unavailable_exit(self): + for name in ("run-uwsgi.sh", "run-uwsgilog.sh", "run-nginx.sh"): + with self.subTest(name=name), tempfile.TemporaryDirectory() as directory: + root = self.fixture(directory) + (root / "webapp/privacy-diagnostic").unlink() + child, stdout, stderr = self.run_launcher(name, root) + self.assertEqual(child.returncode, 69) + self.assertEqual(stdout + stderr, b"") + + def test_nginx_readonly_bootstrap_handoff_and_failure_codes(self): + # The su/config/native boundaries are synthetic; readonly=true prevents + # the real launcher from writing nginx or certificate paths on this host. + for failure in (None, "config", "config_eval", "preflight", "launch"): + with self.subTest(failure=failure), tempfile.TemporaryDirectory() as directory: + root = self.fixture(directory) + extra = {} if failure is None else {"SYNTHETIC_" + failure.upper() + "_FAIL": "1"} + child, stdout, stderr = self.run_launcher("run-nginx.sh", root, **extra) + self.assertEqual(stderr, b"") + self.assertNotIn(PRIVATE.encode(), stdout) + record = json.loads(stdout) + if failure is None: + self.assertEqual(child.returncode, 0) + self.assertEqual(record["pid"], child.pid) + self.assertTrue(record["extra_sink_closed"]) + self.assertEqual(record["argv"][:3], ["--component", "nginx", "--"]) + else: + self.assertEqual(child.returncode, 70) + self.assertEqual(record, {"schema": 1, "component": "nginx", + "event": "startup_failed", "phase": failure}) + + def test_broken_output_pipe_stops_bootstrap_with_sink_failure(self): + for name in ("run-uwsgi.sh", "run-uwsgilog.sh"): + with self.subTest(name=name), tempfile.TemporaryDirectory() as directory: + root = self.fixture(directory, activation_failure=True) + read_fd, write_fd = os.pipe() + os.close(read_fd) + try: + env = dict(os.environ, DA_ROOT=str(root), DA_PYTHON=str(root / "runtime")) + result = subprocess.run(["bash", str(ROOT / "Docker" / name)], env=env, + stdout=write_fd, stderr=subprocess.PIPE, timeout=5) + self.assertEqual(result.returncode, 74) + self.assertEqual(result.stderr, b"") + finally: + os.close(write_fd) + + def test_full_output_pipe_has_a_bounded_failure_deadline(self): + for name in ("run-uwsgi.sh", "run-uwsgilog.sh"): + with self.subTest(name=name), tempfile.TemporaryDirectory() as directory: + root = self.fixture(directory, activation_failure=True) + read_fd, write_fd = os.pipe() + try: + os.set_blocking(write_fd, False) + for chunk in (b"x" * 4096, b"x"): + while True: + try: + os.write(write_fd, chunk) + except BlockingIOError: + break + os.set_blocking(write_fd, True) + env = dict(os.environ, DA_ROOT=str(root), DA_PYTHON=str(root / "runtime")) + started = time.monotonic() + result = subprocess.run(["bash", str(ROOT / "Docker" / name)], env=env, + stdout=write_fd, stderr=subprocess.PIPE, timeout=4) + elapsed = time.monotonic() - started + self.assertEqual(result.returncode, 74) + self.assertEqual(result.stderr, b"") + self.assertGreater(elapsed, 0.8) + self.assertLess(elapsed, 3) + finally: + os.close(read_fd) + os.close(write_fd) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/privacy_native/test_nginx_preflight_image.sh b/tests/privacy_native/test_nginx_preflight_image.sh new file mode 100644 index 000000000..f5b7dd33a --- /dev/null +++ b/tests/privacy_native/test_nginx_preflight_image.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# Check the compiled Go preflight against real nginx -T in a disposable image. +# The caller supplies an already-present image; this script never pulls one. +set -euo pipefail +if [ "$#" -ne 1 ]; then + printf '%s\n' 'usage: bash tests/privacy_native/test_nginx_preflight_image.sh IMAGE' >&2 + exit 64 +fi +REVIEW_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +REVIEW_IMAGE=$(docker image inspect --format '{{.Id}}' "$1") +REVIEW_ARCH=$(docker image inspect --format '{{.Architecture}}' "$REVIEW_IMAGE") +case "$REVIEW_ARCH" in amd64|arm64) ;; *) exit 64 ;; esac +REVIEW_BUILD="$REVIEW_ROOT/tests/.privacy-build" +REVIEW_BINARY="$REVIEW_BUILD/privacy-preflight-linux-$REVIEW_ARCH" +test -x "$REVIEW_BINARY" +REVIEW_TEMP=$(mktemp -d "$REVIEW_BUILD/nginx-preflight.XXXXXX") +trap 'rmdir "$REVIEW_TEMP"' EXIT +for REVIEW_CASE in safe unsafe; do + REVIEW_CONTAINER=(--rm --init --pull=never --network none --read-only --cap-drop ALL + --security-opt no-new-privileges --user "$(id -u):$(id -g)" + --tmpfs /tmp:rw,nosuid,nodev + --mount "type=bind,src=$REVIEW_BINARY,dst=/review/privacy-preflight,readonly" + --mount "type=bind,src=$REVIEW_ROOT/tests/privacy_native/fixtures/nginx-preflight.conf,dst=/etc/nginx/nginx.conf,readonly" + --mount "type=bind,src=$REVIEW_ROOT/tests/privacy_native/fixtures/nginx-preflight-$REVIEW_CASE.conf,dst=/etc/nginx/preflight-override.conf,readonly") + # Both fixtures must pass native inspection. Otherwise a generic exit 70 + # could hide a syntax/installation failure instead of policy rejection. + REVIEW_STATUS=0 + docker run "${REVIEW_CONTAINER[@]}" --entrypoint /usr/sbin/nginx "$REVIEW_IMAGE" -T -e stderr \ + > "$REVIEW_TEMP/native-output" 2> "$REVIEW_TEMP/native-error" || REVIEW_STATUS=$? + if [ "$REVIEW_STATUS" -ne 0 ]; then + printf 'nginx native %s control failed: exit %s; inspect %s\n' \ + "$REVIEW_CASE" "$REVIEW_STATUS" "$REVIEW_TEMP" >&2 + trap - EXIT + exit 1 + fi + rm "$REVIEW_TEMP/native-output" "$REVIEW_TEMP/native-error" + REVIEW_STATUS=0 + docker run "${REVIEW_CONTAINER[@]}" --entrypoint /review/privacy-preflight "$REVIEW_IMAGE" nginx \ + > "$REVIEW_TEMP/output" 2> "$REVIEW_TEMP/error" || REVIEW_STATUS=$? + REVIEW_EXPECTED=0 + if [ "$REVIEW_CASE" = unsafe ]; then REVIEW_EXPECTED=70; fi + if [ "$REVIEW_STATUS" -ne "$REVIEW_EXPECTED" ] || [ -s "$REVIEW_TEMP/output" ] || [ -s "$REVIEW_TEMP/error" ]; then + printf 'nginx preflight %s failed: exit %s, expected %s; inspect %s\n' \ + "$REVIEW_CASE" "$REVIEW_STATUS" "$REVIEW_EXPECTED" "$REVIEW_TEMP" >&2 + trap - EXIT + exit 1 + fi + rm "$REVIEW_TEMP/output" "$REVIEW_TEMP/error" + printf 'nginx preflight %s: native control passed, exit %s, no output\n' "$REVIEW_CASE" "$REVIEW_STATUS" +done diff --git a/tests/privacy_native/test_nginx_request_counts.py b/tests/privacy_native/test_nginx_request_counts.py new file mode 100644 index 000000000..470d1e707 --- /dev/null +++ b/tests/privacy_native/test_nginx_request_counts.py @@ -0,0 +1,40 @@ +import json +import unittest +from native_support import LogAggregator + + +class NginxRequestCounts(unittest.TestCase): + def test_real_nginx_decimal_seconds_have_exact_latency_buckets(self): + cases = (("0.000", "latency_fast"), ("0.099", "latency_fast"), + ("0.100", "latency_medium"), ("0.999", "latency_medium"), + ("1.000", "latency_slow"), ("29.999", "latency_slow"), + ("30.000", "latency_timeout"), ("600.000", "latency_timeout")) + for seconds, bucket in cases: + with self.subTest(seconds=seconds): + agg = LogAggregator("nginx") + line = ("PRIVACY_REQUEST status=204 seconds=" + seconds + "\n").encode() + for byte in line: + agg.feed("stdout", bytes([byte])) + self.assertEqual(agg.snapshot()["status_2xx"], 1) + self.assertEqual(agg.snapshot()[bucket], 1) + self.assertEqual(agg.snapshot()["rejected"], 0) + + def test_malformed_decimal_or_out_of_range_is_rejected_without_markers(self): + values = ("00.001", "+0.100", "-0.100", "0.1", "0.0000", "1e3", + "600.001", "999999999999.000", "0.000 SYNTHETIC_PRIVATE", "nan", "١.000") + for value in values: + with self.subTest(value=value): + agg = LogAggregator("nginx") + agg.feed("stdout", ("PRIVACY_REQUEST status=200 seconds=" + value + "\n").encode()) + self.assertEqual(agg.snapshot()["rejected"], 1) + self.assertEqual(agg.snapshot()["status_2xx"], 0) + self.assertNotIn("SYNTHETIC_PRIVATE", json.dumps(agg.snapshot())) + + def test_nginx_seconds_not_misread_as_uwsgi_milliseconds(self): + agg = LogAggregator("uwsgi") + agg.feed("stderr", b"PRIVACY_REQUEST status=200 seconds=0.010\n") + self.assertEqual(agg.snapshot()["rejected"], 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/privacy_native/test_rotation_image.sh b/tests/privacy_native/test_rotation_image.sh new file mode 100644 index 000000000..5318f9c20 --- /dev/null +++ b/tests/privacy_native/test_rotation_image.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# Real native rotation/forwarding acceptance with synthetic application output. +set -euo pipefail +if [ "$#" -ne 1 ]; then + printf '%s\n' 'usage: bash tests/privacy_native/test_rotation_image.sh IMAGE' >&2 + exit 64 +fi +REVIEW_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +REVIEW_BUILD="$REVIEW_ROOT/tests/.privacy-build" +mkdir -p "$REVIEW_BUILD/go-cache" "$REVIEW_BUILD/go-mod" "$REVIEW_BUILD/go-tmp" "$REVIEW_BUILD/tmp" +export GOCACHE="$REVIEW_BUILD/go-cache" GOMODCACHE="$REVIEW_BUILD/go-mod" +export GOTMPDIR="$REVIEW_BUILD/go-tmp" TMPDIR="$REVIEW_BUILD/tmp" +export TEST_TELEMETRY_DIR="$REVIEW_BUILD/telemetry" GOTOOLCHAIN=local GOPROXY=off +export PYTHONDONTWRITEBYTECODE=1 +REVIEW_IMAGE=$(docker image inspect --format '{{.Id}}' "$1") +REVIEW_ARCH=$(docker image inspect --format '{{.Architecture}}' "$REVIEW_IMAGE") +case "$REVIEW_ARCH" in amd64|arm64) ;; *) exit 64 ;; esac +REVIEW_RUN=$(mktemp -d "$REVIEW_BUILD/rotation-run.XXXXXXXX") +REVIEW_NAME="jos81-$(basename "$REVIEW_RUN")" +cleanup() { + REVIEW_STATUS=$? + trap - EXIT + if ! python3.14 -B - "$REVIEW_NAME" <<'PY' +import subprocess +import sys +try: + subprocess.run(['docker', 'rm', '--force', sys.argv[1]], timeout=10, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + remaining = subprocess.run(['docker', 'ps', '-aq', '--filter', 'name=^/' + sys.argv[1] + '$'], + check=True, capture_output=True, timeout=10) + assert not remaining.stdout.strip(), 'test container survived cleanup' +except (subprocess.SubprocessError, AssertionError): + print('rotation test container cleanup failed', file=sys.stderr) + sys.exit(125) +PY + then + if [ "$REVIEW_STATUS" -eq 0 ]; then REVIEW_STATUS=125; fi + fi + exit "$REVIEW_STATUS" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM +# Rebuild from this checkout so a standalone run cannot validate stale binaries. +CGO_ENABLED=0 GOOS=linux GOARCH="$REVIEW_ARCH" go -C "$REVIEW_ROOT/Docker/privacy-diagnostic" build \ + -trimpath -buildvcs=false -o "$REVIEW_RUN/privacy-process" ./cmd/privacy-process +printf 'rotation test image: %s\n' "$REVIEW_IMAGE" +go version +shasum -a 256 "$REVIEW_RUN"/privacy-* +REVIEW_COMMAND=(docker run --name "$REVIEW_NAME" --rm --init --pull=never --network none --read-only --cap-drop ALL \ + --hostname privacy-rotation-test --add-host privacy-rotation-test:127.0.0.1 \ + --log-driver none --memory 512m --memory-swap 512m --pids-limit 128 \ + --security-opt no-new-privileges --user www-data \ + --tmpfs /tmp:rw,nosuid,nodev,noexec,size=64m \ + --mount "type=bind,src=$REVIEW_ROOT,dst=/review,readonly" \ + --mount "type=bind,src=$REVIEW_RUN/privacy-process,dst=/usr/share/docassemble/webapp/privacy-process,readonly" \ + --entrypoint /usr/bin/env "$REVIEW_IMAGE" PYTHONDONTWRITEBYTECODE=1 \ + timeout -k 5 60 python3.14 -B /review/tests/privacy_native/check_rotation_image.py) +python3.14 -B - "${REVIEW_COMMAND[@]}" <<'PY' +import subprocess +import sys +try: + result = subprocess.run(sys.argv[1:], timeout=70) + sys.exit(result.returncode if result.returncode >= 0 else 128 - result.returncode) +except subprocess.TimeoutExpired: + print('rotation test exceeded its outer deadline', file=sys.stderr) + sys.exit(124) +PY diff --git a/tests/privacy_native/test_rotation_ownership.py b/tests/privacy_native/test_rotation_ownership.py new file mode 100644 index 000000000..a241bf325 --- /dev/null +++ b/tests/privacy_native/test_rotation_ownership.py @@ -0,0 +1,78 @@ +"""Local counter streams must have one writer/rotator and preserve forwarding.""" +import configparser +import fnmatch +from pathlib import Path +import re +import unittest + +ROOT = Path(__file__).resolve().parents[2] +FORWARDED = {'celery': 'worker.log', 'celerysingle': 'single_worker.log', + 'uwsgi': 'uwsgi.log', 'websockets': 'websockets.log'} + + +def supervisor(): + config = configparser.ConfigParser(interpolation=None) + config.read(ROOT / 'Docker/docassemble-supervisor.conf') + return config + + +class RotationOwnershipTests(unittest.TestCase): + def test_counter_files_have_no_external_writer_or_rotator(self): + config = supervisor() + paths = [config['program:' + name]['stdout_logfile'] + for name in (*FORWARDED, 'uwsgilog', 'nginx')] + paths.append(config['eventlistener:privacy-monitor']['stderr_logfile']) + self.assertEqual(len(paths), len(set(paths))) + collector = (ROOT / 'Docker/syslog-ng.conf').read_text() + collector_paths = re.findall(r'file\("([^"]+)"', collector) + patterns = [] + for name in ('docassemble', 'nginx', 'apache'): + patterns.extend(re.findall(r'^(/\S+)', (ROOT / 'Docker' / (name + '.logrotate')).read_text(), re.M)) + for path in paths: + with self.subTest(path=path): + self.assertNotIn(path, collector_paths) + self.assertFalse(any(fnmatch.fnmatchcase(path, pattern) for pattern in patterns)) + + def test_forwarding_tracks_capture_files_and_preserves_collector_labels(self): + source = (ROOT / 'Docker/docassemble-syslog-ng.conf').read_text() + collector = (ROOT / 'Docker/syslog-ng.conf').read_text() + rotation = (ROOT / 'Docker/docassemble.logrotate').read_text() + config = supervisor() + for component, legacy in FORWARDED.items(): + with self.subTest(component=component): + path = config['program:' + component]['stdout_logfile'] + self.assertEqual(Path(path).name, 'privacy-' + component + '.log') + self.assertRegex(source, r'file\("' + re.escape(path) + r'"[^\n]+program-override\("' + component + r'"\)') + self.assertIn('file("/usr/share/docassemble/log/' + legacy + '"', collector) + self.assertIn('/usr/share/docassemble/log/' + legacy + '\n', rotation) + + def test_forwarded_counter_files_are_precreated_before_ownership_pass(self): + text = (ROOT / 'Docker/initialize.sh').read_text() + start = text.index('touch /usr/share/docassemble/log/') + end = text.index('chown -R www-data:www-data /usr/share/docassemble/log', start) + precreation = text[start:end] + for component in FORWARDED: + self.assertIn(supervisor()['program:' + component]['stdout_logfile'], precreation) + + def test_supervisor_keeps_bounded_rotation(self): + config = supervisor() + for name in (*FORWARDED, 'uwsgilog', 'nginx'): + with self.subTest(component=name): + section = config['program:' + name] + self.assertEqual(section['stdout_logfile_maxbytes'], '5MB') + self.assertEqual(section['stdout_logfile_backups'], '7') + self.assertEqual(section['redirect_stderr'], 'true') + monitor = config['eventlistener:privacy-monitor'] + self.assertEqual(monitor['stderr_logfile_maxbytes'], '5MB') + self.assertEqual(monitor['stderr_logfile_backups'], '7') + + def test_celery_receiver_does_not_duplicate_single_queue_records(self): + text = (ROOT / 'Docker/syslog-ng.conf').read_text() + expression = re.search(r'filter f_daworker \{ program\("([^"]+)"\); \};', text).group(1) + self.assertIsNotNone(re.search(expression, 'celery')) + self.assertIsNone(re.search(expression, 'celerysingle')) + self.assertIsNone(re.search(expression, 'other-celery')) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/privacy_native/test_supervisor_monitor_image.sh b/tests/privacy_native/test_supervisor_monitor_image.sh new file mode 100644 index 000000000..f07727520 --- /dev/null +++ b/tests/privacy_native/test_supervisor_monitor_image.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Test the compiled monitor using an already-present Supervisor/Python 3.14 image. +set -euo pipefail +if [ "$#" -ne 1 ]; then + printf '%s\n' 'usage: bash tests/privacy_native/test_supervisor_monitor_image.sh IMAGE' >&2 + exit 64 +fi +REVIEW_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +REVIEW_IMAGE=$(docker image inspect --format '{{.Id}}' "$1") +REVIEW_ARCH=$(docker image inspect --format '{{.Architecture}}' "$REVIEW_IMAGE") +case "$REVIEW_ARCH" in amd64|arm64) ;; *) exit 64 ;; esac +REVIEW_BINARY="$REVIEW_ROOT/tests/.privacy-build/privacy-monitor-linux-$REVIEW_ARCH" +test -x "$REVIEW_BINARY" +REVIEW_PROCESS="$REVIEW_ROOT/tests/.privacy-build/privacy-process-linux-$REVIEW_ARCH" +REVIEW_DIAGNOSTIC="$REVIEW_ROOT/tests/.privacy-build/privacy-diagnostic-linux-$REVIEW_ARCH" +test -x "$REVIEW_PROCESS" +test -x "$REVIEW_DIAGNOSTIC" +docker run --rm --init --pull=never --network none --read-only --cap-drop ALL \ + --security-opt no-new-privileges --user www-data \ + --tmpfs /tmp:rw,nosuid,nodev \ + --mount "type=bind,src=$REVIEW_ROOT,dst=/review,readonly" \ + --mount "type=bind,src=$REVIEW_BINARY,dst=/usr/share/docassemble/webapp/privacy-monitor,readonly" \ + --mount "type=bind,src=$REVIEW_PROCESS,dst=/usr/share/docassemble/webapp/privacy-process,readonly" \ + --mount "type=bind,src=$REVIEW_DIAGNOSTIC,dst=/usr/share/docassemble/webapp/privacy-diagnostic,readonly" \ + --entrypoint /usr/bin/env "$REVIEW_IMAGE" PYTHONDONTWRITEBYTECODE=1 \ + python3.14 -B /review/tests/privacy_native/check_supervisor_monitor_image.py diff --git a/tests/privacy_native/test_uwsgi_image.sh b/tests/privacy_native/test_uwsgi_image.sh new file mode 100644 index 000000000..30d20e84d --- /dev/null +++ b/tests/privacy_native/test_uwsgi_image.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# Real uWSGI acceptance only; uses synthetic application/configuration modules. +set -euo pipefail +if [ "$#" -ne 1 ]; then + printf '%s\n' 'usage: bash tests/privacy_native/test_uwsgi_image.sh IMAGE' >&2 + exit 64 +fi +REVIEW_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +REVIEW_BUILD="$REVIEW_ROOT/tests/.privacy-build" +mkdir -p "$REVIEW_BUILD/go-cache" "$REVIEW_BUILD/go-mod" "$REVIEW_BUILD/go-tmp" "$REVIEW_BUILD/tmp" +export GOCACHE="$REVIEW_BUILD/go-cache" GOMODCACHE="$REVIEW_BUILD/go-mod" +export GOTMPDIR="$REVIEW_BUILD/go-tmp" TMPDIR="$REVIEW_BUILD/tmp" +export TEST_TELEMETRY_DIR="$REVIEW_BUILD/telemetry" GOTOOLCHAIN=local GOPROXY=off +export PYTHONDONTWRITEBYTECODE=1 +REVIEW_IMAGE=$(docker image inspect --format '{{.Id}}' "$1") +REVIEW_ARCH=$(docker image inspect --format '{{.Architecture}}' "$REVIEW_IMAGE") +case "$REVIEW_ARCH" in amd64|arm64) ;; *) exit 64 ;; esac +REVIEW_RUN=$(mktemp -d "$REVIEW_BUILD/uwsgi-run.XXXXXXXX") +REVIEW_NAME="jos81-$(basename "$REVIEW_RUN")" +cleanup() { + REVIEW_STATUS=$? + trap - EXIT + if ! python3.14 -B - "$REVIEW_NAME" <<'PY' +import subprocess +import sys +try: + subprocess.run(['docker', 'rm', '--force', sys.argv[1]], timeout=10, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + remaining = subprocess.run(['docker', 'ps', '-aq', '--filter', 'name=^/' + sys.argv[1] + '$'], + check=True, capture_output=True, timeout=10) + assert not remaining.stdout.strip(), 'test container survived cleanup' +except (subprocess.SubprocessError, AssertionError): + print('uWSGI test container cleanup failed', file=sys.stderr) + sys.exit(125) +PY + then + if [ "$REVIEW_STATUS" -eq 0 ]; then REVIEW_STATUS=125; fi + fi + exit "$REVIEW_STATUS" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM +# Rebuild from this checkout so a standalone run cannot validate stale binaries. +for REVIEW_COMPONENT in process preflight diagnostic; do + REVIEW_PACKAGE="./cmd/privacy-$REVIEW_COMPONENT" + if [ "$REVIEW_COMPONENT" = diagnostic ]; then REVIEW_PACKAGE=.; fi + CGO_ENABLED=0 GOOS=linux GOARCH="$REVIEW_ARCH" go -C "$REVIEW_ROOT/Docker/privacy-diagnostic" build \ + -trimpath -buildvcs=false -o "$REVIEW_RUN/privacy-$REVIEW_COMPONENT" "$REVIEW_PACKAGE" +done +printf 'uWSGI test image: %s\n' "$REVIEW_IMAGE" +go version +shasum -a 256 "$REVIEW_RUN"/privacy-* +REVIEW_COMMAND=(docker run --name "$REVIEW_NAME" --rm --init --pull=never --network none --read-only --cap-drop ALL \ + --log-driver none --memory 512m --memory-swap 512m --pids-limit 128 \ + --security-opt no-new-privileges --user www-data \ + --tmpfs /tmp:rw,nosuid,nodev,noexec,size=64m \ + --tmpfs /var/run/uwsgi:rw,nosuid,nodev,noexec,size=1m,uid=82,gid=82,mode=0700 \ + --tmpfs /usr/share/docassemble/config:rw,nosuid,nodev,noexec,size=1m,uid=82,gid=82,mode=0700 \ + --mount "type=bind,src=$REVIEW_ROOT,dst=/review,readonly" \ + --mount "type=bind,src=$REVIEW_RUN/privacy-process,dst=/usr/share/docassemble/webapp/privacy-process,readonly" \ + --mount "type=bind,src=$REVIEW_RUN/privacy-preflight,dst=/usr/share/docassemble/webapp/privacy-preflight,readonly" \ + --mount "type=bind,src=$REVIEW_RUN/privacy-diagnostic,dst=/usr/share/docassemble/webapp/privacy-diagnostic,readonly" \ + --entrypoint /usr/bin/env "$REVIEW_IMAGE" PYTHONDONTWRITEBYTECODE=1 \ + timeout -k 5 45 python3.14 -B /review/tests/privacy_native/check_uwsgi_image.py) +python3.14 -B - "${REVIEW_COMMAND[@]}" <<'PY' +import subprocess +import sys +try: + result = subprocess.run(sys.argv[1:], timeout=55) + sys.exit(result.returncode if result.returncode >= 0 else 128 - result.returncode) +except subprocess.TimeoutExpired: + print('uWSGI test exceeded its outer deadline', file=sys.stderr) + sys.exit(124) +PY diff --git a/tests/verify_privacy.sh b/tests/verify_privacy.sh new file mode 100644 index 000000000..8897469a9 --- /dev/null +++ b/tests/verify_privacy.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# Focused candidate checks only; this does not establish full-install readiness. +set -euo pipefail +REVIEW_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$REVIEW_ROOT" +REVIEW_BUILD="$REVIEW_ROOT/tests/.privacy-build" +mkdir -p "$REVIEW_BUILD/go-cache" "$REVIEW_BUILD/go-mod" "$REVIEW_BUILD/go-tmp" "$REVIEW_BUILD/tmp" +export GOCACHE="$REVIEW_BUILD/go-cache" GOMODCACHE="$REVIEW_BUILD/go-mod" +export GOTMPDIR="$REVIEW_BUILD/go-tmp" TMPDIR="$REVIEW_BUILD/tmp" +export TEST_TELEMETRY_DIR="$REVIEW_BUILD/telemetry" +export GOTOOLCHAIN=local GOPROXY=off PYTHONDONTWRITEBYTECODE=1 +# Keep race detection enabled without its artificial one-second fixture exit +# sleep, which would exceed the shortened synthetic native shutdown deadlines. +export GORACE=atexit_sleep_ms=0 +unset PYTHONPATH +test -z "$(gofmt -l Docker/privacy-diagnostic)" +go -C Docker/privacy-diagnostic vet ./... +go -C Docker/privacy-diagnostic test -race -count=1 -timeout 30s ./... +go -C Docker/privacy-diagnostic test -run '^$' -fuzz FuzzParseHasFixedBoundedOutput \ + -fuzztime=10s -parallel=2 +go -C Docker/privacy-diagnostic test ./internal/aggregate -run '^$' \ + -fuzz FuzzFragmentationPreservesCounters -fuzztime=10s -parallel=2 +go -C Docker/privacy-diagnostic test ./internal/preflight -run '^$' \ + -fuzz FuzzParsersReturnOnlyFixedErrors -fuzztime=10s -parallel=2 +go -C Docker/privacy-diagnostic test ./internal/monitor -run '^$' \ + -fuzz FuzzEventRecordsHaveOnlyFixedOutput -fuzztime=10s -parallel=2 +go -C Docker/privacy-diagnostic mod verify +go -C Docker/privacy-diagnostic build -trimpath -buildvcs=false -o "$REVIEW_BUILD/privacy-diagnostic" . +go -C Docker/privacy-diagnostic build -trimpath -buildvcs=false -o "$REVIEW_BUILD/privacy-process" ./cmd/privacy-process +go -C Docker/privacy-diagnostic build -trimpath -buildvcs=false -o "$REVIEW_BUILD/privacy-preflight" ./cmd/privacy-preflight +go -C Docker/privacy-diagnostic build -trimpath -buildvcs=false -o "$REVIEW_BUILD/privacy-monitor" ./cmd/privacy-monitor +CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go -C Docker/privacy-diagnostic build \ + -trimpath -buildvcs=false -o "$REVIEW_BUILD/privacy-diagnostic-linux-amd64" . +CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go -C Docker/privacy-diagnostic build \ + -trimpath -buildvcs=false -o "$REVIEW_BUILD/privacy-diagnostic-linux-arm64" . +CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go -C Docker/privacy-diagnostic build \ + -trimpath -buildvcs=false -o "$REVIEW_BUILD/privacy-process-linux-amd64" ./cmd/privacy-process +CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go -C Docker/privacy-diagnostic build \ + -trimpath -buildvcs=false -o "$REVIEW_BUILD/privacy-process-linux-arm64" ./cmd/privacy-process +CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go -C Docker/privacy-diagnostic build \ + -trimpath -buildvcs=false -o "$REVIEW_BUILD/privacy-preflight-linux-amd64" ./cmd/privacy-preflight +CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go -C Docker/privacy-diagnostic build \ + -trimpath -buildvcs=false -o "$REVIEW_BUILD/privacy-preflight-linux-arm64" ./cmd/privacy-preflight +CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go -C Docker/privacy-diagnostic build \ + -trimpath -buildvcs=false -o "$REVIEW_BUILD/privacy-monitor-linux-amd64" ./cmd/privacy-monitor +CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go -C Docker/privacy-diagnostic build \ + -trimpath -buildvcs=false -o "$REVIEW_BUILD/privacy-monitor-linux-arm64" ./cmd/privacy-monitor +# On non-Linux hosts the lifecycle tests are run separately in a Linux sandbox. +CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go -C Docker/privacy-diagnostic build ./... +CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go -C Docker/privacy-diagnostic build ./... +python3.14 -B -m unittest discover -s tests/privacy_native -v +# Historical Python draft assertions are retained against pinned references. +# Current application logger behavior runs through Go in the Linux suite. +python3.14 -B -m unittest discover -s tests/privacy_logging -v +bash -n Docker/run-nginx.sh +bash -n Docker/run-uwsgi.sh +bash -n Docker/run-uwsgilog.sh +bash -n Docker/run-celery.sh +bash -n Docker/run-celery-single.sh +bash -n Docker/run-websockets.sh +bash -n Docker/run-cron.sh +bash -n Docker/sync.sh +bash -n Docker/restart-post-logrotate.sh +for REVIEW_SCRIPT in Docker/cron/docassemble-cron-*.sh; do bash -n "$REVIEW_SCRIPT"; done +bash -n Docker/process-email.sh +bash -n Docker/initialize.sh +bash -n tests/verify_privacy.sh +bash -n tests/privacy_native/test_nginx_preflight_image.sh +bash -n tests/privacy_native/test_supervisor_monitor_image.sh +bash -n tests/privacy_native/test_uwsgi_image.sh +bash -n tests/privacy_native/test_rotation_image.sh +bash -n tests/privacy_native/test_install_image.sh +bash -n tests/privacy_native/fixtures/application-command.sh +bash -n tests/privacy_native/fixtures/rotation-command.sh +git diff --check