From 893701dfc51273d50a2062f59586defac5a3974d Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 14 Jul 2026 09:21:18 -0400 Subject: [PATCH] fix(ngmix): release FitModel observations after each metacal fit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each resdict value returned by MetacalBootstrapper.go is an ngmix FitModel — a dict subclass that retains the full per-epoch metacal observations on the plain attribute .obs (set by FitModel._set_obs) plus the pixel arrays derived from them in ._pixels_list, ~5 MB per object. do_ngmix_metacal results accumulate in final_res until the SAVE_BATCH flush, so process RSS ramps ~4.8 MB/object (to ~6 GB at SAVE_BATCH=1000 on a 3000-object test) even though compile_results reads only scalar dict keys. Null .obs and ._pixels_list immediately after the fit (and drop the obsdict reference once average_multiepoch_psf has consumed it). With the fix, RSS stays flat at ~0.9 GB over the same run and the output catalog is numerically identical to the unpatched baseline. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WuMuqutUtBCgRxCoWjoebV --- src/shapepipe/modules/ngmix_package/ngmix.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/shapepipe/modules/ngmix_package/ngmix.py b/src/shapepipe/modules/ngmix_package/ngmix.py index 1c8c2c8a2..d4f64a74e 100644 --- a/src/shapepipe/modules/ngmix_package/ngmix.py +++ b/src/shapepipe/modules/ngmix_package/ngmix.py @@ -1630,4 +1630,18 @@ def do_ngmix_metacal( ) resdict, obsdict = boot.go(gal_obs_list) psf_res = average_multiepoch_psf(obsdict) + del obsdict + + # Each FitModel in resdict retains the full metacal observations + # (``.obs``, a plain attribute set by FitModel._set_obs) plus the pixel + # arrays derived from them (``._pixels_list``) — several MB per object. + # Results accumulate until the SAVE_BATCH flush, and compile_results + # reads only scalar dict keys, so release the arrays now to keep + # resident memory flat. + for _fm in resdict.values(): + if hasattr(_fm, 'obs'): + _fm.obs = None + if hasattr(_fm, '_pixels_list'): + _fm._pixels_list = None + return MetacalResult(resdict=resdict, reconv=psf_res, orig=psf_orig_res)