%run ~/notebooks/helpers.py

from IPython.display import Image

Before fix

collections = "u/wtg/run20/detectorMaps"

butler = Butler('/work/datastore', collections=collections)

dataId = dict(visit=119621, arm='b', spectrograph=3, instrument='PFS')

Image(filename=butler.getURI('dmQaResidualPlot', dataId).unquoted_path)

[l['message'] for l in butler.get('dmResiduals_log', dataId).model_dump()]

["Preparing execution of quantum for label=dmResiduals dataId={instrument: 'PFS', arm: 'b', spectrograph: 3, visit: 119621, ...}.",
 "Constructing task and executing quantum for label=dmResiduals dataId={instrument: 'PFS', arm: 'b', spectrograph: 3, visit: 119621, ...}.",
 'Getting and scrubbing the data',
 "Scrubbing data using config={'lineFlags': ['BAD'], 'traceIterations': 2, 'iterations': 3, 'rejection': 4.0, 'order': 2, 'reserveFraction': 0.1, 'soften': 0.003, 'lsqThreshold': 1e-06, 'doSlitOffsets': False, 'slitOffsetIterations': 3, 'base': '/work/stack-2024-07-11/conda/envs/lsst-scipipe-7.0.1/share/eups/Linux64/drp_pfs_data/w.2025.03/detectorMap/detectorMap-sim-%(arm)s%(spectrograph)s.fits', 'minSignalToNoise': 20.0, 'maxCentroidError': 0.15, 'maxRejectionFrac': 0.3, 'minNumWavelengths': 3, 'weightings': {}, 'qaNumFibers': 5, 'exclusionRadius': 4.0, 'doRejectBadLines': False, 'forced': [], 'parameters': [], 'spatialOffsets': {}, 'spectralOffsets': {}, 'chipGap': 69.33333333333334}",
 '2312509 line centroids for 585 traces',
 '2312509 lines in list',
 '2312509 good lines after initial flags (Trace: 2312509)',
 '2312509 good lines after line flags (Trace: 2312509)',
 '2312314 good lines after finite positions (Trace: 2312314)',
 '2312314 good lines after finite intensities (Trace: 2312314)',
 'Using mean SN=114.42 instead of config 20.0',
 '1375368 good lines after min SN=114.42 (Trace: 1375368)',
 '1375366 good lines after maxCentroidError=0.15 centroid errors (Trace: 1375366)',
 'Masking outliers',
 'Adding fiber information',
 'Removing outliers',
 'Getting residual stats',
 'Making residual plots',
 "Execution of task 'dmResiduals' on quantum {instrument: 'PFS', arm: 'b', spectrograph: 3, visit: 119621, ...} took 33.616 seconds"]

After fix

-d "combination = 'run20/detectorMaps/input'" \

!pipetask \
    --no-log-tty \
    --long-log \
    --log-level pfs=DEBUG \
    run \
    -b /work/datastore \
    --instrument lsst.obs.pfs.PrimeFocusSpectrograph \
    -i "PFS/calib/pipe2d-1622/verify.20250122b,PFS/defaults" \
    -o u/wtg/tickets/PIPE2D-1651 \
    -p "/work/wtg/drp_qa_dev/pipelines/drpQA.yaml#dmResiduals,dmCombinedResiduals" \
    -d "combination = 'run20/detectorMaps/input' and visit = 119621 and arm = 'b'" \
    --fail-fast \
    -j 32

butler = Butler('/work/datastore', collections='u/wtg/tickets/PIPE2D-1651')

[l['message'] for l in butler.get('dmResiduals_log', dataId).model_dump()]

["Preparing execution of quantum for label=dmResiduals dataId={instrument: 'PFS', arm: 'b', spectrograph: 3, visit: 119621, dither: -13, pfs_design_id: 6577301261601167266}.",
 "Constructing task and executing quantum for label=dmResiduals dataId={instrument: 'PFS', arm: 'b', spectrograph: 3, visit: 119621, dither: -13, pfs_design_id: 6577301261601167266}.",
 'Getting and scrubbing the data',
 "Scrubbing data using config={'lineFlags': ['BAD'], 'traceIterations': 2, 'iterations': 3, 'rejection': 4.0, 'order': 2, 'reserveFraction': 0.1, 'soften': 0.003, 'lsqThreshold': 1e-06, 'doSlitOffsets': False, 'slitOffsetIterations': 3, 'base': '/work/stack-2024-07-11/conda/envs/lsst-scipipe-7.0.1/share/eups/Linux64/drp_pfs_data/w.2025.03/detectorMap/detectorMap-sim-%(arm)s%(spectrograph)s.fits', 'minSignalToNoise': 20.0, 'maxCentroidError': 0.15, 'maxRejectionFrac': 0.3, 'minNumWavelengths': 3, 'weightings': {}, 'qaNumFibers': 5, 'exclusionRadius': 4.0, 'doRejectBadLines': False, 'forced': [], 'parameters': [], 'spatialOffsets': {}, 'spectralOffsets': {}, 'chipGap': 69.33333333333334}",
 '2312509 line centroids for 585 traces',
 '2312509 lines in list',
 '2312509 good lines after initial flags (Trace: 2312509)',
 '2312509 good lines after line flags (Trace: 2312509)',
 '2312314 good lines after finite positions (Trace: 2312314)',
 '2312314 good lines after finite intensities (Trace: 2312314)',
 'Filtering SN < sn_cut=20.00',
 '2312314 good lines after SN filtering (Trace: 2312314)',
 '2306119 good lines after maxCentroidError=0.15 centroid errors (Trace: 2306119)',
 '/work/wtg/drp_qa_dev/python/pfs/drp/qa/dmResiduals.py:468: FutureWarning: Logical ops (and, or, xor) between Pandas objects and dtype-less sequences (e.g. list, tuple) are deprecated and will raise in a future version. Wrap the object in a Series, Index, or np.array before operating instead.\n  is_reserved = (arc_data.status & ReferenceLineStatus.DETECTORMAP_RESERVED) != 0\n',
 '/work/wtg/drp_qa_dev/python/pfs/drp/qa/dmResiduals.py:469: FutureWarning: Logical ops (and, or, xor) between Pandas objects and dtype-less sequences (e.g. list, tuple) are deprecated and will raise in a future version. Wrap the object in a Series, Index, or np.array before operating instead.\n  is_used = (arc_data.status & ReferenceLineStatus.DETECTORMAP_USED) != 0\n',
 'Masking outliers',
 'Adding fiber information',
 'Removing outliers',
 'Getting residual stats',
 'Making residual plots',
 '/work/wtg/drp_qa_dev/python/pfs/drp/qa/dmResiduals.py:1001: FutureWarning: The default of observed=False is deprecated and will be changed to True in a future version of pandas. Pass observed=False to retain current behavior or observed=True to adopt the future default and silence this warning.\n  binned_data = plotData.dropna(subset=["wavelength", column]).groupby(["bin", "status", "isOutlier"])[\n',
 "Execution of task 'dmResiduals' on quantum {instrument: 'PFS', arm: 'b', spectrograph: 3, visit: 119621, dither: -13, pfs_design_id: 6577301261601167266} took 38.106 seconds"]

Image(filename=butler.getURI('dmQaResidualPlot', dataId).unquoted_path)