PR #472 Validation-Script Fixes — Implementation Plan¶

git fetch origin 05-14-test_validate_expand_smoke_script_with_happy-path_promote_rollback_sqs
git switch -c fix/pr472-validation-coverage \
    origin/05-14-test_validate_expand_smoke_script_with_happy-path_promote_rollback_sqs

grep -n 'failed=1' scripts/validate_issue_458.sh        # the dead filter (defect 1)
grep -n 'purge it from DLQ' scripts/validate_issue_458.sh # the stale comment (defect 3)
grep -nc 'echo "  SKIP:' scripts/validate_issue_458.sh   # uncounted skips (defect 2)

mkdir -p scripts/lib tests/scripts

#!/usr/bin/env bash
# Dependency-free unit tests for scripts/lib/validate_lib.sh.
# Run: bash tests/scripts/validate_lib_test.sh   (no bats required)
set -uo pipefail

LIB="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../scripts/lib" && pwd)/validate_lib.sh"
# shellcheck source=/dev/null
source "$LIB"

tests_run=0
tests_failed=0
check() {  # check "<desc>" "<actual>" "<expected>"
    local desc="$1" actual="$2" expected="$3"
    tests_run=$((tests_run + 1))
    if [[ "$actual" == "$expected" ]]; then
        echo "ok   - $desc"
    else
        echo "FAIL - $desc (expected '$expected', got '$actual')"
        tests_failed=$((tests_failed + 1))
    fi
}

# --- Task 1: counters ---
SKIP=0; skip "env down" >/dev/null;          check "skip increments SKIP" "$SKIP" "1"
SKIP=0; inconclusive "no data" >/dev/null;   check "inconclusive increments SKIP" "$SKIP" "1"
PASS=0; FAIL=0; SKIP=0
pass "a" >/dev/null; skip "b" >/dev/null
check "pass unaffected by skip" "$PASS" "1"
check "fail unaffected by skip" "$FAIL" "0"

echo ""
echo "Ran $tests_run tests, $tests_failed failed"
[[ $tests_failed -eq 0 ]]

#!/usr/bin/env bash
# Sourceable helpers for scripts/validate_issue_458.sh.
#
# Pure, side-effect-free logic (no AWS calls) lives here so it can be
# unit-tested by tests/scripts/validate_lib_test.sh without a deployed stack.
# Mirrors the scripts/demo/lib.sh sourcing convention.

# Outcome counters. Default to 0 unless the caller already set them.
PASS="${PASS:-0}"
FAIL="${FAIL:-0}"
SKIP="${SKIP:-0}"

pass() { echo "  PASS: $1"; PASS=$((PASS + 1)); }
fail() { echo "  FAIL: $1"; FAIL=$((FAIL + 1)); }

# skip / inconclusive record a NON-assertion outcome. They increment SKIP so
# the summary can report incomplete coverage instead of silently passing.
# Use skip() when a precondition makes a check impossible (e.g. a dependency
# is down); use inconclusive() when a check ran but could not decide.
skip()         { echo "  SKIP: $1"; SKIP=$((SKIP + 1)); }
inconclusive() { echo "  INCONCLUSIVE: $1"; SKIP=$((SKIP + 1)); }

chmod +x scripts/lib/validate_lib.sh tests/scripts/validate_lib_test.sh
git add scripts/lib/validate_lib.sh tests/scripts/validate_lib_test.sh
git commit -m "test(validate): add validate_lib with counted skip/inconclusive outcomes"

# --- Task 2: render_summary ---
PASS=2; FAIL=0; SKIP=1
out="$(render_summary)"
echo "$out" | grep -q "2 passed, 0 failed, 1 skipped/inconclusive" && r=yes || r=no
check "summary line includes skip count" "$r" "yes"
echo "$out" | grep -q "coverage is INCOMPLETE" && r=yes || r=no
check "summary warns when something was skipped" "$r" "yes"

PASS=3; FAIL=0; SKIP=0
out="$(render_summary)"
echo "$out" | grep -q "INCOMPLETE" && r=yes || r=no
check "no incomplete-coverage warning when nothing skipped" "$r" "no"

# Render the results summary, including skipped/inconclusive coverage. When any
# check was skipped or inconclusive, print a prominent warning so a reader does
# NOT mistake a partially-run validation for full coverage of the fix.
render_summary() {
    echo "  Results: ${PASS} passed, ${FAIL} failed, ${SKIP} skipped/inconclusive"
    if [[ "${SKIP}" -gt 0 ]]; then
        echo "  WARNING: ${SKIP} check(s) skipped or inconclusive — coverage is INCOMPLETE."
        echo "           Do NOT read this run as a full validation of the fix."
    fi
}

git add scripts/lib/validate_lib.sh tests/scripts/validate_lib_test.sh
git commit -m "feat(validate): summary surfaces incomplete coverage from skips"

# --- Task 3: exit_code_for_results ---
FAIL=0; SKIP=0; ALLOW_INCOMPLETE=0; exit_code_for_results; rc=$?
check "rc 0 when no failures and full coverage" "$rc" "0"
FAIL=2; SKIP=0; ALLOW_INCOMPLETE=0; exit_code_for_results; rc=$?
check "rc 1 when failures present" "$rc" "1"
FAIL=0; SKIP=3; ALLOW_INCOMPLETE=0; exit_code_for_results; rc=$?
check "rc 1 on incomplete coverage by default" "$rc" "1"
FAIL=0; SKIP=3; ALLOW_INCOMPLETE=1; exit_code_for_results; rc=$?
check "rc 0 on incomplete coverage when explicitly allowed" "$rc" "0"

# Exit-code policy. A hard FAIL always fails the run. Incomplete coverage (any
# skip/inconclusive) ALSO fails by default so an automated run cannot exit 0
# while having validated nothing — the exact false-positive PR #472 was flagged
# for. Set ALLOW_INCOMPLETE=1 to downgrade incomplete coverage to a pass for
# interactive runs against a partial dev environment.
exit_code_for_results() {
    [[ "${FAIL}" -gt 0 ]] && return 1
    if [[ "${SKIP}" -gt 0 && "${ALLOW_INCOMPLETE:-0}" != "1" ]]; then
        return 1
    fi
    return 0
}

git add scripts/lib/validate_lib.sh tests/scripts/validate_lib_test.sh
git commit -m "feat(validate): exit-code policy (skips do not fail the run)"

# --- Task 4: is_partial_batch_failure_line ---
is_partial_batch_failure_line "0 succeeded, 0 skipped, 1 failed" && r=yes || r=no
check "1 failed -> partial-batch failure" "$r" "yes"
is_partial_batch_failure_line "5 succeeded, 0 skipped, 0 failed" && r=yes || r=no
check "0 failed -> not a failure" "$r" "no"
is_partial_batch_failure_line "3 succeeded, 1 skipped, 2 failed" && r=yes || r=no
check "2 failed -> partial-batch failure" "$r" "yes"
is_partial_batch_failure_line "12 succeeded, 0 skipped, 10 failed" && r=yes || r=no
check "multi-digit 10 failed -> partial-batch failure" "$r" "yes"
is_partial_batch_failure_line "INFO 2026-06-02 0 succeeded, 0 skipped, 4 failed" && r=yes || r=no
check "log-prefixed summary still classified" "$r" "yes"
is_partial_batch_failure_line "Lambda init: cold start complete" && r=yes || r=no
check "unrelated line -> not a failure" "$r" "no"
is_partial_batch_failure_line "job skipped, then 2 failed" && r=yes || r=no
check "non-summary line ending in 'N failed' -> not a failure" "$r" "no"
is_partial_batch_failure_line "1 failed" && r=yes || r=no
check "bare '1 failed' (not the summary) -> not a failure" "$r" "no"
is_partial_batch_failure_line "" && r=yes || r=no
check "empty line -> not a failure" "$r" "no"

# Return 0 (true) iff the given backtest-consumer log line is the batch summary
# AND its failed-count is >= 1. Matches the full shape emitted by
# lambdas/backtest-consumer/handler.py (with any leading log prefix):
#   "{N} succeeded, {N} skipped, {N} failed"
# Matching the whole shape (not just "<n> failed") avoids false positives on
# unrelated lines. Replaces the CloudWatch "failed=1" filter, which never
# matched this format and could not express "failed >= 1" anyway.
is_partial_batch_failure_line() {
    local line="$1"
    local failed
    failed=$(printf '%s\n' "$line" \
        | sed -nE 's/.*[0-9]+ succeeded, [0-9]+ skipped, ([0-9]+) failed.*/\1/p')
    [[ -n "$failed" && "$failed" -ge 1 ]]
}

git add scripts/lib/validate_lib.sh tests/scripts/validate_lib_test.sh
git commit -m "feat(validate): classify partial-batch failures by log content"

# --- Task 4b: per-section assertion guard ---
PASS=0; FAIL=0; SKIP=0
section_begin; pass "did something" >/dev/null
require_section_assertions "demo" >/dev/null
check "section with a pass does not add a fail" "$FAIL" "0"

PASS=0; FAIL=0; SKIP=0
section_begin; skip "dep down" >/dev/null
require_section_assertions "demo" >/dev/null
check "section with only a skip does not add a fail" "$FAIL" "0"

PASS=0; FAIL=0; SKIP=0
section_begin
require_section_assertions "demo" >/dev/null
check "zero-outcome section becomes a fail" "$FAIL" "1"

# Per-section guard against a SILENT zero-outcome section (no pass/fail/skip at
# all) — the exact false-positive PR #472 was flagged for. Call section_begin()
# at the top of a validation section and require_section_assertions "<name>" at
# the end; a section that recorded nothing becomes a hard fail.
_SECTION_BASE=0
section_begin() { _SECTION_BASE=$((PASS + FAIL + SKIP)); }
require_section_assertions() {
    local name="$1"
    if [[ $((PASS + FAIL + SKIP)) -eq "${_SECTION_BASE}" ]]; then
        fail "${name}: section recorded ZERO outcomes — it validated nothing"
    fi
}

git add scripts/lib/validate_lib.sh tests/scripts/validate_lib_test.sh
git commit -m "feat(validate): per-section guard fails silent zero-outcome sections"

#!/usr/bin/env bash
# Smoke checks for scripts/validate_issue_458.sh that need NO AWS:
#   - the script parses (bash -n)
#   - it sources the shared library
#   - the dead "failed=1" filter is gone
#   - the stale "purge it from DLQ" comment is gone
#   - skips/inconclusives are counted (no bare 'echo "  SKIP:' / "  INCONCLUSIVE:')
set -uo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
SCRIPT="$ROOT/scripts/validate_issue_458.sh"
rc=0
bash -n "$SCRIPT"                              || { echo "FAIL - bash -n"; rc=1; }
grep -q 'source .*validate_lib.sh' "$SCRIPT"   || { echo "FAIL - does not source validate_lib.sh"; rc=1; }
grep -q 'failed=1' "$SCRIPT"                    && { echo "FAIL - dead failed=1 filter still present"; rc=1; }
grep -q 'purge it from DLQ' "$SCRIPT"           && { echo "FAIL - stale DLQ-purge comment still present"; rc=1; }
grep -q 'echo "  SKIP:' "$SCRIPT"               && { echo "FAIL - uncounted bare SKIP echo still present"; rc=1; }
grep -q 'echo "  INCONCLUSIVE:' "$SCRIPT"       && { echo "FAIL - uncounted bare INCONCLUSIVE echo still present"; rc=1; }
[[ $rc -eq 0 ]] && echo "ok - validate_issue_458.sh smoke checks passed"
exit $rc

PASS=0
FAIL=0

pass() { echo "  PASS: $1"; PASS=$((PASS + 1)); }
fail() { echo "  FAIL: $1"; FAIL=$((FAIL + 1)); }

# shellcheck source=scripts/lib/validate_lib.sh
source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib/validate_lib.sh"

        echo "  SKIP: data-collection service unhealthy (FunctionError=$DC_HEALTH); a real backtest cannot succeed."
        echo "  Note: this is an environmental issue, not a fix regression."
        echo ""
        # Skip the rest of section 4
        SKIP_HAPPY_PATH=1

        skip "data-collection service unhealthy (FunctionError=$DC_HEALTH); a real backtest cannot succeed — environmental, not a fix regression."
        echo ""
        # Skip the rest of section 4
        SKIP_HAPPY_PATH=1

            echo "  INCONCLUSIVE: EnsureData failed with upstream service error ($(echo "$cause" | head -c 80)...)"
            echo "  This means data-collection /sync is unhealthy in $ENVIRONMENT — env issue, not a fix regression."
            echo "  The fix IS working: it propagated the upstream failure correctly through Catch."

            inconclusive "EnsureData failed with upstream service error ($(echo "$cause" | head -c 80)...) — data-collection /sync unhealthy in $ENVIRONMENT; the fix propagated it correctly through Catch."

    # Check 2: recent partial-batch failures in CloudWatch (last 1h)
    SINCE_TS=$(($(date +%s) - 3600))000
    # filter-log-events with --output text + length(@) can return multiple
    # paginated responses joined with newlines. Sum them.
    PARTIAL_BATCH_HITS=$(aws logs filter-log-events \
        --profile "$AWS_PROFILE" --region "$AWS_REGION" \
        --log-group-name "$CONSUMER_LOG_GROUP" \
        --start-time "$SINCE_TS" \
        --filter-pattern "failed=1" \
        --query "events | length(@)" --output text 2>/dev/null \
        | awk '{s+=$1} END {print s+0}' || echo "0")
    if [[ "$PARTIAL_BATCH_HITS" -gt 0 ]]; then
        pass "consumer logged $PARTIAL_BATCH_HITS partial-batch-failure events in last hour (contract is exercised)"
    else
        echo "  INCONCLUSIVE: no recent partial-batch failures in CloudWatch (last 1h)."
        echo "  This is normal in a quiet environment. Send a malformed message manually to verify."
    fi

    # Check 2: recent partial-batch failures in CloudWatch (last 1h).
    # CloudWatch filter patterns cannot express "failed >= 1", and the consumer
    # logs "<n> succeeded, <n> skipped, <n> failed" (never "failed=1"). Pull the
    # candidate lines with the broad term filter "failed", then classify each
    # locally with the strict is_partial_batch_failure_line().
    SINCE_TS=$(($(date +%s) - 3600))000
    LOG_EVENTS_RC=0
    LOG_MESSAGES=$(aws logs filter-log-events \
        --profile "$AWS_PROFILE" --region "$AWS_REGION" \
        --log-group-name "$CONSUMER_LOG_GROUP" \
        --start-time "$SINCE_TS" \
        --filter-pattern "failed" \
        --query "events[].message" --output text 2>"$TMPDIR_VALIDATE/logs_err.txt") || LOG_EVENTS_RC=$?

    if [[ "$LOG_EVENTS_RC" -ne 0 ]]; then
        # A real AWS error must NOT become a quiet "0 hits" — surface the cause.
        inconclusive "could not read consumer logs (aws rc=$LOG_EVENTS_RC): $(head -c 120 "$TMPDIR_VALIDATE/logs_err.txt")"
    else
        PARTIAL_BATCH_HITS=0
        while IFS= read -r line; do
            [[ -z "$line" ]] && continue
            is_partial_batch_failure_line "$line" && PARTIAL_BATCH_HITS=$((PARTIAL_BATCH_HITS + 1))
        done < <(printf '%s\n' "$LOG_MESSAGES" | tr '\t' '\n')
        if [[ "$PARTIAL_BATCH_HITS" -gt 0 ]]; then
            pass "consumer logged $PARTIAL_BATCH_HITS partial-batch-failure event(s) in last hour (contract is exercised)"
        else
            inconclusive "no partial-batch failures in CloudWatch (last 1h) — normal in a quiet env; send a malformed message manually to verify."
        fi
    fi

    # ---------------------------------------------------------------------------
    # Gap #3: SQS partial-batch (ReportBatchItemFailures) contract — READ-ONLY.
    #
    # This section performs NO writes to the queue or DLQ. It verifies the
    # deployed contract two ways, without racing a fresh message:
    #   1. The EventSourceMapping has FunctionResponseTypes=ReportBatchItemFailures
    #      (a missing flag is a deploy regression).
    #   2. Recent backtest-consumer logs are scanned for partial-batch failures
    #      (failed >= 1), confirming the handler emits the partial-batch shape at
    #      runtime. Inconclusive (not a failure) when the environment is quiet.
    #
    # We deliberately do NOT send a message: BatchSize=1 + VisibilityTimeout=900s
    # + FIFO ordering means a fresh message can sit behind backlog for minutes —
    # that is queue shape, not fix correctness. The cleanup-isolation invariant
    # is covered directly by the unit tests in tests/unit/lambdas/.
    # ---------------------------------------------------------------------------

if [[ "${SKIP_SQS:-0}" != "1" ]]; then
    echo "[7/N] SQS partial-batch contract verification..."
    section_begin
    # ... check 1 (EventSourceMapping flag) — records pass/fail ...
    # ... check 2 (CloudWatch classify) — records pass/inconclusive ...
    require_section_assertions "SQS partial-batch contract"
else
    skip "SQS partial-batch contract section disabled via SKIP_SQS=1"
fi

echo ""
echo "========================================================================"
echo "  Results: $PASS passed, $FAIL failed"
echo "========================================================================"

[[ $FAIL -eq 0 ]] || exit 1

echo ""
echo "========================================================================"
render_summary
echo "========================================================================"

exit_code_for_results || exit 1

chmod +x tests/scripts/validate_script_smoke_test.sh
git add scripts/validate_issue_458.sh tests/scripts/validate_script_smoke_test.sh
git commit -m "fix(validate): count skips, fix partial-batch log check, correct Gap #3 comment

Resolves PR #472 review: skips/inconclusives now reach the summary and no
longer read as a clean pass; the dead 'failed=1' CloudWatch filter is replaced
by local classification of the real log format; the Gap #3 comment matches the
read-only code."

git add justfile
git commit -m "chore(validate): add just test-scripts recipe for shell unit tests"