repro/repro-sentinel-zero.sh

1#!/usr/bin/env bash

2# Off-device reproduction of the SENTINEL-ZERO storm (2026-07-02 live failure)

3# and proof of the fix. NOTHING here touches real Telegram: the poller is

4# pointed at the local mock via COURIER_TELEGRAM_API_URL.

5#

6# The bug: a count=0 cold-start drain (empty backlog) persisted offset=0.

7# A later restart that trusts that 0 restores it and the normal poll loop

8# issues getUpdates offset=0 -> Telegram re-returns the entire backlog ->

9# every message is re-delivered -> the leader acks each -> outbound storm.

10#

11# Scenario for BOTH binaries:

12# 1. empty backlog, cold start (no offset file) -> drain, count=0

13# 2. a backlog accumulates (e.g. ~24h of inbound)

14# 3. RESTART the poller

15# 4. assert step 3 delivers ZERO message events

16#

17# CONTROL (models the deployed buggy binary: load accepts 0) -> step 3

18# RESTORES offset 0 and the normal loop RE-DELIVERS -> storm.

19# FIXED -> step 1 leaves NO offset file (0 is never persisted), so

20# step 3 is a cold start that DRAINS the backlog -> no delivery.

21#

22# Usage: repro-sentinel-zero.sh <fixed-courier-bin> [control-courier-bin]

23# Without a control bin, only the FIXED assertions run.

24set -u

25HERE="$(cd "$(dirname "$0")" && pwd)"

26FIXED="${1:?usage: repro-sentinel-zero.sh <fixed-bin> [control-bin]}"

27CONTROL="${2:-}"

28PORT="${PORT:-19877}"

29WORK="$(mktemp -d)"

30RELAY="$WORK/relays"

31OFF="$WORK/telegram-offset" # = <dirname(relay-dir)>/telegram-offset

32mkdir -p "$RELAY" "$WORK/logs"

34cleanup () { [ -n "${MOCK:-}" ] && kill -9 "$MOCK" 2>/dev/null; rm -rf "$WORK"; }

35trap cleanup EXIT

37python3 "$HERE/mock_telegram.py" "$PORT" "$WORK/logs" >"$WORK/mock.out" 2>&1 &

38MOCK=$!

39sleep 1.5

40curl -sf -m2 "http://127.0.0.1:$PORT/_stats" >/dev/null || { echo "FAIL: mock not up"; cat "$WORK/mock.out"; exit 1; }

42seed_empty () { curl -s -m2 -X POST "http://127.0.0.1:$PORT/_seed" -d '{"updates":[]}' >/dev/null; }

43add () { curl -s -m2 -X POST "http://127.0.0.1:$PORT/_add" -d "{\"text\":\"$1\"}" >/dev/null; }

44run () { # $1=bin $2=tag

45 env COURIER_TELEGRAM_TOKEN=tok COURIER_TELEGRAM_API_URL="http://127.0.0.1:$PORT" \

46 COURIER_RELAY_DIR="$RELAY" \

47 timeout 6 "$1" --telegram-poller < <(sleep 12) >"$WORK/$2.out" 2>"$WORK/$2.err"

48}

49msgs () { local c; c=$(grep -c '"type":"message"' "$WORK/$1.out" 2>/dev/null); echo "${c:-0}"; }

51scenario () { # $1=bin $2=label ; echo message-event count on restart

52 seed_empty; rm -f "$OFF"

53 run "$1" "${2}_cold"

54 add OLD-1; add OLD-2; add OLD-3

55 run "$1" "${2}_restart"

56 echo "$(msgs ${2}_restart)"

57}

59rc=0

60echo "=== FIXED ($FIXED) ==="

61n=$(scenario "$FIXED" fixed)

62coldfile_exists=$([ -e "$OFF" ] && echo yes || echo no)

63echo " empty cold start left offset file? (expect no): note file removed by restart drain"

64echo " restart message events: $n (expect 0)"

if [ "$n" -eq 0 ]; then echo "  PASS: fixed binary does not re-deliver"; else echo "  FAIL: fixed binary re-delivered $n"; rc=1; fi

67if [ -n "$CONTROL" ]; then

68 echo "=== CONTROL ($CONTROL) — models the deployed sentinel-0 binary ==="

69 n=$(scenario "$CONTROL" control)

70 echo " restart message events: $n (expect >0 = storm reproduced)"

  if [ "$n" -gt 0 ]; then echo "  PASS: control reproduces the storm ($n re-delivered)"; else echo "  FAIL: control did not storm"; rc=1; fi

72fi

74echo "=== $( [ $rc -eq 0 ] && echo ALL CHECKS PASSED || echo CHECKS FAILED ) ==="

75exit $rc