AtlatestRepositoryapiary

apiary / tree / testsmoke-keepalive-integration.sh

1#!/usr/bin/env bash
2#
3# Cross-repo keepalive integration smoke. Boots a v0.2.8+
4# enclave-server with TIGHT ping-interval / ping-timeout knobs
5# (5 s / 5 s) so the test finishes in ~30 s, then runs the v0.1.2+
6# apiary binary against it and verifies the two-sided
7# PING/PONG handshake actually composes.
8#
9# Scenarios (each <30 s):
11# 1. Server-PING / apiary-PONG: with tight ping-interval the
12# server PINGs apiary within 5-15 s of connect. apiary's
13# `handle-server-ping` emits a matching PONG. We assert
14# the apiary log shows "Apiary responded to server PING"
15# AND the apiary connection stays alive (no
16# ping-timeout-disconnect on the server side).
18# 2. Force-disconnect detection + reconnect: kill -STOP the
19# enclave to freeze writes. Apiary sees no incoming traffic
20# and (after its 60 s idle threshold + 30 s PONG timeout)
21# would trigger reconnect — but to keep the test under a
22# minute, we instead kill -9 the enclave (drops the socket
23# cleanly), restart it, and assert apiary's reconnect-
24# attempt path runs and re-establishes registration.
26# A separate, longer-running manual procedure (see
27# `procedures/apiary-enclave-keepalive-soak`) covers the full
28# 15-min idle window + iptables-based network-kill scenario.
30set -u
31PORT=${PORT:-49870}
32ENCLAVE_REPO=${ENCLAVE_REPO:-/home/daviwil/Projects/Code/sigil/task-apiary-and-enclave-keepalive-pair-enclave}
33APIARY_REPO=${APIARY_REPO:-$(pwd)}
34ENCLAVE_BIN=${ENCLAVE_BIN:-$ENCLAVE_REPO/build/dev/bin/enclave}
35APIARY_BIN=${APIARY_BIN:-$APIARY_REPO/build/dev/bin/apiary}
36DB=${DB:-/tmp/apiary-keepalive-smoke.db}
37PING_INTERVAL=${PING_INTERVAL:-5}
38PING_TIMEOUT=${PING_TIMEOUT:-5}
40if [ ! -x "$ENCLAVE_BIN" ]; then
41 echo "FATAL: enclave binary missing at $ENCLAVE_BIN" >&2
42 exit 1
43fi
44if [ ! -x "$APIARY_BIN" ]; then
45 echo "FATAL: apiary binary missing at $APIARY_BIN" >&2
46 exit 1
47fi
49SERVER_LOG=$(mktemp)
50APIARY_LOG=$(mktemp)
51SERVER=
52APIARY_PID=
54cleanup() {
55 [ -n "${APIARY_PID:-}" ] && kill "$APIARY_PID" 2>/dev/null
56 [ -n "${SERVER:-}" ] && kill "$SERVER" 2>/dev/null
57 wait 2>/dev/null
58 rm -f "$SERVER_LOG" "$APIARY_LOG"
59 rm -f "$DB" "$DB-shm" "$DB-wal"
61trap cleanup EXIT
63# ============================================================
64# Phase 1 — server-PING / apiary-PONG composition.
65# ============================================================
66echo "==> seeding DB at $DB"
67rm -f "$DB" "$DB-shm" "$DB-wal"
69DAVIWIL_PASS=admin
70ENCLAVE_DB_PATH="$DB" "$ENCLAVE_BIN" \
71 user add daviwil --password "$DAVIWIL_PASS" --role admin > /dev/null
73LEADER_OUT=$(ENCLAVE_DB_PATH="$DB" "$ENCLAVE_BIN" \
74 bot register leader-quinn --owner daviwil --allow-services yes --auto-join "#hive")
75LEADER_TOKEN=$(echo "$LEADER_OUT" | grep -oE '[a-f0-9]{64}' | head -1)
76if [ -z "$LEADER_TOKEN" ]; then
77 echo "FATAL: no leader-quinn token" >&2; exit 1
78fi
79echo " leader-quinn token: ${LEADER_TOKEN:0:8}…(${#LEADER_TOKEN})"
81start_server() {
82 ENCLAVE_LOG_LEVEL=debug \
83 ENCLAVE_PORT="$PORT" \
84 ENCLAVE_DB_PATH="$DB" \
85 ENCLAVE_PING_INTERVAL="$PING_INTERVAL" \
86 ENCLAVE_PING_TIMEOUT="$PING_TIMEOUT" \
87 "$ENCLAVE_BIN" serve >> "$SERVER_LOG" 2>&1 &
88 SERVER=$!
89 sleep 2
90 if ! kill -0 "$SERVER" 2>/dev/null; then
91 echo "FAIL: server died at startup" >&2
92 cat "$SERVER_LOG" >&2
93 exit 1
94 fi
97echo "==> starting enclave-server on :$PORT (ping-interval=${PING_INTERVAL}s ping-timeout=${PING_TIMEOUT}s)"
98start_server
100echo "==> launching apiary"
101export APIARY_ENCLAVE_HOST=127.0.0.1
102export APIARY_ENCLAVE_PORT="$PORT"
103export APIARY_ENCLAVE_TLS=no
104export APIARY_USER=leader-quinn
105export APIARY_TOKEN="$LEADER_TOKEN"
106export APIARY_CHANNEL=#hive
107export APIARY_OWNER_NICK=daviwil
108export APIARY_REPORTS_TO=daviwil
109export APIARY_SERVICES_TIMEOUT=8.0
110export APIARY_MODE=leader
111INIT_JSON='{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{},"clientInfo":{"name":"smoke","version":"0.0"}}}'
112INITED_JSON='{"jsonrpc":"2.0","method":"notifications/initialized","params":{}}'
113(printf '%s\n%s\n' "$INIT_JSON" "$INITED_JSON"; tail -f /dev/null) \
114 | "$APIARY_BIN" --log-level debug > "$APIARY_LOG" 2>&1 &
115APIARY_PID=$!
117# Wait up to 6 s for the bridge to come up.
118for _ in $(seq 1 60); do
119 if grep -q "Apiary bridge connected" "$APIARY_LOG"; then
120 break
121 fi
122 sleep 0.1
123done
125if ! grep -q "Apiary bridge connected" "$APIARY_LOG"; then
126 echo "FAIL: apiary did not connect"
127 echo "==> apiary log:"
128 sed 's/^/ /' "$APIARY_LOG"
129 exit 1
130fi
131echo " apiary connected"
133# Sleep through one ping-interval+slack so the server has time to
134# send at least one PING and apiary has time to respond.
135echo "==> idling for $((PING_INTERVAL * 3)) s to observe server PING / apiary PONG"
136sleep $((PING_INTERVAL * 3))
138# ============================================================
139# Assertions for Phase 1
140# ============================================================
141fail=0
142check() {
143 local desc=$1 pattern=$2 file=$3
144 if grep -qE "$pattern" "$file"; then
145 echo "PASS: $desc"
146 else
147 echo "FAIL: $desc"
148 echo " expected: $pattern"
149 echo " in: $file"
150 fail=1
151 fi
154check_absent() {
155 local desc=$1 pattern=$2 file=$3
156 if ! grep -qE "$pattern" "$file"; then
157 echo "PASS: $desc"
158 else
159 echo "FAIL: $desc"
160 echo " unexpected: $pattern"
161 grep -E "$pattern" "$file" | head -3 | sed 's/^/ /'
162 fail=1
163 fi
166echo
167echo "==> phase 1 assertions (server-PING → apiary-PONG)"
168check "server emitted at least one PING out to apiary" \
169 'ping-out.*nick=leader-quinn' "$SERVER_LOG"
170check "apiary's handle-server-ping responded to the PING" \
171 'Apiary responded to server PING' "$APIARY_LOG"
172check_absent "no Ping timeout disconnect of apiary on the server side" \
173 'ping-timeout-disconnect.*nick=leader-quinn' "$SERVER_LOG"
174check "apiary's keepalive-tick has run at least once" \
175 'Apiary keepalive PING out|Apiary responded to server PING' "$APIARY_LOG"
177# ============================================================
178# Phase 2 — force-disconnect + reconnect.
179# ============================================================
180echo
181echo "==> phase 2: kill enclave, verify apiary detects + reconnects"
182kill -9 "$SERVER"
183wait "$SERVER" 2>/dev/null
184SERVER=
186# Apiary's irc-tick will see EOF on the next read and the event
187# loop's reconnect-trigger path should kick in. Give it a moment
188# to log the reconnect attempt.
189sleep 3
191check "apiary logged reconnect-trigger after server died" \
192 'Apiary reconnecting' "$APIARY_LOG"
194echo "==> restarting enclave"
195start_server
196# Apiary's reconnect loop has exponential backoff starting at 1s.
197# Let it cycle a few times to actually reach the now-running server.
198sleep 12
200check "apiary reconnected after server came back" \
201 'Apiary reconnected' "$APIARY_LOG"
202check_absent "apiary did not segfault" \
203 'segfault|SIGSEGV' "$APIARY_LOG"
205# ============================================================
206# Phase 3 — extended outage spanning multiple reconnect attempts.
208# The brief's failing scenario: enclave restarts (typical
209# downtime ~5-10s during which the listener is down or up but
210# not yet accepting registrations). Apiary's reconnect attempt
211# 1 fires at the 1s backoff window, finds the server unreachable
212# OR partially-reachable, and the registration handshake hangs.
213# Pre-fix, the registration timeout (30s) raised an exception
214# that escaped the goroutine and crashed apiary.
216# Post-fix, do-reconnect! returns 'retry on registration timeout
217# (or any handshake error), the partial irc is cleaned up, and
218# the backoff schedule cycles to attempt 2, 3, etc. With the
219# tightened 10s registration timeout each failed attempt cycles
220# fast enough that recovery happens within ~30s of the server
221# coming back.
223# This phase verifies the no-crash invariant by holding the
224# server down through MULTIPLE reconnect attempts, then bringing
225# it back. Apiary should reach a healthy state without any
226# "Apiary crashed" log line.
227# ============================================================
228echo
229echo "==> phase 3: extended outage — server down for ~12s spanning >=3 reconnect attempts"
230kill -9 "$SERVER"
231wait "$SERVER" 2>/dev/null
232SERVER=
234# Hold the server down for 12s. With backoff 1s/2s/4s/8s, apiary
235# should fire attempts 1, 2, 3 (and possibly 4) within this
236# window — all of which will fail with TCP-refused. Then we
237# bring the server back and let the next attempt succeed.
238sleep 12
240check "apiary logged at least 3 reconnect attempts during outage" \
241 'attempt=3' "$APIARY_LOG"
242check_absent "apiary did not crash with registration-timeout error" \
243 'Apiary crashed.*registration timed out|Apiary crashed.*do-reconnect!' "$APIARY_LOG"
244check_absent "apiary did not crash at all during the outage" \
245 'Apiary crashed mode=' "$APIARY_LOG"
247echo "==> restarting enclave (phase 3)"
248start_server
249# Backoff caps at 60s but should already be at 8-16s by now.
250# Wait long enough for one of the next attempts to land.
251sleep 25
253check "apiary reconnected after extended outage" \
254 'Apiary reconnected after-attempts' "$APIARY_LOG"
256# ============================================================
257# Final result
258# ============================================================
259if [ $fail -eq 0 ]; then
260 echo
261 echo "ALL CHECKS PASSED"
262 exit 0
263else
264 echo
265 echo "SOME CHECKS FAILED"
266 echo
267 echo "==> server log (last 40 lines):"
268 tail -40 "$SERVER_LOG" | sed 's/^/ /'
269 echo
270 echo "==> apiary log (last 40 lines):"
271 tail -40 "$APIARY_LOG" | sed 's/^/ /'
272 exit 1
273fi