Commit33a15c33Recorded25 Jun 2026Repositoryfolio

Fix note-edit append clobber and note-create double-wrap data loss

Message

Appending to a note could replace its entire body with only the newly appended block, leaving just the frontmatter. Three compounding defects:

1. read-frontmatter mis-handled the frontmatter boundary. It paired the opening --- with the first subsequent --- line without checking the lines between were actually frontmatter. Folio notes routinely carry a nested ---...--- block in the body (note-create wraps content that itself has frontmatter -- daily notes and task briefs both do). If a note's top closing fence was ever malformed, the parser overran it, absorbed real body content (headings, whole sections) into "frontmatter", and returned a truncated body -- which the mutators then persisted. Now bail safely when a markdown heading appears before the closing fence: treat the whole document as body so nothing is destroyed.

2. note-create double-wrapped frontmatter when the supplied content itself began with a ---...--- block, producing back-to-back / nested frontmatter on disk (the malformed state defect 1 mishandles). Now lift the content's frontmatter fields up and merge them, using the remainder as the body, so a single frontmatter block is emitted.

3. note-append round-tripped through read/write-frontmatter, so any misparse became permanent loss and the frontmatter was needlessly re-encoded (which also mangled empty-list tags). Rewrite it as a raw append: keep the file byte-for-byte and bump last-updated in place. Body loss is now structurally impossible.

Also normalize tags to a list in note-create (the MCP layer passes arrays; an empty array is not null? and YAML-encoded to a malformed tags: block, which corrupted the frontmatter on the next read).

Adds test/test-append-clobber.sgl: 14 regression tests; the three boundary/clobber/double-wrap cases fail before this change and pass after.

Changed
 src/folio/frontmatter.sgl    |  30 +++++++++++++++++
 src/folio/note.sgl           | 121 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---------
 test/test-append-clobber.sgl | 288 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 424 insertions(+), 15 deletions(-)
Diff
src/folio/frontmatter.sglmodified
@@ -47,12 +47,42 @@
47
(yaml-decode yaml-text)))
48
(body (string-join (cdr rest) "\n")))
49
(values (or fm (dict)) body)))
+50
;; DATA-LOSS GUARD: a markdown heading line can never be part of
+51
;; real YAML frontmatter. If we reach one before finding the
+52
;; closing fence, the opening `---` was NOT a frontmatter
+53
;; delimiter (the note's closing fence is missing or malformed,
+54
;; e.g. a note with a nested `---...---` block in its body whose
+55
;; top fence was lost). Without this guard we would keep scanning
+56
;; and pair the opening `---` with a `---` deeper in the body,
+57
;; silently absorbing real body content (headings, whole
+58
;; sections) into "frontmatter" and truncating the body -- a
+59
;; clobber that the mutators then persist. Bail safely: treat the
+60
;; entire document as body so nothing is destroyed.
+61
((markdown-heading-line? (car rest))
+62
(values (dict) content))
63
(else
64
(loop (cdr rest)
65
(cons (car rest) fm-lines)))))
66
;; No frontmatter
67
(values (dict) content))))
68
+69
;; A line that, ignoring surrounding whitespace, is a Markdown ATX heading
+70
;; (`#`, `##`, ... up to 6, followed by a space or end of line). Such a line
+71
;; is body content, never YAML frontmatter, so its presence before a closing
+72
;; `---` fence signals that the leading `---` was not a frontmatter opener.
+73
(define (markdown-heading-line? line)
+74
(let* ((trimmed (string-trim line))
+75
(len (string-length trimmed)))
+76
(and (> len 0)
+77
(char=? (string-ref trimmed 0) #\#)
+78
(let count ((i 0))
+79
(cond
+80
((and (< i len) (< i 6) (char=? (string-ref trimmed i) #\#))
+81
(count (+ i 1)))
+82
;; After the run of `#`, require a space or end-of-line.
+83
((>= i len) #t)
+84
(else (char=? (string-ref trimmed i) #\space)))))))
+85
86
;;; Combine frontmatter dict and body into a complete file string.
87
;;;
88
;;; Returns a string with YAML frontmatter wrapped in `---` delimiters
src/folio/note.sglmodified
@@ -190,6 +190,18 @@
190
;; Write Operations
191
;; ============================================================
192
+193
;; Normalize a tags-like value (Scheme list, array, #f, or stray scalar)
+194
;; into a plain list of strings. The MCP layer hands arrays through from
+195
;; JSON; an empty array is truthy (not `null?`) and YAML-encodes badly, so
+196
;; folding everything to a list keeps the create/merge logic predictable.
+197
(define (as-string-list v)
+198
(cond
+199
((not v) '())
+200
((null? v) '())
+201
((pair? v) (filter string? v))
+202
((array? v) (filter string? (array->list v)))
+203
(else '())))
+204
205
;;; Create a new note file.
206
;;;
207
;;; Name can include a subdirectory path like "procedures/email-triage".
@@ -210,15 +222,40 @@
222
(substring name (+ slash 1) (string-length name))
223
name)))
224
(today (format-date (current-second)))
−213
(base-fm (dict tags: tags
−214
created: today
−215
last-updated: today))
−216
(fm (if (null? tasks)
−217
base-fm
−218
(dict-merge base-fm (dict tasks: tasks))))
−219
(body (string-append "\n# " title "\n\n" content "\n")))
−220
(write-file-string file-path (write-frontmatter fm body))
−221
file-path))
+225
;; Tags may arrive as a Scheme list OR (from the MCP/JSON layer) as
+226
;; an array. Normalize to a list so emptiness checks and YAML
+227
;; encoding behave consistently (an empty array is not `null?`, and
+228
;; encoding one emitted a malformed `tags:` block).
+229
(tags (as-string-list tags)))
+230
;; If the supplied content ITSELF begins with a `---...---` frontmatter
+231
;; block, do NOT wrap it again -- that produces a note with two
+232
;; back-to-back / nested frontmatter blocks on disk, a malformed state
+233
;; that downstream parsing and appends mishandle (the data-loss bug).
+234
;; Instead, lift the content's frontmatter fields up and merge them into
+235
;; the note's own frontmatter, using the remaining content as the body.
+236
(let-values (((content-fm content-body) (read-frontmatter content)))
+237
(let* ((has-leading-fm? (not (equal? content-body content)))
+238
;; Tags: caller-provided tags win when non-empty; otherwise
+239
;; fall back to any tags declared in the content's frontmatter.
+240
(merged-tags (if (null? tags)
+241
(as-string-list (dict-ref content-fm tags: '()))
+242
tags))
+243
;; Start from the content's frontmatter (preserves status/goal/
+244
;; custom fields), then let folio-managed fields win.
+245
(base-fm (dict-merge (if has-leading-fm? content-fm (dict))
+246
(dict tags: merged-tags
+247
created: today
+248
last-updated: today)))
+249
(fm (if (null? tasks)
+250
base-fm
+251
(dict-merge base-fm (dict tasks: tasks))))
+252
(body (if has-leading-fm?
+253
;; Content carried its own frontmatter + (usually) its
+254
;; own heading; use it verbatim as the body.
+255
(string-append "\n# " title "\n" content-body "\n")
+256
(string-append "\n# " title "\n\n" content "\n"))))
+257
(write-file-string file-path (write-frontmatter fm body))
+258
file-path))))
259
260
;;; Replace a note's body content, preserving frontmatter.
261
(define (note-edit! file-path new-body)
@@ -241,14 +278,68 @@
278
(write-frontmatter updated-fm body))))))
279
280
;;; Append text to a note's body, preserving frontmatter.
+281
;;;
+282
;;; This is intentionally a RAW append: the existing file content is kept
+283
;;; byte-for-byte and the new text is concatenated at the end, after only
+284
;;; bumping the `last-updated` frontmatter field in place. It deliberately
+285
;;; does NOT round-trip the note through (read-frontmatter -> write-
+286
;;; -frontmatter), because that path can destroy data: if the parser
+287
;;; mis-attributes body content to frontmatter (e.g. a note with a nested
+288
;;; `---...---` block in its body whose top fence is malformed), the
+289
;;; re-serialized body would be truncated and the whole note clobbered.
+290
;;; Appending to the raw content makes body loss structurally impossible and
+291
;;; also avoids re-encoding the frontmatter (which could itself mangle values
+292
;;; such as empty lists). See `note-append!` regression tests.
293
(define (note-append! file-path text)
294
(: string? string? -> void?)
−246
(let ((content (read-file-string file-path)))
−247
(let-values (((fm body) (read-frontmatter content)))
−248
(let ((updated-fm (dict-merge fm (dict last-updated: (format-date (current-second))))))
−249
(write-file-string file-path
−250
(write-frontmatter updated-fm
−251
(string-append body text)))))))
+295
(let* ((content (read-file-string file-path))
+296
(bumped (bump-last-updated-in-text content (format-date (current-second)))))
+297
(write-file-string file-path (string-append bumped text))))
+298
+299
;; Replace the `last-updated:` value inside the leading `---...---`
+300
;; frontmatter block, returning the updated content. Only the first
+301
;; frontmatter block is touched; everything else (including any nested
+302
;; `---` blocks in the body) is preserved exactly. If there is no leading
+303
;; frontmatter block, or no closing fence, the content is returned
+304
;; unchanged -- we never restructure a malformed file.
+305
(define (bump-last-updated-in-text content today)
+306
(let ((lines (string-split content "\n")))
+307
(if (and (pair? lines)
+308
(equal? (string-trim (car lines)) "---"))
+309
(let loop ((rest (cdr lines))
+310
(fm-acc '())
+311
(saw-lu #f))
+312
(cond
+313
;; No closing fence: malformed; leave content untouched.
+314
((null? rest) content)
+315
;; Closing fence: rebuild the file with the patched frontmatter.
+316
((equal? (string-trim (car rest)) "---")
+317
(let* ((fm-lines (reverse fm-acc))
+318
(fm-lines (if saw-lu
+319
fm-lines
+320
;; No last-updated field present: add one.
+321
(append fm-lines
+322
(list (string-append "last-updated: " today)))))
+323
(rest-lines (cdr rest)))
+324
(string-join
+325
(append (list "---")
+326
fm-lines
+327
(list "---")
+328
rest-lines)
+329
"\n")))
+330
;; A last-updated line: replace its value.
+331
((last-updated-line? (car rest))
+332
(loop (cdr rest)
+333
(cons (string-append "last-updated: " today) fm-acc)
+334
#t))
+335
(else
+336
(loop (cdr rest) (cons (car rest) fm-acc) saw-lu))))
+337
;; No frontmatter: nothing to bump, return as-is.
+338
content)))
+339
+340
;; Does a frontmatter line declare the `last-updated` field?
+341
(define (last-updated-line? line)
+342
(string-starts-with? (string-trim line) "last-updated:"))
343
344
;;; Apply a list of old/new string patches to a note's body.
345
;;;
test/test-append-clobber.sgladded
@@ -0,0 +1,288 @@
+1
;; Regression tests for the folio note-edit APPEND CLOBBER data-loss bug.
+2
;;
+3
;; Symptom: appending to a note replaced its entire body with only the newly
+4
;; appended block, leaving just the frontmatter + new section.
+5
;;
+6
;; Three compounding defects were involved:
+7
;; 1. read-frontmatter mis-handled the frontmatter boundary on notes with a
+8
;; nested `---...---` block in the body whose top fence was malformed,
+9
;; absorbing real body content into "frontmatter" and truncating the body.
+10
;; 2. note-create double-wrapped frontmatter when the supplied content itself
+11
;; began with a `---...---` block, producing the malformed on-disk state.
+12
;; 3. note-append round-tripped through read/write-frontmatter, persisting any
+13
;; misparse (and re-encoding frontmatter) -- so a parse glitch became
+14
;; permanent data loss.
+15
;;
+16
;; The fixes: harden read-frontmatter, merge leading frontmatter in note-create,
+17
;; and make note-append a raw, body-preserving append.
+18
+19
(import (sigil test)
+20
(sigil string)
+21
(sigil dict)
+22
(sigil fs)
+23
(sigil io)
+24
(sigil path)
+25
(folio store)
+26
(folio note)
+27
(folio frontmatter))
+28
+29
;; A note whose BODY legitimately contains a nested `---...---` frontmatter-style
+30
;; block (this is what folio produces when note-create wraps content that itself
+31
;; had frontmatter -- daily notes and task briefs both look like this).
+32
(define (well-formed-nested)
+33
(string-append
+34
"---\n"
+35
"tags: []\n"
+36
"created: 2026-06-25\n"
+37
"last-updated: 2026-06-25\n"
+38
"---\n"
+39
"\n"
+40
"# 2026-06-25\n"
+41
"\n"
+42
"---\n"
+43
"last-updated: 2026-06-25\n"
+44
"---\n"
+45
"\n"
+46
"## PRIMER\n"
+47
"Keep this primer text.\n"
+48
"\n"
+49
"## Briefing\n"
+50
"Keep this briefing text.\n"))
+51
+52
;; ============================================================
+53
;; Fix 1: read-frontmatter boundary hardening
+54
;; ============================================================
+55
+56
(test-group "read-frontmatter boundary hardening"
+57
;; Well-formed note with a nested block: body fully preserved, frontmatter ok.
+58
(test "well-formed nested block keeps full body"
+59
(let-values (((fm body) (read-frontmatter (well-formed-nested))))
+60
(assert-true (string-contains? body "# 2026-06-25"))
+61
(assert-true (string-contains? body "## PRIMER"))
+62
(assert-true (string-contains? body "## Briefing"))
+63
(assert-equal "2026-06-25" (dict-ref fm created: #f))))
+64
+65
;; Malformed: the TOP closing fence is missing, so the first `---` the old
+66
;; parser would find is the nested block's opener. Before the fix it absorbed
+67
;; the `# 2026-06-25` heading + primer into "frontmatter" and truncated the
+68
;; body. After the fix it bails on the heading and keeps everything as body.
+69
(test "missing top fence does not absorb body (no data loss)"
+70
(let ((malformed
+71
(string-append
+72
"---\n"
+73
"tags: []\n"
+74
"created: 2026-06-25\n"
+75
;; <- no closing --- here
+76
"\n"
+77
"# 2026-06-25\n"
+78
"\n"
+79
"---\n"
+80
"last-updated: 2026-06-25\n"
+81
"---\n"
+82
"\n"
+83
"## PRIMER\n"
+84
"Keep this primer text.\n")))
+85
(let-values (((fm body) (read-frontmatter malformed)))
+86
;; All real content must survive in the body, nothing destroyed.
+87
(assert-true (string-contains? body "# 2026-06-25"))
+88
(assert-true (string-contains? body "## PRIMER"))
+89
(assert-true (string-contains? body "Keep this primer text."))))))
+90
+91
;; ============================================================
+92
;; Fix 3: note-append is raw and body-preserving
+93
;; ============================================================
+94
+95
(test-group "note-append! preserves body"
+96
(test "append to nested-block note keeps all prior content"
+97
(call-with-temp-directory
+98
(lambda (dir)
+99
(let ((file (path-join dir "daily.md")))
+100
(write-file-string file (well-formed-nested))
+101
(note-append! file "\n## New Section\nFresh appended content.\n")
+102
(let ((after (read-file-string file)))
+103
(assert-true (string-contains? after "## PRIMER"))
+104
(assert-true (string-contains? after "## Briefing"))
+105
(assert-true (string-contains? after "Keep this primer text."))
+106
(assert-true (string-contains? after "Fresh appended content.")))))))
+107
+108
(test "three successive appends never clobber"
+109
(call-with-temp-directory
+110
(lambda (dir)
+111
(let ((file (path-join dir "daily.md")))
+112
(write-file-string file (well-formed-nested))
+113
(note-append! file "\n## A\nAlpha.\n")
+114
(note-append! file "\n## B\nBeta.\n")
+115
(note-append! file "\n## C\nGamma.\n")
+116
(let ((after (read-file-string file)))
+117
(assert-true (string-contains? after "Keep this primer text."))
+118
(assert-true (string-contains? after "Alpha."))
+119
(assert-true (string-contains? after "Beta."))
+120
(assert-true (string-contains? after "Gamma.")))))))
+121
+122
;; Even against a malformed on-disk note (missing top fence), append must not
+123
;; destroy existing content -- the raw append preserves every byte.
+124
(test "append to malformed note does not shrink it"
+125
(call-with-temp-directory
+126
(lambda (dir)
+127
(let* ((file (path-join dir "n.md"))
+128
(malformed (string-append
+129
"---\n"
+130
"tags: []\n"
+131
"created: 2026-06-25\n"
+132
"\n"
+133
"# Heading\n"
+134
"\n"
+135
"---\n"
+136
"k: v\n"
+137
"---\n"
+138
"\n"
+139
"## Section\n"
+140
"Important body that must survive.\n")))
+141
(write-file-string file malformed)
+142
(note-append! file "\n## Appended\nNew line.\n")
+143
(let ((after (read-file-string file)))
+144
;; Every original line preserved verbatim, plus the new text.
+145
(assert-true (string-contains? after "# Heading"))
+146
(assert-true (string-contains? after "## Section"))
+147
(assert-true (string-contains? after "Important body that must survive."))
+148
(assert-true (string-contains? after "New line."))
+149
;; The append never makes the file smaller.
+150
(assert-true (> (string-length after) (string-length malformed))))))))
+151
+152
(test "append bumps last-updated in place"
+153
(call-with-temp-directory
+154
(lambda (dir)
+155
(let ((file (path-join dir "n.md")))
+156
(write-file-string file
+157
"---\ntags: []\ncreated: 2026-01-01\nlast-updated: 2026-01-01\n---\n\n# Note\n\nBody.\n")
+158
(note-append! file "\nMore.\n")
+159
(let ((note (read-note file)))
+160
;; created untouched, last-updated bumped to today (not 2026-01-01),
+161
;; body fully preserved.
+162
(assert-equal "2026-01-01" (dict-ref (folio-note-frontmatter note) created: #f))
+163
(assert-false (equal? "2026-01-01" (note-last-updated note)))
+164
(assert-true (string-contains? (folio-note-body note) "Body."))
+165
(assert-true (string-contains? (folio-note-body note) "More.")))))))
+166
+167
(test "append adds last-updated when frontmatter lacks it"
+168
(call-with-temp-directory
+169
(lambda (dir)
+170
(let ((file (path-join dir "n.md")))
+171
(write-file-string file "---\ntags: []\ncreated: 2026-01-01\n---\n\n# Note\n\nBody.\n")
+172
(note-append! file "\nMore.\n")
+173
(let ((note (read-note file)))
+174
(assert-true (string? (note-last-updated note)))
+175
(assert-true (string-contains? (folio-note-body note) "Body."))
+176
(assert-true (string-contains? (folio-note-body note) "More."))))))))
+177
+178
;; ============================================================
+179
;; Fix 2: note-create merges leading frontmatter (no double-wrap)
+180
;; ============================================================
+181
+182
(test-group "note-create! merges leading frontmatter"
+183
(test "content with its own frontmatter does not double-wrap"
+184
(call-with-temp-directory
+185
(lambda (dir)
+186
(let* ((store (make-folio-store dir))
+187
(content (string-append
+188
"---\n"
+189
"tags:\n - task-brief\n - folio\n"
+190
"status: assigned\n"
+191
"---\n"
+192
"\n"
+193
"# Real Title\n"
+194
"\n"
+195
"Real body content.\n"))
+196
(file (note-create! store "tasks/example" '() content)))
+197
(let ((raw (read-file-string file)))
+198
;; Exactly ONE frontmatter block: the file has a single `---\n...\n---`
+199
;; opener/closer pair at the top, never two adjacent or nested-as-fm.
+200
(assert-false (string-contains? raw "---\n---\n"))
+201
;; The note must parse cleanly: frontmatter fields lifted up.
+202
(let ((note (read-note file)))
+203
(assert-true (member "task-brief" (note-tags note)))
+204
(assert-true (member "folio" (note-tags note)))
+205
(assert-equal "assigned"
+206
(dict-ref (folio-note-frontmatter note) status: #f))
+207
(assert-true (string? (note-created note)))
+208
;; Body keeps the real content; the inner `---` block is gone
+209
;; (merged), so the body has no leftover frontmatter fences.
+210
(assert-true (string-contains? (folio-note-body note) "Real body content."))))))))
+211
+212
(test "create-then-append on frontmatter content never clobbers"
+213
(call-with-temp-directory
+214
(lambda (dir)
+215
(let* ((store (make-folio-store dir))
+216
(content "---\ntags:\n - brief\n---\n\n# Title\n\nFirst body.\n")
+217
(file (note-create! store "tasks/cascade" '() content)))
+218
(note-append! file "\n## Update One\nSecond body.\n")
+219
(note-append! file "\n## Update Two\nThird body.\n")
+220
(let ((after (read-file-string file)))
+221
(assert-true (string-contains? after "First body."))
+222
(assert-true (string-contains? after "Second body."))
+223
(assert-true (string-contains? after "Third body.")))))))
+224
+225
(test "plain content (no leading frontmatter) is unchanged behavior"
+226
(call-with-temp-directory
+227
(lambda (dir)
+228
(let* ((store (make-folio-store dir))
+229
(file (note-create! store "Plain Note" '("x") "Just text.")))
+230
(let ((note (read-note file)))
+231
(assert-true (string-contains? (folio-note-body note) "Just text."))
+232
(assert-true (member "x" (note-tags note)))))))))
+233
+234
;; ============================================================
+235
;; Tags arrive from the MCP/JSON layer as an *array*, not a list.
+236
;; An empty array is not `null?`, and YAML-encoding it produced a malformed
+237
;; `tags:` block; it also wrongly suppressed the content's own tags.
+238
;; ============================================================
+239
+240
(test-group "note-create! normalizes array tags"
+241
(test "empty array tags falls back to content frontmatter tags"
+242
(call-with-temp-directory
+243
(lambda (dir)
+244
(let* ((store (make-folio-store dir))
+245
(content "---\ntags:\n - alpha\n - beta\n---\n\n# Title\n\nBody.\n")
+246
(file (note-create! store "tasks/arr" (list->array '()) content))
+247
(note (read-note file))
+248
(raw (read-file-string file)))
+249
(assert-true (member "alpha" (note-tags note)))
+250
(assert-true (member "beta" (note-tags note)))
+251
;; And no malformed empty-list frontmatter line on disk.
+252
(assert-false (string-contains? raw "\n[]\n"))))))
+253
+254
(test "non-empty array tags are used"
+255
(call-with-temp-directory
+256
(lambda (dir)
+257
(let* ((store (make-folio-store dir))
+258
(file (note-create! store "tasks/arr2" (list->array '("one" "two"))
+259
"Plain body."))
+260
(note (read-note file)))
+261
(assert-true (member "one" (note-tags note)))
+262
(assert-true (member "two" (note-tags note))))))))
+263
+264
;; ============================================================
+265
;; prepend/replace/patch sanity (must remain unaffected)
+266
;; ============================================================
+267
+268
(test-group "other mutators unaffected"
+269
(test "note-edit! (replace) still replaces body, keeps frontmatter"
+270
(call-with-temp-directory
+271
(lambda (dir)
+272
(let* ((store (make-folio-store dir))
+273
(file (note-create! store "R" '("t") "Original.")))
+274
(note-edit! file "\n# R\n\nReplaced body.\n")
+275
(let ((note (read-note file)))
+276
(assert-true (string-contains? (folio-note-body note) "Replaced body."))
+277
(assert-false (string-contains? (folio-note-body note) "Original."))
+278
(assert-true (member "t" (note-tags note))))))))
+279
+280
(test "note-patch! still preserves frontmatter and body"
+281
(call-with-temp-directory
+282
(lambda (dir)
+283
(let* ((store (make-folio-store dir))
+284
(file (note-create! store "P" '("a" "b") "Hello world.")))
+285
(note-patch! file (list (cons "Hello" "Hi")))
+286
(let ((note (read-note file)))
+287
(assert-true (string-contains? (folio-note-body note) "Hi world."))
+288
(assert-equal 2 (length (note-tags note)))))))))