test/test-tokenize.sgl — sigil-format

1;;; Tests for the io-free (sigil format tokenize) sub-library.

2;;;

3;;; These import the tokenizer DIRECTLY (not via (sigil format)) to prove the

4;;; sub-library stands alone with the full tokenizer API and no formatter deps.

6(import (sigil test)

7 (sigil format tokenize))

9(test-group "tokenize (direct sub-library import)"

10 (test "simple expression: types and values"

11 (let* ((result (tokenize "(+ 1 2)" "test.sgl"))

12 (chars (tokenize-result-chars result))

13 (tokens (tokenize-result-tokens result)))

14 (assert-true (token? (car tokens)))

15 ;; ( + <ws> 1 <ws> 2 ) eof

16 (assert-equal 'lparen (token-type (list-ref tokens 0)))

17 (assert-equal "(" (token-value (list-ref tokens 0) chars))

18 (assert-equal 'symbol (token-type (list-ref tokens 1)))

19 (assert-equal "+" (token-value (list-ref tokens 1) chars))

20 (assert-equal 'number (token-type (list-ref tokens 3)))

21 (assert-equal "1" (token-value (list-ref tokens 3) chars))

22 (assert-equal 'rparen (token-type (list-ref tokens 6)))

23 (assert-equal 'eof (token-type (list-ref tokens 7)))))

25 (test "line/column/indent positions on a multi-line snippet"

26 (let* ((result (tokenize "(define (foo x)\n (+ x 1))" "test.sgl"))

27 (tokens (tokenize-result-tokens result))

28 (tok0 (car tokens)))

29 ;; First token: the opening paren at line 1, column 1, indent 0.

30 (assert-equal 'lparen (token-type tok0))

31 (assert-equal 1 (token-line tok0))

32 (assert-equal 1 (token-column tok0))

33 (assert-equal 0 (token-indent tok0))

34 ;; Find the lparen that opens the second line's (+ ...) form.

35 (let loop ((ts tokens))

36 (cond

37 ((null? ts) (assert-true #f)) ; must exist

38 ((and (eq? (token-type (car ts)) 'lparen)

39 (= (token-line (car ts)) 2))

40 (assert-equal 3 (token-column (car ts)))

41 (assert-equal 2 (token-indent (car ts))))

42 (else (loop (cdr ts)))))))

44 (test "start/end accessors delimit the source slice"

45 (let* ((result (tokenize "abc" "test.sgl"))

46 (chars (tokenize-result-chars result))

47 (tok (car (tokenize-result-tokens result))))

48 (assert-equal 'symbol (token-type tok))

49 (assert-equal 0 (token-start tok))

50 (assert-equal 3 (token-end tok))

51 (assert-equal "abc" (token-value tok chars))))

53 (test "string, comment, and hash tokens"

54 (let* ((sresult (tokenize "\"hi\"" "t.sgl"))

55 (stoks (tokenize-result-tokens sresult))

56 (cresult (tokenize "; note\n" "t.sgl"))

57 (ctoks (tokenize-result-tokens cresult))

58 (hresult (tokenize "#t" "t.sgl"))

59 (htoks (tokenize-result-tokens hresult)))

60 (assert-equal 'string (token-type (car stoks)))

61 (assert-equal 'comment (token-type (car ctoks)))

62 (assert-equal 'hash-t (token-type (car htoks))))))

64(run-tests)