Extract io-free (sigil format tokenize) sub-library

Message

Move the positioned Sigil tokenizer (token type + accessors, tokenize, tokenize-result-*) out of the monolithic (sigil format) library into a standalone (sigil format tokenize) module that depends only on (sigil core). Downstream consumers (e.g. Slate Tier-B highlighting, compiled to WASM) can now import just the tokenizer without pulling in the formatter or its deps (sigil-json, io/ports/fs).

(sigil format) imports and re-exports the same tokenizer bindings, so existing consumers keep working unchanged with no API break and no duplicate definitions. Drop the unused char-whitespace? helper in the move.

Add test/test-tokenize.sgl exercising the sub-library via a direct import; existing test-format.sgl continues to cover the re-export path.

Bump to 0.16.2 (additive patch).

Changed

 package.sgl                   |   2 +-
 src/sigil/format.sgl          | 375 +++++----------------------------------------------------------------------------------------------------------------------------------------------------
 src/sigil/format/tokenize.sgl | 395 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 test/test-tokenize.sgl        |  64 +++++++++++++++++++++++++++
 4 files changed, 472 insertions(+), 364 deletions(-)

Diff

package.sglmodified

@@ -5,7 +5,7 @@

(package

name: "sigil-format"

−8

version: "0.16.1"

version: "0.16.2"

sigil: "^0.17"

description: "Sigil code formatter (paren-inference + AST-aware reflow)"

url: "https://codeberg.org/sigil/sigil-format"

src/sigil/format.sglmodified

@@ -15,7 +15,8 @@

(import (sigil struct)

(sigil string)

(sigil io)

−18

(sigil fs))

+18

(sigil fs)

+19

(sigil format tokenize))

(export

;; Core API

@@ -54,7 +55,8 @@

paren-inference-confidence

paren-inference-reason

−57

;; Token inspection (for debugging/testing)

+58

;; Positioned tokenizer — re-exported from (sigil format tokenize)

+59

;; so existing (sigil format) consumers keep working unchanged.

token?

token-type

token-value

@@ -62,7 +64,6 @@

token-column

token-indent

−65

;; Tokenizer (for testing)

tokenize

tokenize-result-chars

tokenize-result-tokens)

@@ -101,367 +102,15 @@

102

(reason)) ; explanation string

103

104

;; ============================================================

−104

;;; Tokens

+105

;;; Tokens & Tokenizer

106

;; ============================================================

−106

−107

;; Token layout: #(type start end line column indent)

−108

;; Using raw vectors avoids keyword-arg overhead in the hot path.

−109

−110

;;; Check if value is a token

−111

(define (token? x) (and (vector? x) (= (vector-length x) 6)))

−112

−113

;;; Token type accessor

−114

(define (token-type tok) (vector-ref tok 0))

−115

−116

;;; Token start position accessor

−117

(define (token-start tok) (vector-ref tok 1))

−118

−119

;;; Token end position accessor

−120

(define (token-end tok) (vector-ref tok 2))

−121

−122

;;; Token line accessor

−123

(define (token-line tok) (vector-ref tok 3))

−124

−125

;;; Token column accessor

−126

(define (token-column tok) (vector-ref tok 4))

−127

−128

;;; Token indent accessor

−129

(define (token-indent tok) (vector-ref tok 5))

−130

−131

;;; Extract the text value of a token from the source chars vector

−132

(define (token-value tok chars)

−133

(let* ((s (vector-ref tok 1))

−134

(e (vector-ref tok 2))

−135

(n (- e s))

−136

(v (make-string n)))

−137

(let loop ((i 0))

−138

(if (>= i n)

−139

v

−140

(begin

−141

(string-set! v i (vector-ref chars (+ s i)))

−142

(loop (+ i 1)))))))

−143

−144

;;; Get the chars vector from a tokenize result

−145

(define (tokenize-result-chars result) (car result))

−146

−147

;;; Get the token list from a tokenize result

−148

(define (tokenize-result-tokens result) (cdr result))

−149

−150

;; ============================================================

−151

;;; Tokenizer Implementation

−152

;; ============================================================

−153

−154

;;; Check if character is whitespace

−155

(define (char-whitespace? c)

−156

(and c

−157

(or (char=? c #\space)

−158

(char=? c #\tab)

−159

(char=? c #\newline)

−160

(char=? c #\return))))

−161

−162

;;; Check if character is a digit

−163

(define (char-digit? c)

−164

(and c

−165

(char>=? c #\0)

−166

(char<=? c #\9)))

−167

−168

;; Delimiter lookup table — indexed by char code, #t if delimiter

−169

(define delimiter-table

−170

(let ((t (make-vector 128 #f)))

−171

(for-each (lambda (c) (vector-set! t (char->integer c) #t))

−172

'(#\( #\) #\[ #\] #\{ #\} #\" #\; #\' #\` #\,

−173

#\space #\tab #\newline #\return))

−174

t))

−175

−176

;;; Check if character is a delimiter

−177

(define (delimiter? c)

−178

(or (not c)

−179

(let ((code (char->integer c)))

−180

(and (< code 128) (vector-ref delimiter-table code)))))

−181

−182

;;; Check if string looks like a number

−183

(define (looks-like-number? s)

−184

(let ((len (string-length s)))

−185

(if (= len 0)

−186

#f

−187

(let ((first (string-ref s 0)))

−188

(cond

−189

((char-digit? first) #t)

−190

((and (or (char=? first #\+) (char=? first #\-))

−191

(> len 1)

−192

(char-digit? (string-ref s 1))) #t)

−193

((and (char=? first #\.)

−194

(> len 1)

−195

(char-digit? (string-ref s 1))) #t)

−196

(else #f))))))

−197

−198

;;; Tokenize entire source into a list of tokens

−199

;;; Returns (cons chars tokens) where chars is the source as a vector

−200

(define (tokenize source filename)

−201

(let* ((chars (string->vector source))

−202

(len (vector-length chars))

−203

(pos 0)

−204

(cur-line 1)

−205

(col 1)

−206

(indent 0))

−207

−208

;; Advance one non-newline char (hot path)

−209

(define (advance-one!)

−210

(set! pos (+ pos 1))

−211

(set! col (+ col 1)))

−212

−213

;; Advance past a known newline, recalculate indent

−214

(define (advance-newline!)

−215

(set! pos (+ pos 1))

−216

(set! cur-line (+ cur-line 1))

−217

(set! col 1)

−218

(let loop ((off 0))

−219

(if (>= (+ pos off) len)

−220

(set! indent off)

−221

(let ((ch (vector-ref chars (+ pos off))))

−222

(cond

−223

((char=? ch #\space) (loop (+ off 1)))

−224

((char=? ch #\tab) (loop (+ off 2)))

−225

(else (set! indent off)))))))

−226

−227

;; Full advance with newline handling (for block comments only)

−228

(define (advance!)

−229

(when (< pos len)

−230

(let ((c (vector-ref chars pos)))

−231

(if (char=? c #\newline)

−232

(advance-newline!)

−233

(advance-one!)))))

−234

−235

;; Skip whitespace (not newlines) — inlined, no closure calls

−236

(define (skip-whitespace start-pos start-line start-col start-indent)

−237

(let loop ()

−238

(if (< pos len)

−239

(let ((c (vector-ref chars pos)))

−240

(cond

−241

((or (char=? c #\space) (char=? c #\tab))

−242

(set! pos (+ pos 1))

−243

(set! col (+ col 1))

−244

(loop))

−245

((char=? c #\return)

−246

(set! pos (+ pos 1))

−247

(loop))

−248

(else

−249

(vector 'whitespace start-pos pos start-line start-col start-indent))))

−250

(vector 'whitespace start-pos pos start-line start-col start-indent))))

−251

−252

;; Skip to delimiter — inlined with direct table lookup

−253

(define (skip-to-delimiter)

−254

(let loop ()

−255

(if (>= pos len)

−256

pos

−257

(let ((code (char->integer (vector-ref chars pos))))

−258

(if (and (< code 128) (vector-ref delimiter-table code))

−259

pos

−260

(begin

−261

(set! pos (+ pos 1))

−262

(set! col (+ col 1))

−263

(loop)))))))

−264

−265

;; Skip until newline — inlined

−266

(define (skip-until-newline)

−267

(let loop ()

−268

(if (>= pos len)

−269

pos

−270

(if (char=? (vector-ref chars pos) #\newline)

−271

pos

−272

(begin

−273

(set! pos (+ pos 1))

−274

(set! col (+ col 1))

−275

(loop))))))

−276

−277

;; Count leading spaces from current position

−278

(define (count-leading-spaces)

−279

(let loop ((off 0))

−280

(if (>= (+ pos off) len)

−281

off

−282

(let ((c (vector-ref chars (+ pos off))))

−283

(cond

−284

((char=? c #\space) (loop (+ off 1)))

−285

((char=? c #\tab) (loop (+ off 2)))

−286

(else off))))))

−287

−288

;; Read a comment token — inlined semicolons and skip

−289

(define (read-comment start-pos start-line start-col start-indent)

−290

(let sloop ((count 0))

−291

(if (and (< pos len) (char=? (vector-ref chars pos) #\;))

−292

(begin

−293

(set! pos (+ pos 1))

−294

(set! col (+ col 1))

−295

(sloop (+ count 1)))

−296

(begin

−297

(skip-until-newline)

−298

(vector (if (>= count 3) 'doc-comment 'comment)

−299

start-pos pos start-line start-col start-indent)))))

−300

−301

;; Read a string token — inlined with fast path for regular chars

−302

(define (read-string-token start-pos start-line start-col start-indent)

−303

(advance-one!) ; consume opening quote

−304

(let loop ()

−305

(if (>= pos len)

−306

(vector 'string start-pos pos start-line start-col start-indent)

−307

(let ((c (vector-ref chars pos)))

−308

(cond

−309

((char=? c #\")

−310

(advance-one!)

−311

(vector 'string start-pos pos start-line start-col start-indent))

−312

((char=? c #\\)

−313

(advance-one!) ; backslash

−314

(when (< pos len)

−315

(if (char=? (vector-ref chars pos) #\newline)

−316

(advance-newline!)

−317

(advance-one!)))

−318

(loop))

−319

((char=? c #\newline)

−320

(advance-newline!)

−321

(loop))

−322

(else

−323

(set! pos (+ pos 1))

−324

(set! col (+ col 1))

−325

(loop)))))))

−326

−327

;; Read a hash token — advance-one! for non-newline chars

−328

(define (read-hash-token start-pos start-line start-col start-indent)

−329

(advance-one!) ; consume #

−330

(if (>= pos len)

−331

(vector 'hash-other start-pos pos start-line start-col start-indent)

−332

(let ((c (vector-ref chars pos)))

−333

(cond

−334

((or (char=? c #\t) (char=? c #\T))

−335

(advance-one!)

−336

(vector 'hash-t start-pos pos start-line start-col start-indent))

−337

((or (char=? c #\f) (char=? c #\F))

−338

(advance-one!)

−339

(vector 'hash-f start-pos pos start-line start-col start-indent))

−340

((char=? c #\\)

−341

;; Character literal

−342

(advance-one!) ; consume backslash

−343

(if (>= pos len)

−344

(vector 'char start-pos pos start-line start-col start-indent)

−345

(begin

−346

(advance-one!) ; consume first char after backslash

−347

(skip-to-delimiter)

−348

(vector 'char start-pos pos start-line start-col start-indent))))

−349

((char=? c #\|)

−350

;; Block comment — uses full advance! since it can span lines

−351

(advance-one!) ; consume |

−352

(let loop ((depth 1))

−353

(if (= depth 0)

−354

(vector 'block-comment start-pos pos start-line start-col start-indent)

−355

(if (>= pos len)

−356

(vector 'block-comment start-pos pos start-line start-col start-indent)

−357

(let ((c (vector-ref chars pos)))

−358

(cond

−359

((char=? c #\|)

−360

(advance-one!) ; | is not newline

−361

(if (and (< pos len) (char=? (vector-ref chars pos) #\#))

−362

(begin (advance-one!) (loop (- depth 1)))

−363

(loop depth)))

−364

((char=? c #\#)

−365

(advance-one!) ; # is not newline

−366

(if (and (< pos len) (char=? (vector-ref chars pos) #\|))

−367

(begin (advance-one!) (loop (+ depth 1)))

−368

(loop depth)))

−369

(else

−370

(advance!) ; could be newline

−371

(loop depth))))))))

−372

((char=? c #\{)

−373

(advance-one!)

−374

(vector 'hash-lbrace start-pos pos start-line start-col start-indent))

−375

((char=? c #\[)

−376

(advance-one!)

−377

(vector 'hash-lbracket start-pos pos start-line start-col start-indent))

−378

(else

−379

;; Hash datum like #:keyword

−380

(skip-to-delimiter)

−381

(vector 'hash-other start-pos pos start-line start-col start-indent))))))

−382

−383

;; Initialize indent for first line

−384

(set! indent (count-leading-spaces))

−385

−386

;; Main tokenization loop

−387

(let loop ((tokens '()))

−388

(if (>= pos len)

−389

;; EOF token

−390

(let ((eof-tok (vector 'eof pos pos cur-line col indent)))

−391

(cons chars (reverse (cons eof-tok tokens))))

−392

(let ((c (vector-ref chars pos))

−393

(start-pos pos)

−394

(start-line cur-line)

−395

(start-col col)

−396

(start-indent indent))

−397

(cond

−398

;; Whitespace (not newline)

−399

((or (char=? c #\space) (char=? c #\tab) (char=? c #\return))

−400

(loop (cons (skip-whitespace start-pos start-line start-col start-indent) tokens)))

−401

−402

;; Newline — inline advance-newline!

−403

((char=? c #\newline)

−404

(advance-newline!)

−405

(loop (cons (vector 'newline start-pos pos start-line start-col start-indent) tokens)))

−406

−407

;; Comment

−408

((char=? c #\;)

−409

(loop (cons (read-comment start-pos start-line start-col start-indent) tokens)))

−410

−411

;; Parens and brackets — advance-one! (none are newlines)

−412

                  ((char=? c #\() (advance-one!) (loop (cons (vector 'lparen start-pos pos start-line start-col start-indent) tokens)))

−413

                  ((char=? c #\)) (advance-one!) (loop (cons (vector 'rparen start-pos pos start-line start-col start-indent) tokens)))

−414

                  ((char=? c #\[) (advance-one!) (loop (cons (vector 'lbracket start-pos pos start-line start-col start-indent) tokens)))

−415

                  ((char=? c #\]) (advance-one!) (loop (cons (vector 'rbracket start-pos pos start-line start-col start-indent) tokens)))

−416

                  ((char=? c #\{) (advance-one!) (loop (cons (vector 'lbrace start-pos pos start-line start-col start-indent) tokens)))

−417

                  ((char=? c #\}) (advance-one!) (loop (cons (vector 'rbrace start-pos pos start-line start-col start-indent) tokens)))

−418

−419

;; Quote forms

−420

                  ((char=? c #\') (advance-one!) (loop (cons (vector 'quote start-pos pos start-line start-col start-indent) tokens)))

−421

                  ((char=? c #\`) (advance-one!) (loop (cons (vector 'quasiquote start-pos pos start-line start-col start-indent) tokens)))

−422

((char=? c #\,)

−423

(advance-one!)

−424

(if (and (< pos len) (char=? (vector-ref chars pos) #\@))

−425

                       (begin (advance-one!) (loop (cons (vector 'unquote-splicing start-pos pos start-line start-col start-indent) tokens)))

−426

(loop (cons (vector 'unquote start-pos pos start-line start-col start-indent) tokens))))

−427

−428

;; String

−429

((char=? c #\")

−430

(loop (cons (read-string-token start-pos start-line start-col start-indent) tokens)))

−431

−432

;; Hash forms

−433

((char=? c #\#)

−434

(loop (cons (read-hash-token start-pos start-line start-col start-indent) tokens)))

−435

−436

;; Dot

−437

((char=? c #\.)

−438

(let ((next (if (< (+ pos 1) len) (vector-ref chars (+ pos 1)) #f)))

−439

(if (delimiter? next)

−440

(begin

−441

(advance-one!)

−442

(loop (cons (vector 'dot start-pos pos start-line start-col start-indent) tokens)))

−443

(begin

−444

(skip-to-delimiter)

−445

                           (let* ((text (token-value (vector 'symbol start-pos pos start-line start-col start-indent) chars))

−446

(tok-type (if (looks-like-number? text) 'number 'symbol)))

−447

(loop (cons (vector tok-type start-pos pos start-line start-col start-indent) tokens)))))))

−448

−449

;; Number or +/-

−450

((or (char=? c #\+) (char=? c #\-))

−451

(skip-to-delimiter)

−452

(let* ((text (token-value (vector 'symbol start-pos pos start-line start-col start-indent) chars))

−453

(tok-type (if (looks-like-number? text) 'number 'symbol)))

−454

(loop (cons (vector tok-type start-pos pos start-line start-col start-indent) tokens))))

−455

−456

;; Number

−457

((and (char>=? c #\0) (char<=? c #\9))

−458

(skip-to-delimiter)

−459

(loop (cons (vector 'number start-pos pos start-line start-col start-indent) tokens)))

−460

−461

;; Symbol (everything else)

−462

(else

−463

(skip-to-delimiter)

−464

(loop (cons (vector 'symbol start-pos pos start-line start-col start-indent) tokens)))))))))

+107

;;

+108

;; The positioned tokenizer (token type + accessors, `tokenize`, and the

+109

;; `tokenize-result-*` accessors) lives in the io-free sub-library

+110

;; (sigil format tokenize). It is imported above and re-exported below so

+111

;; existing (sigil format) consumers keep working unchanged, while WASM

+112

;; consumers (e.g. Slate Tier-B highlighting) can import just the

+113

;; tokenizer without pulling in the formatter or its deps.

114

115

;; ============================================================

116

;;; Paren Stack for Tracking

src/sigil/format/tokenize.sgladded

@@ -0,0 +1,395 @@

;;; (sigil format tokenize) - Positioned Sigil tokenizer (io-free)

;;;

;;; The positioned tokenizer extracted from (sigil format) so downstream

;;; consumers (e.g. Slate's Tier-B syntax highlighting, compiled to WASM)

;;; can import JUST the tokenizer without pulling in the formatter or its

;;; dependencies (sigil-json, io/ports/fs).

;;;

;;; This library is deliberately io-free: it depends only on (sigil core).

;;; `tokenize` takes a source string + filename and returns a

+10

;;; `(cons chars tokens)` result; a token is a raw 6-slot vector.

+11

;;;

+12

;;; Example:

+13

;;; (let* ((r (tokenize "(+ 1 2)" "test.sgl"))

+14

;;; (chars (tokenize-result-chars r))

+15

;;; (toks (tokenize-result-tokens r)))

+16

;;; (token-type (car toks)) ; => lparen

+17

;;; (token-value (car toks) chars)) ; => "("

+18

+19

(define-library (sigil format tokenize)

+20

(import (sigil core))

+21

+22

(export

+23

;; Token predicate + accessors

+24

token?

+25

token-type

+26

token-start

+27

token-end

+28

token-line

+29

token-column

+30

token-indent

+31

token-value

+32

+33

;; Tokenize result accessors

+34

tokenize-result-chars

+35

tokenize-result-tokens

+36

+37

;; Tokenizer entry point

+38

tokenize)

+39

+40

(begin

+41

+42

;; ============================================================

+43

;;; Tokens

+44

;; ============================================================

+45

+46

;; Token layout: #(type start end line column indent)

+47

;; Using raw vectors avoids keyword-arg overhead in the hot path.

+48

+49

;;; Check if value is a token

+50

(define (token? x) (and (vector? x) (= (vector-length x) 6)))

+51

+52

;;; Token type accessor

+53

(define (token-type tok) (vector-ref tok 0))

+54

+55

;;; Token start position accessor

+56

(define (token-start tok) (vector-ref tok 1))

+57

+58

;;; Token end position accessor

+59

(define (token-end tok) (vector-ref tok 2))

+60

+61

;;; Token line accessor

+62

(define (token-line tok) (vector-ref tok 3))

+63

+64

;;; Token column accessor

+65

(define (token-column tok) (vector-ref tok 4))

+66

+67

;;; Token indent accessor

+68

(define (token-indent tok) (vector-ref tok 5))

+69

+70

;;; Extract the text value of a token from the source chars vector

+71

(define (token-value tok chars)

+72

(let* ((s (vector-ref tok 1))

+73

(e (vector-ref tok 2))

+74

(n (- e s))

+75

(v (make-string n)))

+76

(let loop ((i 0))

+77

(if (>= i n)

+78

v

+79

(begin

+80

(string-set! v i (vector-ref chars (+ s i)))

+81

(loop (+ i 1)))))))

+82

+83

;;; Get the chars vector from a tokenize result

+84

(define (tokenize-result-chars result) (car result))

+85

+86

;;; Get the token list from a tokenize result

+87

(define (tokenize-result-tokens result) (cdr result))

+88

+89

;; ============================================================

+90

;;; Tokenizer Implementation

+91

;; ============================================================

+92

+93

;;; Check if character is a digit

+94

(define (char-digit? c)

+95

(and c

+96

(char>=? c #\0)

+97

(char<=? c #\9)))

+98

+99

;; Delimiter lookup table — indexed by char code, #t if delimiter

+100

(define delimiter-table

+101

(let ((t (make-vector 128 #f)))

+102

(for-each (lambda (c) (vector-set! t (char->integer c) #t))

+103

'(#\( #\) #\[ #\] #\{ #\} #\" #\; #\' #\` #\,

+104

#\space #\tab #\newline #\return))

+105

t))

+106

+107

;;; Check if character is a delimiter

+108

(define (delimiter? c)

+109

(or (not c)

+110

(let ((code (char->integer c)))

+111

(and (< code 128) (vector-ref delimiter-table code)))))

+112

+113

;;; Check if string looks like a number

+114

(define (looks-like-number? s)

+115

(let ((len (string-length s)))

+116

(if (= len 0)

+117

#f

+118

(let ((first (string-ref s 0)))

+119

(cond

+120

((char-digit? first) #t)

+121

((and (or (char=? first #\+) (char=? first #\-))

+122

(> len 1)

+123

(char-digit? (string-ref s 1))) #t)

+124

((and (char=? first #\.)

+125

(> len 1)

+126

(char-digit? (string-ref s 1))) #t)

+127

(else #f))))))

+128

+129

;;; Tokenize entire source into a list of tokens

+130

;;; Returns (cons chars tokens) where chars is the source as a vector

+131

(define (tokenize source filename)

+132

(let* ((chars (string->vector source))

+133

(len (vector-length chars))

+134

(pos 0)

+135

(cur-line 1)

+136

(col 1)

+137

(indent 0))

+138

+139

;; Advance one non-newline char (hot path)

+140

(define (advance-one!)

+141

(set! pos (+ pos 1))

+142

(set! col (+ col 1)))

+143

+144

;; Advance past a known newline, recalculate indent

+145

(define (advance-newline!)

+146

(set! pos (+ pos 1))

+147

(set! cur-line (+ cur-line 1))

+148

(set! col 1)

+149

(let loop ((off 0))

+150

(if (>= (+ pos off) len)

+151

(set! indent off)

+152

(let ((ch (vector-ref chars (+ pos off))))

+153

(cond

+154

((char=? ch #\space) (loop (+ off 1)))

+155

((char=? ch #\tab) (loop (+ off 2)))

+156

(else (set! indent off)))))))

+157

+158

;; Full advance with newline handling (for block comments only)

+159

(define (advance!)

+160

(when (< pos len)

+161

(let ((c (vector-ref chars pos)))

+162

(if (char=? c #\newline)

+163

(advance-newline!)

+164

(advance-one!)))))

+165

+166

;; Skip whitespace (not newlines) — inlined, no closure calls

+167

(define (skip-whitespace start-pos start-line start-col start-indent)

+168

(let loop ()

+169

(if (< pos len)

+170

(let ((c (vector-ref chars pos)))

+171

(cond

+172

((or (char=? c #\space) (char=? c #\tab))

+173

(set! pos (+ pos 1))

+174

(set! col (+ col 1))

+175

(loop))

+176

((char=? c #\return)

+177

(set! pos (+ pos 1))

+178

(loop))

+179

(else

+180

(vector 'whitespace start-pos pos start-line start-col start-indent))))

+181

(vector 'whitespace start-pos pos start-line start-col start-indent))))

+182

+183

;; Skip to delimiter — inlined with direct table lookup

+184

(define (skip-to-delimiter)

+185

(let loop ()

+186

(if (>= pos len)

+187

pos

+188

(let ((code (char->integer (vector-ref chars pos))))

+189

(if (and (< code 128) (vector-ref delimiter-table code))

+190

pos

+191

(begin

+192

(set! pos (+ pos 1))

+193

(set! col (+ col 1))

+194

(loop)))))))

+195

+196

;; Skip until newline — inlined

+197

(define (skip-until-newline)

+198

(let loop ()

+199

(if (>= pos len)

+200

pos

+201

(if (char=? (vector-ref chars pos) #\newline)

+202

pos

+203

(begin

+204

(set! pos (+ pos 1))

+205

(set! col (+ col 1))

+206

(loop))))))

+207

+208

;; Count leading spaces from current position

+209

(define (count-leading-spaces)

+210

(let loop ((off 0))

+211

(if (>= (+ pos off) len)

+212

off

+213

(let ((c (vector-ref chars (+ pos off))))

+214

(cond

+215

((char=? c #\space) (loop (+ off 1)))

+216

((char=? c #\tab) (loop (+ off 2)))

+217

(else off))))))

+218

+219

;; Read a comment token — inlined semicolons and skip

+220

(define (read-comment start-pos start-line start-col start-indent)

+221

(let sloop ((count 0))

+222

(if (and (< pos len) (char=? (vector-ref chars pos) #\;))

+223

(begin

+224

(set! pos (+ pos 1))

+225

(set! col (+ col 1))

+226

(sloop (+ count 1)))

+227

(begin

+228

(skip-until-newline)

+229

(vector (if (>= count 3) 'doc-comment 'comment)

+230

start-pos pos start-line start-col start-indent)))))

+231

+232

;; Read a string token — inlined with fast path for regular chars

+233

(define (read-string-token start-pos start-line start-col start-indent)

+234

(advance-one!) ; consume opening quote

+235

(let loop ()

+236

(if (>= pos len)

+237

(vector 'string start-pos pos start-line start-col start-indent)

+238

(let ((c (vector-ref chars pos)))

+239

(cond

+240

((char=? c #\")

+241

(advance-one!)

+242

(vector 'string start-pos pos start-line start-col start-indent))

+243

((char=? c #\\)

+244

(advance-one!) ; backslash

+245

(when (< pos len)

+246

(if (char=? (vector-ref chars pos) #\newline)

+247

(advance-newline!)

+248

(advance-one!)))

+249

(loop))

+250

((char=? c #\newline)

+251

(advance-newline!)

+252

(loop))

+253

(else

+254

(set! pos (+ pos 1))

+255

(set! col (+ col 1))

+256

(loop)))))))

+257

+258

;; Read a hash token — advance-one! for non-newline chars

+259

(define (read-hash-token start-pos start-line start-col start-indent)

+260

(advance-one!) ; consume #

+261

(if (>= pos len)

+262

(vector 'hash-other start-pos pos start-line start-col start-indent)

+263

(let ((c (vector-ref chars pos)))

+264

(cond

+265

((or (char=? c #\t) (char=? c #\T))

+266

(advance-one!)

+267

(vector 'hash-t start-pos pos start-line start-col start-indent))

+268

((or (char=? c #\f) (char=? c #\F))

+269

(advance-one!)

+270

(vector 'hash-f start-pos pos start-line start-col start-indent))

+271

((char=? c #\\)

+272

;; Character literal

+273

(advance-one!) ; consume backslash

+274

(if (>= pos len)

+275

(vector 'char start-pos pos start-line start-col start-indent)

+276

(begin

+277

(advance-one!) ; consume first char after backslash

+278

(skip-to-delimiter)

+279

(vector 'char start-pos pos start-line start-col start-indent))))

+280

((char=? c #\|)

+281

;; Block comment — uses full advance! since it can span lines

+282

(advance-one!) ; consume |

+283

(let loop ((depth 1))

+284

(if (= depth 0)

+285

(vector 'block-comment start-pos pos start-line start-col start-indent)

+286

(if (>= pos len)

+287

(vector 'block-comment start-pos pos start-line start-col start-indent)

+288

(let ((c (vector-ref chars pos)))

+289

(cond

+290

((char=? c #\|)

+291

(advance-one!) ; | is not newline

+292

(if (and (< pos len) (char=? (vector-ref chars pos) #\#))

+293

(begin (advance-one!) (loop (- depth 1)))

+294

(loop depth)))

+295

((char=? c #\#)

+296

(advance-one!) ; # is not newline

+297

(if (and (< pos len) (char=? (vector-ref chars pos) #\|))

+298

(begin (advance-one!) (loop (+ depth 1)))

+299

(loop depth)))

+300

(else

+301

(advance!) ; could be newline

+302

(loop depth))))))))

+303

((char=? c #\{)

+304

(advance-one!)

+305

(vector 'hash-lbrace start-pos pos start-line start-col start-indent))

+306

((char=? c #\[)

+307

(advance-one!)

+308

(vector 'hash-lbracket start-pos pos start-line start-col start-indent))

+309

(else

+310

;; Hash datum like #:keyword

+311

(skip-to-delimiter)

+312

(vector 'hash-other start-pos pos start-line start-col start-indent))))))

+313

+314

;; Initialize indent for first line

+315

(set! indent (count-leading-spaces))

+316

+317

;; Main tokenization loop

+318

(let loop ((tokens '()))

+319

(if (>= pos len)

+320

;; EOF token

+321

(let ((eof-tok (vector 'eof pos pos cur-line col indent)))

+322

(cons chars (reverse (cons eof-tok tokens))))

+323

(let ((c (vector-ref chars pos))

+324

(start-pos pos)

+325

(start-line cur-line)

+326

(start-col col)

+327

(start-indent indent))

+328

(cond

+329

;; Whitespace (not newline)

+330

((or (char=? c #\space) (char=? c #\tab) (char=? c #\return))

+331

(loop (cons (skip-whitespace start-pos start-line start-col start-indent) tokens)))

+332

+333

;; Newline — inline advance-newline!

+334

((char=? c #\newline)

+335

(advance-newline!)

+336

(loop (cons (vector 'newline start-pos pos start-line start-col start-indent) tokens)))

+337

+338

;; Comment

+339

((char=? c #\;)

+340

(loop (cons (read-comment start-pos start-line start-col start-indent) tokens)))

+341

+342

;; Parens and brackets — advance-one! (none are newlines)

+343

                  ((char=? c #\() (advance-one!) (loop (cons (vector 'lparen start-pos pos start-line start-col start-indent) tokens)))

+344

                  ((char=? c #\)) (advance-one!) (loop (cons (vector 'rparen start-pos pos start-line start-col start-indent) tokens)))

+345

                  ((char=? c #\[) (advance-one!) (loop (cons (vector 'lbracket start-pos pos start-line start-col start-indent) tokens)))

+346

                  ((char=? c #\]) (advance-one!) (loop (cons (vector 'rbracket start-pos pos start-line start-col start-indent) tokens)))

+347

                  ((char=? c #\{) (advance-one!) (loop (cons (vector 'lbrace start-pos pos start-line start-col start-indent) tokens)))

+348

                  ((char=? c #\}) (advance-one!) (loop (cons (vector 'rbrace start-pos pos start-line start-col start-indent) tokens)))

+349

+350

;; Quote forms

+351

                  ((char=? c #\') (advance-one!) (loop (cons (vector 'quote start-pos pos start-line start-col start-indent) tokens)))

+352

                  ((char=? c #\`) (advance-one!) (loop (cons (vector 'quasiquote start-pos pos start-line start-col start-indent) tokens)))

+353

((char=? c #\,)

+354

(advance-one!)

+355

(if (and (< pos len) (char=? (vector-ref chars pos) #\@))

+356

                       (begin (advance-one!) (loop (cons (vector 'unquote-splicing start-pos pos start-line start-col start-indent) tokens)))

+357

(loop (cons (vector 'unquote start-pos pos start-line start-col start-indent) tokens))))

+358

+359

;; String

+360

((char=? c #\")

+361

(loop (cons (read-string-token start-pos start-line start-col start-indent) tokens)))

+362

+363

;; Hash forms

+364

((char=? c #\#)

+365

(loop (cons (read-hash-token start-pos start-line start-col start-indent) tokens)))

+366

+367

;; Dot

+368

((char=? c #\.)

+369

(let ((next (if (< (+ pos 1) len) (vector-ref chars (+ pos 1)) #f)))

+370

(if (delimiter? next)

+371

(begin

+372

(advance-one!)

+373

(loop (cons (vector 'dot start-pos pos start-line start-col start-indent) tokens)))

+374

(begin

+375

(skip-to-delimiter)

+376

                           (let* ((text (token-value (vector 'symbol start-pos pos start-line start-col start-indent) chars))

+377

(tok-type (if (looks-like-number? text) 'number 'symbol)))

+378

(loop (cons (vector tok-type start-pos pos start-line start-col start-indent) tokens)))))))

+379

+380

;; Number or +/-

+381

((or (char=? c #\+) (char=? c #\-))

+382

(skip-to-delimiter)

+383

(let* ((text (token-value (vector 'symbol start-pos pos start-line start-col start-indent) chars))

+384

(tok-type (if (looks-like-number? text) 'number 'symbol)))

+385

(loop (cons (vector tok-type start-pos pos start-line start-col start-indent) tokens))))

+386

+387

;; Number

+388

((and (char>=? c #\0) (char<=? c #\9))

+389

(skip-to-delimiter)

+390

(loop (cons (vector 'number start-pos pos start-line start-col start-indent) tokens)))

+391

+392

;; Symbol (everything else)

+393

(else

+394

(skip-to-delimiter)

+395

(loop (cons (vector 'symbol start-pos pos start-line start-col start-indent) tokens)))))))))))

test/test-tokenize.sgladded

@@ -0,0 +1,64 @@

;;; Tests for the io-free (sigil format tokenize) sub-library.

;;;

;;; These import the tokenizer DIRECTLY (not via (sigil format)) to prove the

;;; sub-library stands alone with the full tokenizer API and no formatter deps.

(import (sigil test)

(sigil format tokenize))

(test-group "tokenize (direct sub-library import)"

+10

(test "simple expression: types and values"

+11

(let* ((result (tokenize "(+ 1 2)" "test.sgl"))

+12

(chars (tokenize-result-chars result))

+13

(tokens (tokenize-result-tokens result)))

+14

(assert-true (token? (car tokens)))

+15

;; ( + <ws> 1 <ws> 2 ) eof

+16

(assert-equal 'lparen (token-type (list-ref tokens 0)))

+17

(assert-equal "(" (token-value (list-ref tokens 0) chars))

+18

(assert-equal 'symbol (token-type (list-ref tokens 1)))

+19

(assert-equal "+" (token-value (list-ref tokens 1) chars))

+20

(assert-equal 'number (token-type (list-ref tokens 3)))

+21

(assert-equal "1" (token-value (list-ref tokens 3) chars))

+22

(assert-equal 'rparen (token-type (list-ref tokens 6)))

+23

(assert-equal 'eof (token-type (list-ref tokens 7)))))

+24

+25

(test "line/column/indent positions on a multi-line snippet"

+26

(let* ((result (tokenize "(define (foo x)\n (+ x 1))" "test.sgl"))

+27

(tokens (tokenize-result-tokens result))

+28

(tok0 (car tokens)))

+29

;; First token: the opening paren at line 1, column 1, indent 0.

+30

(assert-equal 'lparen (token-type tok0))

+31

(assert-equal 1 (token-line tok0))

+32

(assert-equal 1 (token-column tok0))

+33

(assert-equal 0 (token-indent tok0))

+34

;; Find the lparen that opens the second line's (+ ...) form.

+35

(let loop ((ts tokens))

+36

(cond

+37

((null? ts) (assert-true #f)) ; must exist

+38

((and (eq? (token-type (car ts)) 'lparen)

+39

(= (token-line (car ts)) 2))

+40

(assert-equal 3 (token-column (car ts)))

+41

(assert-equal 2 (token-indent (car ts))))

+42

(else (loop (cdr ts)))))))

+43

+44

(test "start/end accessors delimit the source slice"

+45

(let* ((result (tokenize "abc" "test.sgl"))

+46

(chars (tokenize-result-chars result))

+47

(tok (car (tokenize-result-tokens result))))

+48

(assert-equal 'symbol (token-type tok))

+49

(assert-equal 0 (token-start tok))

+50

(assert-equal 3 (token-end tok))

+51

(assert-equal "abc" (token-value tok chars))))

+52

+53

(test "string, comment, and hash tokens"

+54

(let* ((sresult (tokenize "\"hi\"" "t.sgl"))

+55

(stoks (tokenize-result-tokens sresult))

+56

(cresult (tokenize "; note\n" "t.sgl"))

+57

(ctoks (tokenize-result-tokens cresult))

+58

(hresult (tokenize "#t" "t.sgl"))

+59

(htoks (tokenize-result-tokens hresult)))

+60

(assert-equal 'string (token-type (car stoks)))

+61

(assert-equal 'comment (token-type (car ctoks)))

+62

(assert-equal 'hash-t (token-type (car htoks))))))

+63

+64

(run-tests)