Optimize formatter: vector tokens and inlined tokenizer hot loops

Message

Replace define-struct tokens with raw vectors (#(type start end line col indent)) to avoid keyword-arg construction overhead. Inline all tokenizer hot paths (skip-whitespace, skip-to-delimiter, skip-until-newline, read-comment, read-string-token) to use direct vector-ref instead of closure-based peek/advance! calls. Add delimiter lookup table for O(1) checks.

Tokenize phase: 30s -> 1.7s on build.sgl (2829 lines, 135KB). Total format time: 30s -> 4.5s.

Changed

 src/sigil/format.sgl | 787 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---------------------------------------------------------------------------------------------
 test/test-format.sgl |  45 ++++++----
 2 files changed, 387 insertions(+), 445 deletions(-)

Diff

src/sigil/format.sglmodified

@@ -15,7 +15,8 @@

(import (sigil struct)

(sigil string)

(sigil io)

−18

(sigil json))

+18

(sigil json)

+19

(sigil fs))

(export

;; Core API

@@ -64,7 +65,9 @@

token-indent

;; Tokenizer (for testing)

−67

tokenize)

+68

tokenize

+69

tokenize-result-chars

+70

tokenize-result-tokens)

(begin

@@ -103,94 +106,52 @@

106

;;; Tokens

107

;; ============================================================

108

−106

(define-struct token

−107

(type) ; 'lparen 'rparen 'lbracket 'rbracket

−108

; 'lbrace 'rbrace

−109

; 'hash-lbrace 'hash-lbracket

−110

; 'symbol 'string 'number 'char

−111

; 'quote 'quasiquote 'unquote 'unquote-splicing

−112

; 'comment 'doc-comment 'block-comment

−113

; 'hash-t 'hash-f 'hash-other

−114

; 'dot 'eof 'whitespace 'newline

−115

(value) ; the actual text

−116

(line) ; 1-based line number

−117

(column) ; 1-based column

−118

(indent)) ; indentation at start of this line

+109

;; Token layout: #(type start end line column indent)

+110

;; Using raw vectors avoids keyword-arg overhead in the hot path.

111

−120

;; ============================================================

−121

;;; Tokenizer State

−122

;; ============================================================

+112

;;; Check if value is a token

+113

(define (token? x) (and (vector? x) (= (vector-length x) 6)))

114

−124

(define-struct tokenizer

−125

(source)

−126

(filename)

−127

(pos mutable: #t)

−128

(line mutable: #t)

−129

(column mutable: #t)

−130

(line-indent mutable: #t))

+115

;;; Token type accessor

+116

(define (token-type tok) (vector-ref tok 0))

117

−132

;; ============================================================

−133

;;; Tokenizer Implementation

−134

;; ============================================================

+118

;;; Token start position accessor

+119

(define (token-start tok) (vector-ref tok 1))

120

−136

;;; Check if at end of source

−137

(define (tokenizer-eof? t)

−138

(>= (tokenizer-pos t) (string-length (tokenizer-source t))))

+121

;;; Token end position accessor

+122

(define (token-end tok) (vector-ref tok 2))

123

−140

;;; Peek at current character without advancing

−141

(define (tokenizer-peek t)

−142

(if (tokenizer-eof? t)

−143

#f

−144

(string-ref (tokenizer-source t) (tokenizer-pos t))))

+124

;;; Token line accessor

+125

(define (token-line tok) (vector-ref tok 3))

126

−146

;;; Peek at character at offset from current position

−147

(define (tokenizer-peek-ahead t offset)

−148

(let ((new-pos (+ (tokenizer-pos t) offset)))

−149

(if (>= new-pos (string-length (tokenizer-source t)))

−150

#f

−151

(string-ref (tokenizer-source t) new-pos))))

−152

−153

;;; Advance position by one character

−154

(define (tokenizer-advance! t)

−155

(let ((c (tokenizer-peek t)))

−156

(if c

−157

(begin

−158

(set-tokenizer-pos! t (+ (tokenizer-pos t) 1))

−159

(if (char=? c #\newline)

−160

(begin

−161

(set-tokenizer-line! t (+ (tokenizer-line t) 1))

−162

(set-tokenizer-column! t 1)

−163

;; Calculate indentation of next line

−164

(set-tokenizer-line-indent! t (count-leading-spaces t)))

−165

(set-tokenizer-column! t (+ (tokenizer-column t) 1)))

−166

c)

−167

#f)))

−168

−169

;;; Count leading spaces from current position (for indent calculation)

−170

(define (count-leading-spaces t)

−171

(let loop ((offset 0))

−172

(let ((c (tokenizer-peek-ahead t offset)))

−173

(cond

−174

((not c) offset)

−175

((char=? c #\space) (loop (+ offset 1)))

−176

((char=? c #\tab) (loop (+ offset 2))) ; treat tab as 2 spaces

−177

(else offset)))))

+127

;;; Token column accessor

+128

(define (token-column tok) (vector-ref tok 4))

129

−179

;;; Check if character is a delimiter

−180

(define (delimiter? c)

−181

(or (not c)

−182

(char=? c #\()

−183

(char=? c #\))

−184

(char=? c #\[)

−185

(char=? c #\])

−186

(char=? c #\{)

−187

(char=? c #\})

−188

(char=? c #\")

−189

(char=? c #\;)

−190

(char=? c #\')

−191

(char=? c #\`)

−192

(char=? c #\,)

−193

(char-whitespace? c)))

+130

;;; Token indent accessor

+131

(define (token-indent tok) (vector-ref tok 5))

+132

+133

;;; Extract the text value of a token from the source chars vector

+134

(define (token-value tok chars)

+135

(let* ((s (vector-ref tok 1))

+136

(e (vector-ref tok 2))

+137

(n (- e s))

+138

(v (make-string n)))

+139

(let loop ((i 0))

+140

(if (>= i n)

+141

v

+142

(begin

+143

(string-set! v i (vector-ref chars (+ s i)))

+144

(loop (+ i 1)))))))

+145

+146

;;; Get the chars vector from a tokenize result

+147

(define (tokenize-result-chars result) (car result))

+148

+149

;;; Get the token list from a tokenize result

+150

(define (tokenize-result-tokens result) (cdr result))

+151

+152

;; ============================================================

+153

;;; Tokenizer Implementation

+154

;; ============================================================

155

156

;;; Check if character is whitespace

157

(define (char-whitespace? c)

@@ -206,319 +167,19 @@

167

(char>=? c #\0)

168

(char<=? c #\9)))

169

−209

;;; Check if character can start a number

−210

(define (number-start? c next)

−211

(or (char-digit? c)

−212

(and (or (and c (char=? c #\-))

−213

(and c (char=? c #\+)))

−214

(char-digit? next))))

−215

−216

;;; Make a token at current position

−217

(define (make-token-here t tok-type value)

−218

(token

−219

type: tok-type

−220

value: value

−221

line: (tokenizer-line t)

−222

column: (tokenizer-column t)

−223

indent: (tokenizer-line-indent t)))

−224

−225

;;; Tokenize the next token from source

−226

(define (tokenize-next t)

−227

(if (tokenizer-eof? t)

−228

(make-token-here t 'eof "")

−229

(let ((c (tokenizer-peek t))

−230

(start-line (tokenizer-line t))

−231

(start-col (tokenizer-column t))

−232

(start-indent (tokenizer-line-indent t)))

−233

(cond

−234

;; Whitespace (but not newline)

−235

((and c (or (char=? c #\space) (char=? c #\tab) (char=? c #\return)))

−236

(let ((ws (read-whitespace t)))

−237

(token type: 'whitespace value: ws

−238

line: start-line column: start-col indent: start-indent)))

−239

−240

;; Newline

−241

((and c (char=? c #\newline))

−242

(tokenizer-advance! t)

−243

(token type: 'newline value: "\n"

−244

line: start-line column: start-col indent: start-indent))

−245

−246

;; Comments

−247

((and c (char=? c #\;))

−248

(read-comment t start-line start-col start-indent))

−249

−250

;; Parentheses and brackets

−251

((and c (char=? c #\())

−252

(tokenizer-advance! t)

−253

(token type: 'lparen value: "("

−254

line: start-line column: start-col indent: start-indent))

−255

((and c (char=? c #\)))

−256

(tokenizer-advance! t)

−257

(token type: 'rparen value: ")"

−258

line: start-line column: start-col indent: start-indent))

−259

((and c (char=? c #\[))

−260

(tokenizer-advance! t)

−261

(token type: 'lbracket value: "["

−262

line: start-line column: start-col indent: start-indent))

−263

((and c (char=? c #\]))

−264

(tokenizer-advance! t)

−265

(token type: 'rbracket value: "]"

−266

line: start-line column: start-col indent: start-indent))

−267

((and c (char=? c #\{))

−268

(tokenizer-advance! t)

−269

(token type: 'lbrace value: "{"

−270

line: start-line column: start-col indent: start-indent))

−271

((and c (char=? c #\}))

−272

(tokenizer-advance! t)

−273

(token type: 'rbrace value: "}"

−274

line: start-line column: start-col indent: start-indent))

−275

−276

;; Quote forms

−277

((and c (char=? c #\'))

−278

(tokenizer-advance! t)

−279

(token type: 'quote value: "'"

−280

line: start-line column: start-col indent: start-indent))

−281

((and c (char=? c #\`))

−282

(tokenizer-advance! t)

−283

(token type: 'quasiquote value: "`"

−284

line: start-line column: start-col indent: start-indent))

−285

((and c (char=? c #\,))

−286

(tokenizer-advance! t)

−287

(if (and (not (tokenizer-eof? t))

−288

(char=? (tokenizer-peek t) #\@))

−289

(begin

−290

(tokenizer-advance! t)

−291

(token type: 'unquote-splicing value: ",@"

−292

line: start-line column: start-col indent: start-indent))

−293

(token type: 'unquote value: ","

−294

line: start-line column: start-col indent: start-indent)))

−295

−296

;; String

−297

((and c (char=? c #\"))

−298

(read-string-token t start-line start-col start-indent))

−299

−300

;; Hash forms

−301

((and c (char=? c #\#))

−302

(read-hash-token t start-line start-col start-indent))

−303

−304

;; Dot

−305

((and c (char=? c #\.))

−306

(let ((next (tokenizer-peek-ahead t 1)))

−307

(if (delimiter? next)

−308

(begin

−309

(tokenizer-advance! t)

−310

(token type: 'dot value: "."

−311

line: start-line column: start-col indent: start-indent))

−312

(read-symbol-or-number t start-line start-col start-indent))))

−313

−314

;; Number or symbol starting with +/-

−315

((and c (or (char=? c #\+) (char=? c #\-)))

−316

(read-symbol-or-number t start-line start-col start-indent))

−317

−318

;; Number

−319

((char-digit? c)

−320

(read-number t start-line start-col start-indent))

−321

−322

;; Symbol

−323

(else

−324

(read-symbol t start-line start-col start-indent))))))

−325

−326

;;; Read whitespace (not including newlines)

−327

(define (read-whitespace t)

−328

(let loop ((chars '()))

−329

(let ((c (tokenizer-peek t)))

−330

(if (and c (or (char=? c #\space) (char=? c #\tab) (char=? c #\return)))

−331

(begin

−332

(tokenizer-advance! t)

−333

(loop (cons c chars)))

−334

(list->string (reverse chars))))))

−335

−336

;;; Read a comment (line comment or doc comment)

−337

(define (read-comment t start-line start-col start-indent)

−338

;; Count semicolons

−339

(let loop ((count 0))

−340

(if (and (not (tokenizer-eof? t))

−341

(char=? (tokenizer-peek t) #\;))

−342

(begin

−343

(tokenizer-advance! t)

−344

(loop (+ count 1)))

−345

(let ((text (read-until-newline t)))

−346

(token type: (if (>= count 3) 'doc-comment 'comment)

−347

value: (string-append (string-repeat ";" count) text)

−348

line: start-line column: start-col indent: start-indent)))))

−349

−350

;;; Read until end of line

−351

(define (read-until-newline t)

−352

(let loop ((chars '()))

−353

(let ((c (tokenizer-peek t)))

−354

(if (or (not c) (char=? c #\newline))

−355

(list->string (reverse chars))

−356

(begin

−357

(tokenizer-advance! t)

−358

(loop (cons c chars)))))))

−359

−360

;;; Read a string token

−361

(define (read-string-token t start-line start-col start-indent)

−362

(tokenizer-advance! t) ; consume opening quote

−363

(let loop ((chars '(#\")))

−364

(let ((c (tokenizer-peek t)))

−365

(cond

−366

((not c)

−367

;; Unterminated string

−368

(token type: 'string value: (list->string (reverse chars))

−369

line: start-line column: start-col indent: start-indent))

−370

((char=? c #\")

−371

(tokenizer-advance! t)

−372

(token type: 'string value: (list->string (reverse (cons #\" chars)))

−373

line: start-line column: start-col indent: start-indent))

−374

((char=? c #\\)

−375

(tokenizer-advance! t)

−376

(let ((escaped (tokenizer-peek t)))

−377

(if escaped

−378

(begin

−379

(tokenizer-advance! t)

−380

(loop (cons escaped (cons #\\ chars))))

−381

(loop (cons #\\ chars)))))

−382

(else

−383

(tokenizer-advance! t)

−384

(loop (cons c chars)))))))

−385

−386

;;; Read a hash token (#t, #f, #\char, #|...|#, #{...}, #[...], etc.)

−387

(define (read-hash-token t start-line start-col start-indent)

−388

(tokenizer-advance! t) ; consume #

−389

(let ((c (tokenizer-peek t)))

−390

(cond

−391

((not c)

−392

(token type: 'hash-other value: "#"

−393

line: start-line column: start-col indent: start-indent))

−394

((or (char=? c #\t) (char=? c #\T))

−395

(tokenizer-advance! t)

−396

(token type: 'hash-t value: "#t"

−397

line: start-line column: start-col indent: start-indent))

−398

((or (char=? c #\f) (char=? c #\F))

−399

(tokenizer-advance! t)

−400

(token type: 'hash-f value: "#f"

−401

line: start-line column: start-col indent: start-indent))

−402

((char=? c #\\)

−403

(read-char-token t start-line start-col start-indent))

−404

((char=? c #\|)

−405

(read-block-comment t start-line start-col start-indent))

−406

((char=? c #\{)

−407

;; Dict literal: #{...}

−408

(tokenizer-advance! t)

−409

(token type: 'hash-lbrace value: "#{"

−410

line: start-line column: start-col indent: start-indent))

−411

((char=? c #\[)

−412

;; Array literal: #[...]

−413

(tokenizer-advance! t)

−414

(token type: 'hash-lbracket value: "#["

−415

line: start-line column: start-col indent: start-indent))

−416

(else

−417

(let ((text (read-hash-datum t)))

−418

(token type: 'hash-other value: (string-append "#" text)

−419

line: start-line column: start-col indent: start-indent))))))

−420

−421

;;; Read character literal

−422

(define (read-char-token t start-line start-col start-indent)

−423

(tokenizer-advance! t) ; consume backslash

−424

(let ((chars (list #\\ #\#)))

−425

;; Always consume the first character after #\ (it IS the char value,

−426

;; even if it's a delimiter like " ( ) ;)

−427

(let ((c (tokenizer-peek t)))

−428

(if (not c)

−429

(token type: 'char value: (list->string (reverse chars))

−430

line: start-line column: start-col indent: start-indent)

−431

(begin

−432

(tokenizer-advance! t)

−433

;; For named chars like #\space, continue reading non-delimiter chars

−434

(let loop ((cs (cons c chars)))

−435

(let ((c (tokenizer-peek t)))

−436

(if (delimiter? c)

−437

(token type: 'char value: (list->string (reverse cs))

−438

line: start-line column: start-col indent: start-indent)

−439

(begin

−440

(tokenizer-advance! t)

−441

(loop (cons c cs)))))))))))

−442

−443

;;; Read block comment #| ... |#

−444

(define (read-block-comment t start-line start-col start-indent)

−445

(tokenizer-advance! t) ; consume |

−446

(let loop ((chars (list #\| #\#)) (depth 1))

−447

(if (= depth 0)

−448

(token type: 'block-comment value: (list->string (reverse chars))

−449

line: start-line column: start-col indent: start-indent)

−450

(let ((c (tokenizer-peek t)))

−451

(cond

−452

((not c)

−453

(token type: 'block-comment value: (list->string (reverse chars))

−454

line: start-line column: start-col indent: start-indent))

−455

((char=? c #\|)

−456

(tokenizer-advance! t)

−457

(if (and (not (tokenizer-eof? t))

−458

(char=? (tokenizer-peek t) #\#))

−459

(begin

−460

(tokenizer-advance! t)

−461

(loop (cons #\# (cons #\| chars)) (- depth 1)))

−462

(loop (cons c chars) depth)))

−463

((char=? c #\#)

−464

(tokenizer-advance! t)

−465

(if (and (not (tokenizer-eof? t))

−466

(char=? (tokenizer-peek t) #\|))

−467

(begin

−468

(tokenizer-advance! t)

−469

(loop (cons #\| (cons #\# chars)) (+ depth 1)))

−470

(loop (cons c chars) depth)))

−471

(else

−472

(tokenizer-advance! t)

−473

(loop (cons c chars) depth)))))))

−474

−475

;;; Read hash datum (like #:keyword or similar)

−476

(define (read-hash-datum t)

−477

(let loop ((chars '()))

−478

(let ((c (tokenizer-peek t)))

−479

(if (delimiter? c)

−480

(list->string (reverse chars))

−481

(begin

−482

(tokenizer-advance! t)

−483

(loop (cons c chars)))))))

−484

−485

;;; Read a symbol

−486

(define (read-symbol t start-line start-col start-indent)

−487

(let loop ((chars '()))

−488

(let ((c (tokenizer-peek t)))

−489

(if (delimiter? c)

−490

(token type: 'symbol value: (list->string (reverse chars))

−491

line: start-line column: start-col indent: start-indent)

−492

(begin

−493

(tokenizer-advance! t)

−494

(loop (cons c chars)))))))

−495

−496

;;; Read a number

−497

(define (read-number t start-line start-col start-indent)

−498

(let loop ((chars '()))

−499

(let ((c (tokenizer-peek t)))

−500

(if (delimiter? c)

−501

(token type: 'number value: (list->string (reverse chars))

−502

line: start-line column: start-col indent: start-indent)

−503

(begin

−504

(tokenizer-advance! t)

−505

(loop (cons c chars)))))))

−506

−507

;;; Read symbol or number (for things starting with +/-)

−508

(define (read-symbol-or-number t start-line start-col start-indent)

−509

(let loop ((chars '()))

−510

(let ((c (tokenizer-peek t)))

−511

(if (delimiter? c)

−512

(let ((text (list->string (reverse chars))))

−513

;; Determine if it's a number or symbol

−514

(if (looks-like-number? text)

−515

(token type: 'number value: text

−516

line: start-line column: start-col indent: start-indent)

−517

(token type: 'symbol value: text

−518

line: start-line column: start-col indent: start-indent)))

−519

(begin

−520

(tokenizer-advance! t)

−521

(loop (cons c chars)))))))

+170

;; Delimiter lookup table — indexed by char code, #t if delimiter

+171

(define delimiter-table

+172

(let ((t (make-vector 128 #f)))

+173

(for-each (lambda (c) (vector-set! t (char->integer c) #t))

+174

'(#\( #\) #\[ #\] #\{ #\} #\" #\; #\' #\` #\,

+175

#\space #\tab #\newline #\return))

+176

t))

+177

+178

;;; Check if character is a delimiter

+179

(define (delimiter? c)

+180

(or (not c)

+181

(let ((code (char->integer c)))

+182

(and (< code 128) (vector-ref delimiter-table code)))))

183

184

;;; Check if string looks like a number

185

(define (looks-like-number? s)

@@ -537,15 +198,276 @@

198

(else #f))))))

199

200

;;; Tokenize entire source into a list of tokens

+201

;;; Returns (cons chars tokens) where chars is the source as a vector

202

(define (tokenize source filename)

−541

(let ((t (tokenizer source: source filename: filename pos: 0 line: 1 column: 1 line-indent: 0)))

−542

;; Initialize line indent for first line

−543

(set-tokenizer-line-indent! t (count-leading-spaces t))

+203

(let* ((chars (string->vector source))

+204

(len (vector-length chars))

+205

(pos 0)

Showing the first 500 of 949 diff lines for this file. This diff is INCOMPLETE; read the file or clone the repository for the rest.

test/test-format.sglmodified

@@ -7,48 +7,63 @@

(test-group "tokenize"

(test "simple expression tokens"

−10

(let ((tokens (tokenize "(+ 1 2)" "test.sgl")))

+10

(let* ((result (tokenize "(+ 1 2)" "test.sgl"))

+11

(chars (tokenize-result-chars result))

+12

(tokens (tokenize-result-tokens result)))

;; Should have lparen, symbol, whitespace, number, whitespace, number, rparen, eof

(assert-equal 'lparen (token-type (car tokens)))

−13

(assert-equal "(" (token-value (car tokens)))))

+15

(assert-equal "(" (token-value (car tokens) chars))))

(test "string token"

−16

(let ((tokens (tokenize "\"hello\"" "test.sgl")))

+18

(let* ((result (tokenize "\"hello\"" "test.sgl"))

+19

(chars (tokenize-result-chars result))

+20

(tokens (tokenize-result-tokens result)))

(assert-equal 'string (token-type (car tokens)))

−18

(assert-equal "\"hello\"" (token-value (car tokens)))))

+22

(assert-equal "\"hello\"" (token-value (car tokens) chars))))

(test "hash forms"

−21

(let ((tokens (tokenize "#t #f" "test.sgl")))

+25

(let* ((result (tokenize "#t #f" "test.sgl"))

+26

(tokens (tokenize-result-tokens result)))

(assert-equal 'hash-t (token-type (car tokens)))))

(test "comment token"

−25

(let ((tokens (tokenize "; a comment\n" "test.sgl")))

+30

(let* ((result (tokenize "; a comment\n" "test.sgl"))

+31

(tokens (tokenize-result-tokens result)))

(assert-equal 'comment (token-type (car tokens)))))

(test "line/column tracking"

−29

(let ((tokens (tokenize "(define x 1)" "test.sgl")))

+35

(let* ((result (tokenize "(define x 1)" "test.sgl"))

+36

(tokens (tokenize-result-tokens result)))

(assert-equal 1 (token-line (car tokens)))

(assert-equal 1 (token-column (car tokens)))))

(test "char literal with double-quote delimiter"

−34

(let ((tokens (tokenize "#\\\"" "test.sgl")))

+41

(let* ((result (tokenize "#\\\"" "test.sgl"))

+42

(chars (tokenize-result-chars result))

+43

(tokens (tokenize-result-tokens result)))

(assert-equal 'char (token-type (car tokens)))

−36

(assert-equal "#\\\"" (token-value (car tokens)))))

+45

(assert-equal "#\\\"" (token-value (car tokens) chars))))

(test "char literal with open-paren delimiter"

−39

(let ((tokens (tokenize "#\\(" "test.sgl")))

+48

(let* ((result (tokenize "#\\(" "test.sgl"))

+49

(chars (tokenize-result-chars result))

+50

(tokens (tokenize-result-tokens result)))

(assert-equal 'char (token-type (car tokens)))

−41

(assert-equal "#\\(" (token-value (car tokens)))))

+52

(assert-equal "#\\(" (token-value (car tokens) chars))))

(test "char literal with close-paren delimiter"

−44

(let ((tokens (tokenize "#\\)" "test.sgl")))

+55

(let* ((result (tokenize "#\\)" "test.sgl"))

+56

(chars (tokenize-result-chars result))

+57

(tokens (tokenize-result-tokens result)))

(assert-equal 'char (token-type (car tokens)))

−46

(assert-equal "#\\)" (token-value (car tokens)))))

+59

(assert-equal "#\\)" (token-value (car tokens) chars))))

(test "char literal with semicolon delimiter"

−49

(let ((tokens (tokenize "#\\;" "test.sgl")))

+62

(let* ((result (tokenize "#\\;" "test.sgl"))

+63

(chars (tokenize-result-chars result))

+64

(tokens (tokenize-result-tokens result)))

(assert-equal 'char (token-type (car tokens)))

−51

(assert-equal "#\\;" (token-value (car tokens))))))

+66

(assert-equal "#\\;" (token-value (car tokens) chars)))))

;; ============================================================

;; format-string