797d7812cc
To make it possible to have more tokens than 255. Parsers operators (i.e. tokens) is one thing and opcodes for the virtual machine of sqlite3 is another. Unfortunately they are not completely separate, but some of the opcodes in <build-directory>/opcodes.h are the same as the tokens in <build-directory>/parse.h. And while the parser tokens are now 16-bit, the VM opcodes are 8-bit. However, this is probably not a problem even if some of the parser tokens that are duplicated in the opcodes are > 256 as we only use sqlite3 for parsing and not for executing anything (on the sqlite3 VM).
64 lines
1.4 KiB
Tcl
64 lines
1.4 KiB
Tcl
#!/usr/bin/tclsh
|
|
#
|
|
# This script appends additional token codes to the end of the
|
|
# parse.h file that lemon generates. These extra token codes are
|
|
# not used by the parser. But they are used by the tokenizer and/or
|
|
# the code generator.
|
|
#
|
|
#
|
|
set in [open [lindex $argv 0] rb]
|
|
set max 0
|
|
while {![eof $in]} {
|
|
set line [gets $in]
|
|
if {[regexp {^#define TK_} $line]} {
|
|
puts $line
|
|
set x [lindex $line 2]
|
|
if {$x>$max} {set max $x}
|
|
}
|
|
}
|
|
close $in
|
|
|
|
# The following are the extra token codes to be added. SPACE and
|
|
# ILLEGAL *must* be the last two token codes and they must be in that order.
|
|
#
|
|
set extras {
|
|
TO_TEXT
|
|
TO_BLOB
|
|
TO_NUMERIC
|
|
TO_INT
|
|
TO_REAL
|
|
ISNOT
|
|
END_OF_FILE
|
|
UNCLOSED_STRING
|
|
FUNCTION
|
|
COLUMN
|
|
AGG_FUNCTION
|
|
AGG_COLUMN
|
|
UMINUS
|
|
UPLUS
|
|
REGISTER
|
|
ASTERISK
|
|
SPACE
|
|
ILLEGAL
|
|
}
|
|
if {[lrange $extras end-1 end]!="SPACE ILLEGAL"} {
|
|
error "SPACE and ILLEGAL must be the last two token codes and they\
|
|
must be in that order"
|
|
}
|
|
foreach x $extras {
|
|
incr max
|
|
puts [format "#define TK_%-29s %4d" $x $max]
|
|
}
|
|
|
|
# Some additional #defines related to token codes.
|
|
#
|
|
puts "\n/* The token codes above must all fit in 12 bits */"
|
|
puts [format "#define %-20s %-6s" TKFLG_MASK 0xfff]
|
|
puts "\n/* Flags that can be added to a token code when it is not"
|
|
puts "** being stored in a u8: */"
|
|
foreach {fg val comment} {
|
|
TKFLG_DONTFOLD 0x1000 {/* Omit constant folding optimizations */}
|
|
} {
|
|
puts [format "#define %-20s %-6s %s" $fg $val $comment]
|
|
}
|