tokenize (version Ka-Ping Yee, 29 March 1997)
index
/web/lfw/python/tokenize.py
 

Tokenization help for Python programs.
 
This module exports a function called 'tokenize()' that breaks a stream of
text into Python tokens.  It accepts a readline-like method which is called
repeatedly to get the next line of input (or "" for EOF) and a "token-eater"
function which is called once for each token found.  The latter function is
passed the token type, a string containing the token, the starting and
ending (row, column) coordinates of the token, and the original line.  It is
designed to match the working of the Python tokenizer exactly, except that
it produces COMMENT tokens for comments and gives type OP for all operators.
 
For compatibility with the older 'tokenize' module, this also compiles a
regular expression into 'tokenprog' that matches Python tokens in individual
lines of text, leaving the token in 'tokenprog.group(3)', but does not
handle indentation, continuations, or multi-line strings.

   
Modules
                        
regex
string

   
Functions
                        
group(*choices)
printtoken(type, token, (srow, scol), (erow, ecol), line)
tokenize(readline, tokeneater=<function printtoken at 81f8b70>)

   
Constants
                        
AMPER = 19
BACKQUOTE = 25
Bracket = '[][(){}]'
CIRCUMFLEX = 33
COLON = 11
COMMA = 12
COMMENT = 39
Comment = '\\(#[^\015\012]*\\)'
ContStr = '\\(\'\\([\\].\\|[^\012\'\\]\\)*\\(\'...[^\012"\\]\\)*\\("\\|[\\]\015?\012\\)\\)'
DEDENT = 6
DOT = 23
DOUBLESTAR = 36
Decnumber = '[1-9][0-9]*[lL]?'
Double = r'\([^"\]\|[\].\)*"'
Double3 = r'\([^"\]\|[\].\|"[^"\]\|....\|""[^"\]\|""[\].\)*"""'
ENDMARKER = 0
EQEQUAL = 28
EQUAL = 22
ERRORTOKEN = 38
Expfloat = '[1-9][0-9]*[eE][-+]?[0-9]+'
Exponent = '[eE][-+]?[0-9]+'
Floatnumber = r'\(\([0-9]+\.[0-9]*\|\.[...][0-9]*[eE][-+]?[0-9]+\)'
Funny = '\\(\\(\\+\\|\\-\\|\\*\\*\\|\\*\\|\\...){}]\\|\\(\015?\012\\|[:;.,`]\\)\\)'
GREATER = 21
GREATEREQUAL = 31
Hexnumber = '0[xX][0-9a-fA-F]*[lL]?'
INDENT = 5
Ignore = '[ \014\011]*\\([\\]\015?\012[ \014\011]*\\)*\\(#[^\015\012]*\\)?'
Imagnumber = r'\(0[jJ]\|[1-9][0-9]*[jJ...*[eE][-+]?[0-9]+\)[jJ]\)'
Intnumber = r'\(0[xX][0-9a-fA-F]*[lL]...lL]?\|[1-9][0-9]*[lL]?\)'
LBRACE = 26
LEFTSHIFT = 34
LESS = 20
LESSEQUAL = 30
LPAR = 7
LSQB = 9
MINUS = 15
NAME = 1
NEWLINE = 4
NOTEQUAL = 29
NT_OFFSET = 256
NUMBER = 2
N_TOKENS = 39
Name = '[a-zA-Z_][a-zA-Z0-9_]*'
Number = r'\(\(0[jJ]\|[1-9][0-9]*[...]?\|[1-9][0-9]*[lL]?\)\)'
OP = 37
Octnumber = '0[0-7]*[lL]?'
Operator = r'\(\+\|\-\|\*\*\|\*\|\^\...=\|<>\|!=\|>=\|=\|<\|>\)'
PERCENT = 24
PLUS = 14
PlainToken = '\\([a-zA-Z_][a-zA-Z0-9_]...}]\\|\\(\015?\012\\|[:;.,`]\\)\\)\\)'
Pointfloat = r'\([0-9]+\.[0-9]*\|\.[0-9]+\)\([eE][-+]?[0-9]+\)?'
PseudoExtras = '\\([\\]\015?\012\\|\\(#[^\015\012]*\\)\\|\\(\'\'\'\\|"""\\)\\)'
PseudoToken = '[ \014\011]*\\(\\([\\]\015?\012\\|\\(#[^...}]\\|\\(\015?\012\\|[:;.,`]\\)\\)\\)'
RBRACE = 27
RIGHTSHIFT = 35
RPAR = 8
RSQB = 10
SEMI = 13
SLASH = 17
STAR = 16
STRING = 3
Single = r"\([^'\]\|[\].\)*'"
Single3 = r"\([^'\]\|[\].\|'[^'\]\|....\|''[^'\]\|''[\].\)*'''"
Special = '\\(\015?\012\\|[:;.,`]\\)'
String = '\\(\'\\([^\012\'\\]\\|[\\].\\)*\'\\|"\\([^\012"\\]\\|[\\].\\)*"\\)'
TILDE = 32
Token = '[ \014\011]*\\([\\]\015?\012[ \014\011]*\\)*...}]\\|\\(\015?\012\\|[:;.,`]\\)\\)\\)'
TokenError = 'TokenError'
Triple = '\\(\'\'\'\\|"""\\)'
VBAR = 18
Whitespace = '[ \014\011]*'
saved_syntax = 0
tabsize = 8