"""token_utils.py
------------------
A collection of useful functions and methods to deal with tokenizing
source code.
"""
from token_utils import py_tokenize
from io import StringIO as _StringIO
from token_utils.token_class import Token
[docs]
def find_token_by_position(tokens, row, column):
"""Given a list of tokens, a specific row (linenumber) and column,
a two-tuple is returned that includes the token
found at that position as well as its list index.
If no such token can be found, ``None, None`` is returned.
"""
for index, tok in enumerate(tokens):
if (
tok.start_row <= row <= tok.end_row
and tok.start_col <= column < tok.end_col
):
return tok, index
return None, None
[docs]
def fix_empty_line(source, tokens):
"""Prior to version 3.12, Python's tokenizer drops entirely a last line
if it consists only of space characters and/or tab characters.
To ensure that we can always have::
untokenize(tokenize(source)) == source
we correct the last token content if needed.
"""
nb = 0
for char in reversed(source):
if char in (" ", "\t"):
nb += 1
else:
break
tokens[-1].string = source[-nb:]
[docs]
def tokenize(source, warning=True):
"""Transforms a source (string) into a list of Tokens.
If an exception is raised by Python's tokenize module, the list of tokens
accumulated up to that point is returned.
"""
tokens = []
for tok in py_tokenize.generate_tokens(_StringIO(source).readline):
try:
token = Token(tok)
tokens.append(token)
except (py_tokenize.TokenError, Exception) as exc:
if warning:
print(
"WARNING: the following error was raised in ",
f"{__name__}.tokenize",
)
print(exc)
return tokens
if source.endswith((" ", "\t")):
if not tokens[-2].line.endswith((" ", "\t")):
fix_empty_line(source, tokens)
return tokens
[docs]
def get_significant_tokens(source):
"""Gets a list of tokens from a source (str), ignoring comments
as well as any token whose string value is either null or
consists of spaces, newline or tab characters.
If an exception is raised by Python's tokenize module, the list of tokens
accumulated up to that point is returned.
"""
tokens = []
try:
for tok in py_tokenize.generate_tokens(_StringIO(source).readline):
token = Token(tok)
if not token.string.strip():
continue
if token.is_comment():
continue
tokens.append(token)
except py_tokenize.TokenError:
return tokens
return tokens
[docs]
def get_lines(source):
"""Transforms a source (string) into a list of Tokens, with each
(inner) list containing all the tokens found on a given line of code.
"""
lines = []
current_row = -1
new_line = []
for tok in py_tokenize.generate_tokens(_StringIO(source).readline):
try:
token = Token(tok)
if token.start_row != current_row:
current_row = token.start_row
if new_line:
lines.append(new_line)
new_line = []
new_line.append(token)
except (py_tokenize.TokenError, Exception) as exc:
print(
"WARNING: the following tokenize error was raised in "
f"{__name__}.get_lines"
)
print(exc)
if new_line:
lines.append(new_line)
if source.endswith((" ", "\t")):
if len(lines) > 1:
penultimate_line = lines[-2]
if not penultimate_line[-1].line.endswith((" ", "\t")):
fix_empty_line(source, lines[-1])
return lines
[docs]
def get_number(tokens, exclude_comment=True):
"""Given a list of tokens, gives a count of the number of
tokens which are not space tokens (such as ``NEWLINE``, ``INDENT``,
``DEDENT``, etc.)
By default, ``COMMMENT`` tokens are not included in the count.
If you wish to include them, set ``exclude_comment`` to ``False``.
"""
nb = len(tokens)
for token in tokens:
if token.is_space():
nb -= 1
elif exclude_comment and token.is_comment():
nb -= 1
return nb
# TODO: add unit test for this
[docs]
def find_substring_index(main, substring):
"""Somewhat similar to the find() method for strings,
this function determines if the tokens for substring appear
as a subsequence of the tokens for main. If so, the index
of the first token in returned, otherwise -1 is returned.
"""
main_tokens = [tok.string for tok in get_significant_tokens(main)]
sub_tokens = [tok.string for tok in get_significant_tokens(substring)]
for index, token in enumerate(main_tokens):
if (
token == sub_tokens[0]
and main_tokens[index : index + len(sub_tokens)] == sub_tokens
):
return index
return -1
[docs]
def get_first(tokens, exclude_comment=True):
"""Given a list of tokens, find the first token which is not a space token
(such as a ``NEWLINE``, ``INDENT``, ``DEDENT``, etc.) and,
by default, also not a ``COMMMENT``.
``COMMMENT`` tokens can be included by setting ``exclude_comment`` to ``False``.
Returns ``None`` if none is found.
"""
for token in tokens:
if token.is_space() or (exclude_comment and token.is_comment()):
continue
return token
return None
[docs]
def get_first_index(tokens, exclude_comment=True):
"""Given a list of tokens, find the index of the first token which is
not a space token (such as a ``NEWLINE``, ``INDENT``, ``DEDENT``, etc.) nor
a ``COMMMENT``. If it is desired to include COMMENT, set ``exclude_comment``
to ``True``.
Returns ``None`` if none is found.
"""
for index, token in enumerate(tokens):
if token.is_space() or (exclude_comment and token.is_comment()):
continue
return index
return None
[docs]
def get_last(tokens, exclude_comment=True):
"""Given a list of tokens, find the last token which is not a space token
(such as a ``NEWLINE``, ``INDENT``, ``DEDENT``, etc.) and, by default,
also not a ``COMMMENT``.
``COMMMENT`` tokens can be included by setting``exclude_comment``
to ``False``.
Returns ``None`` if none is found.
"""
return get_first(reversed(tokens), exclude_comment=exclude_comment)
[docs]
def get_last_index(tokens, exclude_comment=True):
"""Given a list of tokens, find the index of the last token which is
not a space token (such as a ``NEWLINE``, ``INDENT``, ``DEDENT``, etc.) nor
a ``COMMMENT``. If it is desired to include COMMENT, set ``exclude_comment``
to True.
Returns ``None`` if none is found.
"""
return (
len(tokens)
- 1
- get_first_index(reversed(tokens), exclude_comment=exclude_comment)
)
[docs]
def dedent(tokens, nb):
"""Given a list of tokens, produces an equivalent list corresponding
to a line of code with the first nb characters removed.
"""
line = untokenize(tokens)
line = line[nb:]
return tokenize(line)
[docs]
def indent(tokens, nb, tab=False):
"""Given a list of tokens, produces an equivalent list corresponding
to a line of code with nb space characters inserted at the beginning.
If ``tab`` is specified to be ``True``, ``nb`` tab characters are inserted
instead of spaces.
"""
line = untokenize(tokens)
if tab:
line = "\t" * nb + line
else:
line = " " * nb + line
return tokenize(line)
[docs]
def untokenize(tokens):
"""Return source code based on tokens.
Adapted from https://github.com/myint/untokenize,
Copyright (C) 2013-2018 Steven Myint, MIT License (same as this project).
This is similar to Python's own tokenize.untokenize(), except that it
preserves spacing between tokens, by using the line
information recorded by Python's tokenize.generate_tokens.
As a result, if the original soure code had multiple spaces between
some tokens or if escaped newlines were used or if tab characters
were present in the original source, those will also be present
in the source code produced by untokenize.
Thus ``source == untokenize(tokenize(source))``.
Note: if you you modifying tokens from an original source:
Instead of full token object, ``untokenize`` will accept simple
strings; however, it will only insert them *as is* without taking them
into account when it comes with figuring out spacing between tokens.
"""
words = []
previous_line = ""
last_row = 0
last_column = -1
last_non_whitespace_token_type = None
for token in tokens:
if isinstance(token, str):
words.append(token)
continue
if token.type == py_tokenize.ENCODING:
continue
# Preserve escaped newlines.
if (
last_non_whitespace_token_type != py_tokenize.COMMENT
and token.start_row > last_row
):
if previous_line.endswith(("\\\n", "\\\r\n", "\\\r")):
words.append(previous_line[len(previous_line.rstrip(" \t\n\r\\")) :])
# Preserve spacing.
if token.start_row > last_row:
last_column = 0
if token.start_col > last_column:
words.append(token.line[last_column : token.start_col])
words.append(token.string)
previous_line = token.line
last_row = token.end_row
last_column = token.end_col
if not token.is_space():
last_non_whitespace_token_type = token.type
return "".join(words)
[docs]
def print_tokens(source):
"""Prints tokens found in source, excluding spaces and comments.
``source`` is either a string to be tokenized, or a list of Token objects.
This is occasionally useful as a debugging tool.
"""
if isinstance(source[0], Token):
source = untokenize(source)
for lines in get_lines(source):
for token in lines:
print(repr(token))
print()
__all__ = ["__all__"]
_names = dir()
def _make_all():
for name in _names:
if not name.startswith("_") and not name.startswith("py"):
__all__.append(name)
_make_all()