diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..b508dca8 --- /dev/null +++ b/.flake8 @@ -0,0 +1,8 @@ +[flake8] +exclude = + tests, + docs, + dist +max-complexity = 31 +statistics = True +show-source = True \ No newline at end of file diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 5acaa67d..b560fd65 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -43,7 +43,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -54,7 +54,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@v3 # ℹ️ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -68,4 +68,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index ff7269bc..0608cc38 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -15,36 +15,33 @@ on: jobs: test: name: Run tests on ${{ matrix.py }} - runs-on: ubuntu-20.04 # keep it on 20.04 to have Python 3.6 available + runs-on: ubuntu-latest strategy: matrix: py: + - "3.13" - "3.12" - "3.11" - "3.10" - "3.9" - "3.8" - - "3.7" - - "3.6" + - "pypy-3.10" - "pypy-3.9" - - "pypy-3.8" - - "pypy-3.7" steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.py }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.py }} allow-prereleases: true check-latest: true - name: Install dependencies run: | - python -m pip install --upgrade pip flit - flit install --deps=develop + python -m pip install --upgrade pip hatch - name: Lint with flake8 - run: flake8 sqlparse --count --max-complexity=31 --show-source --statistics - - name: Test with pytest - run: pytest --cov=sqlparse + run: hatch run flake8 + - name: Test with pytest and coverage + run: hatch run cov - name: Publish to codecov - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 diff --git a/.gitignore b/.gitignore index e5953853..77479f17 100644 --- a/.gitignore +++ b/.gitignore @@ -1,22 +1,10 @@ -# PyCharm -.idea/ - *.py[co] docs/build dist/ build/ MANIFEST .coverage -.tox/ .cache/ *.egg-info/ htmlcov/ -extras/appengine/sqlparse -extras/appengine/lib/ -extras/py3k/sqlparse -extras/py3k/tests -extras/py3k/sqlparse.diff -extras/py3k/tests.diff -coverage.xml -*.class .pytest_cache \ No newline at end of file diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 00000000..6dffd85a --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,35 @@ +# Read the Docs configuration file for Sphinx projects +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the OS, Python version and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.12" + # You can also specify other tool versions: + # nodejs: "20" + # rust: "1.70" + # golang: "1.20" + +# Build documentation in the "docs/" directory with Sphinx +sphinx: + configuration: docs/source/conf.py + # You can configure Sphinx to use a different builder, for instance use the dirhtml builder for simpler URLs + # builder: "dirhtml" + # Fail on all warnings to avoid broken references + # fail_on_warning: true + +# Optionally build your docs in additional formats such as PDF and ePub +# formats: +# - pdf +# - epub + +# Optional but recommended, declare the Python requirements required +# to build your documentation +# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +# python: +# install: +# - requirements: docs/requirements.txt \ No newline at end of file diff --git a/AUTHORS b/AUTHORS index 4617b7d7..ba46db80 100644 --- a/AUTHORS +++ b/AUTHORS @@ -7,74 +7,81 @@ This module contains code (Python 2/3 compatibility) from the six project: https://bitbucket.org/gutworth/six. Alphabetical list of contributors: -* Adam Greenhall -* Aki Ariga -* Alexander Beedie -* Alexey Malyshev -* ali-tny -* andrew deryabin -* Andrew Tipton -* atronah -* casey -* Cauê Beloni -* Christian Clauss -* circld -* Corey Zumar -* Cristian Orellana -* Dag Wieers -* Daniel Harding -* Darik Gamble -* Demetrio92 -* Dennis Taylor -* Dvořák Václav -* Erik Cederstrand -* Florian Bauer -* Fredy Wijaya -* Gavin Wahl -* Hugo van Kemenade -* hurcy -* Ian Robertson -* JacekPliszka -* JavierPan -* Jean-Martin Archer -* Jesús Leganés Combarro "Piranna" -* Johannes Hoff -* John Bodley -* Jon Dufresne -* Josh Soref -* Kevin Jing Qiu -* koljonen -* Likai Liu -* Long Le Xich -* mathilde.oustlant -* Michael Schuller -* Mike Amy -* mulos -* Oleg Broytman -* osmnv <80402144+osmnv@users.noreply.github.com> -* Patrick Schemitz -* Pi Delport -* Prudhvi Vatala -* quest -* Robert Nix -* Rocky Meza -* Romain Rigaux -* Rowan Seymour -* Ryan Wooden -* saaj -* Shen Longxing -* Simon Heisterkamp -* Sjoerd Job Postmus -* Soloman Weng -* spigwitmer -* Tao Wang -* Tenghuan -* Tim Graham -* Victor Hahn -* Victor Uriarte -* Ville Skyttä -* vthriller -* wayne.wuw -* Will Jones -* William Ivanski -* Yago Riveiro + +- Adam Greenhall +- Adam Johnson +- Aki Ariga +- Alexander Beedie +- Alexey Malyshev +- ali-tny +- andrew deryabin +- Andrew Tipton +- atronah +- casey +- Cauê Beloni +- Christian Clauss +- circld +- Corey Zumar +- Cristian Orellana +- Dag Wieers +- Daniel Harding +- Darik Gamble +- Demetrio92 +- Dennis Taylor +- Dvořák Václav +- Erik Cederstrand +- Florian Bauer +- Fredy Wijaya +- Gavin Wahl +- Georg Traar +- griff <70294474+griffatrasgo@users.noreply.github.com> +- Hugo van Kemenade +- hurcy +- Ian Robertson +- Igor Khrol +- JacekPliszka +- JavierPan +- Jean-Martin Archer +- Jesús Leganés Combarro "Piranna" +- Johannes Hoff +- John Bodley +- Jon Dufresne +- Josh Soref +- Kevin Jing Qiu +- koljonen +- Likai Liu +- Long Le Xich +- mathilde.oustlant +- Michael Schuller +- Mike Amy +- mulos +- Oleg Broytman +- osmnv <80402144+osmnv@users.noreply.github.com> +- Patrick Schemitz +- Pi Delport +- Prudhvi Vatala +- quest +- Robert Nix +- Rocky Meza +- Romain Rigaux +- Rowan Seymour +- Ryan Wooden +- saaj +- Shen Longxing +- Simon Heisterkamp +- Sjoerd Job Postmus +- skryzh +- Soloman Weng +- spigwitmer +- Tao Wang +- Tenghuan +- Tim Graham +- Victor Hahn +- Victor Uriarte +- Ville Skyttä +- vthriller +- wayne.wuw +- Will Jones +- William Ivanski +- Yago Riveiro +- Zi-Xuan Fu diff --git a/CHANGELOG b/CHANGELOG index 525918a2..b90076c9 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -61,6 +61,136 @@ Other * Update github actions to test on Python 3.10 as well (pr661, by cclaus). +Release 0.5.3 (Dez 10, 2024) +---------------------------- + +Bug Fixes + +* This version introduces a more generalized handling of potential denial of + service attack (DOS) due to recursion errors for deeply nested statements. + Brought up and fixed by @living180. Thanks a lot! + + +Release 0.5.2 (Nov 14, 2024) +---------------------------- + +Bug Fixes + +* EXTENSION is now recognized as a keyword (issue785). +* SQL hints are not removed when removing comments (issue262, by skryzh). + + +Release 0.5.1 (Jul 15, 2024) +---------------------------- + +Enhancements + +* New "compact" option for formatter. If set, the formatter tries to produce + a more compact output by avoiding some line breaks (issue783). + +Bug Fixes + +* The strip comments filter was a bit greedy and removed too much + whitespace (issue772). + Note: In some cases you might want to add `strip_whitespace=True` where you + previously used just `strip_comments=True`. `strip_comments` did some of the + work that `strip_whitespace` should do. +* Fix error when splitting statements that contain multiple CASE clauses + within a BEGIN block (issue784). +* Fix whitespace removal with nested expressions (issue782). +* Fix parsing and formatting of ORDER clauses containing NULLS FIRST or + NULLS LAST (issue532). + + +Release 0.5.0 (Apr 13, 2024) +---------------------------- + +Notable Changes + +* Drop support for Python 3.5, 3.6, and 3.7. +* Python 3.12 is now supported (pr725, by hugovk). +* IMPORTANT: Fixes a potential denial of service attack (DOS) due to recursion + error for deeply nested statements. Instead of recursion error a generic + SQLParseError is raised. See the security advisory for details: + https://github.com/andialbrecht/sqlparse/security/advisories/GHSA-2m57-hf25-phgg + The vulnerability was discovered by @uriyay-jfrog. Thanks for reporting! + +Enhancements + +* Splitting statements now allows to remove the semicolon at the end. + Some database backends love statements without semicolon (issue742). +* Support TypedLiterals in get_parameters (pr749, by Khrol). +* Improve splitting of Transact SQL when using GO keyword (issue762). +* Support for some JSON operators (issue682). +* Improve formatting of statements containing JSON operators (issue542). +* Support for BigQuery and Snowflake keywords (pr699, by griffatrasgo). +* Support parsing of OVER clause (issue701, pr768 by r33s3n6). + +Bug Fixes + +* Ignore dunder attributes when creating Tokens (issue672). +* Allow operators to precede dollar-quoted strings (issue763). +* Fix parsing of nested order clauses (issue745, pr746 by john-bodley). +* Thread-safe initialization of Lexer class (issue730). +* Classify TRUNCATE as DDL and GRANT/REVOKE as DCL keywords (based on pr719 + by josuc1, thanks for bringing this up!). +* Fix parsing of PRIMARY KEY (issue740). + +Other + +* Optimize performance of matching function (pr799, by admachainz). + + +Release 0.4.4 (Apr 18, 2023) +---------------------------- + +Notable Changes + +* IMPORTANT: This release fixes a security vulnerability in the + parser where a regular expression vulnerable to ReDOS (Regular + Expression Denial of Service) was used. See the security advisory + for details: https://github.com/andialbrecht/sqlparse/security/advisories/GHSA-rrm6-wvj7-cwh2 + The vulnerability was discovered by @erik-krogh from GitHub + Security Lab (GHSL). Thanks for reporting! + +Bug Fixes + +* Revert a change from 0.4.0 that changed IN to be a comparison (issue694). + The primary expectation is that IN is treated as a keyword and not as a + comparison operator. That also follows the definition of reserved keywords + for the major SQL syntax definitions. +* Fix regular expressions for string parsing. + +Other + +* sqlparse now uses pyproject.toml instead of setup.cfg (issue685). + + +Release 0.4.3 (Sep 23, 2022) +---------------------------- + +Enhancements + +* Add support for DIV operator (pr664, by chezou). +* Add support for additional SPARK keywords (pr643, by mrmasterplan). +* Avoid tokens copy (pr622, by living180). +* Add REGEXP as a comparision (pr647, by PeterSandwich). +* Add DISTINCTROW keyword for MS Access (issue677). +* Improve parsing of CREATE TABLE AS SELECT (pr662, by chezou). + +Bug Fixes + +* Fix spelling of INDICATOR keyword (pr653, by ptld). +* Fix formatting error in EXTRACT function (issue562, issue670, pr676, by ecederstrand). +* Fix bad parsing of create table statements that use lower case (issue217, pr642, by mrmasterplan). +* Handle backtick as valid quote char (issue628, pr629, by codenamelxl). +* Allow any unicode character as valid identifier name (issue641). + +Other + +* Update github actions to test on Python 3.10 as well (pr661, by cclaus). + + Release 0.4.2 (Sep 10, 2021) ---------------------------- diff --git a/Makefile b/Makefile index b3db8e3d..bc5144d2 100644 --- a/Makefile +++ b/Makefile @@ -23,5 +23,5 @@ clean: release: @rm -rf dist/ python -m build - twine upload --sign --identity E0B84F81 dist/* + hatch publish @echo "Reminder: Add release on github https://github.com/andialbrecht/sqlparse/releases" diff --git a/README.rst b/README.rst index 67ddaf96..3eaf0efb 100644 --- a/README.rst +++ b/README.rst @@ -11,7 +11,7 @@ python-sqlparse - Parse SQL statements sqlparse is a non-validating SQL parser for Python. It provides support for parsing, splitting and formatting SQL statements. -The module is compatible with Python 3.6+ and released under the terms of the +The module is compatible with Python 3.8+ and released under the terms of the `New BSD license `_. Visit the project page at https://github.com/andialbrecht/sqlparse for diff --git a/docs/source/api.rst b/docs/source/api.rst index 40193d0b..08da3220 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -62,6 +62,9 @@ The :meth:`~sqlparse.format` function accepts the following keyword arguments. The column limit (in characters) for wrapping comma-separated lists. If unspecified, it puts every item in the list on its own line. +``compact`` + If ``True`` the formatter tries to produce more compact output. + ``output_format`` If given the output is additionally formatted to be used as a variable in a programming language. Allowed values are "python" and "php". @@ -69,4 +72,4 @@ The :meth:`~sqlparse.format` function accepts the following keyword arguments. ``comma_first`` If ``True`` comma-first notation for column names is used. - + diff --git a/docs/source/extending.rst b/docs/source/extending.rst index 0c10924b..866303b7 100644 --- a/docs/source/extending.rst +++ b/docs/source/extending.rst @@ -70,7 +70,7 @@ a keyword to the lexer: lex.add_keywords(keywords.KEYWORDS) # add a custom keyword dictionary - lex.add_keywords({'BAR', sqlparse.tokens.Keyword}) + lex.add_keywords({'BAR': sqlparse.tokens.Keyword}) # no configuration is passed here. The lexer is used as a singleton. sqlparse.parse("select * from foo zorder by bar;") diff --git a/examples/column_defs_lowlevel.py b/examples/column_defs_lowlevel.py index 11ff4f38..5b43945c 100644 --- a/examples/column_defs_lowlevel.py +++ b/examples/column_defs_lowlevel.py @@ -27,7 +27,7 @@ def extract_definitions(token_list): if par_level == 0: break else: - par_level += 1 + par_level -= 1 elif token.match(sqlparse.tokens.Punctuation, ','): if tmp: definitions.append(tmp) diff --git a/pyproject.toml b/pyproject.toml index d9a921f1..229f344d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [build-system] -requires = ["flit_core >=3.2,<4"] -build-backend = "flit_core.buildapi" +requires = ["hatchling"] +build-backend = "hatchling.build" [project] name = "sqlparse" @@ -16,24 +16,23 @@ classifiers = [ "Programming Language :: Python", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Database", "Topic :: Software Development", ] -requires-python = ">=3.6" +requires-python = ">=3.8" [project.urls] Home = "https://github.com/andialbrecht/sqlparse" Documentation = "https://sqlparse.readthedocs.io/" -"Release Notes" = "https://sqlparse.readthedocs.io/en/latest/changes/" +"Release Notes" = "https://sqlparse.readthedocs.io/en/latest/changes.html" Source = "https://github.com/andialbrecht/sqlparse" Tracker = "https://github.com/andialbrecht/sqlparse/issues" @@ -42,35 +41,64 @@ sqlformat = "sqlparse.__main__:main" [project.optional-dependencies] dev = [ - "importlib_metadata<5; python_version <= '3.7'", - "flake8", + "hatch", "build", ] -test = [ - "pytest", - "pytest-cov", -] doc = [ "sphinx", ] -tox = [ - "virtualenv<20.22.0", # 20.22.0 dropped Python 3.6 support - "tox<4.5.0", # >=4.5.0 requires virtualenv>=20.22 + +[tool.hatch.version] +path = "sqlparse/__init__.py" + +[tool.hatch.envs.default] +dependencies = [ + "coverage[toml]>=6.5", + "pytest", + # switch to ruff, but fix problems first + # but check defaults! + # https://hatch.pypa.io/1.9/config/static-analysis/#default-settings + "flake8", +] +[tool.hatch.envs.default.scripts] +unittest = "pytest {args:tests}" +test-cov = "coverage run -m pytest {args:tests}" +cov-report = [ + "- coverage combine", + "coverage report", +] +cov = [ + "test-cov", + "cov-report", ] +check = "flake8 sqlparse/" +test = ["check", "unittest"] -[tool.flit.sdist] -include = [ - "docs/source/", - "docs/sqlformat.1", - "docs/Makefile", - "tests/*.py", "tests/files/*.sql", - "LICENSE", - "TODO", - "AUTHORS", - "CHANGELOG", - "Makefile", - "tox.ini", +[[tool.hatch.envs.all.matrix]] +python = ["3.8", "3.9", "3.10", "3.11", "3.12"] + +[tool.hatch.envs.types] +dependencies = [ + "mypy>=1.0.0", ] +[tool.hatch.envs.types.scripts] +check = "mypy --install-types --non-interactive {args:sqlparse tests}" [tool.coverage.run] -omit = ["sqlparse/__main__.py"] +source_pkgs = ["sqlparse", "tests"] +branch = true +parallel = true +omit = [ + "sqlparse/__main__.py", +] + +[tool.coverage.paths] +sqlparse = ["sqlparse"] +tests = ["tests"] + +[tool.coverage.report] +exclude_lines = [ + "no cov", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", +] diff --git a/sqlparse/__init__.py b/sqlparse/__init__.py index 38202080..88781b2e 100644 --- a/sqlparse/__init__.py +++ b/sqlparse/__init__.py @@ -17,8 +17,8 @@ from sqlparse import joins # noqa: F401 -__version__ = '0.5.0.dev0' -__all__ = ['engine', 'filters', 'formatter', 'sql', 'tokens', 'cli'] +__version__ = "0.5.4.dev0" +__all__ = ["engine", "filters", "formatter", "sql", "tokens", "cli"] def parse(sql, encoding=None): @@ -57,15 +57,17 @@ def format(sql, encoding=None, **options): options = formatter.validate_options(options) stack = formatter.build_filter_stack(stack, options) stack.postprocess.append(filters.SerializerUnicode()) - return ''.join(stack.run(sql, encoding)) + return "".join(stack.run(sql, encoding)) -def split(sql, encoding=None): +def split(sql, encoding=None, strip_semicolon=False): """Split *sql* into single statements. :param sql: A string containing one or more SQL statements. :param encoding: The encoding of the statement (optional). + :param strip_semicolon: If True, remove trainling semicolons + (default: False). :returns: A list of strings. """ - stack = engine.FilterStack() + stack = engine.FilterStack(strip_semicolon=strip_semicolon) return [str(stmt).strip() for stmt in stack.run(sql, encoding)] diff --git a/sqlparse/cli.py b/sqlparse/cli.py index 7a8aacbf..5bbfd9f1 100755 --- a/sqlparse/cli.py +++ b/sqlparse/cli.py @@ -60,7 +60,7 @@ def create_parser(): dest='keyword_case', choices=_CASE_CHOICES, help='change case of keywords, CHOICE is one of {}'.format( - ', '.join('"{}"'.format(x) for x in _CASE_CHOICES))) + ', '.join(f'"{x}"' for x in _CASE_CHOICES))) group.add_argument( '-i', '--identifiers', @@ -68,7 +68,7 @@ def create_parser(): dest='identifier_case', choices=_CASE_CHOICES, help='change case of identifiers, CHOICE is one of {}'.format( - ', '.join('"{}"'.format(x) for x in _CASE_CHOICES))) + ', '.join(f'"{x}"' for x in _CASE_CHOICES))) group.add_argument( '-l', '--language', @@ -139,6 +139,13 @@ def create_parser(): type=bool, help='Insert linebreak before comma (default False)') + group.add_argument( + '--compact', + dest='compact', + default=False, + type=bool, + help='Try to produce more compact output (default False)') + group.add_argument( '--encoding', dest='encoding', @@ -150,7 +157,7 @@ def create_parser(): def _error(msg): """Print msg and optionally exit with return code exit_.""" - sys.stderr.write('[ERROR] {}\n'.format(msg)) + sys.stderr.write(f'[ERROR] {msg}\n') return 1 @@ -170,7 +177,7 @@ def main(args=None): data = ''.join(f.readlines()) except OSError as e: return _error( - 'Failed to read {}: {}'.format(args.filename, e)) + f'Failed to read {args.filename}: {e}') close_stream = False if args.outfile: @@ -178,7 +185,7 @@ def main(args=None): stream = open(args.outfile, 'w', encoding=args.encoding) close_stream = True except OSError as e: - return _error('Failed to open {}: {}'.format(args.outfile, e)) + return _error(f'Failed to open {args.outfile}: {e}') else: stream = sys.stdout @@ -186,7 +193,7 @@ def main(args=None): try: formatter_opts = sqlparse.formatter.validate_options(formatter_opts) except SQLParseError as e: - return _error('Invalid options: {}'.format(e)) + return _error(f'Invalid options: {e}') s = sqlparse.format(data, **formatter_opts) stream.write(s) diff --git a/sqlparse/engine/filter_stack.py b/sqlparse/engine/filter_stack.py index 9665a224..415d3fc9 100644 --- a/sqlparse/engine/filter_stack.py +++ b/sqlparse/engine/filter_stack.py @@ -10,35 +10,42 @@ from sqlparse import lexer from sqlparse.engine import grouping from sqlparse.engine.statement_splitter import StatementSplitter +from sqlparse.exceptions import SQLParseError +from sqlparse.filters import StripTrailingSemicolonFilter class FilterStack: - def __init__(self): + def __init__(self, strip_semicolon=False): self.preprocess = [] self.stmtprocess = [] self.postprocess = [] self._grouping = False + if strip_semicolon: + self.stmtprocess.append(StripTrailingSemicolonFilter()) def enable_grouping(self): self._grouping = True def run(self, sql, encoding=None): - stream = lexer.tokenize(sql, encoding) - # Process token stream - for filter_ in self.preprocess: - stream = filter_.process(stream) + try: + stream = lexer.tokenize(sql, encoding) + # Process token stream + for filter_ in self.preprocess: + stream = filter_.process(stream) - stream = StatementSplitter().process(stream) + stream = StatementSplitter().process(stream) - # Output: Stream processed Statements - for stmt in stream: - if self._grouping: - stmt = grouping.group(stmt) + # Output: Stream processed Statements + for stmt in stream: + if self._grouping: + stmt = grouping.group(stmt) - for filter_ in self.stmtprocess: - filter_.process(stmt) + for filter_ in self.stmtprocess: + filter_.process(stmt) - for filter_ in self.postprocess: - stmt = filter_.process(stmt) + for filter_ in self.postprocess: + stmt = filter_.process(stmt) - yield stmt + yield stmt + except RecursionError as err: + raise SQLParseError('Maximum recursion depth exceeded') from err diff --git a/sqlparse/engine/grouping.py b/sqlparse/engine/grouping.py index 57d257e2..a63f4da2 100644 --- a/sqlparse/engine/grouping.py +++ b/sqlparse/engine/grouping.py @@ -139,7 +139,12 @@ def post(tlist, pidx, tidx, nidx): def group_period(tlist): def match(token): - return token.match(T.Punctuation, '.') + for ttype, value in ((T.Punctuation, '.'), + (T.Operator, '->'), + (T.Operator, '->>')): + if token.match(ttype, value): + return True + return False def valid_prev(token): sqlcls = sql.SquareBrackets, sql.Identifier @@ -153,7 +158,7 @@ def valid_next(token): def post(tlist, pidx, tidx, nidx): # next_ validation is being performed here. issue261 sqlcls = sql.SquareBrackets, sql.Function - ttypes = T.Name, T.String.Symbol, T.Wildcard + ttypes = T.Name, T.String.Symbol, T.Wildcard, T.String.Single next_ = tlist[nidx] if nidx is not None else None valid_next = imt(next_, i=sqlcls, t=ttypes) @@ -230,6 +235,16 @@ def group_identifier(tlist): tidx, token = tlist.token_next_by(t=ttypes, idx=tidx) +@recurse(sql.Over) +def group_over(tlist): + tidx, token = tlist.token_next_by(m=sql.Over.M_OPEN) + while token: + nidx, next_ = tlist.token_next(tidx) + if imt(next_, i=sql.Parenthesis, t=T.Name): + tlist.group_tokens(sql.Over, tidx, nidx) + tidx, token = tlist.token_next_by(m=sql.Over.M_OPEN, idx=tidx) + + def group_arrays(tlist): sqlcls = sql.SquareBrackets, sql.Identifier, sql.Function ttypes = T.Name, T.String.Symbol @@ -299,7 +314,7 @@ def group_comments(tlist): tidx, token = tlist.token_next_by(t=T.Comment) while token: eidx, end = tlist.token_not_matching( - lambda tk: imt(tk, t=T.Comment) or tk.is_whitespace, idx=tidx) + lambda tk: imt(tk, t=T.Comment) or tk.is_newline, idx=tidx) if end is not None: eidx, end = tlist.token_prev(eidx, skip_ws=False) tlist.group_tokens(sql.Comment, tidx, eidx) @@ -356,10 +371,16 @@ def group_functions(tlist): while token: nidx, next_ = tlist.token_next(tidx) if isinstance(next_, sql.Parenthesis): - tlist.group_tokens(sql.Function, tidx, nidx) + over_idx, over = tlist.token_next(nidx) + if over and isinstance(over, sql.Over): + eidx = over_idx + else: + eidx = nidx + tlist.group_tokens(sql.Function, tidx, eidx) tidx, token = tlist.token_next_by(t=T.Name, idx=tidx) +@recurse(sql.Identifier) def group_order(tlist): """Group together Identifier and Asc/Desc token""" tidx, token = tlist.token_next_by(t=T.Keyword.Order) @@ -406,6 +427,7 @@ def group(stmt): group_for, group_begin, + group_over, group_functions, group_where, group_period, diff --git a/sqlparse/engine/statement_splitter.py b/sqlparse/engine/statement_splitter.py index 57bef806..9d3a9a83 100644 --- a/sqlparse/engine/statement_splitter.py +++ b/sqlparse/engine/statement_splitter.py @@ -17,6 +17,7 @@ def __init__(self): def _reset(self): """Set the filter attributes to its default values""" self._in_declare = False + self._in_case = False self._is_create = False self._begin_depth = 0 @@ -54,20 +55,22 @@ def _change_splitlevel(self, ttype, value): if unified == 'BEGIN': self._begin_depth += 1 if self._is_create: - # FIXME(andi): This makes no sense. + # FIXME(andi): This makes no sense. ## this comment neither return 1 return 0 - # Should this respect a preceding BEGIN? - # In CASE ... WHEN ... END this results in a split level -1. - # Would having multiple CASE WHEN END and a Assignment Operator - # cause the statement to cut off prematurely? + # BEGIN and CASE/WHEN both end with END if unified == 'END': - self._begin_depth = max(0, self._begin_depth - 1) + if not self._in_case: + self._begin_depth = max(0, self._begin_depth - 1) + else: + self._in_case = False return -1 if (unified in ('IF', 'FOR', 'WHILE', 'CASE') and self._is_create and self._begin_depth > 0): + if unified == 'CASE': + self._in_case = True return 1 if unified in ('END IF', 'END FOR', 'END WHILE'): @@ -99,7 +102,12 @@ def process(self, stream): self.tokens.append(sql.Token(ttype, value, position)) # Check if we get the end of a statement - if self.level <= 0 and ttype is T.Punctuation and value == ';': + # Issue762: Allow GO (or "GO 2") as statement splitter. + # When implementing a language toggle, it's not only to add + # keywords it's also to change some rules, like this splitting + # rule. + if (self.level <= 0 and ttype is T.Punctuation and value == ';') \ + or (ttype is T.Keyword and value.split()[0] == 'GO'): self.consume_ws = True # Yield pending statement (if any) diff --git a/sqlparse/filters/__init__.py b/sqlparse/filters/__init__.py index 5bd6b325..06169460 100644 --- a/sqlparse/filters/__init__.py +++ b/sqlparse/filters/__init__.py @@ -8,6 +8,7 @@ from sqlparse.filters.others import SerializerUnicode from sqlparse.filters.others import StripCommentsFilter from sqlparse.filters.others import StripWhitespaceFilter +from sqlparse.filters.others import StripTrailingSemicolonFilter from sqlparse.filters.others import SpacesAroundOperatorsFilter from sqlparse.filters.output import OutputPHPFilter @@ -25,6 +26,7 @@ 'SerializerUnicode', 'StripCommentsFilter', 'StripWhitespaceFilter', + 'StripTrailingSemicolonFilter', 'SpacesAroundOperatorsFilter', 'OutputPHPFilter', diff --git a/sqlparse/filters/aligned_indent.py b/sqlparse/filters/aligned_indent.py index dc609263..553483bf 100644 --- a/sqlparse/filters/aligned_indent.py +++ b/sqlparse/filters/aligned_indent.py @@ -126,7 +126,7 @@ def _process_default(self, tlist): self._process(sgroup) def _process(self, tlist): - func_name = '_process_{cls}'.format(cls=type(tlist).__name__) + func_name = f'_process_{type(tlist).__name__}' func = getattr(self, func_name.lower(), self._process_default) func(tlist) diff --git a/sqlparse/filters/others.py b/sqlparse/filters/others.py index 9e617c37..40a808df 100644 --- a/sqlparse/filters/others.py +++ b/sqlparse/filters/others.py @@ -15,9 +15,9 @@ class StripCommentsFilter: @staticmethod def _process(tlist): - def get_next_comment(): + def get_next_comment(idx=-1): # TODO(andi) Comment types should be unified, see related issue38 - return tlist.token_next_by(i=sql.Comment, t=T.Comment) + return tlist.token_next_by(i=sql.Comment, t=T.Comment, idx=idx) def _get_insert_token(token): """Returns either a whitespace or the line breaks from token.""" @@ -31,15 +31,35 @@ def _get_insert_token(token): else: return sql.Token(T.Whitespace, ' ') + sql_hints = (T.Comment.Multiline.Hint, T.Comment.Single.Hint) tidx, token = get_next_comment() while token: + # skipping token remove if token is a SQL-Hint. issue262 + is_sql_hint = False + if token.ttype in sql_hints: + is_sql_hint = True + elif isinstance(token, sql.Comment): + comment_tokens = token.tokens + if len(comment_tokens) > 0: + if comment_tokens[0].ttype in sql_hints: + is_sql_hint = True + + if is_sql_hint: + # using current index as start index to search next token for + # preventing infinite loop in cases when token type is a + # "SQL-Hint" and has to be skipped + tidx, token = get_next_comment(idx=tidx) + continue + pidx, prev_ = tlist.token_prev(tidx, skip_ws=False) nidx, next_ = tlist.token_next(tidx, skip_ws=False) # Replace by whitespace if prev and next exist and if they're not # whitespaces. This doesn't apply if prev or next is a parenthesis. - if (prev_ is None or next_ is None - or prev_.is_whitespace or prev_.match(T.Punctuation, '(') - or next_.is_whitespace or next_.match(T.Punctuation, ')')): + if ( + prev_ is None or next_ is None + or prev_.is_whitespace or prev_.match(T.Punctuation, '(') + or next_.is_whitespace or next_.match(T.Punctuation, ')') + ): # Insert a whitespace to ensure the following SQL produces # a valid SQL (see #425). if prev_ is not None and not prev_.match(T.Punctuation, '('): @@ -48,7 +68,10 @@ def _get_insert_token(token): else: tlist.tokens[tidx] = _get_insert_token(token) - tidx, token = get_next_comment() + # using current index as start index to search next token for + # preventing infinite loop in cases when token type is a + # "SQL-Hint" and has to be skipped + tidx, token = get_next_comment(idx=tidx) def process(self, stmt): [self.process(sgroup) for sgroup in stmt.get_sublists()] @@ -58,7 +81,7 @@ def process(self, stmt): class StripWhitespaceFilter: def _stripws(self, tlist): - func_name = '_stripws_{cls}'.format(cls=type(tlist).__name__) + func_name = f'_stripws_{type(tlist).__name__}' func = getattr(self, func_name.lower(), self._stripws_default) func(tlist) @@ -91,6 +114,10 @@ def _stripws_parenthesis(self, tlist): tlist.tokens.pop(1) while tlist.tokens[-2].is_whitespace: tlist.tokens.pop(-2) + if tlist.tokens[-2].is_group: + # save to remove the last whitespace + while tlist.tokens[-2].tokens[-1].is_whitespace: + tlist.tokens[-2].tokens.pop(-1) self._stripws_default(tlist) def process(self, stmt, depth=0): @@ -126,6 +153,15 @@ def process(self, stmt): return stmt +class StripTrailingSemicolonFilter: + + def process(self, stmt): + while stmt.tokens and (stmt.tokens[-1].is_whitespace + or stmt.tokens[-1].value == ';'): + stmt.tokens.pop() + return stmt + + # --------------------------- # postprocess diff --git a/sqlparse/filters/reindent.py b/sqlparse/filters/reindent.py index 9fb232f0..e8cc3b59 100644 --- a/sqlparse/filters/reindent.py +++ b/sqlparse/filters/reindent.py @@ -12,7 +12,7 @@ class ReindentFilter: def __init__(self, width=2, char=' ', wrap_after=0, n='\n', comma_first=False, indent_after_first=False, - indent_columns=False): + indent_columns=False, compact=False): self.n = n self.width = width self.char = char @@ -21,6 +21,7 @@ def __init__(self, width=2, char=' ', wrap_after=0, n='\n', self.wrap_after = wrap_after self.comma_first = comma_first self.indent_columns = indent_columns + self.compact = compact self._curr_stmt = None self._last_stmt = None self._last_func = None @@ -96,7 +97,7 @@ def _split_statements(self, tlist): tidx, token = tlist.token_next_by(t=ttypes, idx=tidx) def _process(self, tlist): - func_name = '_process_{cls}'.format(cls=type(tlist).__name__) + func_name = f'_process_{type(tlist).__name__}' func = getattr(self, func_name.lower(), self._process_default) func(tlist) @@ -196,15 +197,19 @@ def _process_case(self, tlist): with offset(self, self._get_offset(tlist[0])): with offset(self, self._get_offset(first)): for cond, value in iterable: - token = value[0] if cond is None else cond[0] - tlist.insert_before(token, self.nl()) + str_cond = ''.join(str(x) for x in cond or []) + str_value = ''.join(str(x) for x in value) + end_pos = self.offset + 1 + len(str_cond) + len(str_value) + if (not self.compact and end_pos > self.wrap_after): + token = value[0] if cond is None else cond[0] + tlist.insert_before(token, self.nl()) # Line breaks on group level are done. let's add an offset of # len "when ", "then ", "else " with offset(self, len("WHEN ")): self._process_default(tlist) end_idx, end = tlist.token_next_by(m=sql.Case.M_CLOSE) - if end_idx is not None: + if end_idx is not None and not self.compact: tlist.insert_before(end_idx, self.nl()) def _process_values(self, tlist): diff --git a/sqlparse/filters/right_margin.py b/sqlparse/filters/right_margin.py index 3e670562..5a8e6f87 100644 --- a/sqlparse/filters/right_margin.py +++ b/sqlparse/filters/right_margin.py @@ -37,7 +37,7 @@ def _process(self, group, stream): indent = match.group() else: indent = '' - yield sql.Token(T.Whitespace, '\n{}'.format(indent)) + yield sql.Token(T.Whitespace, f'\n{indent}') self.line = indent self.line += val yield token diff --git a/sqlparse/formatter.py b/sqlparse/formatter.py index 1d1871cf..72f2c19e 100644 --- a/sqlparse/formatter.py +++ b/sqlparse/formatter.py @@ -11,7 +11,7 @@ from sqlparse.exceptions import SQLParseError -def validate_options(options): +def validate_options(options): # noqa: C901 """Validates options.""" kwcase = options.get('keyword_case') if kwcase not in [None, 'upper', 'lower', 'capitalize']: @@ -116,6 +116,11 @@ def validate_options(options): raise SQLParseError('comma_first requires a boolean value') options['comma_first'] = comma_first + compact = options.get('compact', False) + if compact not in [True, False]: + raise SQLParseError('compact requires a boolean value') + options['compact'] = compact + right_margin = options.get('right_margin') if right_margin is not None: try: @@ -171,7 +176,8 @@ def build_filter_stack(stack, options): indent_after_first=options['indent_after_first'], indent_columns=options['indent_columns'], wrap_after=options['wrap_after'], - comma_first=options['comma_first'])) + comma_first=options['comma_first'], + compact=options['compact'],)) if options.get('reindent_aligned', False): stack.enable_grouping() diff --git a/sqlparse/keywords.py b/sqlparse/keywords.py index a56f73b8..e0a8a958 100644 --- a/sqlparse/keywords.py +++ b/sqlparse/keywords.py @@ -14,52 +14,41 @@ SQL_REGEX = [ - (r'(--|# )\+.*?(\r\n|\r|\n|$)', tokens.Comment.Single.Hint), - (r'/\*\+[\s\S]*?\*/', tokens.Comment.Multiline.Hint), - - (r'(--|# ).*?(\r\n|\r|\n|$)', tokens.Comment.Single), - (r'/\*[\s\S]*?\*/', tokens.Comment.Multiline), - - (r'(\r\n|\r|\n)', tokens.Newline), - (r'\s+?', tokens.Whitespace), - - (r':=', tokens.Assignment), - (r'::', tokens.Punctuation), - - (r'\*', tokens.Wildcard), - + (r"(--|# )\+.*?(\r\n|\r|\n|$)", tokens.Comment.Single.Hint), + (r"/\*\+[\s\S]*?\*/", tokens.Comment.Multiline.Hint), + (r"(--|# ).*?(\r\n|\r|\n|$)", tokens.Comment.Single), + (r"/\*[\s\S]*?\*/", tokens.Comment.Multiline), + (r"(\r\n|\r|\n)", tokens.Newline), + (r"\s+?", tokens.Whitespace), + (r":=", tokens.Assignment), + (r"::", tokens.Punctuation), + (r"\*", tokens.Wildcard), (r"`(``|[^`])*`", tokens.Name), (r"´(´´|[^´])*´", tokens.Name), - (r'((?=~!]+', tokens.Operator.Comparison), - (r'[+/@#%^&|^-]+', tokens.Operator), + (r"\w[$#\w]*", PROCESS_AS_KEYWORD), + (r"[;:()\[\],\.]", tokens.Punctuation), + # JSON operators + (r"(\->>?|#>>?|@>|<@|\?\|?|\?&|\-|#\-)", tokens.Operator), + (r"[<>=~!]+", tokens.Operator.Comparison), + (r"[+/@#%^&|^-]+", tokens.Operator), ] KEYWORDS = { - 'ABORT': tokens.Keyword, - 'ABS': tokens.Keyword, - 'ABSOLUTE': tokens.Keyword, - 'ACCESS': tokens.Keyword, - 'ADA': tokens.Keyword, - 'ADD': tokens.Keyword, - 'ADMIN': tokens.Keyword, - 'AFTER': tokens.Keyword, - 'AGGREGATE': tokens.Keyword, - 'ALIAS': tokens.Keyword, - 'ALL': tokens.Keyword, - 'ALLOCATE': tokens.Keyword, - 'ANALYSE': tokens.Keyword, - 'ANALYZE': tokens.Keyword, - 'ANY': tokens.Keyword, - 'ARRAYLEN': tokens.Keyword, - 'ARE': tokens.Keyword, - 'ASC': tokens.Keyword.Order, - 'ASENSITIVE': tokens.Keyword, - 'ASSERTION': tokens.Keyword, - 'ASSIGNMENT': tokens.Keyword, - 'ASYMMETRIC': tokens.Keyword, - 'AT': tokens.Keyword, - 'ATOMIC': tokens.Keyword, - 'AUDIT': tokens.Keyword, - 'AUTHORIZATION': tokens.Keyword, - 'AUTO_INCREMENT': tokens.Keyword, - 'AVG': tokens.Keyword, - - 'BACKWARD': tokens.Keyword, - 'BEFORE': tokens.Keyword, - 'BEGIN': tokens.Keyword, - 'BETWEEN': tokens.Keyword, - 'BITVAR': tokens.Keyword, - 'BIT_LENGTH': tokens.Keyword, - 'BOTH': tokens.Keyword, - 'BREADTH': tokens.Keyword, - + "ABORT": tokens.Keyword, + "ABS": tokens.Keyword, + "ABSOLUTE": tokens.Keyword, + "ACCESS": tokens.Keyword, + "ADA": tokens.Keyword, + "ADD": tokens.Keyword, + "ADMIN": tokens.Keyword, + "AFTER": tokens.Keyword, + "AGGREGATE": tokens.Keyword, + "ALIAS": tokens.Keyword, + "ALL": tokens.Keyword, + "ALLOCATE": tokens.Keyword, + "ANALYSE": tokens.Keyword, + "ANALYZE": tokens.Keyword, + "ANY": tokens.Keyword, + "ARRAYLEN": tokens.Keyword, + "ARE": tokens.Keyword, + "ASENSITIVE": tokens.Keyword, + "ASSERTION": tokens.Keyword, + "ASSIGNMENT": tokens.Keyword, + "ASYMMETRIC": tokens.Keyword, + "AT": tokens.Keyword, + "ATOMIC": tokens.Keyword, + "AUDIT": tokens.Keyword, + "AUTHORIZATION": tokens.Keyword, + "AUTO_INCREMENT": tokens.Keyword, + "AVG": tokens.Keyword, + "BACKWARD": tokens.Keyword, + "BEFORE": tokens.Keyword, + "BEGIN": tokens.Keyword, + "BETWEEN": tokens.Keyword, + "BITVAR": tokens.Keyword, + "BIT_LENGTH": tokens.Keyword, + "BOTH": tokens.Keyword, + "BREADTH": tokens.Keyword, # 'C': tokens.Keyword, # most likely this is an alias - 'CACHE': tokens.Keyword, - 'CALL': tokens.Keyword, - 'CALLED': tokens.Keyword, - 'CARDINALITY': tokens.Keyword, - 'CASCADE': tokens.Keyword, - 'CASCADED': tokens.Keyword, - 'CAST': tokens.Keyword, - 'CATALOG': tokens.Keyword, - 'CATALOG_NAME': tokens.Keyword, - 'CHAIN': tokens.Keyword, - 'CHARACTERISTICS': tokens.Keyword, - 'CHARACTER_LENGTH': tokens.Keyword, - 'CHARACTER_SET_CATALOG': tokens.Keyword, - 'CHARACTER_SET_NAME': tokens.Keyword, - 'CHARACTER_SET_SCHEMA': tokens.Keyword, - 'CHAR_LENGTH': tokens.Keyword, - 'CHARSET': tokens.Keyword, - 'CHECK': tokens.Keyword, - 'CHECKED': tokens.Keyword, - 'CHECKPOINT': tokens.Keyword, - 'CLASS': tokens.Keyword, - 'CLASS_ORIGIN': tokens.Keyword, - 'CLOB': tokens.Keyword, - 'CLOSE': tokens.Keyword, - 'CLUSTER': tokens.Keyword, - 'COALESCE': tokens.Keyword, - 'COBOL': tokens.Keyword, - 'COLLATE': tokens.Keyword, - 'COLLATION': tokens.Keyword, - 'COLLATION_CATALOG': tokens.Keyword, - 'COLLATION_NAME': tokens.Keyword, - 'COLLATION_SCHEMA': tokens.Keyword, - 'COLLECT': tokens.Keyword, - 'COLUMN': tokens.Keyword, - 'COLUMN_NAME': tokens.Keyword, - 'COMPRESS': tokens.Keyword, - 'COMMAND_FUNCTION': tokens.Keyword, - 'COMMAND_FUNCTION_CODE': tokens.Keyword, - 'COMMENT': tokens.Keyword, - 'COMMIT': tokens.Keyword.DML, - 'COMMITTED': tokens.Keyword, - 'COMPLETION': tokens.Keyword, - 'CONCURRENTLY': tokens.Keyword, - 'CONDITION_NUMBER': tokens.Keyword, - 'CONNECT': tokens.Keyword, - 'CONNECTION': tokens.Keyword, - 'CONNECTION_NAME': tokens.Keyword, - 'CONSTRAINT': tokens.Keyword, - 'CONSTRAINTS': tokens.Keyword, - 'CONSTRAINT_CATALOG': tokens.Keyword, - 'CONSTRAINT_NAME': tokens.Keyword, - 'CONSTRAINT_SCHEMA': tokens.Keyword, - 'CONSTRUCTOR': tokens.Keyword, - 'CONTAINS': tokens.Keyword, - 'CONTINUE': tokens.Keyword, - 'CONVERSION': tokens.Keyword, - 'CONVERT': tokens.Keyword, - 'COPY': tokens.Keyword, - 'CORRESPONDING': tokens.Keyword, - 'COUNT': tokens.Keyword, - 'CREATEDB': tokens.Keyword, - 'CREATEUSER': tokens.Keyword, - 'CROSS': tokens.Keyword, - 'CUBE': tokens.Keyword, - 'CURRENT': tokens.Keyword, - 'CURRENT_DATE': tokens.Keyword, - 'CURRENT_PATH': tokens.Keyword, - 'CURRENT_ROLE': tokens.Keyword, - 'CURRENT_TIME': tokens.Keyword, - 'CURRENT_TIMESTAMP': tokens.Keyword, - 'CURRENT_USER': tokens.Keyword, - 'CURSOR': tokens.Keyword, - 'CURSOR_NAME': tokens.Keyword, - 'CYCLE': tokens.Keyword, - - 'DATA': tokens.Keyword, - 'DATABASE': tokens.Keyword, - 'DATETIME_INTERVAL_CODE': tokens.Keyword, - 'DATETIME_INTERVAL_PRECISION': tokens.Keyword, - 'DAY': tokens.Keyword, - 'DEALLOCATE': tokens.Keyword, - 'DECLARE': tokens.Keyword, - 'DEFAULT': tokens.Keyword, - 'DEFAULTS': tokens.Keyword, - 'DEFERRABLE': tokens.Keyword, - 'DEFERRED': tokens.Keyword, - 'DEFINED': tokens.Keyword, - 'DEFINER': tokens.Keyword, - 'DELIMITER': tokens.Keyword, - 'DELIMITERS': tokens.Keyword, - 'DEREF': tokens.Keyword, - 'DESC': tokens.Keyword.Order, - 'DESCRIBE': tokens.Keyword, - 'DESCRIPTOR': tokens.Keyword, - 'DESTROY': tokens.Keyword, - 'DESTRUCTOR': tokens.Keyword, - 'DETERMINISTIC': tokens.Keyword, - 'DIAGNOSTICS': tokens.Keyword, - 'DICTIONARY': tokens.Keyword, - 'DISABLE': tokens.Keyword, - 'DISCONNECT': tokens.Keyword, - 'DISPATCH': tokens.Keyword, - 'DIV': tokens.Operator, - 'DO': tokens.Keyword, - 'DOMAIN': tokens.Keyword, - 'DYNAMIC': tokens.Keyword, - 'DYNAMIC_FUNCTION': tokens.Keyword, - 'DYNAMIC_FUNCTION_CODE': tokens.Keyword, - - 'EACH': tokens.Keyword, - 'ENABLE': tokens.Keyword, - 'ENCODING': tokens.Keyword, - 'ENCRYPTED': tokens.Keyword, - 'END-EXEC': tokens.Keyword, - 'ENGINE': tokens.Keyword, - 'EQUALS': tokens.Keyword, - 'ESCAPE': tokens.Keyword, - 'EVERY': tokens.Keyword, - 'EXCEPT': tokens.Keyword, - 'EXCEPTION': tokens.Keyword, - 'EXCLUDING': tokens.Keyword, - 'EXCLUSIVE': tokens.Keyword, - 'EXEC': tokens.Keyword, - 'EXECUTE': tokens.Keyword, - 'EXISTING': tokens.Keyword, - 'EXISTS': tokens.Keyword, - 'EXPLAIN': tokens.Keyword, - 'EXTERNAL': tokens.Keyword, - 'EXTRACT': tokens.Keyword, - - 'FALSE': tokens.Keyword, - 'FETCH': tokens.Keyword, - 'FILE': tokens.Keyword, - 'FIRST': tokens.Keyword, - 'FORCE': tokens.Keyword, - 'FOREACH': tokens.Keyword, - 'FOREIGN': tokens.Keyword, - 'FORTRAN': tokens.Keyword, - 'FORWARD': tokens.Keyword, - 'FOUND': tokens.Keyword, - 'FREE': tokens.Keyword, - 'FREEZE': tokens.Keyword, - 'FULL': tokens.Keyword, - 'FUNCTION': tokens.Keyword, - + "CACHE": tokens.Keyword, + "CALL": tokens.Keyword, + "CALLED": tokens.Keyword, + "CARDINALITY": tokens.Keyword, + "CASCADE": tokens.Keyword, + "CASCADED": tokens.Keyword, + "CAST": tokens.Keyword, + "CATALOG": tokens.Keyword, + "CATALOG_NAME": tokens.Keyword, + "CHAIN": tokens.Keyword, + "CHARACTERISTICS": tokens.Keyword, + "CHARACTER_LENGTH": tokens.Keyword, + "CHARACTER_SET_CATALOG": tokens.Keyword, + "CHARACTER_SET_NAME": tokens.Keyword, + "CHARACTER_SET_SCHEMA": tokens.Keyword, + "CHAR_LENGTH": tokens.Keyword, + "CHARSET": tokens.Keyword, + "CHECK": tokens.Keyword, + "CHECKED": tokens.Keyword, + "CHECKPOINT": tokens.Keyword, + "CLASS": tokens.Keyword, + "CLASS_ORIGIN": tokens.Keyword, + "CLOB": tokens.Keyword, + "CLOSE": tokens.Keyword, + "CLUSTER": tokens.Keyword, + "COALESCE": tokens.Keyword, + "COBOL": tokens.Keyword, + "COLLATE": tokens.Keyword, + "COLLATION": tokens.Keyword, + "COLLATION_CATALOG": tokens.Keyword, + "COLLATION_NAME": tokens.Keyword, + "COLLATION_SCHEMA": tokens.Keyword, + "COLLECT": tokens.Keyword, + "COLUMN": tokens.Keyword, + "COLUMN_NAME": tokens.Keyword, + "COMPRESS": tokens.Keyword, + "COMMAND_FUNCTION": tokens.Keyword, + "COMMAND_FUNCTION_CODE": tokens.Keyword, + "COMMENT": tokens.Keyword, + "COMMIT": tokens.Keyword.DML, + "COMMITTED": tokens.Keyword, + "COMPLETION": tokens.Keyword, + "CONCURRENTLY": tokens.Keyword, + "CONDITION_NUMBER": tokens.Keyword, + "CONNECT": tokens.Keyword, + "CONNECTION": tokens.Keyword, + "CONNECTION_NAME": tokens.Keyword, + "CONSTRAINT": tokens.Keyword, + "CONSTRAINTS": tokens.Keyword, + "CONSTRAINT_CATALOG": tokens.Keyword, + "CONSTRAINT_NAME": tokens.Keyword, + "CONSTRAINT_SCHEMA": tokens.Keyword, + "CONSTRUCTOR": tokens.Keyword, + "CONTAINS": tokens.Keyword, + "CONTINUE": tokens.Keyword, + "CONVERSION": tokens.Keyword, + "CONVERT": tokens.Keyword, + "COPY": tokens.Keyword, + "CORRESPONDING": tokens.Keyword, + "COUNT": tokens.Keyword, + "CREATEDB": tokens.Keyword, + "CREATEUSER": tokens.Keyword, + "CROSS": tokens.Keyword, + "CUBE": tokens.Keyword, + "CURRENT": tokens.Keyword, + "CURRENT_DATE": tokens.Keyword, + "CURRENT_PATH": tokens.Keyword, + "CURRENT_ROLE": tokens.Keyword, + "CURRENT_TIME": tokens.Keyword, + "CURRENT_TIMESTAMP": tokens.Keyword, + "CURRENT_USER": tokens.Keyword, + "CURSOR": tokens.Keyword, + "CURSOR_NAME": tokens.Keyword, + "CYCLE": tokens.Keyword, + "DATA": tokens.Keyword, + "DATABASE": tokens.Keyword, + "DATETIME_INTERVAL_CODE": tokens.Keyword, + "DATETIME_INTERVAL_PRECISION": tokens.Keyword, + "DAY": tokens.Keyword, + "DEALLOCATE": tokens.Keyword, + "DECLARE": tokens.Keyword, + "DEFAULT": tokens.Keyword, + "DEFAULTS": tokens.Keyword, + "DEFERRABLE": tokens.Keyword, + "DEFERRED": tokens.Keyword, + "DEFINED": tokens.Keyword, + "DEFINER": tokens.Keyword, + "DELIMITER": tokens.Keyword, + "DELIMITERS": tokens.Keyword, + "DEREF": tokens.Keyword, + "DESCRIBE": tokens.Keyword, + "DESCRIPTOR": tokens.Keyword, + "DESTROY": tokens.Keyword, + "DESTRUCTOR": tokens.Keyword, + "DETERMINISTIC": tokens.Keyword, + "DIAGNOSTICS": tokens.Keyword, + "DICTIONARY": tokens.Keyword, + "DISABLE": tokens.Keyword, + "DISCONNECT": tokens.Keyword, + "DISPATCH": tokens.Keyword, + "DIV": tokens.Operator, + "DO": tokens.Keyword, + "DOMAIN": tokens.Keyword, + "DYNAMIC": tokens.Keyword, + "DYNAMIC_FUNCTION": tokens.Keyword, + "DYNAMIC_FUNCTION_CODE": tokens.Keyword, + "EACH": tokens.Keyword, + "ENABLE": tokens.Keyword, + "ENCODING": tokens.Keyword, + "ENCRYPTED": tokens.Keyword, + "END-EXEC": tokens.Keyword, + "ENGINE": tokens.Keyword, + "EQUALS": tokens.Keyword, + "ESCAPE": tokens.Keyword, + "EVERY": tokens.Keyword, + "EXCEPT": tokens.Keyword, + "EXCEPTION": tokens.Keyword, + "EXCLUDING": tokens.Keyword, + "EXCLUSIVE": tokens.Keyword, + "EXEC": tokens.Keyword, + "EXECUTE": tokens.Keyword, + "EXISTING": tokens.Keyword, + "EXISTS": tokens.Keyword, + "EXPLAIN": tokens.Keyword, + "EXTERNAL": tokens.Keyword, + "EXTRACT": tokens.Keyword, + "FALSE": tokens.Keyword, + "FETCH": tokens.Keyword, + "FILE": tokens.Keyword, + "FIRST": tokens.Keyword, + "FORCE": tokens.Keyword, + "FOREACH": tokens.Keyword, + "FOREIGN": tokens.Keyword, + "FORTRAN": tokens.Keyword, + "FORWARD": tokens.Keyword, + "FOUND": tokens.Keyword, + "FREE": tokens.Keyword, + "FREEZE": tokens.Keyword, + "FULL": tokens.Keyword, + "FUNCTION": tokens.Keyword, # 'G': tokens.Keyword, - 'GENERAL': tokens.Keyword, - 'GENERATED': tokens.Keyword, - 'GET': tokens.Keyword, - 'GLOBAL': tokens.Keyword, - 'GO': tokens.Keyword, - 'GOTO': tokens.Keyword, - 'GRANT': tokens.Keyword, - 'GRANTED': tokens.Keyword, - 'GROUPING': tokens.Keyword, - - 'HAVING': tokens.Keyword, - 'HIERARCHY': tokens.Keyword, - 'HOLD': tokens.Keyword, - 'HOUR': tokens.Keyword, - 'HOST': tokens.Keyword, - - 'IDENTIFIED': tokens.Keyword, - 'IDENTITY': tokens.Keyword, - 'IGNORE': tokens.Keyword, - 'ILIKE': tokens.Keyword, - 'IMMEDIATE': tokens.Keyword, - 'IMMUTABLE': tokens.Keyword, - - 'IMPLEMENTATION': tokens.Keyword, - 'IMPLICIT': tokens.Keyword, - 'INCLUDING': tokens.Keyword, - 'INCREMENT': tokens.Keyword, - 'INDEX': tokens.Keyword, - - 'INDICATOR': tokens.Keyword, - 'INFIX': tokens.Keyword, - 'INHERITS': tokens.Keyword, - 'INITIAL': tokens.Keyword, - 'INITIALIZE': tokens.Keyword, - 'INITIALLY': tokens.Keyword, - 'INOUT': tokens.Keyword, - 'INPUT': tokens.Keyword, - 'INSENSITIVE': tokens.Keyword, - 'INSTANTIABLE': tokens.Keyword, - 'INSTEAD': tokens.Keyword, - 'INTERSECT': tokens.Keyword, - 'INTO': tokens.Keyword, - 'INVOKER': tokens.Keyword, - 'IS': tokens.Keyword, - 'ISNULL': tokens.Keyword, - 'ISOLATION': tokens.Keyword, - 'ITERATE': tokens.Keyword, - + "GENERAL": tokens.Keyword, + "GENERATED": tokens.Keyword, + "GET": tokens.Keyword, + "GLOBAL": tokens.Keyword, + "GO": tokens.Keyword, + "GOTO": tokens.Keyword, + "GRANTED": tokens.Keyword, + "GROUPING": tokens.Keyword, + "HAVING": tokens.Keyword, + "HIERARCHY": tokens.Keyword, + "HOLD": tokens.Keyword, + "HOUR": tokens.Keyword, + "HOST": tokens.Keyword, + "IDENTIFIED": tokens.Keyword, + "IDENTITY": tokens.Keyword, + "IGNORE": tokens.Keyword, + "ILIKE": tokens.Keyword, + "IMMEDIATE": tokens.Keyword, + "IMMUTABLE": tokens.Keyword, + "IMPLEMENTATION": tokens.Keyword, + "IMPLICIT": tokens.Keyword, + "INCLUDING": tokens.Keyword, + "INCREMENT": tokens.Keyword, + "INDEX": tokens.Keyword, + "INDICATOR": tokens.Keyword, + "INFIX": tokens.Keyword, + "INHERITS": tokens.Keyword, + "INITIAL": tokens.Keyword, + "INITIALIZE": tokens.Keyword, + "INITIALLY": tokens.Keyword, + "INOUT": tokens.Keyword, + "INPUT": tokens.Keyword, + "INSENSITIVE": tokens.Keyword, + "INSTANTIABLE": tokens.Keyword, + "INSTEAD": tokens.Keyword, + "INTERSECT": tokens.Keyword, + "INTO": tokens.Keyword, + "INVOKER": tokens.Keyword, + "IS": tokens.Keyword, + "ISNULL": tokens.Keyword, + "ISOLATION": tokens.Keyword, + "ITERATE": tokens.Keyword, # 'K': tokens.Keyword, - 'KEY': tokens.Keyword, - 'KEY_MEMBER': tokens.Keyword, - 'KEY_TYPE': tokens.Keyword, - - 'LANCOMPILER': tokens.Keyword, - 'LANGUAGE': tokens.Keyword, - 'LARGE': tokens.Keyword, - 'LAST': tokens.Keyword, - 'LATERAL': tokens.Keyword, - 'LEADING': tokens.Keyword, - 'LENGTH': tokens.Keyword, - 'LESS': tokens.Keyword, - 'LEVEL': tokens.Keyword, - 'LIMIT': tokens.Keyword, - 'LISTEN': tokens.Keyword, - 'LOAD': tokens.Keyword, - 'LOCAL': tokens.Keyword, - 'LOCALTIME': tokens.Keyword, - 'LOCALTIMESTAMP': tokens.Keyword, - 'LOCATION': tokens.Keyword, - 'LOCATOR': tokens.Keyword, - 'LOCK': tokens.Keyword, - 'LOWER': tokens.Keyword, - + "KEY": tokens.Keyword, + "KEY_MEMBER": tokens.Keyword, + "KEY_TYPE": tokens.Keyword, + "LANCOMPILER": tokens.Keyword, + "LANGUAGE": tokens.Keyword, + "LARGE": tokens.Keyword, + "LAST": tokens.Keyword, + "LATERAL": tokens.Keyword, + "LEADING": tokens.Keyword, + "LENGTH": tokens.Keyword, + "LESS": tokens.Keyword, + "LEVEL": tokens.Keyword, + "LIMIT": tokens.Keyword, + "LISTEN": tokens.Keyword, + "LOAD": tokens.Keyword, + "LOCAL": tokens.Keyword, + "LOCALTIME": tokens.Keyword, + "LOCALTIMESTAMP": tokens.Keyword, + "LOCATION": tokens.Keyword, + "LOCATOR": tokens.Keyword, + "LOCK": tokens.Keyword, + "LOWER": tokens.Keyword, # 'M': tokens.Keyword, - 'MAP': tokens.Keyword, - 'MATCH': tokens.Keyword, - 'MAXEXTENTS': tokens.Keyword, - 'MAXVALUE': tokens.Keyword, - 'MESSAGE_LENGTH': tokens.Keyword, - 'MESSAGE_OCTET_LENGTH': tokens.Keyword, - 'MESSAGE_TEXT': tokens.Keyword, - 'METHOD': tokens.Keyword, - 'MINUTE': tokens.Keyword, - 'MINUS': tokens.Keyword, - 'MINVALUE': tokens.Keyword, - 'MOD': tokens.Keyword, - 'MODE': tokens.Keyword, - 'MODIFIES': tokens.Keyword, - 'MODIFY': tokens.Keyword, - 'MONTH': tokens.Keyword, - 'MORE': tokens.Keyword, - 'MOVE': tokens.Keyword, - 'MUMPS': tokens.Keyword, - - 'NAMES': tokens.Keyword, - 'NATIONAL': tokens.Keyword, - 'NATURAL': tokens.Keyword, - 'NCHAR': tokens.Keyword, - 'NCLOB': tokens.Keyword, - 'NEW': tokens.Keyword, - 'NEXT': tokens.Keyword, - 'NO': tokens.Keyword, - 'NOAUDIT': tokens.Keyword, - 'NOCOMPRESS': tokens.Keyword, - 'NOCREATEDB': tokens.Keyword, - 'NOCREATEUSER': tokens.Keyword, - 'NONE': tokens.Keyword, - 'NOT': tokens.Keyword, - 'NOTFOUND': tokens.Keyword, - 'NOTHING': tokens.Keyword, - 'NOTIFY': tokens.Keyword, - 'NOTNULL': tokens.Keyword, - 'NOWAIT': tokens.Keyword, - 'NULL': tokens.Keyword, - 'NULLABLE': tokens.Keyword, - 'NULLIF': tokens.Keyword, - - 'OBJECT': tokens.Keyword, - 'OCTET_LENGTH': tokens.Keyword, - 'OF': tokens.Keyword, - 'OFF': tokens.Keyword, - 'OFFLINE': tokens.Keyword, - 'OFFSET': tokens.Keyword, - 'OIDS': tokens.Keyword, - 'OLD': tokens.Keyword, - 'ONLINE': tokens.Keyword, - 'ONLY': tokens.Keyword, - 'OPEN': tokens.Keyword, - 'OPERATION': tokens.Keyword, - 'OPERATOR': tokens.Keyword, - 'OPTION': tokens.Keyword, - 'OPTIONS': tokens.Keyword, - 'ORDINALITY': tokens.Keyword, - 'OUT': tokens.Keyword, - 'OUTPUT': tokens.Keyword, - 'OVERLAPS': tokens.Keyword, - 'OVERLAY': tokens.Keyword, - 'OVERRIDING': tokens.Keyword, - 'OWNER': tokens.Keyword, - - 'QUARTER': tokens.Keyword, - - 'PAD': tokens.Keyword, - 'PARAMETER': tokens.Keyword, - 'PARAMETERS': tokens.Keyword, - 'PARAMETER_MODE': tokens.Keyword, - 'PARAMETER_NAME': tokens.Keyword, - 'PARAMETER_ORDINAL_POSITION': tokens.Keyword, - 'PARAMETER_SPECIFIC_CATALOG': tokens.Keyword, - 'PARAMETER_SPECIFIC_NAME': tokens.Keyword, - 'PARAMETER_SPECIFIC_SCHEMA': tokens.Keyword, - 'PARTIAL': tokens.Keyword, - 'PASCAL': tokens.Keyword, - 'PCTFREE': tokens.Keyword, - 'PENDANT': tokens.Keyword, - 'PLACING': tokens.Keyword, - 'PLI': tokens.Keyword, - 'POSITION': tokens.Keyword, - 'POSTFIX': tokens.Keyword, - 'PRECISION': tokens.Keyword, - 'PREFIX': tokens.Keyword, - 'PREORDER': tokens.Keyword, - 'PREPARE': tokens.Keyword, - 'PRESERVE': tokens.Keyword, - 'PRIMARY': tokens.Keyword, - 'PRIOR': tokens.Keyword, - 'PRIVILEGES': tokens.Keyword, - 'PROCEDURAL': tokens.Keyword, - 'PROCEDURE': tokens.Keyword, - 'PUBLIC': tokens.Keyword, - - 'RAISE': tokens.Keyword, - 'RAW': tokens.Keyword, - 'READ': tokens.Keyword, - 'READS': tokens.Keyword, - 'RECHECK': tokens.Keyword, - 'RECURSIVE': tokens.Keyword, - 'REF': tokens.Keyword, - 'REFERENCES': tokens.Keyword, - 'REFERENCING': tokens.Keyword, - 'REINDEX': tokens.Keyword, - 'RELATIVE': tokens.Keyword, - 'RENAME': tokens.Keyword, - 'REPEATABLE': tokens.Keyword, - 'RESET': tokens.Keyword, - 'RESOURCE': tokens.Keyword, - 'RESTART': tokens.Keyword, - 'RESTRICT': tokens.Keyword, + "MAP": tokens.Keyword, + "MATCH": tokens.Keyword, + "MAXEXTENTS": tokens.Keyword, + "MAXVALUE": tokens.Keyword, + "MESSAGE_LENGTH": tokens.Keyword, + "MESSAGE_OCTET_LENGTH": tokens.Keyword, + "MESSAGE_TEXT": tokens.Keyword, + "METHOD": tokens.Keyword, + "MINUTE": tokens.Keyword, + "MINUS": tokens.Keyword, + "MINVALUE": tokens.Keyword, + "MOD": tokens.Keyword, + "MODE": tokens.Keyword, + "MODIFIES": tokens.Keyword, + "MODIFY": tokens.Keyword, + "MONTH": tokens.Keyword, + "MORE": tokens.Keyword, + "MOVE": tokens.Keyword, + "MUMPS": tokens.Keyword, + "NAMES": tokens.Keyword, + "NATIONAL": tokens.Keyword, + "NATURAL": tokens.Keyword, + "NCHAR": tokens.Keyword, + "NCLOB": tokens.Keyword, + "NEW": tokens.Keyword, + "NEXT": tokens.Keyword, + "NO": tokens.Keyword, + "NOAUDIT": tokens.Keyword, + "NOCOMPRESS": tokens.Keyword, + "NOCREATEDB": tokens.Keyword, + "NOCREATEUSER": tokens.Keyword, + "NONE": tokens.Keyword, + "NOT": tokens.Keyword, + "NOTFOUND": tokens.Keyword, + "NOTHING": tokens.Keyword, + "NOTIFY": tokens.Keyword, + "NOTNULL": tokens.Keyword, + "NOWAIT": tokens.Keyword, + "NULL": tokens.Keyword, + "NULLABLE": tokens.Keyword, + "NULLIF": tokens.Keyword, + "OBJECT": tokens.Keyword, + "OCTET_LENGTH": tokens.Keyword, + "OF": tokens.Keyword, + "OFF": tokens.Keyword, + "OFFLINE": tokens.Keyword, + "OFFSET": tokens.Keyword, + "OIDS": tokens.Keyword, + "OLD": tokens.Keyword, + "ONLINE": tokens.Keyword, + "ONLY": tokens.Keyword, + "OPEN": tokens.Keyword, + "OPERATION": tokens.Keyword, + "OPERATOR": tokens.Keyword, + "OPTION": tokens.Keyword, + "OPTIONS": tokens.Keyword, + "ORDINALITY": tokens.Keyword, + "OUT": tokens.Keyword, + "OUTPUT": tokens.Keyword, + "OVERLAPS": tokens.Keyword, + "OVERLAY": tokens.Keyword, + "OVERRIDING": tokens.Keyword, + "OWNER": tokens.Keyword, + "QUARTER": tokens.Keyword, + "PAD": tokens.Keyword, + "PARAMETER": tokens.Keyword, + "PARAMETERS": tokens.Keyword, + "PARAMETER_MODE": tokens.Keyword, + "PARAMETER_NAME": tokens.Keyword, + "PARAMETER_ORDINAL_POSITION": tokens.Keyword, + "PARAMETER_SPECIFIC_CATALOG": tokens.Keyword, + "PARAMETER_SPECIFIC_NAME": tokens.Keyword, + "PARAMETER_SPECIFIC_SCHEMA": tokens.Keyword, + "PARTIAL": tokens.Keyword, + "PASCAL": tokens.Keyword, + "PCTFREE": tokens.Keyword, + "PENDANT": tokens.Keyword, + "PLACING": tokens.Keyword, + "PLI": tokens.Keyword, + "POSITION": tokens.Keyword, + "POSTFIX": tokens.Keyword, + "PRECISION": tokens.Keyword, + "PREFIX": tokens.Keyword, + "PREORDER": tokens.Keyword, + "PREPARE": tokens.Keyword, + "PRESERVE": tokens.Keyword, + "PRIMARY": tokens.Keyword, + "PRIOR": tokens.Keyword, + "PRIVILEGES": tokens.Keyword, + "PROCEDURAL": tokens.Keyword, + "PROCEDURE": tokens.Keyword, + "PUBLIC": tokens.Keyword, + "RAISE": tokens.Keyword, + "RAW": tokens.Keyword, + "READ": tokens.Keyword, + "READS": tokens.Keyword, + "RECHECK": tokens.Keyword, + "RECURSIVE": tokens.Keyword, + "REF": tokens.Keyword, + "REFERENCES": tokens.Keyword, + "REFERENCING": tokens.Keyword, + "REINDEX": tokens.Keyword, + "RELATIVE": tokens.Keyword, + "RENAME": tokens.Keyword, + "REPEATABLE": tokens.Keyword, + "RESET": tokens.Keyword, + "RESOURCE": tokens.Keyword, + "RESTART": tokens.Keyword, + "RESTRICT": tokens.Keyword, # 'RESULT': tokens.Keyword, - 'RETURN': tokens.Keyword, - 'RETURNED_LENGTH': tokens.Keyword, - 'RETURNED_OCTET_LENGTH': tokens.Keyword, - 'RETURNED_SQLSTATE': tokens.Keyword, - 'RETURNING': tokens.Keyword, - 'RETURNS': tokens.Keyword, - 'REVOKE': tokens.Keyword, - 'RIGHT': tokens.Keyword, - 'ROLE': tokens.Keyword, - 'ROLLBACK': tokens.Keyword.DML, - 'ROLLUP': tokens.Keyword, - 'ROUTINE': tokens.Keyword, - 'ROUTINE_CATALOG': tokens.Keyword, - 'ROUTINE_NAME': tokens.Keyword, - 'ROUTINE_SCHEMA': tokens.Keyword, - 'ROW': tokens.Keyword, - 'ROWS': tokens.Keyword, - 'ROW_COUNT': tokens.Keyword, - 'RULE': tokens.Keyword, - - 'SAVE_POINT': tokens.Keyword, - 'SCALE': tokens.Keyword, - 'SCHEMA': tokens.Keyword, - 'SCHEMA_NAME': tokens.Keyword, - 'SCOPE': tokens.Keyword, - 'SCROLL': tokens.Keyword, - 'SEARCH': tokens.Keyword, - 'SECOND': tokens.Keyword, - 'SECURITY': tokens.Keyword, - 'SELF': tokens.Keyword, - 'SENSITIVE': tokens.Keyword, - 'SEQUENCE': tokens.Keyword, - 'SERIALIZABLE': tokens.Keyword, - 'SERVER_NAME': tokens.Keyword, - 'SESSION': tokens.Keyword, - 'SESSION_USER': tokens.Keyword, - 'SETOF': tokens.Keyword, - 'SETS': tokens.Keyword, - 'SHARE': tokens.Keyword, - 'SHOW': tokens.Keyword, - 'SIMILAR': tokens.Keyword, - 'SIMPLE': tokens.Keyword, - 'SIZE': tokens.Keyword, - 'SOME': tokens.Keyword, - 'SOURCE': tokens.Keyword, - 'SPACE': tokens.Keyword, - 'SPECIFIC': tokens.Keyword, - 'SPECIFICTYPE': tokens.Keyword, - 'SPECIFIC_NAME': tokens.Keyword, - 'SQL': tokens.Keyword, - 'SQLBUF': tokens.Keyword, - 'SQLCODE': tokens.Keyword, - 'SQLERROR': tokens.Keyword, - 'SQLEXCEPTION': tokens.Keyword, - 'SQLSTATE': tokens.Keyword, - 'SQLWARNING': tokens.Keyword, - 'STABLE': tokens.Keyword, - 'START': tokens.Keyword.DML, + "RETURN": tokens.Keyword, + "RETURNED_LENGTH": tokens.Keyword, + "RETURNED_OCTET_LENGTH": tokens.Keyword, + "RETURNED_SQLSTATE": tokens.Keyword, + "RETURNING": tokens.Keyword, + "RETURNS": tokens.Keyword, + "RIGHT": tokens.Keyword, + "ROLE": tokens.Keyword, + "ROLLBACK": tokens.Keyword.DML, + "ROLLUP": tokens.Keyword, + "ROUTINE": tokens.Keyword, + "ROUTINE_CATALOG": tokens.Keyword, + "ROUTINE_NAME": tokens.Keyword, + "ROUTINE_SCHEMA": tokens.Keyword, + "ROWS": tokens.Keyword, + "ROW_COUNT": tokens.Keyword, + "RULE": tokens.Keyword, + "SAVE_POINT": tokens.Keyword, + "SCALE": tokens.Keyword, + "SCHEMA": tokens.Keyword, + "SCHEMA_NAME": tokens.Keyword, + "SCOPE": tokens.Keyword, + "SCROLL": tokens.Keyword, + "SEARCH": tokens.Keyword, + "SECOND": tokens.Keyword, + "SECURITY": tokens.Keyword, + "SELF": tokens.Keyword, + "SENSITIVE": tokens.Keyword, + "SEQUENCE": tokens.Keyword, + "SERIALIZABLE": tokens.Keyword, + "SERVER_NAME": tokens.Keyword, + "SESSION": tokens.Keyword, + "SESSION_USER": tokens.Keyword, + "SETOF": tokens.Keyword, + "SETS": tokens.Keyword, + "SHARE": tokens.Keyword, + "SHOW": tokens.Keyword, + "SIMILAR": tokens.Keyword, + "SIMPLE": tokens.Keyword, + "SIZE": tokens.Keyword, + "SOME": tokens.Keyword, + "SOURCE": tokens.Keyword, + "SPACE": tokens.Keyword, + "SPECIFIC": tokens.Keyword, + "SPECIFICTYPE": tokens.Keyword, + "SPECIFIC_NAME": tokens.Keyword, + "SQL": tokens.Keyword, + "SQLBUF": tokens.Keyword, + "SQLCODE": tokens.Keyword, + "SQLERROR": tokens.Keyword, + "SQLEXCEPTION": tokens.Keyword, + "SQLSTATE": tokens.Keyword, + "SQLWARNING": tokens.Keyword, + "STABLE": tokens.Keyword, + "START": tokens.Keyword.DML, # 'STATE': tokens.Keyword, - 'STATEMENT': tokens.Keyword, - 'STATIC': tokens.Keyword, - 'STATISTICS': tokens.Keyword, - 'STDIN': tokens.Keyword, - 'STDOUT': tokens.Keyword, - 'STORAGE': tokens.Keyword, - 'STRICT': tokens.Keyword, - 'STRUCTURE': tokens.Keyword, - 'STYPE': tokens.Keyword, - 'SUBCLASS_ORIGIN': tokens.Keyword, - 'SUBLIST': tokens.Keyword, - 'SUBSTRING': tokens.Keyword, - 'SUCCESSFUL': tokens.Keyword, - 'SUM': tokens.Keyword, - 'SYMMETRIC': tokens.Keyword, - 'SYNONYM': tokens.Keyword, - 'SYSID': tokens.Keyword, - 'SYSTEM': tokens.Keyword, - 'SYSTEM_USER': tokens.Keyword, - - 'TABLE': tokens.Keyword, - 'TABLE_NAME': tokens.Keyword, - 'TEMP': tokens.Keyword, - 'TEMPLATE': tokens.Keyword, - 'TEMPORARY': tokens.Keyword, - 'TERMINATE': tokens.Keyword, - 'THAN': tokens.Keyword, - 'TIMESTAMP': tokens.Keyword, - 'TIMEZONE_HOUR': tokens.Keyword, - 'TIMEZONE_MINUTE': tokens.Keyword, - 'TO': tokens.Keyword, - 'TOAST': tokens.Keyword, - 'TRAILING': tokens.Keyword, - 'TRANSATION': tokens.Keyword, - 'TRANSACTIONS_COMMITTED': tokens.Keyword, - 'TRANSACTIONS_ROLLED_BACK': tokens.Keyword, - 'TRANSATION_ACTIVE': tokens.Keyword, - 'TRANSFORM': tokens.Keyword, - 'TRANSFORMS': tokens.Keyword, - 'TRANSLATE': tokens.Keyword, - 'TRANSLATION': tokens.Keyword, - 'TREAT': tokens.Keyword, - 'TRIGGER': tokens.Keyword, - 'TRIGGER_CATALOG': tokens.Keyword, - 'TRIGGER_NAME': tokens.Keyword, - 'TRIGGER_SCHEMA': tokens.Keyword, - 'TRIM': tokens.Keyword, - 'TRUE': tokens.Keyword, - 'TRUSTED': tokens.Keyword, - 'TYPE': tokens.Keyword, - - 'UID': tokens.Keyword, - 'UNCOMMITTED': tokens.Keyword, - 'UNDER': tokens.Keyword, - 'UNENCRYPTED': tokens.Keyword, - 'UNION': tokens.Keyword, - 'UNIQUE': tokens.Keyword, - 'UNKNOWN': tokens.Keyword, - 'UNLISTEN': tokens.Keyword, - 'UNNAMED': tokens.Keyword, - 'UNNEST': tokens.Keyword, - 'UNTIL': tokens.Keyword, - 'UPPER': tokens.Keyword, - 'USAGE': tokens.Keyword, - 'USE': tokens.Keyword, - 'USER': tokens.Keyword, - 'USER_DEFINED_TYPE_CATALOG': tokens.Keyword, - 'USER_DEFINED_TYPE_NAME': tokens.Keyword, - 'USER_DEFINED_TYPE_SCHEMA': tokens.Keyword, - 'USING': tokens.Keyword, - - 'VACUUM': tokens.Keyword, - 'VALID': tokens.Keyword, - 'VALIDATE': tokens.Keyword, - 'VALIDATOR': tokens.Keyword, - 'VALUES': tokens.Keyword, - 'VARIABLE': tokens.Keyword, - 'VERBOSE': tokens.Keyword, - 'VERSION': tokens.Keyword, - 'VIEW': tokens.Keyword, - 'VOLATILE': tokens.Keyword, - - 'WEEK': tokens.Keyword, - 'WHENEVER': tokens.Keyword, - 'WITH': tokens.Keyword.CTE, - 'WITHOUT': tokens.Keyword, - 'WORK': tokens.Keyword, - 'WRITE': tokens.Keyword, - - 'YEAR': tokens.Keyword, - - 'ZONE': tokens.Keyword, - + "STATEMENT": tokens.Keyword, + "STATIC": tokens.Keyword, + "STATISTICS": tokens.Keyword, + "STDIN": tokens.Keyword, + "STDOUT": tokens.Keyword, + "STORAGE": tokens.Keyword, + "STRICT": tokens.Keyword, + "STRUCTURE": tokens.Keyword, + "STYPE": tokens.Keyword, + "SUBCLASS_ORIGIN": tokens.Keyword, + "SUBLIST": tokens.Keyword, + "SUBSTRING": tokens.Keyword, + "SUCCESSFUL": tokens.Keyword, + "SUM": tokens.Keyword, + "SYMMETRIC": tokens.Keyword, + "SYNONYM": tokens.Keyword, + "SYSID": tokens.Keyword, + "SYSTEM": tokens.Keyword, + "SYSTEM_USER": tokens.Keyword, + "TABLE": tokens.Keyword, + "TABLE_NAME": tokens.Keyword, + "TEMP": tokens.Keyword, + "TEMPLATE": tokens.Keyword, + "TEMPORARY": tokens.Keyword, + "TERMINATE": tokens.Keyword, + "THAN": tokens.Keyword, + "TIMESTAMP": tokens.Keyword, + "TIMEZONE_HOUR": tokens.Keyword, + "TIMEZONE_MINUTE": tokens.Keyword, + "TO": tokens.Keyword, + "TOAST": tokens.Keyword, + "TRAILING": tokens.Keyword, + "TRANSATION": tokens.Keyword, + "TRANSACTIONS_COMMITTED": tokens.Keyword, + "TRANSACTIONS_ROLLED_BACK": tokens.Keyword, + "TRANSATION_ACTIVE": tokens.Keyword, + "TRANSFORM": tokens.Keyword, + "TRANSFORMS": tokens.Keyword, + "TRANSLATE": tokens.Keyword, + "TRANSLATION": tokens.Keyword, + "TREAT": tokens.Keyword, + "TRIGGER": tokens.Keyword, + "TRIGGER_CATALOG": tokens.Keyword, + "TRIGGER_NAME": tokens.Keyword, + "TRIGGER_SCHEMA": tokens.Keyword, + "TRIM": tokens.Keyword, + "TRUE": tokens.Keyword, + "TRUSTED": tokens.Keyword, + "TYPE": tokens.Keyword, + "UID": tokens.Keyword, + "UNCOMMITTED": tokens.Keyword, + "UNDER": tokens.Keyword, + "UNENCRYPTED": tokens.Keyword, + "UNION": tokens.Keyword, + "UNIQUE": tokens.Keyword, + "UNKNOWN": tokens.Keyword, + "UNLISTEN": tokens.Keyword, + "UNNAMED": tokens.Keyword, + "UNNEST": tokens.Keyword, + "UNTIL": tokens.Keyword, + "UPPER": tokens.Keyword, + "USAGE": tokens.Keyword, + "USE": tokens.Keyword, + "USER": tokens.Keyword, + "USER_DEFINED_TYPE_CATALOG": tokens.Keyword, + "USER_DEFINED_TYPE_NAME": tokens.Keyword, + "USER_DEFINED_TYPE_SCHEMA": tokens.Keyword, + "USING": tokens.Keyword, + "VACUUM": tokens.Keyword, + "VALID": tokens.Keyword, + "VALIDATE": tokens.Keyword, + "VALIDATOR": tokens.Keyword, + "VALUES": tokens.Keyword, + "VARIABLE": tokens.Keyword, + "VERBOSE": tokens.Keyword, + "VERSION": tokens.Keyword, + "VIEW": tokens.Keyword, + "VOLATILE": tokens.Keyword, + "WEEK": tokens.Keyword, + "WHENEVER": tokens.Keyword, + "WITH": tokens.Keyword.CTE, + "WITHOUT": tokens.Keyword, + "WORK": tokens.Keyword, + "WRITE": tokens.Keyword, + "YEAR": tokens.Keyword, + "ZONE": tokens.Keyword, # Name.Builtin - 'ARRAY': tokens.Name.Builtin, - 'BIGINT': tokens.Name.Builtin, - 'BINARY': tokens.Name.Builtin, - 'BIT': tokens.Name.Builtin, - 'BLOB': tokens.Name.Builtin, - 'BOOLEAN': tokens.Name.Builtin, - 'CHAR': tokens.Name.Builtin, - 'CHARACTER': tokens.Name.Builtin, - 'DATE': tokens.Name.Builtin, - 'DEC': tokens.Name.Builtin, - 'DECIMAL': tokens.Name.Builtin, - 'FILE_TYPE': tokens.Name.Builtin, - 'FLOAT': tokens.Name.Builtin, - 'INT': tokens.Name.Builtin, - 'INT8': tokens.Name.Builtin, - 'INTEGER': tokens.Name.Builtin, - 'INTERVAL': tokens.Name.Builtin, - 'LONG': tokens.Name.Builtin, - 'NATURALN': tokens.Name.Builtin, - 'NVARCHAR': tokens.Name.Builtin, - 'NUMBER': tokens.Name.Builtin, - 'NUMERIC': tokens.Name.Builtin, - 'PLS_INTEGER': tokens.Name.Builtin, - 'POSITIVE': tokens.Name.Builtin, - 'POSITIVEN': tokens.Name.Builtin, - 'REAL': tokens.Name.Builtin, - 'ROWID': tokens.Name.Builtin, - 'ROWLABEL': tokens.Name.Builtin, - 'ROWNUM': tokens.Name.Builtin, - 'SERIAL': tokens.Name.Builtin, - 'SERIAL8': tokens.Name.Builtin, - 'SIGNED': tokens.Name.Builtin, - 'SIGNTYPE': tokens.Name.Builtin, - 'SIMPLE_DOUBLE': tokens.Name.Builtin, - 'SIMPLE_FLOAT': tokens.Name.Builtin, - 'SIMPLE_INTEGER': tokens.Name.Builtin, - 'SMALLINT': tokens.Name.Builtin, - 'SYS_REFCURSOR': tokens.Name.Builtin, - 'SYSDATE': tokens.Name, - 'TEXT': tokens.Name.Builtin, - 'TINYINT': tokens.Name.Builtin, - 'UNSIGNED': tokens.Name.Builtin, - 'UROWID': tokens.Name.Builtin, - 'UTL_FILE': tokens.Name.Builtin, - 'VARCHAR': tokens.Name.Builtin, - 'VARCHAR2': tokens.Name.Builtin, - 'VARYING': tokens.Name.Builtin, + "ARRAY": tokens.Name.Builtin, + "BIGINT": tokens.Name.Builtin, + "BINARY": tokens.Name.Builtin, + "BIT": tokens.Name.Builtin, + "BLOB": tokens.Name.Builtin, + "BOOLEAN": tokens.Name.Builtin, + "CHAR": tokens.Name.Builtin, + "CHARACTER": tokens.Name.Builtin, + "DATE": tokens.Name.Builtin, + "DEC": tokens.Name.Builtin, + "DECIMAL": tokens.Name.Builtin, + "FILE_TYPE": tokens.Name.Builtin, + "FLOAT": tokens.Name.Builtin, + "INT": tokens.Name.Builtin, + "INT8": tokens.Name.Builtin, + "INTEGER": tokens.Name.Builtin, + "INTERVAL": tokens.Name.Builtin, + "LONG": tokens.Name.Builtin, + "NATURALN": tokens.Name.Builtin, + "NVARCHAR": tokens.Name.Builtin, + "NUMBER": tokens.Name.Builtin, + "NUMERIC": tokens.Name.Builtin, + "PLS_INTEGER": tokens.Name.Builtin, + "POSITIVE": tokens.Name.Builtin, + "POSITIVEN": tokens.Name.Builtin, + "REAL": tokens.Name.Builtin, + "ROWID": tokens.Name.Builtin, + "ROWLABEL": tokens.Name.Builtin, + "ROWNUM": tokens.Name.Builtin, + "SERIAL": tokens.Name.Builtin, + "SERIAL8": tokens.Name.Builtin, + "SIGNED": tokens.Name.Builtin, + "SIGNTYPE": tokens.Name.Builtin, + "SIMPLE_DOUBLE": tokens.Name.Builtin, + "SIMPLE_FLOAT": tokens.Name.Builtin, + "SIMPLE_INTEGER": tokens.Name.Builtin, + "SMALLINT": tokens.Name.Builtin, + "SYS_REFCURSOR": tokens.Name.Builtin, + "SYSDATE": tokens.Name, + "TEXT": tokens.Name.Builtin, + "TINYINT": tokens.Name.Builtin, + "UNSIGNED": tokens.Name.Builtin, + "UROWID": tokens.Name.Builtin, + "UTL_FILE": tokens.Name.Builtin, + "VARCHAR": tokens.Name.Builtin, + "VARCHAR2": tokens.Name.Builtin, + "VARYING": tokens.Name.Builtin, } KEYWORDS_COMMON = { - 'SELECT': tokens.Keyword.DML, - 'INSERT': tokens.Keyword.DML, - 'DELETE': tokens.Keyword.DML, - 'UPDATE': tokens.Keyword.DML, - 'UPSERT': tokens.Keyword.DML, - 'REPLACE': tokens.Keyword.DML, - 'MERGE': tokens.Keyword.DML, - 'TRUNCATE': tokens.Keyword.DML, - 'DROP': tokens.Keyword.DDL, - 'CREATE': tokens.Keyword.DDL, - 'ALTER': tokens.Keyword.DDL, - - 'WHERE': tokens.Keyword, - 'FROM': tokens.Keyword, - 'INNER': tokens.Keyword, - 'JOIN': tokens.Keyword, - 'AND': tokens.Keyword, - 'OR': tokens.Keyword, - 'LIKE': tokens.Keyword, - 'ON': tokens.Keyword, - 'IN': tokens.Keyword, - 'SET': tokens.Keyword, - - 'BY': tokens.Keyword, - 'GROUP': tokens.Keyword, - 'ORDER': tokens.Keyword, - 'LEFT': tokens.Keyword, - 'OUTER': tokens.Keyword, - 'FULL': tokens.Keyword, - - 'IF': tokens.Keyword, - 'END': tokens.Keyword, - 'THEN': tokens.Keyword, - 'LOOP': tokens.Keyword, - 'AS': tokens.Keyword, - 'ELSE': tokens.Keyword, - 'FOR': tokens.Keyword, - 'WHILE': tokens.Keyword, - - 'CASE': tokens.Keyword, - 'WHEN': tokens.Keyword, - 'MIN': tokens.Keyword, - 'MAX': tokens.Keyword, - 'DISTINCT': tokens.Keyword, + "SELECT": tokens.Keyword.DML, + "INSERT": tokens.Keyword.DML, + "DELETE": tokens.Keyword.DML, + "UPDATE": tokens.Keyword.DML, + "UPSERT": tokens.Keyword.DML, + "REPLACE": tokens.Keyword.DML, + "MERGE": tokens.Keyword.DML, + "DROP": tokens.Keyword.DDL, + "CREATE": tokens.Keyword.DDL, + "ALTER": tokens.Keyword.DDL, + "TRUNCATE": tokens.Keyword.DDL, + "GRANT": tokens.Keyword.DCL, + "REVOKE": tokens.Keyword.DCL, + "WHERE": tokens.Keyword, + "FROM": tokens.Keyword, + "INNER": tokens.Keyword, + "JOIN": tokens.Keyword, + "AND": tokens.Keyword, + "OR": tokens.Keyword, + "LIKE": tokens.Keyword, + "ON": tokens.Keyword, + "IN": tokens.Keyword, + "SET": tokens.Keyword, + "BY": tokens.Keyword, + "GROUP": tokens.Keyword, + "ORDER": tokens.Keyword, + "LEFT": tokens.Keyword, + "OUTER": tokens.Keyword, + "FULL": tokens.Keyword, + "IF": tokens.Keyword, + "END": tokens.Keyword, + "THEN": tokens.Keyword, + "LOOP": tokens.Keyword, + "AS": tokens.Keyword, + "ELSE": tokens.Keyword, + "FOR": tokens.Keyword, + "WHILE": tokens.Keyword, + "CASE": tokens.Keyword, + "WHEN": tokens.Keyword, + "MIN": tokens.Keyword, + "MAX": tokens.Keyword, + "DISTINCT": tokens.Keyword, } KEYWORDS_ORACLE = { - 'ARCHIVE': tokens.Keyword, - 'ARCHIVELOG': tokens.Keyword, - - 'BACKUP': tokens.Keyword, - 'BECOME': tokens.Keyword, - 'BLOCK': tokens.Keyword, - 'BODY': tokens.Keyword, - - 'CANCEL': tokens.Keyword, - 'CHANGE': tokens.Keyword, - 'COMPILE': tokens.Keyword, - 'CONTENTS': tokens.Keyword, - 'CONTROLFILE': tokens.Keyword, - - 'DATAFILE': tokens.Keyword, - 'DBA': tokens.Keyword, - 'DISMOUNT': tokens.Keyword, - 'DOUBLE': tokens.Keyword, - 'DUMP': tokens.Keyword, - - 'ELSIF': tokens.Keyword, - 'EVENTS': tokens.Keyword, - 'EXCEPTIONS': tokens.Keyword, - 'EXPLAIN': tokens.Keyword, - 'EXTENT': tokens.Keyword, - 'EXTERNALLY': tokens.Keyword, - - 'FLUSH': tokens.Keyword, - 'FREELIST': tokens.Keyword, - 'FREELISTS': tokens.Keyword, - + "ARCHIVE": tokens.Keyword, + "ARCHIVELOG": tokens.Keyword, + "BACKUP": tokens.Keyword, + "BECOME": tokens.Keyword, + "BLOCK": tokens.Keyword, + "BODY": tokens.Keyword, + "CANCEL": tokens.Keyword, + "CHANGE": tokens.Keyword, + "COMPILE": tokens.Keyword, + "CONTENTS": tokens.Keyword, + "CONTROLFILE": tokens.Keyword, + "DATAFILE": tokens.Keyword, + "DBA": tokens.Keyword, + "DISMOUNT": tokens.Keyword, + "DOUBLE": tokens.Keyword, + "DUMP": tokens.Keyword, + "ELSIF": tokens.Keyword, + "EVENTS": tokens.Keyword, + "EXCEPTIONS": tokens.Keyword, + "EXPLAIN": tokens.Keyword, + "EXTENT": tokens.Keyword, + "EXTERNALLY": tokens.Keyword, + "FLUSH": tokens.Keyword, + "FREELIST": tokens.Keyword, + "FREELISTS": tokens.Keyword, # groups seems too common as table name # 'GROUPS': tokens.Keyword, + "INDICATOR": tokens.Keyword, + "INITRANS": tokens.Keyword, + "INSTANCE": tokens.Keyword, + "LAYER": tokens.Keyword, + "LINK": tokens.Keyword, + "LISTS": tokens.Keyword, + "LOGFILE": tokens.Keyword, + "MANAGE": tokens.Keyword, + "MANUAL": tokens.Keyword, + "MAXDATAFILES": tokens.Keyword, + "MAXINSTANCES": tokens.Keyword, + "MAXLOGFILES": tokens.Keyword, + "MAXLOGHISTORY": tokens.Keyword, + "MAXLOGMEMBERS": tokens.Keyword, + "MAXTRANS": tokens.Keyword, + "MINEXTENTS": tokens.Keyword, + "MODULE": tokens.Keyword, + "MOUNT": tokens.Keyword, + "NOARCHIVELOG": tokens.Keyword, + "NOCACHE": tokens.Keyword, + "NOCYCLE": tokens.Keyword, + "NOMAXVALUE": tokens.Keyword, + "NOMINVALUE": tokens.Keyword, + "NOORDER": tokens.Keyword, + "NORESETLOGS": tokens.Keyword, + "NORMAL": tokens.Keyword, + "NOSORT": tokens.Keyword, + "OPTIMAL": tokens.Keyword, + "OWN": tokens.Keyword, + "PACKAGE": tokens.Keyword, + "PARALLEL": tokens.Keyword, + "PCTINCREASE": tokens.Keyword, + "PCTUSED": tokens.Keyword, + "PLAN": tokens.Keyword, + "PRIVATE": tokens.Keyword, + "PROFILE": tokens.Keyword, + "QUOTA": tokens.Keyword, + "RECOVER": tokens.Keyword, + "RESETLOGS": tokens.Keyword, + "RESTRICTED": tokens.Keyword, + "REUSE": tokens.Keyword, + "ROLES": tokens.Keyword, + "SAVEPOINT": tokens.Keyword, + "SCN": tokens.Keyword, + "SECTION": tokens.Keyword, + "SEGMENT": tokens.Keyword, + "SHARED": tokens.Keyword, + "SNAPSHOT": tokens.Keyword, + "SORT": tokens.Keyword, + "STATEMENT_ID": tokens.Keyword, + "STOP": tokens.Keyword, + "SWITCH": tokens.Keyword, + "TABLES": tokens.Keyword, + "TABLESPACE": tokens.Keyword, + "THREAD": tokens.Keyword, + "TIME": tokens.Keyword, + "TRACING": tokens.Keyword, + "TRANSACTION": tokens.Keyword, + "TRIGGERS": tokens.Keyword, + "UNLIMITED": tokens.Keyword, + "UNLOCK": tokens.Keyword, +} - 'INDICATOR': tokens.Keyword, - 'INITRANS': tokens.Keyword, - 'INSTANCE': tokens.Keyword, - - 'LAYER': tokens.Keyword, - 'LINK': tokens.Keyword, - 'LISTS': tokens.Keyword, - 'LOGFILE': tokens.Keyword, - - 'MANAGE': tokens.Keyword, - 'MANUAL': tokens.Keyword, - 'MAXDATAFILES': tokens.Keyword, - 'MAXINSTANCES': tokens.Keyword, - 'MAXLOGFILES': tokens.Keyword, - 'MAXLOGHISTORY': tokens.Keyword, - 'MAXLOGMEMBERS': tokens.Keyword, - 'MAXTRANS': tokens.Keyword, - 'MINEXTENTS': tokens.Keyword, - 'MODULE': tokens.Keyword, - 'MOUNT': tokens.Keyword, - - 'NOARCHIVELOG': tokens.Keyword, - 'NOCACHE': tokens.Keyword, - 'NOCYCLE': tokens.Keyword, - 'NOMAXVALUE': tokens.Keyword, - 'NOMINVALUE': tokens.Keyword, - 'NOORDER': tokens.Keyword, - 'NORESETLOGS': tokens.Keyword, - 'NORMAL': tokens.Keyword, - 'NOSORT': tokens.Keyword, - - 'OPTIMAL': tokens.Keyword, - 'OWN': tokens.Keyword, - - 'PACKAGE': tokens.Keyword, - 'PARALLEL': tokens.Keyword, - 'PCTINCREASE': tokens.Keyword, - 'PCTUSED': tokens.Keyword, - 'PLAN': tokens.Keyword, - 'PRIVATE': tokens.Keyword, - 'PROFILE': tokens.Keyword, - - 'QUOTA': tokens.Keyword, - - 'RECOVER': tokens.Keyword, - 'RESETLOGS': tokens.Keyword, - 'RESTRICTED': tokens.Keyword, - 'REUSE': tokens.Keyword, - 'ROLES': tokens.Keyword, - - 'SAVEPOINT': tokens.Keyword, - 'SCN': tokens.Keyword, - 'SECTION': tokens.Keyword, - 'SEGMENT': tokens.Keyword, - 'SHARED': tokens.Keyword, - 'SNAPSHOT': tokens.Keyword, - 'SORT': tokens.Keyword, - 'STATEMENT_ID': tokens.Keyword, - 'STOP': tokens.Keyword, - 'SWITCH': tokens.Keyword, - - 'TABLES': tokens.Keyword, - 'TABLESPACE': tokens.Keyword, - 'THREAD': tokens.Keyword, - 'TIME': tokens.Keyword, - 'TRACING': tokens.Keyword, - 'TRANSACTION': tokens.Keyword, - 'TRIGGERS': tokens.Keyword, - - 'UNLIMITED': tokens.Keyword, - 'UNLOCK': tokens.Keyword, +# MySQL +KEYWORDS_MYSQL = { + "ROW": tokens.Keyword, } # PostgreSQL Syntax KEYWORDS_PLPGSQL = { - 'CONFLICT': tokens.Keyword, - 'WINDOW': tokens.Keyword, - 'PARTITION': tokens.Keyword, - 'OVER': tokens.Keyword, - 'PERFORM': tokens.Keyword, - 'NOTICE': tokens.Keyword, - 'PLPGSQL': tokens.Keyword, - 'INHERIT': tokens.Keyword, - 'INDEXES': tokens.Keyword, - 'ON_ERROR_STOP': tokens.Keyword, - - 'BYTEA': tokens.Keyword, - 'BIGSERIAL': tokens.Keyword, - 'BIT VARYING': tokens.Keyword, - 'BOX': tokens.Keyword, - 'CHARACTER': tokens.Keyword, - 'CHARACTER VARYING': tokens.Keyword, - 'CIDR': tokens.Keyword, - 'CIRCLE': tokens.Keyword, - 'DOUBLE PRECISION': tokens.Keyword, - 'INET': tokens.Keyword, - 'JSON': tokens.Keyword, - 'JSONB': tokens.Keyword, - 'LINE': tokens.Keyword, - 'LSEG': tokens.Keyword, - 'MACADDR': tokens.Keyword, - 'MONEY': tokens.Keyword, - 'PATH': tokens.Keyword, - 'PG_LSN': tokens.Keyword, - 'POINT': tokens.Keyword, - 'POLYGON': tokens.Keyword, - 'SMALLSERIAL': tokens.Keyword, - 'TSQUERY': tokens.Keyword, - 'TSVECTOR': tokens.Keyword, - 'TXID_SNAPSHOT': tokens.Keyword, - 'UUID': tokens.Keyword, - 'XML': tokens.Keyword, - - 'FOR': tokens.Keyword, - 'IN': tokens.Keyword, - 'LOOP': tokens.Keyword, + "CONFLICT": tokens.Keyword, + "WINDOW": tokens.Keyword, + "PARTITION": tokens.Keyword, + "OVER": tokens.Keyword, + "PERFORM": tokens.Keyword, + "NOTICE": tokens.Keyword, + "PLPGSQL": tokens.Keyword, + "INHERIT": tokens.Keyword, + "INDEXES": tokens.Keyword, + "ON_ERROR_STOP": tokens.Keyword, + "EXTENSION": tokens.Keyword, + "BYTEA": tokens.Keyword, + "BIGSERIAL": tokens.Keyword, + "BIT VARYING": tokens.Keyword, + "BOX": tokens.Keyword, + "CHARACTER": tokens.Keyword, + "CHARACTER VARYING": tokens.Keyword, + "CIDR": tokens.Keyword, + "CIRCLE": tokens.Keyword, + "DOUBLE PRECISION": tokens.Keyword, + "INET": tokens.Keyword, + "JSON": tokens.Keyword, + "JSONB": tokens.Keyword, + "LINE": tokens.Keyword, + "LSEG": tokens.Keyword, + "MACADDR": tokens.Keyword, + "MONEY": tokens.Keyword, + "PATH": tokens.Keyword, + "PG_LSN": tokens.Keyword, + "POINT": tokens.Keyword, + "POLYGON": tokens.Keyword, + "SMALLSERIAL": tokens.Keyword, + "TSQUERY": tokens.Keyword, + "TSVECTOR": tokens.Keyword, + "TXID_SNAPSHOT": tokens.Keyword, + "UUID": tokens.Keyword, + "XML": tokens.Keyword, + "FOR": tokens.Keyword, + "IN": tokens.Keyword, + "LOOP": tokens.Keyword, } # Hive Syntax KEYWORDS_HQL = { - 'EXPLODE': tokens.Keyword, - 'DIRECTORY': tokens.Keyword, - 'DISTRIBUTE': tokens.Keyword, - 'INCLUDE': tokens.Keyword, - 'LOCATE': tokens.Keyword, - 'OVERWRITE': tokens.Keyword, - 'POSEXPLODE': tokens.Keyword, + "EXPLODE": tokens.Keyword, + "DIRECTORY": tokens.Keyword, + "DISTRIBUTE": tokens.Keyword, + "INCLUDE": tokens.Keyword, + "LOCATE": tokens.Keyword, + "OVERWRITE": tokens.Keyword, + "POSEXPLODE": tokens.Keyword, + "ARRAY_CONTAINS": tokens.Keyword, + "CMP": tokens.Keyword, + "COLLECT_LIST": tokens.Keyword, + "CONCAT": tokens.Keyword, + "CONDITION": tokens.Keyword, + "DATE_ADD": tokens.Keyword, + "DATE_SUB": tokens.Keyword, + "DECODE": tokens.Keyword, + "DBMS_OUTPUT": tokens.Keyword, + "ELEMENTS": tokens.Keyword, + "EXCHANGE": tokens.Keyword, + "EXTENDED": tokens.Keyword, + "FLOOR": tokens.Keyword, + "FOLLOWING": tokens.Keyword, + "FROM_UNIXTIME": tokens.Keyword, + "FTP": tokens.Keyword, + "HOUR": tokens.Keyword, + "INLINE": tokens.Keyword, + "INSTR": tokens.Keyword, + "LEN": tokens.Keyword, + "MAP": tokens.Name.Builtin, + "MAXELEMENT": tokens.Keyword, + "MAXINDEX": tokens.Keyword, + "MAX_PART_DATE": tokens.Keyword, + "MAX_PART_INT": tokens.Keyword, + "MAX_PART_STRING": tokens.Keyword, + "MINELEMENT": tokens.Keyword, + "MININDEX": tokens.Keyword, + "MIN_PART_DATE": tokens.Keyword, + "MIN_PART_INT": tokens.Keyword, + "MIN_PART_STRING": tokens.Keyword, + "NOW": tokens.Keyword, + "NVL": tokens.Keyword, + "NVL2": tokens.Keyword, + "PARSE_URL_TUPLE": tokens.Keyword, + "PART_LOC": tokens.Keyword, + "PART_COUNT": tokens.Keyword, + "PART_COUNT_BY": tokens.Keyword, + "PRINT": tokens.Keyword, + "PUT_LINE": tokens.Keyword, + "RANGE": tokens.Keyword, + "REDUCE": tokens.Keyword, + "REGEXP_REPLACE": tokens.Keyword, + "RESIGNAL": tokens.Keyword, + "RTRIM": tokens.Keyword, + "SIGN": tokens.Keyword, + "SIGNAL": tokens.Keyword, + "SIN": tokens.Keyword, + "SPLIT": tokens.Keyword, + "SQRT": tokens.Keyword, + "STACK": tokens.Keyword, + "STR": tokens.Keyword, + "STRING": tokens.Name.Builtin, + "STRUCT": tokens.Name.Builtin, + "SUBSTR": tokens.Keyword, + "SUMMARY": tokens.Keyword, + "TBLPROPERTIES": tokens.Keyword, + "TIMESTAMP": tokens.Name.Builtin, + "TIMESTAMP_ISO": tokens.Keyword, + "TO_CHAR": tokens.Keyword, + "TO_DATE": tokens.Keyword, + "TO_TIMESTAMP": tokens.Keyword, + "TRUNC": tokens.Keyword, + "UNBOUNDED": tokens.Keyword, + "UNIQUEJOIN": tokens.Keyword, + "UNIX_TIMESTAMP": tokens.Keyword, + "UTC_TIMESTAMP": tokens.Keyword, + "VIEWS": tokens.Keyword, + "EXIT": tokens.Keyword, + "BREAK": tokens.Keyword, + "LEAVE": tokens.Keyword, +} + - 'ARRAY_CONTAINS': tokens.Keyword, - 'CMP': tokens.Keyword, - 'COLLECT_LIST': tokens.Keyword, - 'CONCAT': tokens.Keyword, - 'CONDITION': tokens.Keyword, - 'DATE_ADD': tokens.Keyword, - 'DATE_SUB': tokens.Keyword, - 'DECODE': tokens.Keyword, - 'DBMS_OUTPUT': tokens.Keyword, - 'ELEMENTS': tokens.Keyword, - 'EXCHANGE': tokens.Keyword, - 'EXTENDED': tokens.Keyword, - 'FLOOR': tokens.Keyword, - 'FOLLOWING': tokens.Keyword, - 'FROM_UNIXTIME': tokens.Keyword, - 'FTP': tokens.Keyword, - 'HOUR': tokens.Keyword, - 'INLINE': tokens.Keyword, - 'INSTR': tokens.Keyword, - 'LEN': tokens.Keyword, - 'MAP': tokens.Name.Builtin, - 'MAXELEMENT': tokens.Keyword, - 'MAXINDEX': tokens.Keyword, - 'MAX_PART_DATE': tokens.Keyword, - 'MAX_PART_INT': tokens.Keyword, - 'MAX_PART_STRING': tokens.Keyword, - 'MINELEMENT': tokens.Keyword, - 'MININDEX': tokens.Keyword, - 'MIN_PART_DATE': tokens.Keyword, - 'MIN_PART_INT': tokens.Keyword, - 'MIN_PART_STRING': tokens.Keyword, - 'NOW': tokens.Keyword, - 'NVL': tokens.Keyword, - 'NVL2': tokens.Keyword, - 'PARSE_URL_TUPLE': tokens.Keyword, - 'PART_LOC': tokens.Keyword, - 'PART_COUNT': tokens.Keyword, - 'PART_COUNT_BY': tokens.Keyword, - 'PRINT': tokens.Keyword, - 'PUT_LINE': tokens.Keyword, - 'RANGE': tokens.Keyword, - 'REDUCE': tokens.Keyword, - 'REGEXP_REPLACE': tokens.Keyword, - 'RESIGNAL': tokens.Keyword, - 'RTRIM': tokens.Keyword, - 'SIGN': tokens.Keyword, - 'SIGNAL': tokens.Keyword, - 'SIN': tokens.Keyword, - 'SPLIT': tokens.Keyword, - 'SQRT': tokens.Keyword, - 'STACK': tokens.Keyword, - 'STR': tokens.Keyword, - 'STRING': tokens.Name.Builtin, - 'STRUCT': tokens.Name.Builtin, - 'SUBSTR': tokens.Keyword, - 'SUMMARY': tokens.Keyword, - 'TBLPROPERTIES': tokens.Keyword, - 'TIMESTAMP': tokens.Name.Builtin, - 'TIMESTAMP_ISO': tokens.Keyword, - 'TO_CHAR': tokens.Keyword, - 'TO_DATE': tokens.Keyword, - 'TO_TIMESTAMP': tokens.Keyword, - 'TRUNC': tokens.Keyword, - 'UNBOUNDED': tokens.Keyword, - 'UNIQUEJOIN': tokens.Keyword, - 'UNIX_TIMESTAMP': tokens.Keyword, - 'UTC_TIMESTAMP': tokens.Keyword, - 'VIEWS': tokens.Keyword, +KEYWORDS_MSACCESS = { + "DISTINCTROW": tokens.Keyword, +} - 'EXIT': tokens.Keyword, - 'BREAK': tokens.Keyword, - 'LEAVE': tokens.Keyword, + +KEYWORDS_SNOWFLAKE = { + "ACCOUNT": tokens.Keyword, + "GSCLUSTER": tokens.Keyword, + "ISSUE": tokens.Keyword, + "ORGANIZATION": tokens.Keyword, + "PIVOT": tokens.Keyword, + "QUALIFY": tokens.Keyword, + "REGEXP": tokens.Keyword, + "RLIKE": tokens.Keyword, + "SAMPLE": tokens.Keyword, + "TRY_CAST": tokens.Keyword, + "UNPIVOT": tokens.Keyword, + "VARIANT": tokens.Name.Builtin, } -KEYWORDS_MSACCESS = { - 'DISTINCTROW': tokens.Keyword, +KEYWORDS_BIGQUERY = { + "ASSERT_ROWS_MODIFIED": tokens.Keyword, + "DEFINE": tokens.Keyword, + "ENUM": tokens.Keyword, + "HASH": tokens.Keyword, + "LOOKUP": tokens.Keyword, + "PRECEDING": tokens.Keyword, + "PROTO": tokens.Keyword, + "RESPECT": tokens.Keyword, + "TABLESAMPLE": tokens.Keyword, + "BIGNUMERIC": tokens.Name.Builtin, } diff --git a/sqlparse/lexer.py b/sqlparse/lexer.py index 8df82fed..796c601b 100644 --- a/sqlparse/lexer.py +++ b/sqlparse/lexer.py @@ -7,6 +7,7 @@ """SQL Lexer""" import re +from threading import Lock # This code is based on the SqlLexer in pygments. # http://pygments.org/ @@ -23,20 +24,21 @@ class Lexer: """The Lexer supports configurable syntax. To add support for additional keywords, use the `add_keywords` method.""" - _default_intance = None + _default_instance = None + _lock = Lock() # Development notes: # - This class is prepared to be able to support additional SQL dialects # in the future by adding additional functions that take the place of - # the function default_initialization() + # the function default_initialization(). # - The lexer class uses an explicit singleton behavior with the # instance-getter method get_default_instance(). This mechanism has # the advantage that the call signature of the entry-points to the # sqlparse library are not affected. Also, usage of sqlparse in third - # party code does not need to be adapted. On the other hand, singleton - # behavior is not thread safe, and the current implementation does not - # easily allow for multiple SQL dialects to be parsed in the same - # process. Such behavior can be supported in the future by passing a + # party code does not need to be adapted. On the other hand, the current + # implementation does not easily allow for multiple SQL dialects to be + # parsed in the same process. + # Such behavior can be supported in the future by passing a # suitably initialized lexer object as an additional parameter to the # entry-point functions (such as `parse`). Code will need to be written # to pass down and utilize such an object. The current implementation @@ -47,10 +49,11 @@ class Lexer: def get_default_instance(cls): """Returns the lexer instance used internally by the sqlparse core functions.""" - if cls._default_intance is None: - cls._default_intance = cls() - cls._default_intance.default_initialization() - return cls._default_intance + with cls._lock: + if cls._default_instance is None: + cls._default_instance = cls() + cls._default_instance.default_initialization() + return cls._default_instance def default_initialization(self): """Initialize the lexer with default dictionaries. @@ -59,9 +62,12 @@ def default_initialization(self): self.set_SQL_REGEX(keywords.SQL_REGEX) self.add_keywords(keywords.KEYWORDS_COMMON) self.add_keywords(keywords.KEYWORDS_ORACLE) + self.add_keywords(keywords.KEYWORDS_MYSQL) self.add_keywords(keywords.KEYWORDS_PLPGSQL) self.add_keywords(keywords.KEYWORDS_HQL) self.add_keywords(keywords.KEYWORDS_MSACCESS) + self.add_keywords(keywords.KEYWORDS_SNOWFLAKE) + self.add_keywords(keywords.KEYWORDS_BIGQUERY) self.add_keywords(keywords.KEYWORDS) def clear(self): @@ -75,10 +81,7 @@ def clear(self): def set_SQL_REGEX(self, SQL_REGEX): """Set the list of regex that will parse the SQL.""" FLAGS = re.IGNORECASE | re.UNICODE - self._SQL_REGEX = [ - (re.compile(rx, FLAGS).match, tt) - for rx, tt in SQL_REGEX - ] + self._SQL_REGEX = [(re.compile(rx, FLAGS).match, tt) for rx, tt in SQL_REGEX] def add_keywords(self, keywords): """Add keyword dictionaries. Keywords are looked up in the same order @@ -121,17 +124,20 @@ def get_tokens(self, text, encoding=None): text = text.decode(encoding) else: try: - text = text.decode('utf-8') + text = text.decode("utf-8") except UnicodeDecodeError: - text = text.decode('unicode-escape') + text = text.decode("unicode-escape") else: - raise TypeError("Expected text or file-like object, got {!r}". - format(type(text))) + raise TypeError( + "Expected text or file-like object, got {!r}".format(type(text)) + ) iterable = enumerate(text) for pos, char in iterable: for rexmatch, action in self._SQL_REGEX: m = rexmatch(text, pos) + if m is not None and m.group() == "IFF": + print(m.group(), pos, action) if not m: continue diff --git a/sqlparse/sql.py b/sqlparse/sql.py index ba4945ae..8425a076 100644 --- a/sqlparse/sql.py +++ b/sqlparse/sql.py @@ -19,14 +19,14 @@ class NameAliasMixin: def get_real_name(self): """Returns the real name (object name) of this identifier.""" # a.b - dot_idx, _ = self.token_next_by(m=(T.Punctuation, '.')) + dot_idx, _ = self.token_next_by(m=(T.Punctuation, ".")) return self._get_first_name(dot_idx, real_name=True) def get_alias(self): """Returns the alias for this identifier or ``None``.""" # "name AS alias" - kw_idx, kw = self.token_next_by(m=(T.Keyword, 'AS')) + kw_idx, kw = self.token_next_by(m=(T.Keyword, "AS")) if kw is not None: return self._get_first_name(kw_idx + 1, keywords=True) @@ -44,8 +44,18 @@ class Token: the type of the token. """ - __slots__ = ('value', 'ttype', 'parent', 'normalized', 'is_keyword', - 'is_group', 'is_whitespace', 'position', 'length') + __slots__ = ( + "value", + "ttype", + "parent", + "normalized", + "is_keyword", + "is_group", + "is_whitespace", + "is_newline", + "position", + "length", + ) def __init__(self, ttype, value, position=None): value = str(value) @@ -55,6 +65,7 @@ def __init__(self, ttype, value, position=None): self.is_group = False self.is_keyword = ttype in T.Keyword self.is_whitespace = self.ttype in T.Whitespace + self.is_newline = self.ttype in T.Newline self.normalized = value.upper() if self.is_keyword else value self.position = position self.length = len(value) @@ -71,17 +82,16 @@ def __repr__(self): value = self._get_repr_value() q = '"' if value.startswith("'") and value.endswith("'") else "'" - return "<{cls} {q}{value}{q} at 0x{id:2X}>".format( - id=id(self), **locals()) + return "<{cls} {q}{value}{q} at 0x{id:2X}>".format(id=id(self), **locals()) def _get_repr_name(self): - return str(self.ttype).split('.')[-1] + return str(self.ttype).split(".")[-1] def _get_repr_value(self): raw = str(self) if len(raw) > 7: - raw = raw[:6] + '...' - return re.sub(r'\s+', ' ', raw) + raw = raw[:6] + "..." + return re.sub(r"\s+", " ", raw) def flatten(self): """Resolve subgroups.""" @@ -155,11 +165,11 @@ class TokenList(Token): list of child-tokens. """ - __slots__ = 'tokens' + __slots__ = "tokens" def __init__(self, tokens=None): self.tokens = tokens or [] - [setattr(token, 'parent', self) for token in self.tokens] + [setattr(token, "parent", self) for token in self.tokens] super().__init__(None, str(self)) self.is_group = True if ( @@ -173,7 +183,7 @@ def __init__(self, tokens=None): ) + self.tokens[-1].length def __str__(self): - return ''.join(token.value for token in self.flatten()) + return "".join(token.value for token in self.flatten()) # weird bug # def __len__(self): @@ -188,7 +198,7 @@ def __getitem__(self, item): def _get_repr_name(self): return type(self).__name__ - def _pprint_tree(self, max_depth=None, depth=0, f=None, _pre=''): + def _pprint_tree(self, max_depth=None, depth=0, f=None, _pre=""): """Pretty-print the object tree.""" token_count = len(self.tokens) for idx, token in enumerate(self.tokens): @@ -196,14 +206,13 @@ def _pprint_tree(self, max_depth=None, depth=0, f=None, _pre=''): value = token._get_repr_value() last = idx == (token_count - 1) - pre = '`- ' if last else '|- ' + pre = "`- " if last else "|- " q = '"' if value.startswith("'") and value.endswith("'") else "'" - print("{_pre}{pre}{idx} {cls} {q}{value}{q}" - .format(**locals()), file=f) + print(f"{_pre}{pre}{idx} {cls} {q}{value}{q}", file=f) if token.is_group and (max_depth is None or depth < max_depth): - parent_pre = ' ' if last else '| ' + parent_pre = " " if last else "| " token._pprint_tree(max_depth, depth + 1, f, _pre + parent_pre) def get_token_at_offset(self, offset): @@ -266,10 +275,14 @@ def token_first(self, skip_ws=True, skip_cm=False): if *skip_cm* is ``True`` (default: ``False``), comments are ignored too. """ + # this on is inconsistent, using Comment instead of T.Comment... def matcher(tk): - return not ((skip_ws and tk.is_whitespace) - or (skip_cm and imt(tk, t=T.Comment, i=Comment))) + return not ( + (skip_ws and tk.is_whitespace) + or (skip_cm and imt(tk, t=T.Comment, i=Comment)) + ) + return self._token_matching(matcher)[1] def token_next_by(self, i=None, m=None, t=None, idx=-1, end=None): @@ -306,8 +319,11 @@ def token_next(self, idx, skip_ws=True, skip_cm=False, _reverse=False): idx += 1 # alot of code usage current pre-compensates for this def matcher(tk): - return not ((skip_ws and tk.is_whitespace) - or (skip_cm and imt(tk, t=T.Comment, i=Comment))) + return not ( + (skip_ws and tk.is_whitespace) + or (skip_cm and imt(tk, t=T.Comment, i=Comment)) + ) + return self._token_matching(matcher, idx, reverse=_reverse) def token_index(self, token, start=0): @@ -315,8 +331,7 @@ def token_index(self, token, start=0): start = start if isinstance(start, int) else self.token_index(start) return start + self.tokens[start:].index(token) - def group_tokens(self, grp_cls, start, end, include_end=True, - extend=False): + def group_tokens(self, grp_cls, start, end, include_end=True, extend=False): """Replace tokens by an instance of *grp_cls*.""" start_idx = start start = self.tokens[start_idx] @@ -328,11 +343,11 @@ def group_tokens(self, grp_cls, start, end, include_end=True, # tokens = tokens[:-1] if extend and isinstance(start, grp_cls): - subtokens = self.tokens[start_idx + 1:end_idx] + subtokens = self.tokens[start_idx + 1 : end_idx] grp = start grp.tokens.extend(subtokens) - del self.tokens[start_idx + 1:end_idx] + del self.tokens[start_idx + 1 : end_idx] grp.value = str(start) else: subtokens = self.tokens[start_idx:end_idx] @@ -389,12 +404,11 @@ def get_parent_name(self): A parent object is identified by the first occurring dot. """ - dot_idx, _ = self.token_next_by(m=(T.Punctuation, '.')) + dot_idx, _ = self.token_next_by(m=(T.Punctuation, ".")) _, prev_ = self.token_prev(dot_idx) return remove_quotes(prev_.value) if prev_ is not None else None - def _get_first_name(self, idx=None, reverse=False, keywords=False, - real_name=False): + def _get_first_name(self, idx=None, reverse=False, keywords=False, real_name=False): """Returns the name of the first token with a name""" tokens = self.tokens[idx:] if idx else self.tokens @@ -412,7 +426,7 @@ def _get_first_name(self, idx=None, reverse=False, keywords=False, # duckdb supports FROM-first syntax, PIVOT statements -SELECT_LIKE = {'FROM', 'PIVOT', 'PIVOT_WIDER', 'UNPIVOT'} +SELECT_LIKE = {"FROM", "PIVOT", "PIVOT_WIDER", "UNPIVOT"} class Statement(TokenList): @@ -436,13 +450,13 @@ def get_type(self): if token is None: # An "empty" statement that either has not tokens at all # or only whitespace tokens. - return 'UNKNOWN' + return "UNKNOWN" elif token.ttype in (T.Keyword.DML, T.Keyword.DDL): return token.normalized elif token.normalized in SELECT_LIKE: - return 'SELECT' + return "SELECT" elif token.ttype == T.Keyword.CTE: # The WITH keyword should be followed by either an Identifier or @@ -454,14 +468,13 @@ def get_type(self): if isinstance(token, (Identifier, IdentifierList)): tidx, token = self.token_next(tidx, skip_ws=True) - if token is not None \ - and token.ttype == T.Keyword.DML: + if token is not None and token.ttype == T.Keyword.DML: return token.normalized elif token.normalized in SELECT_LIKE: - return 'SELECT' + return "SELECT" # Hmm, probably invalid syntax, so return unknown. - return 'UNKNOWN' + return "UNKNOWN" class Identifier(NameAliasMixin, TokenList): @@ -477,7 +490,7 @@ def is_wildcard(self): def get_typecast(self): """Returns the typecast or ``None`` of this object as a string.""" - midx, marker = self.token_next_by(m=(T.Punctuation, '::')) + midx, marker = self.token_next_by(m=(T.Punctuation, "::")) nidx, next_ = self.token_next(midx, skip_ws=False) return next_.value if next_ else None @@ -504,12 +517,13 @@ def get_identifiers(self): Whitespaces and punctuations are not included in this generator. """ for token in self.tokens: - if not (token.is_whitespace or token.match(T.Punctuation, ',')): + if not (token.is_whitespace or token.match(T.Punctuation, ",")): yield token class TypedLiteral(TokenList): """A typed literal, such as "date '2001-09-28'" or "interval '2 hours'".""" + M_OPEN = [(T.Name.Builtin, None), (T.Keyword, "TIMESTAMP")] M_CLOSE = T.String.Single, None M_EXTEND = T.Keyword, ("DAY", "HOUR", "MINUTE", "MONTH", "SECOND", "YEAR") @@ -517,8 +531,9 @@ class TypedLiteral(TokenList): class Parenthesis(TokenList): """Tokens between parenthesis.""" - M_OPEN = T.Punctuation, '(' - M_CLOSE = T.Punctuation, ')' + + M_OPEN = T.Punctuation, "(" + M_CLOSE = T.Punctuation, ")" @property def _groupable_tokens(self): @@ -527,8 +542,9 @@ def _groupable_tokens(self): class SquareBrackets(TokenList): """Tokens between square brackets""" - M_OPEN = T.Punctuation, '[' - M_CLOSE = T.Punctuation, ']' + + M_OPEN = T.Punctuation, "[" + M_CLOSE = T.Punctuation, "]" @property def _groupable_tokens(self): @@ -541,14 +557,16 @@ class Assignment(TokenList): class If(TokenList): """An 'if' clause with possible 'else if' or 'else' parts.""" - M_OPEN = T.Keyword, 'IF' - M_CLOSE = T.Keyword, 'END IF' + + M_OPEN = T.Keyword, "IF" + M_CLOSE = T.Keyword, "END IF" class For(TokenList): """A 'FOR' loop.""" - M_OPEN = T.Keyword, ('FOR', 'FOREACH') - M_CLOSE = T.Keyword, 'END LOOP' + + M_OPEN = T.Keyword, ("FOR", "FOREACH") + M_CLOSE = T.Keyword, "END LOOP" class Comparison(TokenList): @@ -572,22 +590,39 @@ def is_multiline(self): class Where(TokenList): """A WHERE clause.""" - M_OPEN = T.Keyword, 'WHERE' + + M_OPEN = T.Keyword, "WHERE" M_CLOSE = T.Keyword, ( - 'ORDER BY', 'GROUP BY', 'LIMIT', 'UNION', 'UNION ALL', 'EXCEPT', - 'HAVING', 'RETURNING', 'INTO') + "ORDER BY", + "GROUP BY", + "LIMIT", + "UNION", + "UNION ALL", + "EXCEPT", + "HAVING", + "RETURNING", + "INTO", + ) + + +class Over(TokenList): + """An OVER clause.""" + + M_OPEN = T.Keyword, "OVER" class Having(TokenList): """A HAVING clause.""" - M_OPEN = T.Keyword, 'HAVING' - M_CLOSE = T.Keyword, ('ORDER BY', 'LIMIT') + + M_OPEN = T.Keyword, "HAVING" + M_CLOSE = T.Keyword, ("ORDER BY", "LIMIT") class Case(TokenList): """A CASE statement with one or more WHEN and possibly an ELSE part.""" - M_OPEN = T.Keyword, 'CASE' - M_CLOSE = T.Keyword, 'END' + + M_OPEN = T.Keyword, "CASE" + M_CLOSE = T.Keyword, "END" def get_cases(self, skip_ws=False): """Returns a list of 2-tuples (condition, value). @@ -602,24 +637,24 @@ def get_cases(self, skip_ws=False): for token in self.tokens: # Set mode from the current statement - if token.match(T.Keyword, 'CASE'): + if token.match(T.Keyword, "CASE"): continue elif skip_ws and token.ttype in T.Whitespace: continue - elif token.match(T.Keyword, 'WHEN'): + elif token.match(T.Keyword, "WHEN"): ret.append(([], [])) mode = CONDITION - elif token.match(T.Keyword, 'THEN'): + elif token.match(T.Keyword, "THEN"): mode = VALUE - elif token.match(T.Keyword, 'ELSE'): + elif token.match(T.Keyword, "ELSE"): ret.append((None, [])) mode = VALUE - elif token.match(T.Keyword, 'END'): + elif token.match(T.Keyword, "END"): mode = None # First condition without preceding WHEN @@ -642,19 +677,28 @@ class Function(NameAliasMixin, TokenList): def get_parameters(self): """Return a list of parameters.""" - parenthesis = self.tokens[-1] + parenthesis = self.token_next_by(i=Parenthesis)[1] + result = [] for token in parenthesis.tokens: if isinstance(token, IdentifierList): return token.get_identifiers() - elif imt(token, i=(Function, Identifier), t=T.Literal): - return [token, ] - return [] + elif imt(token, i=(Function, Identifier, TypedLiteral), t=T.Literal): + result.append(token) + return result + + def get_window(self): + """Return the window if it exists.""" + over_clause = self.token_next_by(i=Over) + if not over_clause: + return None + return over_clause[1].tokens[-1] class Begin(TokenList): """A BEGIN/END block.""" - M_OPEN = T.Keyword, 'BEGIN' - M_CLOSE = T.Keyword, 'END' + + M_OPEN = T.Keyword, "BEGIN" + M_CLOSE = T.Keyword, "END" class Operation(TokenList): diff --git a/sqlparse/utils.py b/sqlparse/utils.py index 512f0385..58c0245a 100644 --- a/sqlparse/utils.py +++ b/sqlparse/utils.py @@ -86,20 +86,23 @@ def imt(token, i=None, m=None, t=None): :param t: TokenType or Tuple/List of TokenTypes :return: bool """ - clss = i - types = [t, ] if t and not isinstance(t, list) else t - mpatterns = [m, ] if m and not isinstance(m, list) else m - if token is None: return False - elif clss and isinstance(token, clss): - return True - elif mpatterns and any(token.match(*pattern) for pattern in mpatterns): + if i and isinstance(token, i): return True - elif types and any(token.ttype in ttype for ttype in types): - return True - else: - return False + if m: + if isinstance(m, list): + if any(token.match(*pattern) for pattern in m): + return True + elif token.match(*m): + return True + if t: + if isinstance(t, list): + if any(token.ttype in ttype for ttype in t): + return True + elif token.ttype in t: + return True + return False def consume(iterator, n): diff --git a/tests/files/multiple_case_in_begin.sql b/tests/files/multiple_case_in_begin.sql new file mode 100644 index 00000000..6cbb3864 --- /dev/null +++ b/tests/files/multiple_case_in_begin.sql @@ -0,0 +1,8 @@ +CREATE TRIGGER mytrig +AFTER UPDATE OF vvv ON mytable +BEGIN + UPDATE aa + SET mycola = (CASE WHEN (A=1) THEN 2 END); + UPDATE bb + SET mycolb = (CASE WHEN (B=1) THEN 5 END); +END; \ No newline at end of file diff --git a/tests/test_cli.py b/tests/test_cli.py index b681a60b..a0c1f2b0 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -69,8 +69,8 @@ def test_stdout(filepath, load_file, capsys): def test_script(): # Call with the --help option as a basic sanity check. - cmd = "{:s} -m sqlparse.cli --help".format(sys.executable) - assert subprocess.call(cmd.split()) == 0 + cmd = [sys.executable, '-m', 'sqlparse.cli', '--help'] + assert subprocess.call(cmd) == 0 @pytest.mark.parametrize('fpath, encoding', ( diff --git a/tests/test_format.py b/tests/test_format.py index 70bb8055..4dae299d 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -73,15 +73,15 @@ def test_strip_comments_multi(self): assert res == 'select' sql = '/* sql starts here */ select' res = sqlparse.format(sql, strip_comments=True) - assert res == 'select' + assert res == ' select' # note whitespace is preserved, see issue 772 sql = '/*\n * sql starts here\n */\nselect' res = sqlparse.format(sql, strip_comments=True) assert res == 'select' sql = 'select (/* sql starts here */ select 2)' - res = sqlparse.format(sql, strip_comments=True) + res = sqlparse.format(sql, strip_comments=True, strip_whitespace=True) assert res == 'select (select 2)' sql = 'select (/* sql /* starts here */ select 2)' - res = sqlparse.format(sql, strip_comments=True) + res = sqlparse.format(sql, strip_comments=True, strip_whitespace=True) assert res == 'select (select 2)' def test_strip_comments_preserves_linebreak(self): @@ -101,6 +101,31 @@ def test_strip_comments_preserves_linebreak(self): res = sqlparse.format(sql, strip_comments=True) assert res == 'select *\n\nfrom foo' + def test_strip_comments_preserves_whitespace(self): + sql = 'SELECT 1/*bar*/ AS foo' # see issue772 + res = sqlparse.format(sql, strip_comments=True) + assert res == 'SELECT 1 AS foo' + + def test_strip_comments_preserves_hint(self): + sql = 'select --+full(u)' + res = sqlparse.format(sql, strip_comments=True) + assert res == sql + sql = '#+ hint\nselect * from foo' + res = sqlparse.format(sql, strip_comments=True) + assert res == sql + sql = 'select --+full(u)\n--comment simple' + res = sqlparse.format(sql, strip_comments=True) + assert res == 'select --+full(u)\n' + sql = '#+ hint\nselect * from foo\n# comment simple' + res = sqlparse.format(sql, strip_comments=True) + assert res == '#+ hint\nselect * from foo\n' + sql = 'SELECT /*+cluster(T)*/* FROM T_EEE T where A >:1' + res = sqlparse.format(sql, strip_comments=True) + assert res == sql + sql = 'insert /*+ DIRECT */ into sch.table_name as select * from foo' + res = sqlparse.format(sql, strip_comments=True) + assert res == sql + def test_strip_ws(self): f = lambda sql: sqlparse.format(sql, strip_whitespace=True) s = 'select\n* from foo\n\twhere ( 1 = 2 )\n' @@ -722,3 +747,28 @@ def test_format_right_margin_invalid_option(right_margin): def test_format_right_margin(): # TODO: Needs better test, only raises exception right now sqlparse.format('foo', right_margin="79") + + +def test_format_json_ops(): # issue542 + formatted = sqlparse.format( + "select foo->'bar', foo->'bar';", reindent=True) + expected = "select foo->'bar',\n foo->'bar';" + assert formatted == expected + + +@pytest.mark.parametrize('sql, expected_normal, expected_compact', [ + ('case when foo then 1 else bar end', + 'case\n when foo then 1\n else bar\nend', + 'case when foo then 1 else bar end')]) +def test_compact(sql, expected_normal, expected_compact): # issue783 + formatted_normal = sqlparse.format(sql, reindent=True) + formatted_compact = sqlparse.format(sql, reindent=True, compact=True) + assert formatted_normal == expected_normal + assert formatted_compact == expected_compact + + +def test_strip_ws_removes_trailing_ws_in_groups(): # issue782 + formatted = sqlparse.format('( where foo = bar ) from', + strip_whitespace=True) + expected = '(where foo = bar) from' + assert formatted == expected diff --git a/tests/test_grouping.py b/tests/test_grouping.py index 03d16c5d..bf278817 100644 --- a/tests/test_grouping.py +++ b/tests/test_grouping.py @@ -17,13 +17,6 @@ def test_grouping_parenthesis(): assert len(parsed.tokens[2].tokens[3].tokens) == 3 -def test_grouping_comments(): - s = '/*\n * foo\n */ \n bar' - parsed = sqlparse.parse(s)[0] - assert str(parsed) == s - assert len(parsed.tokens) == 2 - - @pytest.mark.parametrize('s', ['foo := 1;', 'foo := 1']) def test_grouping_assignment(s): parsed = sqlparse.parse(s)[0] @@ -185,6 +178,20 @@ def test_grouping_identifier_function(): assert isinstance(p.tokens[0], sql.Identifier) assert isinstance(p.tokens[0].tokens[0], sql.Operation) assert isinstance(p.tokens[0].tokens[0].tokens[0], sql.Function) + p = sqlparse.parse('foo(c1) over win1 as bar')[0] + assert isinstance(p.tokens[0], sql.Identifier) + assert isinstance(p.tokens[0].tokens[0], sql.Function) + assert len(p.tokens[0].tokens[0].tokens) == 4 + assert isinstance(p.tokens[0].tokens[0].tokens[3], sql.Over) + assert isinstance(p.tokens[0].tokens[0].tokens[3].tokens[2], + sql.Identifier) + p = sqlparse.parse('foo(c1) over (partition by c2 order by c3) as bar')[0] + assert isinstance(p.tokens[0], sql.Identifier) + assert isinstance(p.tokens[0].tokens[0], sql.Function) + assert len(p.tokens[0].tokens[0].tokens) == 4 + assert isinstance(p.tokens[0].tokens[0].tokens[3], sql.Over) + assert isinstance(p.tokens[0].tokens[0].tokens[3].tokens[2], + sql.Parenthesis) @pytest.mark.parametrize('s', ['foo+100', 'foo + 100', 'foo*100']) @@ -247,6 +254,14 @@ def test_grouping_identifier_list_with_order(): assert str(p.tokens[0].tokens[3]) == '2 desc' +def test_grouping_nested_identifier_with_order(): + # issue745 + p = sqlparse.parse('(a desc)')[0] + assert isinstance(p.tokens[0], sql.Parenthesis) + assert isinstance(p.tokens[0].tokens[1], sql.Identifier) + assert str(p.tokens[0].tokens[1]) == 'a desc' + + def test_grouping_where(): s = 'select * from foo where bar = 1 order by id desc' p = sqlparse.parse(s)[0] @@ -370,6 +385,14 @@ def test_grouping_function(): p = sqlparse.parse('foo(null, bar)')[0] assert isinstance(p.tokens[0], sql.Function) assert len(list(p.tokens[0].get_parameters())) == 2 + p = sqlparse.parse('foo(5) over win1')[0] + assert isinstance(p.tokens[0], sql.Function) + assert len(list(p.tokens[0].get_parameters())) == 1 + assert isinstance(p.tokens[0].get_window(), sql.Identifier) + p = sqlparse.parse('foo(5) over (PARTITION BY c1)')[0] + assert isinstance(p.tokens[0], sql.Function) + assert len(list(p.tokens[0].get_parameters())) == 1 + assert isinstance(p.tokens[0].get_window(), sql.Parenthesis) def test_grouping_function_not_in(): @@ -483,7 +506,7 @@ def test_comparison_with_parenthesis(): )) def test_comparison_with_strings(operator): # issue148 - p = sqlparse.parse("foo {} 'bar'".format(operator))[0] + p = sqlparse.parse(f"foo {operator} 'bar'")[0] assert len(p.tokens) == 1 assert isinstance(p.tokens[0], sql.Comparison) assert p.tokens[0].right.value == "'bar'" @@ -562,7 +585,7 @@ def test_comparison_with_typed_literal(): @pytest.mark.parametrize('start', ['FOR', 'FOREACH']) def test_forloops(start): - p = sqlparse.parse('{} foo in bar LOOP foobar END LOOP'.format(start))[0] + p = sqlparse.parse(f'{start} foo in bar LOOP foobar END LOOP')[0] assert (len(p.tokens)) == 1 assert isinstance(p.tokens[0], sql.For) diff --git a/tests/test_parse.py b/tests/test_parse.py index 7b0e776f..005cc006 100644 --- a/tests/test_parse.py +++ b/tests/test_parse.py @@ -1,4 +1,5 @@ """Tests sqlparse.parse().""" + from io import StringIO import pytest @@ -9,49 +10,54 @@ def test_parse_tokenize(): - s = 'select * from foo;' + s = "select * from foo;" stmts = sqlparse.parse(s) assert len(stmts) == 1 assert str(stmts[0]) == s def test_parse_multistatement(): - sql1 = 'select * from foo;' - sql2 = 'select * from bar;' + sql1 = "select * from foo;" + sql2 = "select * from bar;" stmts = sqlparse.parse(sql1 + sql2) assert len(stmts) == 2 assert str(stmts[0]) == sql1 assert str(stmts[1]) == sql2 -@pytest.mark.parametrize('s', ['select\n*from foo;', - 'select\r\n*from foo', - 'select\r*from foo', - 'select\r\n*from foo\n']) +@pytest.mark.parametrize( + "s", + [ + "select\n*from foo;", + "select\r\n*from foo", + "select\r*from foo", + "select\r\n*from foo\n", + ], +) def test_parse_newlines(s): p = sqlparse.parse(s)[0] assert str(p) == s def test_parse_within(): - s = 'foo(col1, col2)' + s = "foo(col1, col2)" p = sqlparse.parse(s)[0] col1 = p.tokens[0].tokens[1].tokens[1].tokens[0] assert col1.within(sql.Function) def test_parse_child_of(): - s = '(col1, col2)' + s = "(col1, col2)" p = sqlparse.parse(s)[0] assert p.tokens[0].tokens[1].is_child_of(p.tokens[0]) - s = 'select foo' + s = "select foo" p = sqlparse.parse(s)[0] assert not p.tokens[2].is_child_of(p.tokens[0]) assert p.tokens[2].is_child_of(p) def test_parse_has_ancestor(): - s = 'foo or (bar, baz)' + s = "foo or (bar, baz)" p = sqlparse.parse(s)[0] baz = p.tokens[-1].tokens[1].tokens[-1] assert baz.has_ancestor(p.tokens[-1].tokens[1]) @@ -59,19 +65,23 @@ def test_parse_has_ancestor(): assert baz.has_ancestor(p) -@pytest.mark.parametrize('s', ['.5', '.51', '1.5', '12.5']) +@pytest.mark.parametrize("s", [".5", ".51", "1.5", "12.5"]) def test_parse_float(s): t = sqlparse.parse(s)[0].tokens assert len(t) == 1 assert t[0].ttype is sqlparse.tokens.Number.Float -@pytest.mark.parametrize('s, holder', [ - ('select * from foo where user = ?', '?'), - ('select * from foo where user = :1', ':1'), - ('select * from foo where user = :name', ':name'), - ('select * from foo where user = %s', '%s'), - ('select * from foo where user = $a', '$a')]) +@pytest.mark.parametrize( + "s, holder", + [ + ("select * from foo where user = ?", "?"), + ("select * from foo where user = :1", ":1"), + ("select * from foo where user = :name", ":name"), + ("select * from foo where user = %s", "%s"), + ("select * from foo where user = $a", "$a"), + ], +) def test_parse_placeholder(s, holder): t = sqlparse.parse(s)[0].tokens[-1].tokens assert t[-1].ttype is sqlparse.tokens.Name.Placeholder @@ -79,78 +89,91 @@ def test_parse_placeholder(s, holder): def test_parse_modulo_not_placeholder(): - tokens = list(sqlparse.lexer.tokenize('x %3')) + tokens = list(sqlparse.lexer.tokenize("x %3")) assert tokens[2][0] == sqlparse.tokens.Operator def test_parse_access_symbol(): # see issue27 - t = sqlparse.parse('select a.[foo bar] as foo')[0].tokens + t = sqlparse.parse("select a.[foo bar] as foo")[0].tokens assert isinstance(t[-1], sql.Identifier) - assert t[-1].get_name() == 'foo' - assert t[-1].get_real_name() == '[foo bar]' - assert t[-1].get_parent_name() == 'a' + assert t[-1].get_name() == "foo" + assert t[-1].get_real_name() == "[foo bar]" + assert t[-1].get_parent_name() == "a" def test_parse_square_brackets_notation_isnt_too_greedy(): # see issue153 - t = sqlparse.parse('[foo], [bar]')[0].tokens + t = sqlparse.parse("[foo], [bar]")[0].tokens assert isinstance(t[0], sql.IdentifierList) assert len(t[0].tokens) == 4 - assert t[0].tokens[0].get_real_name() == '[foo]' - assert t[0].tokens[-1].get_real_name() == '[bar]' + assert t[0].tokens[0].get_real_name() == "[foo]" + assert t[0].tokens[-1].get_real_name() == "[bar]" def test_parse_square_brackets_notation_isnt_too_greedy2(): # see issue583 - t = sqlparse.parse('[(foo[i])]')[0].tokens + t = sqlparse.parse("[(foo[i])]")[0].tokens assert isinstance(t[0], sql.SquareBrackets) # not Identifier! def test_parse_keyword_like_identifier(): # see issue47 - t = sqlparse.parse('foo.key')[0].tokens + t = sqlparse.parse("foo.key")[0].tokens assert len(t) == 1 assert isinstance(t[0], sql.Identifier) def test_parse_function_parameter(): # see issue94 - t = sqlparse.parse('abs(some_col)')[0].tokens[0].get_parameters() + t = sqlparse.parse("abs(some_col)")[0].tokens[0].get_parameters() assert len(t) == 1 assert isinstance(t[0], sql.Identifier) def test_parse_function_param_single_literal(): - t = sqlparse.parse('foo(5)')[0].tokens[0].get_parameters() + t = sqlparse.parse("foo(5)")[0].tokens[0].get_parameters() assert len(t) == 1 assert t[0].ttype is T.Number.Integer def test_parse_nested_function(): - t = sqlparse.parse('foo(bar(5))')[0].tokens[0].get_parameters() + t = sqlparse.parse("foo(bar(5))")[0].tokens[0].get_parameters() assert len(t) == 1 assert type(t[0]) is sql.Function +def test_parse_casted_params(): + t = ( + sqlparse.parse("foo(DATE '2023-11-14', TIMESTAMP '2023-11-15')")[0] + .tokens[0] + .get_parameters() + ) + assert len(t) == 2 + + def test_parse_div_operator(): - p = sqlparse.parse('col1 DIV 5 AS div_col1')[0].tokens + p = sqlparse.parse("col1 DIV 5 AS div_col1")[0].tokens assert p[0].tokens[0].tokens[2].ttype is T.Operator - assert p[0].get_alias() == 'div_col1' + assert p[0].get_alias() == "div_col1" def test_quoted_identifier(): t = sqlparse.parse('select x.y as "z" from foo')[0].tokens assert isinstance(t[2], sql.Identifier) - assert t[2].get_name() == 'z' - assert t[2].get_real_name() == 'y' - - -@pytest.mark.parametrize('name', [ - 'foo', '_foo', # issue175 - '1_data', # valid MySQL table name, see issue337 - '業者名稱', # valid at least for SQLite3, see issue641 -]) + assert t[2].get_name() == "z" + assert t[2].get_real_name() == "y" + + +@pytest.mark.parametrize( + "name", + [ + "foo", + "_foo", # issue175 + "1_data", # valid MySQL table name, see issue337 + "業者名稱", # valid at least for SQLite3, see issue641 + ], +) def test_valid_identifier_names(name): t = sqlparse.parse(name)[0].tokens assert isinstance(t[0], sql.Identifier) @@ -161,47 +184,62 @@ def test_psql_quotation_marks(): # issue83 # regression: make sure plain $$ work - t = sqlparse.split(""" + t = sqlparse.split( + """ CREATE OR REPLACE FUNCTION testfunc1(integer) RETURNS integer AS $$ .... $$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION testfunc2(integer) RETURNS integer AS $$ .... - $$ LANGUAGE plpgsql;""") + $$ LANGUAGE plpgsql;""" + ) assert len(t) == 2 # make sure $SOMETHING$ works too - t = sqlparse.split(""" + t = sqlparse.split( + """ CREATE OR REPLACE FUNCTION testfunc1(integer) RETURNS integer AS $PROC_1$ .... $PROC_1$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION testfunc2(integer) RETURNS integer AS $PROC_2$ .... - $PROC_2$ LANGUAGE plpgsql;""") + $PROC_2$ LANGUAGE plpgsql;""" + ) + assert len(t) == 2 + + # operators are valid infront of dollar quoted strings + t = sqlparse.split("""UPDATE SET foo =$$bar;SELECT bar$$""") + assert len(t) == 1 + + # identifiers must be separated by whitespace + t = sqlparse.split("""UPDATE SET foo TO$$bar;SELECT bar$$""") assert len(t) == 2 def test_double_precision_is_builtin(): - s = 'DOUBLE PRECISION' + s = "DOUBLE PRECISION" t = sqlparse.parse(s)[0].tokens assert len(t) == 1 assert t[0].ttype == sqlparse.tokens.Name.Builtin - assert t[0].value == 'DOUBLE PRECISION' + assert t[0].value == "DOUBLE PRECISION" -@pytest.mark.parametrize('ph', ['?', ':1', ':foo', '%s', '%(foo)s']) +@pytest.mark.parametrize("ph", ["?", ":1", ":foo", "%s", "%(foo)s"]) def test_placeholder(ph): p = sqlparse.parse(ph)[0].tokens assert len(p) == 1 assert p[0].ttype is T.Name.Placeholder -@pytest.mark.parametrize('num, expected', [ - ('6.67428E-8', T.Number.Float), - ('1.988e33', T.Number.Float), - ('1e-12', T.Number.Float), - ('e1', None), -]) +@pytest.mark.parametrize( + "num, expected", + [ + ("6.67428E-8", T.Number.Float), + ("1.988e33", T.Number.Float), + ("1e-12", T.Number.Float), + ("e1", None), + ], +) def test_scientific_numbers(num, expected): p = sqlparse.parse(num)[0].tokens assert len(p) == 1 @@ -229,59 +267,59 @@ def test_single_quotes_with_linebreaks(): def test_sqlite_identifiers(): # Make sure we still parse sqlite style escapes - p = sqlparse.parse('[col1],[col2]')[0].tokens + p = sqlparse.parse("[col1],[col2]")[0].tokens id_names = [id_.get_name() for id_ in p[0].get_identifiers()] assert len(p) == 1 assert isinstance(p[0], sql.IdentifierList) - assert id_names == ['[col1]', '[col2]'] + assert id_names == ["[col1]", "[col2]"] - p = sqlparse.parse('[col1]+[col2]')[0] + p = sqlparse.parse("[col1]+[col2]")[0] types = [tok.ttype for tok in p.flatten()] assert types == [T.Name, T.Operator, T.Name] def test_simple_1d_array_index(): - p = sqlparse.parse('col[1]')[0].tokens + p = sqlparse.parse("col[1]")[0].tokens assert len(p) == 1 - assert p[0].get_name() == 'col' + assert p[0].get_name() == "col" indices = list(p[0].get_array_indices()) assert len(indices) == 1 # 1-dimensional index assert len(indices[0]) == 1 # index is single token - assert indices[0][0].value == '1' + assert indices[0][0].value == "1" def test_2d_array_index(): - p = sqlparse.parse('col[x][(y+1)*2]')[0].tokens + p = sqlparse.parse("col[x][(y+1)*2]")[0].tokens assert len(p) == 1 - assert p[0].get_name() == 'col' + assert p[0].get_name() == "col" assert len(list(p[0].get_array_indices())) == 2 # 2-dimensional index def test_array_index_function_result(): - p = sqlparse.parse('somefunc()[1]')[0].tokens + p = sqlparse.parse("somefunc()[1]")[0].tokens assert len(p) == 1 assert len(list(p[0].get_array_indices())) == 1 def test_schema_qualified_array_index(): - p = sqlparse.parse('schem.col[1]')[0].tokens + p = sqlparse.parse("schem.col[1]")[0].tokens assert len(p) == 1 - assert p[0].get_parent_name() == 'schem' - assert p[0].get_name() == 'col' - assert list(p[0].get_array_indices())[0][0].value == '1' + assert p[0].get_parent_name() == "schem" + assert p[0].get_name() == "col" + assert list(p[0].get_array_indices())[0][0].value == "1" def test_aliased_array_index(): - p = sqlparse.parse('col[1] x')[0].tokens + p = sqlparse.parse("col[1] x")[0].tokens assert len(p) == 1 - assert p[0].get_alias() == 'x' - assert p[0].get_real_name() == 'col' - assert list(p[0].get_array_indices())[0][0].value == '1' + assert p[0].get_alias() == "x" + assert p[0].get_real_name() == "col" + assert list(p[0].get_array_indices())[0][0].value == "1" def test_array_literal(): # See issue #176 - p = sqlparse.parse('ARRAY[%s, %s]')[0] + p = sqlparse.parse("ARRAY[%s, %s]")[0] assert len(p.tokens) == 2 assert len(list(p.flatten())) == 7 @@ -289,13 +327,12 @@ def test_array_literal(): def test_typed_array_definition(): # array indices aren't grouped with built-ins, but make sure we can extract # identifier names - p = sqlparse.parse('x int, y int[], z int')[0] - names = [x.get_name() for x in p.get_sublists() - if isinstance(x, sql.Identifier)] - assert names == ['x', 'y', 'z'] + p = sqlparse.parse("x int, y int[], z int")[0] + names = [x.get_name() for x in p.get_sublists() if isinstance(x, sql.Identifier)] + assert names == ["x", "y", "z"] -@pytest.mark.parametrize('s', ['select 1 -- foo', 'select 1 # foo']) +@pytest.mark.parametrize("s", ["select 1 -- foo", "select 1 # foo"]) def test_single_line_comments(s): # see issue178 p = sqlparse.parse(s)[0] @@ -303,7 +340,7 @@ def test_single_line_comments(s): assert p.tokens[-1].ttype == T.Comment.Single -@pytest.mark.parametrize('s', ['foo', '@foo', '#foo', '##foo']) +@pytest.mark.parametrize("s", ["foo", "@foo", "#foo", "##foo"]) def test_names_and_special_names(s): # see issue192 p = sqlparse.parse(s)[0] @@ -312,7 +349,7 @@ def test_names_and_special_names(s): def test_get_token_at_offset(): - p = sqlparse.parse('select * from dual')[0] + p = sqlparse.parse("select * from dual")[0] # 0123456789 assert p.get_token_at_offset(0) == p.tokens[0] assert p.get_token_at_offset(1) == p.tokens[0] @@ -324,78 +361,82 @@ def test_get_token_at_offset(): def test_pprint(): - p = sqlparse.parse('select a0, b0, c0, d0, e0 from ' - '(select * from dual) q0 where 1=1 and 2=2')[0] + p = sqlparse.parse( + "select a0, b0, c0, d0, e0 from " "(select * from dual) q0 where 1=1 and 2=2" + )[0] output = StringIO() p._pprint_tree(f=output) - pprint = '\n'.join([ - "|- 0 DML 'select'", - "|- 1 Whitespace ' '", - "|- 2 IdentifierList 'a0, b0...'", - "| |- 0 Identifier 'a0'", - "| | `- 0 Name 'a0'", - "| |- 1 Punctuation ','", - "| |- 2 Whitespace ' '", - "| |- 3 Identifier 'b0'", - "| | `- 0 Name 'b0'", - "| |- 4 Punctuation ','", - "| |- 5 Whitespace ' '", - "| |- 6 Identifier 'c0'", - "| | `- 0 Name 'c0'", - "| |- 7 Punctuation ','", - "| |- 8 Whitespace ' '", - "| |- 9 Identifier 'd0'", - "| | `- 0 Name 'd0'", - "| |- 10 Punctuation ','", - "| |- 11 Whitespace ' '", - "| `- 12 Identifier 'e0'", - "| `- 0 Name 'e0'", - "|- 3 Whitespace ' '", - "|- 4 Keyword 'from'", - "|- 5 Whitespace ' '", - "|- 6 Identifier '(selec...'", - "| |- 0 Parenthesis '(selec...'", - "| | |- 0 Punctuation '('", - "| | |- 1 DML 'select'", - "| | |- 2 Whitespace ' '", - "| | |- 3 Wildcard '*'", - "| | |- 4 Whitespace ' '", - "| | |- 5 Keyword 'from'", - "| | |- 6 Whitespace ' '", - "| | |- 7 Identifier 'dual'", - "| | | `- 0 Name 'dual'", - "| | `- 8 Punctuation ')'", - "| |- 1 Whitespace ' '", - "| `- 2 Identifier 'q0'", - "| `- 0 Name 'q0'", - "|- 7 Whitespace ' '", - "`- 8 Where 'where ...'", - " |- 0 Keyword 'where'", - " |- 1 Whitespace ' '", - " |- 2 Comparison '1=1'", - " | |- 0 Integer '1'", - " | |- 1 Comparison '='", - " | `- 2 Integer '1'", - " |- 3 Whitespace ' '", - " |- 4 Keyword 'and'", - " |- 5 Whitespace ' '", - " `- 6 Comparison '2=2'", - " |- 0 Integer '2'", - " |- 1 Comparison '='", - " `- 2 Integer '2'", - ""]) + pprint = "\n".join( + [ + "|- 0 DML 'select'", + "|- 1 Whitespace ' '", + "|- 2 IdentifierList 'a0, b0...'", + "| |- 0 Identifier 'a0'", + "| | `- 0 Name 'a0'", + "| |- 1 Punctuation ','", + "| |- 2 Whitespace ' '", + "| |- 3 Identifier 'b0'", + "| | `- 0 Name 'b0'", + "| |- 4 Punctuation ','", + "| |- 5 Whitespace ' '", + "| |- 6 Identifier 'c0'", + "| | `- 0 Name 'c0'", + "| |- 7 Punctuation ','", + "| |- 8 Whitespace ' '", + "| |- 9 Identifier 'd0'", + "| | `- 0 Name 'd0'", + "| |- 10 Punctuation ','", + "| |- 11 Whitespace ' '", + "| `- 12 Identifier 'e0'", + "| `- 0 Name 'e0'", + "|- 3 Whitespace ' '", + "|- 4 Keyword 'from'", + "|- 5 Whitespace ' '", + "|- 6 Identifier '(selec...'", + "| |- 0 Parenthesis '(selec...'", + "| | |- 0 Punctuation '('", + "| | |- 1 DML 'select'", + "| | |- 2 Whitespace ' '", + "| | |- 3 Wildcard '*'", + "| | |- 4 Whitespace ' '", + "| | |- 5 Keyword 'from'", + "| | |- 6 Whitespace ' '", + "| | |- 7 Identifier 'dual'", + "| | | `- 0 Name 'dual'", + "| | `- 8 Punctuation ')'", + "| |- 1 Whitespace ' '", + "| `- 2 Identifier 'q0'", + "| `- 0 Name 'q0'", + "|- 7 Whitespace ' '", + "`- 8 Where 'where ...'", + " |- 0 Keyword 'where'", + " |- 1 Whitespace ' '", + " |- 2 Comparison '1=1'", + " | |- 0 Integer '1'", + " | |- 1 Comparison '='", + " | `- 2 Integer '1'", + " |- 3 Whitespace ' '", + " |- 4 Keyword 'and'", + " |- 5 Whitespace ' '", + " `- 6 Comparison '2=2'", + " |- 0 Integer '2'", + " |- 1 Comparison '='", + " `- 2 Integer '2'", + "", + ] + ) assert output.getvalue() == pprint def test_wildcard_multiplication(): - p = sqlparse.parse('select * from dual')[0] + p = sqlparse.parse("select * from dual")[0] assert p.tokens[2].ttype == T.Wildcard - p = sqlparse.parse('select a0.* from dual a0')[0] + p = sqlparse.parse("select a0.* from dual a0")[0] assert p.tokens[2][2].ttype == T.Wildcard - p = sqlparse.parse('select 1 * 2 from dual')[0] + p = sqlparse.parse("select 1 * 2 from dual")[0] assert p.tokens[2][2].ttype == T.Operator @@ -407,14 +448,17 @@ def test_stmt_tokens_parents(): assert token.has_ancestor(stmt) -@pytest.mark.parametrize('sql, is_literal', [ - ('$$foo$$', True), - ('$_$foo$_$', True), - ('$token$ foo $token$', True), - # don't parse inner tokens - ('$_$ foo $token$bar$token$ baz$_$', True), - ('$A$ foo $B$', False) # tokens don't match -]) +@pytest.mark.parametrize( + "sql, is_literal", + [ + ("$$foo$$", True), + ("$_$foo$_$", True), + ("$token$ foo $token$", True), + # don't parse inner tokens + ("$_$ foo $token$bar$token$ baz$_$", True), + ("$A$ foo $B$", False), # tokens don't match + ], +) def test_dbldollar_as_literal(sql, is_literal): # see issue 277 p = sqlparse.parse(sql)[0] @@ -436,8 +480,8 @@ def test_non_ascii(): assert str(statement) == s assert statement._pprint_tree() is None - s = _test_non_ascii.encode('utf-8') - stmts = sqlparse.parse(s, 'utf-8') + s = _test_non_ascii.encode("utf-8") + stmts = sqlparse.parse(s, "utf-8") assert len(stmts) == 1 statement = stmts[0] assert str(statement) == _test_non_ascii @@ -449,47 +493,51 @@ def test_get_real_name(): s = "update a t set t.b=1" stmts = sqlparse.parse(s) assert len(stmts) == 1 - assert 'a' == stmts[0].tokens[2].get_real_name() - assert 't' == stmts[0].tokens[2].get_alias() + assert "a" == stmts[0].tokens[2].get_real_name() + assert "t" == stmts[0].tokens[2].get_alias() def test_from_subquery(): # issue 446 - s = 'from(select 1)' + s = "from(select 1)" stmts = sqlparse.parse(s) assert len(stmts) == 1 assert len(stmts[0].tokens) == 2 - assert stmts[0].tokens[0].value == 'from' + assert stmts[0].tokens[0].value == "from" assert stmts[0].tokens[0].ttype == T.Keyword - s = 'from (select 1)' + s = "from (select 1)" stmts = sqlparse.parse(s) assert len(stmts) == 1 assert len(stmts[0].tokens) == 3 - assert stmts[0].tokens[0].value == 'from' + assert stmts[0].tokens[0].value == "from" assert stmts[0].tokens[0].ttype == T.Keyword assert stmts[0].tokens[1].ttype == T.Whitespace def test_parenthesis(): tokens = sqlparse.parse("(\n\n1\n\n)")[0].tokens[0].tokens - assert list(map(lambda t: t.ttype, tokens)) == [T.Punctuation, - T.Newline, - T.Newline, - T.Number.Integer, - T.Newline, - T.Newline, - T.Punctuation] + assert list(map(lambda t: t.ttype, tokens)) == [ + T.Punctuation, + T.Newline, + T.Newline, + T.Number.Integer, + T.Newline, + T.Newline, + T.Punctuation, + ] tokens = sqlparse.parse("(\n\n 1 \n\n)")[0].tokens[0].tokens - assert list(map(lambda t: t.ttype, tokens)) == [T.Punctuation, - T.Newline, - T.Newline, - T.Whitespace, - T.Number.Integer, - T.Whitespace, - T.Newline, - T.Newline, - T.Punctuation] + assert list(map(lambda t: t.ttype, tokens)) == [ + T.Punctuation, + T.Newline, + T.Newline, + T.Whitespace, + T.Number.Integer, + T.Whitespace, + T.Newline, + T.Newline, + T.Punctuation, + ] def test_configurable_keywords(): @@ -497,9 +545,7 @@ def test_configurable_keywords(): tokens = sqlparse.parse(sql)[0] assert list( - (t.ttype, t.value) - for t in tokens - if t.ttype not in sqlparse.tokens.Whitespace + (t.ttype, t.value) for t in tokens if t.ttype not in sqlparse.tokens.Whitespace ) == [ (sqlparse.tokens.Keyword.DML, "select"), (sqlparse.tokens.Wildcard, "*"), @@ -523,9 +569,7 @@ def test_configurable_keywords(): Lexer.get_default_instance().default_initialization() assert list( - (t.ttype, t.value) - for t in tokens - if t.ttype not in sqlparse.tokens.Whitespace + (t.ttype, t.value) for t in tokens if t.ttype not in sqlparse.tokens.Whitespace ) == [ (sqlparse.tokens.Keyword.DML, "select"), (sqlparse.tokens.Wildcard, "*"), @@ -544,11 +588,7 @@ def test_configurable_regex(): my_regex = (r"ZORDER\s+BY\b", sqlparse.tokens.Keyword) - lex.set_SQL_REGEX( - keywords.SQL_REGEX[:38] - + [my_regex] - + keywords.SQL_REGEX[38:] - ) + lex.set_SQL_REGEX(keywords.SQL_REGEX[:38] + [my_regex] + keywords.SQL_REGEX[38:]) lex.add_keywords(keywords.KEYWORDS_COMMON) lex.add_keywords(keywords.KEYWORDS_ORACLE) lex.add_keywords(keywords.KEYWORDS_PLPGSQL) @@ -562,9 +602,7 @@ def test_configurable_regex(): Lexer.get_default_instance().default_initialization() assert list( - (t.ttype, t.value) - for t in tokens - if t.ttype not in sqlparse.tokens.Whitespace + (t.ttype, t.value) for t in tokens if t.ttype not in sqlparse.tokens.Whitespace )[4] == (sqlparse.tokens.Keyword, "zorder by") @@ -573,3 +611,26 @@ def test_truncate_statement_type(): stmts = sqlparse.parse(s) assert len(stmts) == 1 assert stmts[0].get_type() == "TRUNCATE" + + +@pytest.mark.parametrize( + "sql", + [ + "->", + "->>", + "#>", + "#>>", + "@>", + "<@", + # leaving ? out for now, they're somehow ambiguous as placeholders + # '?', '?|', '?&', + "||", + "-", + "#-", + ], +) +def test_json_operators(sql): + p = sqlparse.parse(sql) + assert len(p) == 1 + assert len(p[0].tokens) == 1 + assert p[0].tokens[0].ttype == sqlparse.tokens.Operator diff --git a/tests/test_regressions.py b/tests/test_regressions.py index fdb9fa59..e1f253a1 100644 --- a/tests/test_regressions.py +++ b/tests/test_regressions.py @@ -1,14 +1,16 @@ import copy +import sys import pytest import sqlparse from sqlparse import sql, tokens as T +from sqlparse.exceptions import SQLParseError def test_issue9(): # make sure where doesn't consume parenthesis - p = sqlparse.parse('(where 1)')[0] + p = sqlparse.parse("(where 1)")[0] assert isinstance(p, sql.Statement) assert len(p.tokens) == 1 assert isinstance(p.tokens[0], sql.Parenthesis) @@ -19,15 +21,12 @@ def test_issue9(): def test_issue13(): - parsed = sqlparse.parse("select 'one';\n" - "select 'two\\'';\n" - "select 'three';") + parsed = sqlparse.parse("select 'one';\n" "select 'two\\'';\n" "select 'three';") assert len(parsed) == 3 assert str(parsed[1]).strip() == "select 'two\\'';" -@pytest.mark.parametrize('s', ['--hello', '-- hello', '--hello\n', - '--', '--\n']) +@pytest.mark.parametrize("s", ["--hello", "-- hello", "--hello\n", "--", "--\n"]) def test_issue26(s): # parse stand-alone comments p = sqlparse.parse(s)[0] @@ -35,7 +34,7 @@ def test_issue26(s): assert p.tokens[0].ttype is T.Comment.Single -@pytest.mark.parametrize('value', ['create', 'CREATE']) +@pytest.mark.parametrize("value", ["create", "CREATE"]) def test_issue34(value): t = sqlparse.parse("create")[0].token_first() assert t.match(T.Keyword.DDL, value) is True @@ -43,13 +42,8 @@ def test_issue34(value): def test_issue35(): # missing space before LIMIT. Updated for #321 - sql = sqlparse.format("select * from foo where bar = 1 limit 1", - reindent=True) - assert sql == "\n".join([ - "select *", - "from foo", - "where bar = 1", - "limit 1"]) + sql = sqlparse.format("select * from foo where bar = 1 limit 1", reindent=True) + assert sql == "\n".join(["select *", "from foo", "where bar = 1", "limit 1"]) def test_issue38(): @@ -60,59 +54,70 @@ def test_issue38(): def test_issue39(): - p = sqlparse.parse('select user.id from user')[0] + p = sqlparse.parse("select user.id from user")[0] assert len(p.tokens) == 7 idt = p.tokens[2] assert idt.__class__ == sql.Identifier assert len(idt.tokens) == 3 - assert idt.tokens[0].match(T.Name, 'user') is True - assert idt.tokens[1].match(T.Punctuation, '.') is True - assert idt.tokens[2].match(T.Name, 'id') is True + assert idt.tokens[0].match(T.Name, "user") is True + assert idt.tokens[1].match(T.Punctuation, ".") is True + assert idt.tokens[2].match(T.Name, "id") is True def test_issue40(): # make sure identifier lists in subselects are grouped - p = sqlparse.parse('SELECT id, name FROM ' - '(SELECT id, name FROM bar) as foo')[0] + p = sqlparse.parse("SELECT id, name FROM " "(SELECT id, name FROM bar) as foo")[0] assert len(p.tokens) == 7 assert p.tokens[2].__class__ == sql.IdentifierList assert p.tokens[-1].__class__ == sql.Identifier - assert p.tokens[-1].get_name() == 'foo' + assert p.tokens[-1].get_name() == "foo" sp = p.tokens[-1].tokens[0] assert sp.tokens[3].__class__ == sql.IdentifierList # make sure that formatting works as expected - s = sqlparse.format('SELECT id == name FROM ' - '(SELECT id, name FROM bar)', reindent=True) - assert s == '\n'.join([ - 'SELECT id == name', - 'FROM', - ' (SELECT id,', - ' name', - ' FROM bar)']) - - s = sqlparse.format('SELECT id == name FROM ' - '(SELECT id, name FROM bar) as foo', reindent=True) - assert s == '\n'.join([ - 'SELECT id == name', - 'FROM', - ' (SELECT id,', - ' name', - ' FROM bar) as foo']) - - -@pytest.mark.parametrize('s', ['select x.y::text as z from foo', - 'select x.y::text as "z" from foo', - 'select x."y"::text as z from foo', - 'select x."y"::text as "z" from foo', - 'select "x".y::text as z from foo', - 'select "x".y::text as "z" from foo', - 'select "x"."y"::text as z from foo', - 'select "x"."y"::text as "z" from foo']) -@pytest.mark.parametrize('func_name, result', [('get_name', 'z'), - ('get_real_name', 'y'), - ('get_parent_name', 'x'), - ('get_alias', 'z'), - ('get_typecast', 'text')]) + s = sqlparse.format( + "SELECT id == name FROM " "(SELECT id, name FROM bar)", reindent=True + ) + assert s == "\n".join( + ["SELECT id == name", "FROM", " (SELECT id,", " name", " FROM bar)"] + ) + + s = sqlparse.format( + "SELECT id == name FROM " "(SELECT id, name FROM bar) as foo", reindent=True + ) + assert s == "\n".join( + [ + "SELECT id == name", + "FROM", + " (SELECT id,", + " name", + " FROM bar) as foo", + ] + ) + + +@pytest.mark.parametrize( + "s", + [ + "select x.y::text as z from foo", + 'select x.y::text as "z" from foo', + 'select x."y"::text as z from foo', + 'select x."y"::text as "z" from foo', + 'select "x".y::text as z from foo', + 'select "x".y::text as "z" from foo', + 'select "x"."y"::text as z from foo', + 'select "x"."y"::text as "z" from foo', + ], +) +@pytest.mark.parametrize( + "func_name, result", + [ + ("get_name", "z"), + ("get_real_name", "y"), + ("get_parent_name", "x"), + ("get_alias", "z"), + ("get_typecast", "text"), + ], +) def test_issue78(s, func_name, result): # the bug author provided this nice examples, let's use them! p = sqlparse.parse(s)[0] @@ -148,7 +153,7 @@ def test_issue83(): def test_comment_encoding_when_reindent(): # There was an UnicodeEncodeError in the reindent filter that # casted every comment followed by a keyword to str. - sql = 'select foo -- Comment containing Ümläuts\nfrom bar' + sql = "select foo -- Comment containing Ümläuts\nfrom bar" formatted = sqlparse.format(sql, reindent=True) assert formatted == sql @@ -156,10 +161,10 @@ def test_comment_encoding_when_reindent(): def test_parse_sql_with_binary(): # See https://github.com/andialbrecht/sqlparse/pull/88 # digest = '‚|ËêŠplL4¡h‘øN{' - digest = '\x82|\xcb\x0e\xea\x8aplL4\xa1h\x91\xf8N{' - sql = "select * from foo where bar = '{}'".format(digest) + digest = "\x82|\xcb\x0e\xea\x8aplL4\xa1h\x91\xf8N{" + sql = f"select * from foo where bar = '{digest}'" formatted = sqlparse.format(sql, reindent=True) - tformatted = "select *\nfrom foo\nwhere bar = '{}'".format(digest) + tformatted = f"select *\nfrom foo\nwhere bar = '{digest}'" assert formatted == tformatted @@ -167,7 +172,7 @@ def test_dont_alias_keywords(): # The _group_left_right function had a bug where the check for the # left side wasn't handled correctly. In one case this resulted in # a keyword turning into an identifier. - p = sqlparse.parse('FROM AS foo')[0] + p = sqlparse.parse("FROM AS foo")[0] assert len(p.tokens) == 5 assert p.tokens[0].ttype is T.Keyword assert p.tokens[2].ttype is T.Keyword @@ -175,9 +180,9 @@ def test_dont_alias_keywords(): def test_format_accepts_encoding(load_file): # issue20 - sql = load_file('test_cp1251.sql', 'cp1251') - formatted = sqlparse.format(sql, reindent=True, encoding='cp1251') - tformatted = 'insert into foo\nvalues (1); -- Песня про надежду' + sql = load_file("test_cp1251.sql", "cp1251") + formatted = sqlparse.format(sql, reindent=True, encoding="cp1251") + tformatted = "insert into foo\nvalues (1); -- Песня про надежду" assert formatted == tformatted @@ -185,59 +190,64 @@ def test_format_accepts_encoding(load_file): def test_stream(get_stream): with get_stream("stream.sql") as stream: p = sqlparse.parse(stream)[0] - assert p.get_type() == 'INSERT' + assert p.get_type() == "INSERT" def test_issue90(): - sql = ('UPDATE "gallery_photo" SET "owner_id" = 4018, "deleted_at" = NULL,' - ' "width" = NULL, "height" = NULL, "rating_votes" = 0,' - ' "rating_score" = 0, "thumbnail_width" = NULL,' - ' "thumbnail_height" = NULL, "price" = 1, "description" = NULL') + sql = ( + 'UPDATE "gallery_photo" SET "owner_id" = 4018, "deleted_at" = NULL,' + ' "width" = NULL, "height" = NULL, "rating_votes" = 0,' + ' "rating_score" = 0, "thumbnail_width" = NULL,' + ' "thumbnail_height" = NULL, "price" = 1, "description" = NULL' + ) formatted = sqlparse.format(sql, reindent=True) - tformatted = '\n'.join([ - 'UPDATE "gallery_photo"', - 'SET "owner_id" = 4018,', - ' "deleted_at" = NULL,', - ' "width" = NULL,', - ' "height" = NULL,', - ' "rating_votes" = 0,', - ' "rating_score" = 0,', - ' "thumbnail_width" = NULL,', - ' "thumbnail_height" = NULL,', - ' "price" = 1,', - ' "description" = NULL']) + tformatted = "\n".join( + [ + 'UPDATE "gallery_photo"', + 'SET "owner_id" = 4018,', + ' "deleted_at" = NULL,', + ' "width" = NULL,', + ' "height" = NULL,', + ' "rating_votes" = 0,', + ' "rating_score" = 0,', + ' "thumbnail_width" = NULL,', + ' "thumbnail_height" = NULL,', + ' "price" = 1,', + ' "description" = NULL', + ] + ) assert formatted == tformatted def test_except_formatting(): - sql = 'SELECT 1 FROM foo WHERE 2 = 3 EXCEPT SELECT 2 FROM bar WHERE 1 = 2' + sql = "SELECT 1 FROM foo WHERE 2 = 3 EXCEPT SELECT 2 FROM bar WHERE 1 = 2" formatted = sqlparse.format(sql, reindent=True) - tformatted = '\n'.join([ - 'SELECT 1', - 'FROM foo', - 'WHERE 2 = 3', - 'EXCEPT', - 'SELECT 2', - 'FROM bar', - 'WHERE 1 = 2']) + tformatted = "\n".join( + [ + "SELECT 1", + "FROM foo", + "WHERE 2 = 3", + "EXCEPT", + "SELECT 2", + "FROM bar", + "WHERE 1 = 2", + ] + ) assert formatted == tformatted def test_null_with_as(): - sql = 'SELECT NULL AS c1, NULL AS c2 FROM t1' + sql = "SELECT NULL AS c1, NULL AS c2 FROM t1" formatted = sqlparse.format(sql, reindent=True) - tformatted = '\n'.join([ - 'SELECT NULL AS c1,', - ' NULL AS c2', - 'FROM t1']) + tformatted = "\n".join(["SELECT NULL AS c1,", " NULL AS c2", "FROM t1"]) assert formatted == tformatted def test_issue190_open_file(filepath): - path = filepath('stream.sql') + path = filepath("stream.sql") with open(path) as stream: p = sqlparse.parse(stream)[0] - assert p.get_type() == 'INSERT' + assert p.get_type() == "INSERT" def test_issue193_splitting_function(): @@ -268,14 +278,14 @@ def test_issue194_splitting_function(): def test_issue186_get_type(): sql = "-- comment\ninsert into foo" p = sqlparse.parse(sql)[0] - assert p.get_type() == 'INSERT' + assert p.get_type() == "INSERT" def test_issue212_py2unicode(): - t1 = sql.Token(T.String, 'schöner ') - t2 = sql.Token(T.String, 'bug') + t1 = sql.Token(T.String, "schöner ") + t2 = sql.Token(T.String, "bug") token_list = sql.TokenList([t1, t2]) - assert str(token_list) == 'schöner bug' + assert str(token_list) == "schöner bug" def test_issue213_leadingws(): @@ -284,40 +294,41 @@ def test_issue213_leadingws(): def test_issue227_gettype_cte(): - select_stmt = sqlparse.parse('SELECT 1, 2, 3 FROM foo;') - assert select_stmt[0].get_type() == 'SELECT' - with_stmt = sqlparse.parse('WITH foo AS (SELECT 1, 2, 3)' - 'SELECT * FROM foo;') - assert with_stmt[0].get_type() == 'SELECT' - with2_stmt = sqlparse.parse(""" + select_stmt = sqlparse.parse("SELECT 1, 2, 3 FROM foo;") + assert select_stmt[0].get_type() == "SELECT" + with_stmt = sqlparse.parse("WITH foo AS (SELECT 1, 2, 3)" "SELECT * FROM foo;") + assert with_stmt[0].get_type() == "SELECT" + with2_stmt = sqlparse.parse( + """ WITH foo AS (SELECT 1 AS abc, 2 AS def), bar AS (SELECT * FROM something WHERE x > 1) - INSERT INTO elsewhere SELECT * FROM foo JOIN bar;""") - assert with2_stmt[0].get_type() == 'INSERT' + INSERT INTO elsewhere SELECT * FROM foo JOIN bar;""" + ) + assert with2_stmt[0].get_type() == "INSERT" def test_issue207_runaway_format(): - sql = 'select 1 from (select 1 as one, 2 as two, 3 from dual) t0' + sql = "select 1 from (select 1 as one, 2 as two, 3 from dual) t0" p = sqlparse.format(sql, reindent=True) - assert p == '\n'.join([ - "select 1", - "from", - " (select 1 as one,", - " 2 as two,", - " 3", - " from dual) t0"]) + assert p == "\n".join( + [ + "select 1", + "from", + " (select 1 as one,", + " 2 as two,", + " 3", + " from dual) t0", + ] + ) def test_token_next_doesnt_ignore_skip_cm(): - sql = '--comment\nselect 1' + sql = "--comment\nselect 1" tok = sqlparse.parse(sql)[0].token_next(-1, skip_cm=True)[1] - assert tok.value == 'select' + assert tok.value == "select" -@pytest.mark.parametrize('s', [ - 'SELECT x AS', - 'AS' -]) +@pytest.mark.parametrize("s", ["SELECT x AS", "AS"]) def test_issue284_as_grouping(s): p = sqlparse.parse(s)[0] assert s == str(p) @@ -329,35 +340,37 @@ def test_issue315_utf8_by_default(): # The digest contains Chinese, Japanese and Korean characters # All in 'utf-8' encoding. digest = ( - '\xe9\xbd\x90\xe5\xa4\xa9\xe5\xa4\xa7\xe5\x9c\xa3.' - '\xe3\x82\xab\xe3\x83\xa9\xe3\x83\x95\xe3\x83\xab\xe3\x81\xaa\xe9' - '\x9b\xb2.' - '\xec\x82\xac\xeb\x9e\x91\xed\x95\xb4\xec\x9a\x94' + "\xe9\xbd\x90\xe5\xa4\xa9\xe5\xa4\xa7\xe5\x9c\xa3." + "\xe3\x82\xab\xe3\x83\xa9\xe3\x83\x95\xe3\x83\xab\xe3\x81\xaa\xe9" + "\x9b\xb2." + "\xec\x82\xac\xeb\x9e\x91\xed\x95\xb4\xec\x9a\x94" ) - sql = "select * from foo where bar = '{}'".format(digest) + sql = f"select * from foo where bar = '{digest}'" formatted = sqlparse.format(sql, reindent=True) - tformatted = "select *\nfrom foo\nwhere bar = '{}'".format(digest) + tformatted = f"select *\nfrom foo\nwhere bar = '{digest}'" assert formatted == tformatted def test_issue322_concurrently_is_keyword(): - s = 'CREATE INDEX CONCURRENTLY myindex ON mytable(col1);' + s = "CREATE INDEX CONCURRENTLY myindex ON mytable(col1);" p = sqlparse.parse(s)[0] assert len(p.tokens) == 12 assert p.tokens[0].ttype is T.Keyword.DDL # CREATE - assert p.tokens[2].ttype is T.Keyword # INDEX - assert p.tokens[4].ttype is T.Keyword # CONCURRENTLY - assert p.tokens[4].value == 'CONCURRENTLY' + assert p.tokens[2].ttype is T.Keyword # INDEX + assert p.tokens[4].ttype is T.Keyword # CONCURRENTLY + assert p.tokens[4].value == "CONCURRENTLY" assert isinstance(p.tokens[6], sql.Identifier) - assert p.tokens[6].value == 'myindex' + assert p.tokens[6].value == "myindex" -@pytest.mark.parametrize('s', [ - 'SELECT @min_price:=MIN(price), @max_price:=MAX(price) FROM shop;', - 'SELECT @min_price:=MIN(price), @max_price:=MAX(price) FROM shop', - -]) +@pytest.mark.parametrize( + "s", + [ + "SELECT @min_price:=MIN(price), @max_price:=MAX(price) FROM shop;", + "SELECT @min_price:=MIN(price), @max_price:=MAX(price) FROM shop", + ], +) def test_issue359_index_error_assignments(s): sqlparse.parse(s) sqlparse.format(s, strip_comments=True) @@ -365,85 +378,118 @@ def test_issue359_index_error_assignments(s): def test_issue469_copy_as_psql_command(): formatted = sqlparse.format( - '\\copy select * from foo', - keyword_case='upper', identifier_case='capitalize') - assert formatted == '\\copy SELECT * FROM Foo' + "\\copy select * from foo", keyword_case="upper", identifier_case="capitalize" + ) + assert formatted == "\\copy SELECT * FROM Foo" -@pytest.mark.xfail(reason='Needs to be fixed') +@pytest.mark.xfail(reason="Needs to be fixed") def test_issue484_comments_and_newlines(): - formatted = sqlparse.format('\n'.join([ - 'Create table myTable', - '(', - ' myId TINYINT NOT NULL, --my special comment', - ' myName VARCHAR2(100) NOT NULL', - ')']), - strip_comments=True) - assert formatted == ('\n'.join([ - 'Create table myTable', - '(', - ' myId TINYINT NOT NULL,', - ' myName VARCHAR2(100) NOT NULL', - ')'])) + formatted = sqlparse.format( + "\n".join( + [ + "Create table myTable", + "(", + " myId TINYINT NOT NULL, --my special comment", + " myName VARCHAR2(100) NOT NULL", + ")", + ] + ), + strip_comments=True, + ) + assert formatted == ( + "\n".join( + [ + "Create table myTable", + "(", + " myId TINYINT NOT NULL,", + " myName VARCHAR2(100) NOT NULL", + ")", + ] + ) + ) def test_issue485_split_multi(): - p_sql = '''CREATE OR REPLACE RULE ruled_tab_2rules AS ON INSERT + p_sql = """CREATE OR REPLACE RULE ruled_tab_2rules AS ON INSERT TO public.ruled_tab DO instead ( select 1; select 2; -);''' +);""" assert len(sqlparse.split(p_sql)) == 1 def test_issue489_tzcasts(): - p = sqlparse.parse('select bar at time zone \'UTC\' as foo')[0] + p = sqlparse.parse("select bar at time zone 'UTC' as foo")[0] assert p.tokens[-1].has_alias() is True - assert p.tokens[-1].get_alias() == 'foo' + assert p.tokens[-1].get_alias() == "foo" def test_issue562_tzcasts(): # Test that whitespace between 'from' and 'bar' is retained formatted = sqlparse.format( - 'SELECT f(HOUR from bar AT TIME ZONE \'UTC\') from foo', reindent=True + "SELECT f(HOUR from bar AT TIME ZONE 'UTC') from foo", reindent=True ) - assert formatted == \ - 'SELECT f(HOUR\n from bar AT TIME ZONE \'UTC\')\nfrom foo' + assert formatted == "SELECT f(HOUR\n from bar AT TIME ZONE 'UTC')\nfrom foo" def test_as_in_parentheses_indents(): # did raise NoneType has no attribute is_group in _process_parentheses - formatted = sqlparse.format('(as foo)', reindent=True) - assert formatted == '(as foo)' + formatted = sqlparse.format("(as foo)", reindent=True) + assert formatted == "(as foo)" def test_format_invalid_where_clause(): # did raise ValueError - formatted = sqlparse.format('where, foo', reindent=True) - assert formatted == 'where, foo' + formatted = sqlparse.format("where, foo", reindent=True) + assert formatted == "where, foo" + def test_cte_named_final(): - p = sqlparse.parse('with final as (select 1) select final')[0] - assert p.get_type() == 'SELECT' + p = sqlparse.parse("with final as (select 1) select final")[0] + assert p.get_type() == "SELECT" + def test_splitting_at_and_backticks_issue588(): splitted = sqlparse.split( - 'grant foo to user1@`myhost`; grant bar to user1@`myhost`;') + "grant foo to user1@`myhost`; grant bar to user1@`myhost`;" + ) assert len(splitted) == 2 - assert splitted[-1] == 'grant bar to user1@`myhost`;' + assert splitted[-1] == "grant bar to user1@`myhost`;" def test_comment_between_cte_clauses_issue632(): - p, = sqlparse.parse(""" + (p,) = sqlparse.parse( + """ WITH foo AS (), -- A comment before baz subquery baz AS () - SELECT * FROM baz;""") + SELECT * FROM baz;""" + ) assert p.get_type() == "SELECT" def test_copy_issue672(): - p = sqlparse.parse('select * from foo')[0] + p = sqlparse.parse("select * from foo")[0] copied = copy.deepcopy(p) assert str(p) == str(copied) + + +def test_primary_key_issue740(): + p = sqlparse.parse("PRIMARY KEY")[0] + assert len(p.tokens) == 1 + assert p.tokens[0].ttype == T.Keyword + + +@pytest.fixture +def limit_recursion(): + curr_limit = sys.getrecursionlimit() + sys.setrecursionlimit(100) + yield + sys.setrecursionlimit(curr_limit) + + +def test_max_recursion(limit_recursion): + with pytest.raises(SQLParseError): + sqlparse.parse("[" * 1000 + "]" * 1000) diff --git a/tests/test_split.py b/tests/test_split.py index e945d4f1..82316082 100644 --- a/tests/test_split.py +++ b/tests/test_split.py @@ -9,24 +9,29 @@ def test_split_semicolon(): - sql1 = 'select * from foo;' + sql1 = "select * from foo;" sql2 = "select * from foo where bar = 'foo;bar';" - stmts = sqlparse.parse(''.join([sql1, sql2])) + stmts = sqlparse.parse("".join([sql1, sql2])) assert len(stmts) == 2 assert str(stmts[0]) == sql1 assert str(stmts[1]) == sql2 def test_split_backslash(): - stmts = sqlparse.parse("select '\'; select '\'';") + stmts = sqlparse.parse("select ''; select ''';") assert len(stmts) == 2 -@pytest.mark.parametrize('fn', ['function.sql', - 'function_psql.sql', - 'function_psql2.sql', - 'function_psql3.sql', - 'function_psql4.sql']) +@pytest.mark.parametrize( + "fn", + [ + "function.sql", + "function_psql.sql", + "function_psql2.sql", + "function_psql3.sql", + "function_psql4.sql", + ], +) def test_split_create_function(load_file, fn): sql = load_file(fn) stmts = sqlparse.parse(sql) @@ -35,86 +40,91 @@ def test_split_create_function(load_file, fn): def test_split_dashcomments(load_file): - sql = load_file('dashcomment.sql') + sql = load_file("dashcomment.sql") stmts = sqlparse.parse(sql) assert len(stmts) == 3 - assert ''.join(str(q) for q in stmts) == sql - - -@pytest.mark.parametrize('s', ['select foo; -- comment\n', - 'select foo; -- comment\r', - 'select foo; -- comment\r\n', - 'select foo; -- comment']) + assert "".join(str(q) for q in stmts) == sql + + +@pytest.mark.parametrize( + "s", + [ + "select foo; -- comment\n", + "select foo; -- comment\r", + "select foo; -- comment\r\n", + "select foo; -- comment", + ], +) def test_split_dashcomments_eol(s): stmts = sqlparse.parse(s) assert len(stmts) == 1 def test_split_begintag(load_file): - sql = load_file('begintag.sql') + sql = load_file("begintag.sql") stmts = sqlparse.parse(sql) assert len(stmts) == 3 - assert ''.join(str(q) for q in stmts) == sql + assert "".join(str(q) for q in stmts) == sql def test_split_begintag_2(load_file): - sql = load_file('begintag_2.sql') + sql = load_file("begintag_2.sql") stmts = sqlparse.parse(sql) assert len(stmts) == 1 - assert ''.join(str(q) for q in stmts) == sql + assert "".join(str(q) for q in stmts) == sql def test_split_dropif(): - sql = 'DROP TABLE IF EXISTS FOO;\n\nSELECT * FROM BAR;' + sql = "DROP TABLE IF EXISTS FOO;\n\nSELECT * FROM BAR;" stmts = sqlparse.parse(sql) assert len(stmts) == 2 - assert ''.join(str(q) for q in stmts) == sql + assert "".join(str(q) for q in stmts) == sql def test_split_comment_with_umlaut(): - sql = ('select * from foo;\n' - '-- Testing an umlaut: ä\n' - 'select * from bar;') + sql = "select * from foo;\n" "-- Testing an umlaut: ä\n" "select * from bar;" stmts = sqlparse.parse(sql) assert len(stmts) == 2 - assert ''.join(str(q) for q in stmts) == sql + assert "".join(str(q) for q in stmts) == sql def test_split_comment_end_of_line(): - sql = ('select * from foo; -- foo\n' - 'select * from bar;') + sql = "select * from foo; -- foo\n" "select * from bar;" stmts = sqlparse.parse(sql) assert len(stmts) == 2 - assert ''.join(str(q) for q in stmts) == sql + assert "".join(str(q) for q in stmts) == sql # make sure the comment belongs to first query - assert str(stmts[0]) == 'select * from foo; -- foo\n' + assert str(stmts[0]) == "select * from foo; -- foo\n" def test_split_casewhen(): - sql = ("SELECT case when val = 1 then 2 else null end as foo;\n" - "comment on table actor is 'The actor table.';") + sql = ( + "SELECT case when val = 1 then 2 else null end as foo;\n" + "comment on table actor is 'The actor table.';" + ) stmts = sqlparse.split(sql) assert len(stmts) == 2 def test_split_casewhen_procedure(load_file): # see issue580 - stmts = sqlparse.split(load_file('casewhen_procedure.sql')) + stmts = sqlparse.split(load_file("casewhen_procedure.sql")) assert len(stmts) == 2 def test_split_cursor_declare(): - sql = ('DECLARE CURSOR "foo" AS SELECT 1;\n' - 'SELECT 2;') + sql = 'DECLARE CURSOR "foo" AS SELECT 1;\n' "SELECT 2;" stmts = sqlparse.split(sql) assert len(stmts) == 2 def test_split_if_function(): # see issue 33 # don't let IF as a function confuse the splitter - sql = ('CREATE TEMPORARY TABLE tmp ' - 'SELECT IF(a=1, a, b) AS o FROM one; ' - 'SELECT t FROM two') + sql = ( + "CREATE TEMPORARY TABLE tmp " + "SELECT IF(a=1, a, b) AS o FROM one; " + "SELECT t FROM two" + ) stmts = sqlparse.split(sql) assert len(stmts) == 2 @@ -133,23 +143,23 @@ def test_split_encoding_parsestream(): def test_split_unicode_parsestream(): - stream = StringIO('SELECT ö') + stream = StringIO("SELECT ö") stmts = list(sqlparse.parsestream(stream)) - assert str(stmts[0]) == 'SELECT ö' + assert str(stmts[0]) == "SELECT ö" def test_split_simple(): - stmts = sqlparse.split('select * from foo; select * from bar;') + stmts = sqlparse.split("select * from foo; select * from bar;") assert len(stmts) == 2 - assert stmts[0] == 'select * from foo;' - assert stmts[1] == 'select * from bar;' + assert stmts[0] == "select * from foo;" + assert stmts[1] == "select * from bar;" def test_split_ignores_empty_newlines(): - stmts = sqlparse.split('select foo;\nselect bar;\n') + stmts = sqlparse.split("select foo;\nselect bar;\n") assert len(stmts) == 2 - assert stmts[0] == 'select foo;' - assert stmts[1] == 'select bar;' + assert stmts[0] == "select foo;" + assert stmts[1] == "select bar;" def test_split_quotes_with_new_line(): @@ -164,10 +174,64 @@ def test_split_quotes_with_new_line(): def test_split_mysql_handler_for(load_file): # see issue581 - stmts = sqlparse.split(load_file('mysql_handler.sql')) + stmts = sqlparse.split(load_file("mysql_handler.sql")) assert len(stmts) == 2 def test_unecessary_parens(): (stmt,) = sqlparse.parse("(select * from foo)") assert stmt.get_type() == "SELECT" + + +@pytest.mark.parametrize( + "sql, expected", + [ + ("select * from foo;", ["select * from foo"]), + ("select * from foo", ["select * from foo"]), + ( + "select * from foo; select * from bar;", + [ + "select * from foo", + "select * from bar", + ], + ), + ( + " select * from foo;\n\nselect * from bar;\n\n\n\n", + [ + "select * from foo", + "select * from bar", + ], + ), + ("select * from foo\n\n; bar", ["select * from foo", "bar"]), + ], +) +def test_split_strip_semicolon(sql, expected): + stmts = sqlparse.split(sql, strip_semicolon=True) + assert len(stmts) == len(expected) + for idx, expectation in enumerate(expected): + assert stmts[idx] == expectation + + +def test_split_strip_semicolon_procedure(load_file): + stmts = sqlparse.split(load_file("mysql_handler.sql"), strip_semicolon=True) + assert len(stmts) == 2 + assert stmts[0].endswith("end") + assert stmts[1].endswith("end") + + +@pytest.mark.parametrize( + "sql, num", + [ + ("USE foo;\nGO\nSELECT 1;\nGO", 4), + ("SELECT * FROM foo;\nGO", 2), + ("USE foo;\nGO 2\nSELECT 1;", 3), + ], +) +def test_split_go(sql, num): # issue762 + stmts = sqlparse.split(sql) + assert len(stmts) == num + + +def test_split_multiple_case_in_begin(load_file): # issue784 + stmts = sqlparse.split(load_file("multiple_case_in_begin.sql")) + assert len(stmts) == 1 diff --git a/tests/test_tokenize.py b/tests/test_tokenize.py index 90b98584..65acfa34 100644 --- a/tests/test_tokenize.py +++ b/tests/test_tokenize.py @@ -149,7 +149,7 @@ def test_stream_error(): 'STRAIGHT_JOIN', 'INNER JOIN']) def test_parse_join(expr): - p = sqlparse.parse('{} foo'.format(expr))[0] + p = sqlparse.parse(f'{expr} foo')[0] assert len(p.tokens) == 3 assert p.tokens[0].ttype is T.Keyword @@ -168,11 +168,16 @@ def test_parse_endifloop(s): assert p.tokens[0].ttype is T.Keyword -@pytest.mark.parametrize('s', ['NULLS FIRST', 'NULLS LAST']) -def test_parse_nulls(s): # issue487 +@pytest.mark.parametrize('s', [ + 'ASC', 'DESC', + 'NULLS FIRST', 'NULLS LAST', + 'ASC NULLS FIRST', 'ASC NULLS LAST', + 'DESC NULLS FIRST', 'DESC NULLS LAST', +]) +def test_parse_order(s): # issue487 p = sqlparse.parse(s)[0] assert len(p.tokens) == 1 - assert p.tokens[0].ttype is T.Keyword + assert p.tokens[0].ttype is T.Keyword.Order @pytest.mark.parametrize('s', [