Edit on GitHub

sqlglot.optimizer.canonicalize

  1from __future__ import annotations
  2
  3import itertools
  4import typing as t
  5
  6from sqlglot import exp
  7from sqlglot.helper import is_date_unit, is_iso_date, is_iso_datetime
  8
  9
 10def canonicalize(expression: exp.Expression) -> exp.Expression:
 11    """Converts a sql expression into a standard form.
 12
 13    This method relies on annotate_types because many of the
 14    conversions rely on type inference.
 15
 16    Args:
 17        expression: The expression to canonicalize.
 18    """
 19
 20    def _canonicalize(expression: exp.Expression) -> exp.Expression:
 21        expression = add_text_to_concat(expression)
 22        expression = replace_date_funcs(expression)
 23        expression = coerce_type(expression)
 24        expression = remove_redundant_casts(expression)
 25        expression = ensure_bools(expression, _replace_int_predicate)
 26        expression = remove_ascending_order(expression)
 27        return expression
 28
 29    return exp.replace_tree(expression, _canonicalize)
 30
 31
 32def add_text_to_concat(node: exp.Expression) -> exp.Expression:
 33    if isinstance(node, exp.Add) and node.type and node.type.this in exp.DataType.TEXT_TYPES:
 34        node = exp.Concat(expressions=[node.left, node.right])
 35    return node
 36
 37
 38def replace_date_funcs(node: exp.Expression) -> exp.Expression:
 39    if isinstance(node, exp.Date) and not node.expressions and not node.args.get("zone"):
 40        return exp.cast(node.this, to=exp.DataType.Type.DATE)
 41    if isinstance(node, exp.Timestamp) and not node.expression:
 42        if not node.type:
 43            from sqlglot.optimizer.annotate_types import annotate_types
 44
 45            node = annotate_types(node)
 46        return exp.cast(node.this, to=node.type or exp.DataType.Type.TIMESTAMP)
 47
 48    return node
 49
 50
 51COERCIBLE_DATE_OPS = (
 52    exp.Add,
 53    exp.Sub,
 54    exp.EQ,
 55    exp.NEQ,
 56    exp.GT,
 57    exp.GTE,
 58    exp.LT,
 59    exp.LTE,
 60    exp.NullSafeEQ,
 61    exp.NullSafeNEQ,
 62)
 63
 64
 65def coerce_type(node: exp.Expression) -> exp.Expression:
 66    if isinstance(node, COERCIBLE_DATE_OPS):
 67        _coerce_date(node.left, node.right)
 68    elif isinstance(node, exp.Between):
 69        _coerce_date(node.this, node.args["low"])
 70    elif isinstance(node, exp.Extract) and not node.expression.type.is_type(
 71        *exp.DataType.TEMPORAL_TYPES
 72    ):
 73        _replace_cast(node.expression, exp.DataType.Type.DATETIME)
 74    elif isinstance(node, (exp.DateAdd, exp.DateSub, exp.DateTrunc)):
 75        _coerce_timeunit_arg(node.this, node.unit)
 76    elif isinstance(node, exp.DateDiff):
 77        _coerce_datediff_args(node)
 78
 79    return node
 80
 81
 82def remove_redundant_casts(expression: exp.Expression) -> exp.Expression:
 83    if (
 84        isinstance(expression, exp.Cast)
 85        and expression.this.type
 86        and expression.to.this == expression.this.type.this
 87    ):
 88        return expression.this
 89    return expression
 90
 91
 92def ensure_bools(
 93    expression: exp.Expression, replace_func: t.Callable[[exp.Expression], None]
 94) -> exp.Expression:
 95    if isinstance(expression, exp.Connector):
 96        replace_func(expression.left)
 97        replace_func(expression.right)
 98    elif isinstance(expression, exp.Not):
 99        replace_func(expression.this)
100        # We can't replace num in CASE x WHEN num ..., because it's not the full predicate
101    elif isinstance(expression, exp.If) and not (
102        isinstance(expression.parent, exp.Case) and expression.parent.this
103    ):
104        replace_func(expression.this)
105    elif isinstance(expression, (exp.Where, exp.Having)):
106        replace_func(expression.this)
107
108    return expression
109
110
111def remove_ascending_order(expression: exp.Expression) -> exp.Expression:
112    if isinstance(expression, exp.Ordered) and expression.args.get("desc") is False:
113        # Convert ORDER BY a ASC to ORDER BY a
114        expression.set("desc", None)
115
116    return expression
117
118
119def _coerce_date(a: exp.Expression, b: exp.Expression) -> None:
120    for a, b in itertools.permutations([a, b]):
121        if isinstance(b, exp.Interval):
122            a = _coerce_timeunit_arg(a, b.unit)
123        if (
124            a.type
125            and a.type.this == exp.DataType.Type.DATE
126            and b.type
127            and b.type.this
128            not in (
129                exp.DataType.Type.DATE,
130                exp.DataType.Type.INTERVAL,
131            )
132        ):
133            _replace_cast(b, exp.DataType.Type.DATE)
134
135
136def _coerce_timeunit_arg(arg: exp.Expression, unit: t.Optional[exp.Expression]) -> exp.Expression:
137    if not arg.type:
138        return arg
139
140    if arg.type.this in exp.DataType.TEXT_TYPES:
141        date_text = arg.name
142        is_iso_date_ = is_iso_date(date_text)
143
144        if is_iso_date_ and is_date_unit(unit):
145            return arg.replace(exp.cast(arg.copy(), to=exp.DataType.Type.DATE))
146
147        # An ISO date is also an ISO datetime, but not vice versa
148        if is_iso_date_ or is_iso_datetime(date_text):
149            return arg.replace(exp.cast(arg.copy(), to=exp.DataType.Type.DATETIME))
150
151    elif arg.type.this == exp.DataType.Type.DATE and not is_date_unit(unit):
152        return arg.replace(exp.cast(arg.copy(), to=exp.DataType.Type.DATETIME))
153
154    return arg
155
156
157def _coerce_datediff_args(node: exp.DateDiff) -> None:
158    for e in (node.this, node.expression):
159        if e.type.this not in exp.DataType.TEMPORAL_TYPES:
160            e.replace(exp.cast(e.copy(), to=exp.DataType.Type.DATETIME))
161
162
163def _replace_cast(node: exp.Expression, to: exp.DataType.Type) -> None:
164    node.replace(exp.cast(node.copy(), to=to))
165
166
167# this was originally designed for presto, there is a similar transform for tsql
168# this is different in that it only operates on int types, this is because
169# presto has a boolean type whereas tsql doesn't (people use bits)
170# with y as (select true as x) select x = 0 FROM y -- illegal presto query
171def _replace_int_predicate(expression: exp.Expression) -> None:
172    if isinstance(expression, exp.Coalesce):
173        for child in expression.iter_expressions():
174            _replace_int_predicate(child)
175    elif expression.type and expression.type.this in exp.DataType.INTEGER_TYPES:
176        expression.replace(expression.neq(0))
def canonicalize( expression: sqlglot.expressions.Expression) -> sqlglot.expressions.Expression:
11def canonicalize(expression: exp.Expression) -> exp.Expression:
12    """Converts a sql expression into a standard form.
13
14    This method relies on annotate_types because many of the
15    conversions rely on type inference.
16
17    Args:
18        expression: The expression to canonicalize.
19    """
20
21    def _canonicalize(expression: exp.Expression) -> exp.Expression:
22        expression = add_text_to_concat(expression)
23        expression = replace_date_funcs(expression)
24        expression = coerce_type(expression)
25        expression = remove_redundant_casts(expression)
26        expression = ensure_bools(expression, _replace_int_predicate)
27        expression = remove_ascending_order(expression)
28        return expression
29
30    return exp.replace_tree(expression, _canonicalize)

Converts a sql expression into a standard form.

This method relies on annotate_types because many of the conversions rely on type inference.

Arguments:
  • expression: The expression to canonicalize.
def add_text_to_concat(node: sqlglot.expressions.Expression) -> sqlglot.expressions.Expression:
33def add_text_to_concat(node: exp.Expression) -> exp.Expression:
34    if isinstance(node, exp.Add) and node.type and node.type.this in exp.DataType.TEXT_TYPES:
35        node = exp.Concat(expressions=[node.left, node.right])
36    return node
def replace_date_funcs(node: sqlglot.expressions.Expression) -> sqlglot.expressions.Expression:
39def replace_date_funcs(node: exp.Expression) -> exp.Expression:
40    if isinstance(node, exp.Date) and not node.expressions and not node.args.get("zone"):
41        return exp.cast(node.this, to=exp.DataType.Type.DATE)
42    if isinstance(node, exp.Timestamp) and not node.expression:
43        if not node.type:
44            from sqlglot.optimizer.annotate_types import annotate_types
45
46            node = annotate_types(node)
47        return exp.cast(node.this, to=node.type or exp.DataType.Type.TIMESTAMP)
48
49    return node
def coerce_type(node: sqlglot.expressions.Expression) -> sqlglot.expressions.Expression:
66def coerce_type(node: exp.Expression) -> exp.Expression:
67    if isinstance(node, COERCIBLE_DATE_OPS):
68        _coerce_date(node.left, node.right)
69    elif isinstance(node, exp.Between):
70        _coerce_date(node.this, node.args["low"])
71    elif isinstance(node, exp.Extract) and not node.expression.type.is_type(
72        *exp.DataType.TEMPORAL_TYPES
73    ):
74        _replace_cast(node.expression, exp.DataType.Type.DATETIME)
75    elif isinstance(node, (exp.DateAdd, exp.DateSub, exp.DateTrunc)):
76        _coerce_timeunit_arg(node.this, node.unit)
77    elif isinstance(node, exp.DateDiff):
78        _coerce_datediff_args(node)
79
80    return node
def remove_redundant_casts( expression: sqlglot.expressions.Expression) -> sqlglot.expressions.Expression:
83def remove_redundant_casts(expression: exp.Expression) -> exp.Expression:
84    if (
85        isinstance(expression, exp.Cast)
86        and expression.this.type
87        and expression.to.this == expression.this.type.this
88    ):
89        return expression.this
90    return expression
def ensure_bools( expression: sqlglot.expressions.Expression, replace_func: Callable[[sqlglot.expressions.Expression], NoneType]) -> sqlglot.expressions.Expression:
 93def ensure_bools(
 94    expression: exp.Expression, replace_func: t.Callable[[exp.Expression], None]
 95) -> exp.Expression:
 96    if isinstance(expression, exp.Connector):
 97        replace_func(expression.left)
 98        replace_func(expression.right)
 99    elif isinstance(expression, exp.Not):
100        replace_func(expression.this)
101        # We can't replace num in CASE x WHEN num ..., because it's not the full predicate
102    elif isinstance(expression, exp.If) and not (
103        isinstance(expression.parent, exp.Case) and expression.parent.this
104    ):
105        replace_func(expression.this)
106    elif isinstance(expression, (exp.Where, exp.Having)):
107        replace_func(expression.this)
108
109    return expression
def remove_ascending_order( expression: sqlglot.expressions.Expression) -> sqlglot.expressions.Expression:
112def remove_ascending_order(expression: exp.Expression) -> exp.Expression:
113    if isinstance(expression, exp.Ordered) and expression.args.get("desc") is False:
114        # Convert ORDER BY a ASC to ORDER BY a
115        expression.set("desc", None)
116
117    return expression