Lex a minus directly after an operand as an operator (issue370)

sqlparse/keywords.py

@@ -158,11 +158,17 @@ def find_delimited_spans(text):
     # because `re.match` doesn't work with look-behind regexp feature
     (r'(?<=\.)[A-ZÀ-Ü]\w*', tokens.Name),  # .'Name'
     (r'[A-ZÀ-Ü]\w*(?=\()', tokens.Name),  # side effect: change kw to func
-    (r'-?0x[\dA-F]+', tokens.Number.Hexadecimal),
-    (r'-?\d+(\.\d+)?E-?\d+', tokens.Number.Float),
-    (r'(?![_A-ZÀ-Ü])-?(\d+(\.\d*)|\.\d+)(?![_A-ZÀ-Ü])',
+    # A leading minus is only part of the number if it can't be a binary
+    # operator, i.e. if it doesn't directly follow an operand (a name, a
+    # number, a closing bracket or a quoted string or name), so that
+    # ``1-1`` is lexed as ``1``, ``-``, ``1``. See issue #370.
+    (r"(?:(?<![\w)\]'\"`])-)?0x[\dA-F]+", tokens.Number.Hexadecimal),
+    (r"(?:(?<![\w)\]'\"`])-)?\d+(\.\d+)?E-?\d+", tokens.Number.Float),
+    (r"(?![_A-ZÀ-Ü])(?:(?<![\w)\]'\"`])-)?(\d+(\.\d*)|\.\d+)"
+     r"(?![_A-ZÀ-Ü])",
      tokens.Number.Float),
-    (r'(?![_A-ZÀ-Ü])-?\d+(?![_A-ZÀ-Ü])', tokens.Number.Integer),
+    (r"(?![_A-ZÀ-Ü])(?:(?<![\w)\]'\"`])-)?\d+(?![_A-ZÀ-Ü])",
+     tokens.Number.Integer),
     (r"'(''|\\'|[^'])*'", tokens.String.Single),
     # not a real string literal in ANSI SQL:
     (r'"(""|\\"|[^"])*"', tokens.String.Symbol),

tests/test_grouping.py

@@ -438,6 +438,13 @@ def test_identifier_with_operators():
     assert isinstance(p.tokens[0], sql.Operation)
 
 
+def test_operation_minus_without_whitespace():
+    # issue370
+    p = sqlparse.parse('select a-1 from foo')[0]
+    assert isinstance(p.tokens[2], sql.Operation)
+    assert p.tokens[2].value == 'a-1'
+
+
 def test_identifier_with_op_trailing_ws():
     # make sure trailing whitespace isn't grouped with identifier
     p = sqlparse.parse('foo || bar ')[0]

tests/test_tokenize.py

@@ -65,6 +65,16 @@ def test_tokenize_negative_numbers():
     assert tokens[2][1] == '-1'
 
 
+@pytest.mark.parametrize('s', ['1-1', 'a-1', 'a-1.5', 'a-.5', 'a-1e3',
+                               'a-0x1F', 'f(a)-1', 'a[1]-1', "'a'-1",
+                               '"a"-1', '`a`-1'])
+def test_tokenize_minus_after_operand(s):
+    # issue370
+    tokens = list(lexer.tokenize(s))
+    assert tokens[-2] == (T.Operator, '-')
+    assert tokens[-1][1] == s.rsplit('-', 1)[1]
+
+
 def test_token_str():
     token = sql.Token(None, 'FoO')
     assert str(token) == 'FoO'