Browse Source

add support for f-strings (including in py3.5)

resolves #23
adam j hartz 1 year ago
parent
commit
ae5028c5c2
4 changed files with 88 additions and 54 deletions
  1. 40
    9
      takoshell/ast.py
  2. 1
    1
      takoshell/built_ins.py
  3. 28
    3
      takoshell/parsers/base.py
  4. 19
    41
      takoshell/tokenize.py

+ 40
- 9
takoshell/ast.py View File

@@ -32,8 +32,9 @@ from ast import (Module, Num, Expr, Str, Bytes, UnaryOp, UAdd, USub, Invert,
32 32
     Del, Pass, Raise, Import, alias, ImportFrom, Continue, Break, Yield,
33 33
     YieldFrom, Return, IfExp, Lambda, arguments, arg, Call, keyword,
34 34
     Attribute, Global, Nonlocal, If, While, For, withitem, With, Try,
35
-    ExceptHandler, FunctionDef, ClassDef, Starred, NodeTransformer,
36
-    Interactive, Expression, Index, literal_eval, dump, walk, increment_lineno)
35
+    ExceptHandler, FunctionDef, ClassDef, Starred, NodeTransformer, NodeVisitor,
36
+    Interactive, Expression, Index, literal_eval, dump, walk, increment_lineno,
37
+    parse)
37 38
 from ast import Ellipsis  # pylint: disable=redefined-builtin
38 39
 # pylint: enable=unused-import
39 40
 import textwrap
@@ -120,13 +121,6 @@ def get_id(node, default=None):
120 121
     return getattr(node, 'id', default)
121 122
 
122 123
 
123
-def gather_names(node):
124
-    """Returns the set of all names present in the node's tree."""
125
-    rtn = set(map(get_id, walk(node)))
126
-    rtn.discard(None)
127
-    return rtn
128
-
129
-
130 124
 def has_elts(x):
131 125
     """Tests if x is an AST node with elements."""
132 126
     return isinstance(x, AST) and hasattr(x, 'elts')
@@ -413,6 +407,43 @@ class CtxAwareTransformer(NodeTransformer):
413 407
         return node
414 408
 
415 409
 
410
+def gather_names(node):
411
+    ng = NameGatherer()
412
+    ng.visit(node)
413
+    ng.names.discard(None)
414
+    return ng.names
415
+
416
+
417
+class NameGatherer(NodeVisitor):
418
+    def __init__(self):
419
+        self.names = set()
420
+        NodeVisitor.__init__(self)
421
+
422
+    def generic_visit(self, node):
423
+        self.names.add(get_id(node))
424
+        NodeVisitor.generic_visit(self, node)
425
+
426
+    def visit_ListComp(self, node):
427
+        defined_names = set()
428
+        needed_names = gather_names(node.elt)
429
+        for g in node.generators:
430
+            defined_names |= gather_names(g)
431
+            for i in g.ifs:
432
+                needed_names |= gather_names(i)
433
+        self.names |= {i for i in needed_names if i not in defined_names}
434
+
435
+    def visit_DictComp(self, node):
436
+        defined_names = set()
437
+        needed_names = gather_names(node.key)
438
+        needed_names |= gather_names(node.value)
439
+        for g in node.generators:
440
+            defined_names |= gather_names(g)
441
+            for i in g.ifs:
442
+                needed_names |= gather_names(i)
443
+        self.names |= {i for i in needed_names if i not in defined_names}
444
+
445
+    visit_SetComp = visit_ListComp
446
+
416 447
 
417 448
 def pdump(s, **kwargs):
418 449
     """performs a pretty dump of an AST node."""

+ 1
- 1
takoshell/built_ins.py View File

@@ -229,7 +229,7 @@ def get_script_subproc_command(fname, args):
229 229
         raise PermissionError
230 230
 
231 231
     if not os.access(fname, os.R_OK):
232
-        # on some systems, some importnat programs (e.g. sudo) will have
232
+        # on some systems, some important programs (e.g. sudo) will have
233 233
         # execute permissions but not read/write permisions. This enables
234 234
         # things with the SUID set to be run. Needs to come before _is_binary()
235 235
         # is called, because that function tries to read the file.

+ 28
- 3
takoshell/parsers/base.py View File

@@ -24,6 +24,7 @@
24 24
 import os
25 25
 import re
26 26
 import sys
27
+import string
27 28
 from collections import Iterable, Sequence, Mapping
28 29
 
29 30
 from takoshell.ply import yacc
@@ -35,6 +36,9 @@ from takoshell.platform import PYTHON_VERSION_INFO
35 36
 from takoshell.tokenize import SearchPath
36 37
 from takoshell.parsers.context_check import check_contexts
37 38
 
39
+_FORMATTER = string.Formatter()
40
+
41
+
38 42
 class Location(object):
39 43
     """Location in a file."""
40 44
 
@@ -1795,9 +1799,30 @@ class BaseParser(object):
1795 1799
     def p_string_literal(self, p):
1796 1800
         """string_literal : string_tok"""
1797 1801
         p1 = p[1]
1798
-        s = ast.literal_eval(p1.value)
1799
-        cls = ast.Bytes if p1.value.startswith('b') else ast.Str
1800
-        p[0] = cls(s=s, lineno=p1.lineno, col_offset=p1.lexpos)
1802
+        for index, val in enumerate(p1.value):
1803
+            if val in {'"', "'"}:
1804
+                break
1805
+        prefix = p1.value[:index]
1806
+        thestr = p1.value[index:]
1807
+        if 'f' in prefix:
1808
+            # this is a hack; maybe come back to improve it later by actually
1809
+            # using 3.6's machinery for f-strings but it is
1810
+            # backward-compatible.
1811
+            prefix = prefix.replace('f', '')
1812
+            s = ast.literal_eval(prefix + thestr)
1813
+            try:
1814
+                included_names = {i[1] for i in _FORMATTER.parse(s) if i[1] is not None}
1815
+            except Exception:
1816
+                included_names = set()
1817
+            # omg this is such a hack.  but it should work, i think.
1818
+            # maybe screws with error reporting
1819
+            thething = '%r.format(**{i: eval(i) for i in %r})'
1820
+            thething = thething % (s, included_names)
1821
+            p[0] = ast.parse(thething, mode='eval').body
1822
+        else:
1823
+            s = ast.literal_eval(p1.value)
1824
+            cls = ast.Bytes if isinstance(s, bytes) else ast.Str
1825
+            p[0] = cls(s=s, lineno=p1.lineno, col_offset=p1.lexpos)
1801 1826
 
1802 1827
     def p_string_literal_list(self, p):
1803 1828
         """string_literal_list : string_literal

+ 19
- 41
takoshell/tokenize.py View File

@@ -273,47 +273,25 @@ PseudoToken = Whitespace + group(PseudoExtras, IORedirect, Number, Funny,
273 273
 def _compile(expr):
274 274
     return re.compile(expr, re.UNICODE)
275 275
 
276
-endpats = {"'": Single, '"': Double,
277
-           "'''": Single3, '"""': Double3,
278
-           "r'''": Single3, 'r"""': Double3,
279
-           "b'''": Single3, 'b"""': Double3,
280
-           "R'''": Single3, 'R"""': Double3,
281
-           "B'''": Single3, 'B"""': Double3,
282
-           "br'''": Single3, 'br"""': Double3,
283
-           "bR'''": Single3, 'bR"""': Double3,
284
-           "Br'''": Single3, 'Br"""': Double3,
285
-           "BR'''": Single3, 'BR"""': Double3,
286
-           "rb'''": Single3, 'rb"""': Double3,
287
-           "Rb'''": Single3, 'Rb"""': Double3,
288
-           "rB'''": Single3, 'rB"""': Double3,
289
-           "RB'''": Single3, 'RB"""': Double3,
290
-           "u'''": Single3, 'u"""': Double3,
291
-           "U'''": Single3, 'U"""': Double3,
292
-           'r': None, 'R': None, 'b': None, 'B': None,
293
-           'u': None, 'U': None}
294
-
295
-triple_quoted = {}
296
-for t in ("'''", '"""',
297
-          "r'''", 'r"""', "R'''", 'R"""',
298
-          "b'''", 'b"""', "B'''", 'B"""',
299
-          "br'''", 'br"""', "Br'''", 'Br"""',
300
-          "bR'''", 'bR"""', "BR'''", 'BR"""',
301
-          "rb'''", 'rb"""', "rB'''", 'rB"""',
302
-          "Rb'''", 'Rb"""', "RB'''", 'RB"""',
303
-          "u'''", 'u"""', "U'''", 'U"""',
304
-          ):
305
-    triple_quoted[t] = t
306
-single_quoted = {}
307
-for t in ("'", '"',
308
-          "r'", 'r"', "R'", 'R"',
309
-          "b'", 'b"', "B'", 'B"',
310
-          "br'", 'br"', "Br'", 'Br"',
311
-          "bR'", 'bR"', "BR'", 'BR"' ,
312
-          "rb'", 'rb"', "rB'", 'rB"',
313
-          "Rb'", 'Rb"', "RB'", 'RB"' ,
314
-          "u'", 'u"', "U'", 'U"',
315
-          ):
316
-    single_quoted[t] = t
276
+# For a given string prefix plus quotes, endpats maps it to a regex
277
+#  to match the remainder of that string. _prefix can be empty, for
278
+#  a normal single or triple quoted string (with no prefix).
279
+endpats = {}
280
+for _prefix in _all_string_prefixes():
281
+    endpats[_prefix + "'"] = Single
282
+    endpats[_prefix + '"'] = Double
283
+    endpats[_prefix + "'''"] = Single3
284
+    endpats[_prefix + '"""'] = Double3
285
+
286
+# A set of all of the single and triple quoted string prefixes,
287
+#  including the opening quotes.
288
+single_quoted = set()
289
+triple_quoted = set()
290
+for t in _all_string_prefixes():
291
+    for u in (t + '"', t + "'"):
292
+        single_quoted.add(u)
293
+    for u in (t + '"""', t + "'''"):
294
+        triple_quoted.add(u)
317 295
 
318 296
 tabsize = 8
319 297
 

Loading…
Cancel
Save