1 | # Grammar for YSH.
|
2 | # Adapted from the Python 3.7 expression grammar, with several changes!
|
3 | #
|
4 | # TODO:
|
5 | # - List comprehensions
|
6 | # - There's also chaining => and maybe implicit vectorization ==>
|
7 | # - But list comprehensions are more familiar, and they are concise
|
8 | # - Generator expressions?
|
9 | # - Do we need lambdas?
|
10 |
|
11 | # Note: trailing commas are allowed:
|
12 | # {k: mydict,}
|
13 | # [mylist,]
|
14 | # mytuple,
|
15 | # f(args,)
|
16 | # func f(params,)
|
17 | #
|
18 | # Kinds used:
|
19 | # VSub, Left, Right, Expr, Op, Arith, Char, Eof, Unknown
|
20 |
|
21 | # YSH patch: removed @=
|
22 | augassign: (
|
23 | '+=' | '-=' | '*=' | '/=' |
|
24 | '**=' | '//=' | '%=' |
|
25 | '&=' | '|=' | '^=' | '<<=' | '>>='
|
26 | )
|
27 |
|
28 | test: or_test ['if' or_test 'else' test] | lambdef
|
29 |
|
30 | # Lambdas follow the same rules as Python:
|
31 | #
|
32 | # |x| 1, 2 == (|x| 1), 2
|
33 | # |x| x if True else 42 == |x| (x if True else 42)
|
34 | #
|
35 | # Python also had a test_nocond production like this: We don't need it because
|
36 | # we can't have multiple ifs.
|
37 | # [x for x in range(3) if lambda x: x if 1]
|
38 | #
|
39 | # The zero arg syntax like || 1 annoys me -- but this also works:
|
40 | # func() { return 1 }
|
41 | #
|
42 | # We used name_type_list rather than param_group because a default value like
|
43 | # x|y (bitwise or) conflicts with the | delimiter!
|
44 | #
|
45 | # TODO: consider this syntax:
|
46 | # fn (x) x # expression
|
47 | # fn (x) ^( echo hi ) # statement
|
48 |
|
49 | lambdef: '|' [name_type_list] '|' test
|
50 |
|
51 | or_test: and_test ('or' and_test)*
|
52 | and_test: not_test ('and' not_test)*
|
53 | not_test: 'not' not_test | comparison
|
54 | comparison: range_expr (comp_op range_expr)*
|
55 |
|
56 | # Unlike slice, beginning and end are required
|
57 | range_expr: expr ['..' expr]
|
58 |
|
59 | # YSH patch: remove legacy <>, add === and more
|
60 | comp_op: (
|
61 | '<'|'>'|'==='|'>='|'<='|'!=='|'in'|'not' 'in'|'is'|'is' 'not'|
|
62 | '~' | '!~' | '~~' | '!~~' | '~=='
|
63 | )
|
64 |
|
65 | # For lists and dicts. Note: In Python this was star_expr *foo
|
66 | splat_expr: '...' expr
|
67 |
|
68 | expr: xor_expr ('|' xor_expr)*
|
69 | xor_expr: and_expr ('^' and_expr)*
|
70 | and_expr: shift_expr ('&' shift_expr)*
|
71 | shift_expr: arith_expr (('<<'|'>>') arith_expr)*
|
72 | # YSH: add concatenation ++ with same precedence as +
|
73 | arith_expr: term (('+'|'-'|'++') term)*
|
74 | # YSH: removed '@' matrix mul
|
75 | term: factor (('*'|'/'|'//'|'%') factor)*
|
76 | factor: ('+'|'-'|'~') factor | power
|
77 | # YSH: removed Python 3 'await'
|
78 | power: atom trailer* ['**' factor]
|
79 |
|
80 | testlist_comp: (test|splat_expr) ( comp_for | (',' (test|splat_expr))* [','] )
|
81 |
|
82 | atom: (
|
83 | '(' [testlist_comp] ')'
|
84 | | '[' [testlist_comp] ']'
|
85 | # Note: newlines are significant inside {}, unlike inside () and []
|
86 | | '{' [Op_Newline] [dict] '}'
|
87 | | '&' Expr_Name place_trailer*
|
88 |
|
89 | # NOTE: These atoms are are allowed in typed array literals
|
90 | | Expr_Name | Expr_Null | Expr_True | Expr_False
|
91 |
|
92 | # Allow suffixes on floats and decimals
|
93 | # e.g. 100 M is a function M which multiplies by 1_000_000
|
94 | # e.g. 100 Mi is a function Mi which multiplies by 1024 * 1024
|
95 | | Expr_Float [Expr_Name]
|
96 | | Expr_DecInt [Expr_Name]
|
97 |
|
98 | | Expr_BinInt | Expr_OctInt | Expr_HexInt
|
99 |
|
100 | | Char_OneChar # char literal \n \\ etc.
|
101 | | Char_UBraced # char literal \u{3bc}
|
102 | | Char_Pound # char literal #'A' etc.
|
103 |
|
104 | | dq_string | sq_string
|
105 | # Expr_Symbol could be %mykey
|
106 |
|
107 | | eggex
|
108 | | literal_expr
|
109 |
|
110 | # $foo is disallowed, but $? is allowed. Should be "$foo" to indicate a
|
111 | # string, or ${foo:-}
|
112 | | simple_var_sub
|
113 | | sh_command_sub | braced_var_sub
|
114 | | sh_array_literal
|
115 | | old_sh_array_literal
|
116 | )
|
117 |
|
118 | literal_expr: '^[' expr ']'
|
119 |
|
120 | place_trailer: (
|
121 | '[' subscriptlist ']'
|
122 | | '.' Expr_Name
|
123 | )
|
124 |
|
125 | # var f = f(x)
|
126 | trailer: (
|
127 | '(' [arglist] ')'
|
128 | | '[' subscriptlist ']'
|
129 |
|
130 | # Is a {} trailing useful for anything? It's not in Python or JS
|
131 |
|
132 | | '.' Expr_Name
|
133 | | '->' Expr_Name
|
134 | | '=>' Expr_Name
|
135 | )
|
136 |
|
137 | # YSH patch: this is 'expr' instead of 'test'
|
138 | # - 1:(3<4) doesn't make any sense.
|
139 | # - And then this allows us to support a[3:] and a[:i] as special cases.
|
140 | # - First class slices have to be written 0:n.
|
141 |
|
142 | subscriptlist: subscript (',' subscript)* [',']
|
143 |
|
144 | # TODO: Add => as low precedence operator, for Func[Str, Int => Str]
|
145 | subscript: expr | [expr] ':' [expr]
|
146 |
|
147 | # TODO: => should be even lower precedence here too
|
148 | testlist: test (',' test)* [',']
|
149 |
|
150 | # Dict syntax resembles JavaScript
|
151 | # https://stackoverflow.com/questions/38948306/what-is-javascript-shorthand-property
|
152 | #
|
153 | # Examples:
|
154 | # {age: 20} is like {'age': 20}
|
155 | #
|
156 | # x = 'age'
|
157 | # d = %{[x]: 20} # Evaluate x as a variable
|
158 | # d = %{["foo$x"]: 20} # Another expression
|
159 | # d = %{[x, y]: 20} # Tuple key
|
160 | # d = %{key1, key1: 123}
|
161 | # Notes:
|
162 | # - Value is optional when the key is a name, because it can be taken from the
|
163 | # environment.
|
164 | # - We don't have:
|
165 | # - dict comprehensions. Maybe wait until LR parsing?
|
166 | # - Splatting with **
|
167 |
|
168 | dict_pair: (
|
169 | Expr_Name [':' test]
|
170 | | '[' testlist ']' ':' test
|
171 | | sq_string ':' test
|
172 | | dq_string ':' test
|
173 | )
|
174 |
|
175 | comma_newline: ',' [Op_Newline] | Op_Newline
|
176 |
|
177 | dict: dict_pair (comma_newline dict_pair)* [comma_newline]
|
178 |
|
179 | # This how Python implemented dict comprehensions. We can probably do the
|
180 | # same.
|
181 | #
|
182 | # dictorsetmaker: ( ((test ':' test | '**' expr)
|
183 | # (comp_for | (',' (test ':' test | '**' expr))* [','])) |
|
184 | # ((test | splat_expr)
|
185 | # (comp_for | (',' (test | splat_expr))* [','])) )
|
186 |
|
187 | # The reason that keywords are test nodes instead of NAME is that using NAME
|
188 | # results in an ambiguity. ast.c makes sure it's a NAME.
|
189 | # "test '=' test" is really "keyword '=' test", but we have no such token.
|
190 | # These need to be in a single rule to avoid grammar that is ambiguous
|
191 | # to our LL(1) parser. Even though 'test' includes '*expr' in splat_expr,
|
192 | # we explicitly match '*' here, too, to give it proper precedence.
|
193 | # Illegal combinations and orderings are blocked in ast.c:
|
194 | # multiple (test comp_for) arguments are blocked; keyword unpackings
|
195 | # that precede iterable unpackings are blocked; etc.
|
196 |
|
197 | argument: (
|
198 | test [comp_for]
|
199 | # named arg
|
200 | | test '=' test
|
201 | # splat. Note we're using prefix syntax to be consistent with Python, JS,
|
202 | # and the prefix @ operator.
|
203 | | '...' test
|
204 | )
|
205 |
|
206 | # The grammar at call sites is less restrictive than at declaration sites.
|
207 | # ... can appear anywhere. Keyword args can appear anywhere too.
|
208 | arg_group: argument (',' argument)* [',']
|
209 | arglist: [arg_group] [';' arg_group]
|
210 |
|
211 |
|
212 | # YSH patch: test_nocond -> or_test. I believe this was trying to prevent the
|
213 | # "double if" ambiguity here:
|
214 | # #
|
215 | # [x for x in range(3) if lambda x: x if 1]
|
216 | #
|
217 | # but YSH doesn't supported "nested loops", so we don't have this problem.
|
218 | comp_for: 'for' name_type_list 'in' or_test ['if' or_test]
|
219 |
|
220 |
|
221 | #
|
222 | # Expressions that are New in YSH
|
223 | #
|
224 |
|
225 | # Notes:
|
226 | # - Most of these occur in 'atom' above
|
227 | # - You can write $mystr but not mystr. It has to be (mystr)
|
228 | array_item: (
|
229 | Expr_Null | Expr_True | Expr_False
|
230 | | Expr_Float | Expr_DecInt | Expr_BinInt | Expr_OctInt | Expr_HexInt
|
231 | | dq_string | sq_string
|
232 | | sh_command_sub | braced_var_sub | simple_var_sub
|
233 | | '(' test ')'
|
234 | )
|
235 | sh_array_literal: ':|' Expr_CastedDummy Op_Pipe
|
236 |
|
237 | # TODO: remove old array
|
238 | old_sh_array_literal: '%(' Expr_CastedDummy Right_ShArrayLiteral
|
239 | sh_command_sub: ( '$(' | '@(' | '^(' ) Expr_CastedDummy Eof_RParen
|
240 |
|
241 | # Note: could add c"" too
|
242 | dq_string: (Left_DoubleQuote | Left_TDoubleQuote | Left_CaretDoubleQuote) Expr_CastedDummy Right_DoubleQuote
|
243 | sq_string: (
|
244 | Left_SingleQuote | Left_TSingleQuote
|
245 | | Left_RSingleQuote | Left_RTSingleQuote
|
246 | | Left_DollarSingleQuote
|
247 | | Left_USingleQuote | Left_UTSingleQuote
|
248 | | Left_BSingleQuote | Left_BTSingleQuote
|
249 | ) Expr_CastedDummy Right_SingleQuote
|
250 |
|
251 | braced_var_sub: '${' Expr_CastedDummy Right_DollarBrace
|
252 |
|
253 | simple_var_sub: (
|
254 | # This is everything in Kind.VSub except VSub_Name, which is braced: ${foo}
|
255 | #
|
256 | # Note: we could allow $foo and $0, but disallow the rest in favor of ${@}
|
257 | # and ${-}? Meh it's too inconsistent.
|
258 | VSub_DollarName | VSub_Number
|
259 | | VSub_Bang | VSub_At | VSub_Pound | VSub_Dollar | VSub_Star | VSub_Hyphen
|
260 | | VSub_QMark
|
261 | # NOTE: $? should be STATUS because it's an integer.
|
262 | )
|
263 |
|
264 | #
|
265 | # Assignment / Type Variables
|
266 | #
|
267 | # Several differences vs. Python:
|
268 | #
|
269 | # - no yield expression on RHS
|
270 | # - no star expressions on either side (Python 3) *x, y = 2, *b
|
271 | # - no multiple assignments like: var x = y = 3
|
272 | # - type annotation syntax is more restrictive # a: (1+2) = 3 is OK in python
|
273 | # - We're validating the lvalue here, instead of doing it in the "transformer".
|
274 | # We have the 'var' prefix which helps.
|
275 |
|
276 | # name_type use cases:
|
277 | # var x Int, y Int = 3, 5
|
278 | # / <capture d+ as date: int> /
|
279 | #
|
280 | # for x Int, y Int
|
281 | # [x for x Int, y Int in ...]
|
282 | #
|
283 | # func(x Int, y Int) - this is separate
|
284 |
|
285 | # Optional colon because we want both
|
286 |
|
287 | # var x: Int = 42 # colon looks nicer
|
288 | # proc p (; x Int, y Int; z Int) { echo hi } # colon gets in the way of ;
|
289 |
|
290 | name_type: Expr_Name [':'] [type_expr]
|
291 | name_type_list: name_type (',' name_type)*
|
292 |
|
293 | type_expr: Expr_Name [ '[' type_expr (',' type_expr)* ']' ]
|
294 |
|
295 | # NOTE: Eof_RParen and Eof_Backtick aren't allowed because we don't want 'var'
|
296 | # in command subs.
|
297 | end_stmt: '}' | ';' | Op_Newline | Eof_Real
|
298 |
|
299 | # TODO: allow -> to denote aliasing/mutation
|
300 | ysh_var_decl: name_type_list ['=' testlist] end_stmt
|
301 |
|
302 | # Note: this is more precise way of writing ysh_mutation, but it's ambiguous :(
|
303 | # ysh_mutation: lhs augassign testlist end_stmt
|
304 | # | lhs_list '=' testlist end_stmt
|
305 |
|
306 | # Note: for YSH (not Tea), we could accept [':'] expr for setvar :out = 'foo'
|
307 | lhs_list: expr (',' expr)*
|
308 |
|
309 | # TODO: allow -> to denote aliasing/mutation
|
310 | ysh_mutation: lhs_list (augassign | '=') testlist end_stmt
|
311 |
|
312 | # For json write (x)
|
313 | ysh_eager_arglist: '(' [arglist] ')'
|
314 | ysh_lazy_arglist: '[' [arglist] ']'
|
315 |
|
316 | #
|
317 | # Other Entry Points
|
318 | #
|
319 |
|
320 | # if (x > 0) etc.
|
321 | ysh_expr: '(' testlist ')'
|
322 |
|
323 | # = 42 + a[i]
|
324 | # call f(x)
|
325 | command_expr: testlist end_stmt
|
326 |
|
327 | # $[d->key] etc.
|
328 | ysh_expr_sub: testlist ']'
|
329 |
|
330 | # Signatures for proc and func.
|
331 |
|
332 | # Note: 'proc name-with-hyphens' is allowed, so we can't parse the name in
|
333 | # expression mode.
|
334 | ysh_proc: (
|
335 | [ '('
|
336 | [ param_group ] # word params, with defaults
|
337 | [ ';' [ param_group ] ] # positional typed params, with defaults
|
338 | [ ';' [ param_group ] ] # named params, with defaults
|
339 | [ ';' [ param_group ] ] # optional block param, with no type or default
|
340 |
|
341 | # This causes a pgen2 error? It doesn't know which branch to take
|
342 | # So we have the extra {block} syntax
|
343 | #[ ';' Expr_Name ] # optional block param, with no type or default
|
344 | ')'
|
345 | ]
|
346 | '{' # opening { for pgen2
|
347 | )
|
348 |
|
349 | ysh_func: (
|
350 | Expr_Name '(' [param_group] [';' param_group] ')' ['=>' type_expr] '{'
|
351 | )
|
352 |
|
353 | param: Expr_Name [type_expr] ['=' expr]
|
354 |
|
355 | # This is an awkward way of writing that '...' has to come last.
|
356 | param_group: (
|
357 | (param ',')*
|
358 | [ (param | '...' Expr_Name) [','] ]
|
359 | )
|
360 |
|
361 | #
|
362 | # Regex Sublanguage
|
363 | #
|
364 |
|
365 | char_literal: Char_OneChar | Char_Hex | Char_UBraced
|
366 |
|
367 | # we allow a-z A-Z 0-9 as ranges, but otherwise they have to be quoted
|
368 | # The parser enforces that they are single strings
|
369 | range_char: Expr_Name | Expr_DecInt | sq_string | char_literal
|
370 |
|
371 | # digit or a-z
|
372 | # We have to do further validation of ranges later.
|
373 | class_literal_term: (
|
374 | # NOTE: range_char has sq_string
|
375 | range_char ['-' range_char ]
|
376 | # splice a literal set of characters
|
377 | | '@' Expr_Name
|
378 | | '!' Expr_Name
|
379 | # Reserved for [[.collating sequences.]] (Unicode)
|
380 | | '.' Expr_Name
|
381 | # Reserved for [[=character equivalents=]] (Unicode)
|
382 | | '=' Expr_Name
|
383 | # TODO: Do these char classes actually work in bash/awk/egrep/sed/etc.?
|
384 |
|
385 | )
|
386 | class_literal: '[' class_literal_term+ ']'
|
387 |
|
388 | # NOTE: Here is an example of where you can put ^ in the middle of a pattern in
|
389 | # Python, and it matters!
|
390 | # >>> r = re.compile('.f[a-z]*', re.DOTALL|re.MULTILINE)
|
391 | # >>> r.findall('z\nfoo\nbeef\nfood\n')
|
392 | # ['\nfoo', 'ef', '\nfood']
|
393 | # >>> r = re.compile('.^f[a-z]*', re.DOTALL|re.MULTILINE)
|
394 | # r.findall('z\nfoo\nbeef\nfood\n')
|
395 | # ['\nfoo', '\nfood']
|
396 |
|
397 | re_atom: (
|
398 | char_literal
|
399 | # builtin regex like 'digit' or a regex reference like 'D'
|
400 | | Expr_Name
|
401 | # %begin or %end
|
402 | | Expr_Symbol
|
403 | | class_literal
|
404 | # !digit or ![a-f]. Note ! %boundary could be \B in Python, but ERE
|
405 | # doesn't have anything like that
|
406 | | '!' (Expr_Name | class_literal)
|
407 |
|
408 | # syntactic space for Perl-style backtracking
|
409 | # !!REF 1 !!REF name
|
410 | # !!AHEAD(d+) !!BEHIND(d+) !!NOT_AHEAD(d+) !!NOT_BEHIND(d+)
|
411 | #
|
412 | # Note: !! conflicts with history
|
413 | | '!' '!' Expr_Name (Expr_Name | Expr_DecInt | '(' regex ')')
|
414 |
|
415 | # Splice another expression
|
416 | | '@' Expr_Name
|
417 | # any %start %end are preferred
|
418 | | '.' | '^' | '$'
|
419 | # In a language-independent spec, backslashes are disallowed within 'sq'.
|
420 | # Write it with char literals outside strings: 'foo' \\ 'bar' \n
|
421 | #
|
422 | # No double-quoted strings because you can write "x = $x" with 'x = ' @x
|
423 | | sq_string
|
424 |
|
425 | # grouping (non-capturing in Perl; capturing in ERE although < > is preferred)
|
426 | | '(' regex ')'
|
427 |
|
428 | # Capturing group, with optional name and conversion function
|
429 | # <capture d+ as date>
|
430 | # <capture d+ as date: int>
|
431 | # <capture d+ : int>
|
432 | | '<' 'capture' regex ['as' Expr_Name] [':' Expr_Name] '>'
|
433 |
|
434 | # Might want this obscure conditional construct. Can't use C-style ternary
|
435 | # because '?' is a regex operator.
|
436 | #| '{' regex 'if' regex 'else' regex '}'
|
437 |
|
438 | # Others:
|
439 | # PCRE has (?R ) for recursion? That could be !RECURSE()
|
440 | # Note: .NET has && in character classes, making it a recursive language
|
441 | )
|
442 |
|
443 | # e.g. a{3} a{3,4} a{3,} a{,4} but not a{,}
|
444 | repeat_range: (
|
445 | Expr_DecInt [',']
|
446 | | ',' Expr_DecInt
|
447 | | Expr_DecInt ',' Expr_DecInt
|
448 | )
|
449 |
|
450 | repeat_op: (
|
451 | '+' | '*' | '?'
|
452 | # In PCRE, ?? *? +? {}? is lazy/nongreedy and ?+ *+ ++ {}+ is "possessive"
|
453 | # We use N and P modifiers within {}.
|
454 | # a{L +} a{P ?} a{P 3,4} a{P ,4}
|
455 | | '{' [Expr_Name] ('+' | '*' | '?' | repeat_range) '}'
|
456 | )
|
457 |
|
458 | re_alt: (re_atom [repeat_op])+
|
459 |
|
460 | regex: [re_alt] (('|'|'or') re_alt)*
|
461 |
|
462 | # e.g. /digit+ ; multiline !ignorecase/
|
463 | #
|
464 | # This can express translation preferences:
|
465 | #
|
466 | # / d+ ; ; ERE / is '[[:digit:]]+'
|
467 | # / d+ ; ; PCRE / is '\d+'
|
468 | # / d+ ; ignorecase ; python / is '(?i)\d+'
|
469 |
|
470 | # Python has the syntax
|
471 | # (?i:myre) to set a flag
|
472 | # (?-i:myre) to remove a flag
|
473 | #
|
474 | # They can apply to portions of the expression, which we don't have here.
|
475 | re_flag: ['!'] Expr_Name
|
476 | eggex: '/' regex [';' re_flag* [';' Expr_Name] ] '/'
|
477 |
|
478 | # Patterns are the start of a case arm. Ie,
|
479 | #
|
480 | # case (foo) {
|
481 | # (40 + 2) | (0) { echo number }
|
482 | # ^^^^^^^^^^^^^^-- This is pattern
|
483 | # }
|
484 | #
|
485 | # Due to limitations created from pgen2/cmd_parser interactions, we also parse
|
486 | # the leading '{' token of the case arm body in pgen2. We do this to help pgen2
|
487 | # figure out when to transfer control back to the cmd_parser. For more details
|
488 | # see #oil-dev > Dev Friction / Smells.
|
489 | #
|
490 | # case (foo) {
|
491 | # (40 + 2) | (0) { echo number }
|
492 | # ^-- End of pattern/beginning of case arm body
|
493 | # }
|
494 |
|
495 | ysh_case_pat: (
|
496 | '(' (pat_else | pat_exprs)
|
497 | | eggex
|
498 | ) [Op_Newline] '{'
|
499 |
|
500 | pat_else: 'else' ')'
|
501 | pat_exprs: expr ')' [Op_Newline] ('|' [Op_Newline] '(' expr ')' [Op_Newline])*
|
502 |
|
503 |
|
504 | # Syntax reserved for PCRE/Python, but that's not in ERE:
|
505 | #
|
506 | # non-greedy a{N *}
|
507 | # non-capturing ( digit+ )
|
508 | # backtracking !!REF 1 !!AHEAD(d+)
|
509 | #
|
510 | # Legacy syntax:
|
511 | #
|
512 | # ^ and $ instead of %start and %end
|
513 | # < and > instead of %start_word and %end_word
|
514 | # . instead of dot
|
515 | # | instead of 'or'
|