1 """
2 args.py - Flag, option, and arg parsing for the shell.
3
4 All existing shells have their own flag parsing, rather than using libc.
5
6 We have 3 types of flag parsing here:
7
8 FlagSpecAndMore() -- e.g. for 'sh +u -o errexit' and 'set +u -o errexit'
9 FlagSpec() -- for echo -en, read -t1.0, etc.
10
11 Examples:
12 set -opipefail # not allowed, space required
13 read -t1.0 # allowed
14
15 Things that getopt/optparse don't support:
16
17 - accepts +o +n for 'set' and bin/osh
18 - pushd and popd also uses +, although it's not an arg.
19 - parses args -- well argparse is supposed to do this
20 - maybe: integrate with usage
21 - maybe: integrate with flags
22
23 optparse:
24 - has option groups (Go flag package has flagset)
25
26 NOTES about builtins:
27 - eval and echo implicitly join their args. We don't want that.
28 - option strict-eval and strict-echo
29 - bash is inconsistent about checking for extra args
30 - exit 1 2 complains, but pushd /lib /bin just ignores second argument
31 - it has a no_args() function that isn't called everywhere. It's not
32 declarative.
33
34 TODO:
35 - Autogenerate help from help='' fields. Usage line like FlagSpec('echo [-en]')
36
37 GNU notes:
38
39 - Consider adding GNU-style option to interleave flags and args?
40 - Not sure I like this.
41 - GNU getopt has fuzzy matching for long flags. I think we should rely
42 on good completion instead.
43
44 Bash notes:
45
46 bashgetopt.c codes:
47 leading +: allow options
48 : requires argument
49 ; argument may be missing
50 # numeric argument
51
52 However I don't see these used anywhere! I only see ':' used.
53 """
54 from __future__ import print_function
55
56 from _devbuild.gen.syntax_asdl import loc, loc_t, CompoundWord
57 from _devbuild.gen.runtime_asdl import value, value_e, value_t
58
59 from core.error import e_usage
60 from mycpp.mylib import log, tagswitch, iteritems
61
62 _ = log
63
64 from typing import (cast, Tuple, Optional, Dict, List, Any, TYPE_CHECKING)
65 if TYPE_CHECKING:
66 from frontend import flag_spec
67 OptChange = Tuple[str, bool]
68
69 # TODO: Move to flag_spec? We use flag_type_t
70 String = 1
71 Int = 2
72 Float = 3 # e.g. for read -t timeout value
73 Bool = 4
74
75
76 class _Attributes(object):
77 """Object to hold flags.
78
79 TODO: FlagSpec doesn't need this; only FlagSpecAndMore.
80 """
81
82 def __init__(self, defaults):
83 # type: (Dict[str, value_t]) -> None
84
85 # New style
86 self.attrs = {} # type: Dict[str, value_t]
87
88 self.opt_changes = [] # type: List[OptChange] # -o errexit +o nounset
89 self.shopt_changes = [
90 ] # type: List[OptChange] # -O nullglob +O nullglob
91 self.show_options = False # 'set -o' without an argument
92 self.actions = [] # type: List[str] # for compgen -A
93 self.saw_double_dash = False # for set --
94 for name, v in iteritems(defaults):
95 self.Set(name, v)
96
97 def SetTrue(self, name):
98 # type: (str) -> None
99 self.Set(name, value.Bool(True))
100
101 def Set(self, name, val):
102 # type: (str, value_t) -> None
103
104 # debug-completion -> debug_completion
105 name = name.replace('-', '_')
106 self.attrs[name] = val
107
108 if 0:
109 # Backward compatibility!
110 with tagswitch(val) as case:
111 if case(value_e.Undef):
112 py_val = None # type: Any
113 elif case(value_e.Bool):
114 py_val = cast(value.Bool, val).b
115 elif case(value_e.Int):
116 py_val = cast(value.Int, val).i
117 elif case(value_e.Float):
118 py_val = cast(value.Float, val).f
119 elif case(value_e.Str):
120 py_val = cast(value.Str, val).s
121 else:
122 raise AssertionError(val)
123
124 setattr(self, name, py_val)
125
126 def __repr__(self):
127 # type: () -> str
128 return '<_Attributes %s>' % self.__dict__
129
130
131 class Reader(object):
132 """Wrapper for argv.
133
134 Modified by both the parsing loop and various actions.
135
136 The caller of the flags parser can continue to use it after flag parsing is
137 done to get args.
138 """
139
140 def __init__(self, argv, locs=None):
141 # type: (List[str], Optional[List[CompoundWord]]) -> None
142 self.argv = argv
143 self.locs = locs
144 self.n = len(argv)
145 self.i = 0
146
147 def __repr__(self):
148 # type: () -> str
149 return '<args.Reader %r %d>' % (self.argv, self.i)
150
151 def Next(self):
152 # type: () -> None
153 """Advance."""
154 self.i += 1
155
156 def Peek(self):
157 # type: () -> Optional[str]
158 """Return the next token, or None if there are no more.
159
160 None is your SENTINEL for parsing.
161 """
162 if self.i >= self.n:
163 return None
164 else:
165 return self.argv[self.i]
166
167 def Peek2(self):
168 # type: () -> Tuple[Optional[str], loc_t]
169 """Return the next token, or None if there are no more.
170
171 None is your SENTINEL for parsing.
172 """
173 if self.i >= self.n:
174 return None, loc.Missing
175 else:
176 return self.argv[self.i], self.locs[self.i]
177
178 def ReadRequired(self, error_msg):
179 # type: (str) -> str
180 arg = self.Peek()
181 if arg is None:
182 # point at argv[0]
183 e_usage(error_msg, self._FirstLocation())
184 self.Next()
185 return arg
186
187 def ReadRequired2(self, error_msg):
188 # type: (str) -> Tuple[str, loc_t]
189 arg = self.Peek()
190 if arg is None:
191 # point at argv[0]
192 e_usage(error_msg, self._FirstLocation())
193 location = self.locs[self.i]
194 self.Next()
195 return arg, location
196
197 def Rest(self):
198 # type: () -> List[str]
199 """Return the rest of the arguments."""
200 return self.argv[self.i:]
201
202 def Rest2(self):
203 # type: () -> Tuple[List[str], List[CompoundWord]]
204 """Return the rest of the arguments."""
205 return self.argv[self.i:], self.locs[self.i:]
206
207 def AtEnd(self):
208 # type: () -> bool
209 return self.i >= self.n # must be >= and not ==
210
211 def _FirstLocation(self):
212 # type: () -> loc_t
213 if self.locs and self.locs[0] is not None:
214 return self.locs[0]
215 else:
216 return loc.Missing
217
218 def Location(self):
219 # type: () -> loc_t
220 if self.locs:
221 if self.i == self.n:
222 i = self.n - 1 # if the last arg is missing, point at the one before
223 else:
224 i = self.i
225 if self.locs[i] is not None:
226 return self.locs[i]
227 else:
228 return loc.Missing
229 else:
230 return loc.Missing
231
232
233 class _Action(object):
234 """What is done when a flag or option is detected."""
235
236 def __init__(self):
237 # type: () -> None
238 """Empty constructor for mycpp."""
239 pass
240
241 def OnMatch(self, attached_arg, arg_r, out):
242 # type: (Optional[str], Reader, _Attributes) -> bool
243 """Called when the flag matches.
244
245 Args:
246 prefix: '-' or '+'
247 suffix: ',' for -d,
248 arg_r: Reader() (rename to Input or InputReader?)
249 out: _Attributes() -- the thing we want to set
250
251 Returns:
252 True if flag parsing should be aborted.
253 """
254 raise NotImplementedError()
255
256
257 class _ArgAction(_Action):
258 def __init__(self, name, quit_parsing_flags, valid=None):
259 # type: (str, bool, Optional[List[str]]) -> None
260 """
261 Args:
262 quit_parsing_flags: Stop parsing args after this one. for sh -c.
263 python -c behaves the same way.
264 """
265 self.name = name
266 self.quit_parsing_flags = quit_parsing_flags
267 self.valid = valid
268
269 def _Value(self, arg, location):
270 # type: (str, loc_t) -> value_t
271 raise NotImplementedError()
272
273 def OnMatch(self, attached_arg, arg_r, out):
274 # type: (Optional[str], Reader, _Attributes) -> bool
275 """Called when the flag matches."""
276 if attached_arg is not None: # for the ',' in -d,
277 arg = attached_arg
278 else:
279 arg_r.Next()
280 arg = arg_r.Peek()
281 if arg is None:
282 e_usage('expected argument to %r' % ('-' + self.name),
283 arg_r.Location())
284
285 val = self._Value(arg, arg_r.Location())
286 out.Set(self.name, val)
287 return self.quit_parsing_flags
288
289
290 class SetToInt(_ArgAction):
291 def __init__(self, name):
292 # type: (str) -> None
293 # repeat defaults for C++ translation
294 _ArgAction.__init__(self, name, False, valid=None)
295
296 def _Value(self, arg, location):
297 # type: (str, loc_t) -> value_t
298 try:
299 i = int(arg)
300 except ValueError:
301 e_usage(
302 'expected integer after %s, got %r' % ('-' + self.name, arg),
303 location)
304
305 # So far all our int values are > 0, so use -1 as the 'unset' value
306 # corner case: this treats -0 as 0!
307 if i < 0:
308 e_usage('got invalid integer for %s: %s' % ('-' + self.name, arg),
309 location)
310 return value.Int(i)
311
312
313 class SetToFloat(_ArgAction):
314 def __init__(self, name):
315 # type: (str) -> None
316 # repeat defaults for C++ translation
317 _ArgAction.__init__(self, name, False, valid=None)
318
319 def _Value(self, arg, location):
320 # type: (str, loc_t) -> value_t
321 try:
322 f = float(arg)
323 except ValueError:
324 e_usage('expected number after %r, got %r' % ('-' + self.name, arg),
325 location)
326 # So far all our float values are > 0, so use -1.0 as the 'unset' value
327 # corner case: this treats -0.0 as 0.0!
328 if f < 0:
329 e_usage('got invalid float for %s: %s' % ('-' + self.name, arg),
330 location)
331 return value.Float(f)
332
333
334 class SetToString(_ArgAction):
335 def __init__(self, name, quit_parsing_flags, valid=None):
336 # type: (str, bool, Optional[List[str]]) -> None
337 _ArgAction.__init__(self, name, quit_parsing_flags, valid=valid)
338
339 def _Value(self, arg, location):
340 # type: (str, loc_t) -> value_t
341 if self.valid is not None and arg not in self.valid:
342 e_usage(
343 'got invalid argument %r to %r, expected one of: %s' %
344 (arg, ('-' + self.name), '|'.join(self.valid)), location)
345 return value.Str(arg)
346
347
348 class SetAttachedBool(_Action):
349 """This is the Go-like syntax of --verbose=1, --verbose, or --verbose=0."""
350
351 def __init__(self, name):
352 # type: (str) -> None
353 self.name = name
354
355 def OnMatch(self, attached_arg, arg_r, out):
356 # type: (Optional[str], Reader, _Attributes) -> bool
357 """Called when the flag matches."""
358
359 if attached_arg is not None: # '0' in --verbose=0
360 if attached_arg in ('0', 'F', 'false',
361 'False'): # TODO: incorrect translation
362 b = False
363 elif attached_arg in ('1', 'T', 'true', 'Talse'):
364 b = True
365 else:
366 e_usage(
367 'got invalid argument to boolean flag: %r' % attached_arg,
368 loc.Missing)
369 else:
370 b = True
371
372 out.Set(self.name, value.Bool(b))
373 return False
374
375
376 class SetToTrue(_Action):
377 def __init__(self, name):
378 # type: (str) -> None
379 self.name = name
380
381 def OnMatch(self, attached_arg, arg_r, out):
382 # type: (Optional[str], Reader, _Attributes) -> bool
383 """Called when the flag matches."""
384 out.SetTrue(self.name)
385 return False
386
387
388 class SetOption(_Action):
389 """Set an option to a boolean, for 'set +e'."""
390
391 def __init__(self, name):
392 # type: (str) -> None
393 self.name = name
394
395 def OnMatch(self, attached_arg, arg_r, out):
396 # type: (Optional[str], Reader, _Attributes) -> bool
397 """Called when the flag matches."""
398 b = (attached_arg == '-')
399 out.opt_changes.append((self.name, b))
400 return False
401
402
403 class SetNamedOption(_Action):
404 """Set a named option to a boolean, for 'set +o errexit'."""
405
406 def __init__(self, shopt=False):
407 # type: (bool) -> None
408 self.names = [] # type: List[str]
409 self.shopt = shopt # is it sh -o (set) or sh -O (shopt)?
410
411 def ArgName(self, name):
412 # type: (str) -> None
413 self.names.append(name)
414
415 def OnMatch(self, attached_arg, arg_r, out):
416 # type: (Optional[str], Reader, _Attributes) -> bool
417 """Called when the flag matches."""
418 b = (attached_arg == '-')
419 #log('SetNamedOption %r %r %r', prefix, suffix, arg_r)
420 arg_r.Next() # always advance
421 arg = arg_r.Peek()
422 if arg is None:
423 # triggers on 'set -O' in addition to 'set -o' (meh OK)
424 out.show_options = True
425 return True # quit parsing
426
427 attr_name = arg # Note: validation is done elsewhere
428 if len(self.names) and attr_name not in self.names:
429 e_usage('Invalid option %r' % arg, loc.Missing)
430 changes = out.shopt_changes if self.shopt else out.opt_changes
431 changes.append((attr_name, b))
432 return False
433
434
435 class SetAction(_Action):
436 """For compgen -f."""
437
438 def __init__(self, name):
439 # type: (str) -> None
440 self.name = name
441
442 def OnMatch(self, attached_arg, arg_r, out):
443 # type: (Optional[str], Reader, _Attributes) -> bool
444 out.actions.append(self.name)
445 return False
446
447
448 class SetNamedAction(_Action):
449 """For compgen -A file."""
450
451 def __init__(self):
452 # type: () -> None
453 self.names = [] # type: List[str]
454
455 def ArgName(self, name):
456 # type: (str) -> None
457 self.names.append(name)
458
459 def OnMatch(self, attached_arg, arg_r, out):
460 # type: (Optional[str], Reader, _Attributes) -> bool
461 """Called when the flag matches."""
462 arg_r.Next() # always advance
463 arg = arg_r.Peek()
464 if arg is None:
465 e_usage('Expected argument for action', loc.Missing)
466
467 attr_name = arg
468 # Validate the option name against a list of valid names.
469 if len(self.names) and attr_name not in self.names:
470 e_usage('Invalid action name %r' % arg, loc.Missing)
471 out.actions.append(attr_name)
472 return False
473
474
475 def Parse(spec, arg_r):
476 # type: (flag_spec._FlagSpec, Reader) -> _Attributes
477
478 # NOTE about -:
479 # 'set -' ignores it, vs set
480 # 'unset -' or 'export -' seems to treat it as a variable name
481 out = _Attributes(spec.defaults)
482
483 while not arg_r.AtEnd():
484 arg = arg_r.Peek()
485 if arg == '--':
486 out.saw_double_dash = True
487 arg_r.Next()
488 break
489
490 # Only accept -- if there are any long flags defined
491 if len(spec.actions_long) and arg.startswith('--'):
492 pos = arg.find('=', 2)
493 if pos == -1:
494 suffix = None # type: Optional[str]
495 flag_name = arg[2:] # strip off --
496 else:
497 suffix = arg[pos + 1:]
498 flag_name = arg[2:pos]
499
500 action = spec.actions_long.get(flag_name)
501 if action is None:
502 e_usage('got invalid flag %r' % arg, arg_r.Location())
503
504 action.OnMatch(suffix, arg_r, out)
505 arg_r.Next()
506 continue
507
508 elif arg.startswith('-') and len(arg) > 1:
509 n = len(arg)
510 for i in xrange(1, n): # parse flag combos like -rx
511 ch = arg[i]
512
513 if ch == '0':
514 ch = 'Z' # hack for read -0
515
516 if ch in spec.plus_flags:
517 out.Set(ch, value.Str('-'))
518 continue
519
520 if ch in spec.arity0: # e.g. read -r
521 out.SetTrue(ch)
522 continue
523
524 if ch in spec.arity1: # e.g. read -t1.0
525 action = spec.arity1[ch]
526 # make sure we don't pass empty string for read -t
527 attached_arg = arg[i + 1:] if i < n - 1 else None
528 action.OnMatch(attached_arg, arg_r, out)
529 break
530
531 e_usage("doesn't accept flag %s" % ('-' + ch), arg_r.Location())
532
533 arg_r.Next() # next arg
534
535 # Only accept + if there are ANY options defined, e.g. for declare +rx.
536 elif len(spec.plus_flags) and arg.startswith('+') and len(arg) > 1:
537 n = len(arg)
538 for i in xrange(1, n): # parse flag combos like -rx
539 ch = arg[i]
540 if ch in spec.plus_flags:
541 out.Set(ch, value.Str('+'))
542 continue
543
544 e_usage("doesn't accept option %s" % ('+' + ch),
545 arg_r.Location())
546
547 arg_r.Next() # next arg
548
549 else: # a regular arg
550 break
551
552 return out
553
554
555 def ParseLikeEcho(spec, arg_r):
556 # type: (flag_spec._FlagSpec, Reader) -> _Attributes
557 """Echo is a special case. These work: echo -n echo -en.
558
559 - But don't respect --
560 - doesn't fail when an invalid flag is passed
561 """
562 out = _Attributes(spec.defaults)
563
564 while not arg_r.AtEnd():
565 arg = arg_r.Peek()
566 chars = arg[1:]
567 if arg.startswith('-') and len(chars):
568 # Check if it looks like -en or not. TODO: could optimize this.
569 done = False
570 for c in chars:
571 if c not in spec.arity0:
572 done = True
573 break
574 if done:
575 break
576
577 for ch in chars:
578 out.SetTrue(ch)
579
580 else:
581 break # Looks like an arg
582
583 arg_r.Next() # next arg
584
585 return out
586
587
588 def ParseMore(spec, arg_r):
589 # type: (flag_spec._FlagSpecAndMore, Reader) -> _Attributes
590 """Return attributes and an index.
591
592 Respects +, like set +eu
593
594 We do NOT respect:
595
596 WRONG: sh -cecho OK: sh -c echo
597 WRONG: set -opipefail OK: set -o pipefail
598
599 But we do accept these
600
601 set -euo pipefail
602 set -oeu pipefail
603 set -oo pipefail errexit
604 """
605 out = _Attributes(spec.defaults)
606
607 quit = False
608 while not arg_r.AtEnd():
609 arg = arg_r.Peek()
610 if arg == '--':
611 out.saw_double_dash = True
612 arg_r.Next()
613 break
614
615 if arg.startswith('--'):
616 action = spec.actions_long.get(arg[2:])
617 if action is None:
618 e_usage('got invalid flag %r' % arg, arg_r.Location())
619
620 # TODO: attached_arg could be 'bar' for --foo=bar
621 action.OnMatch(None, arg_r, out)
622 arg_r.Next()
623 continue
624
625 # corner case: sh +c is also accepted!
626 if (arg.startswith('-') or arg.startswith('+')) and len(arg) > 1:
627 # note: we're not handling sh -cecho (no space) as an argument
628 # It complains about a missing argument
629
630 char0 = arg[0]
631
632 # TODO: set - - empty
633 for ch in arg[1:]:
634 #log('ch %r arg_r %s', ch, arg_r)
635 action = spec.actions_short.get(ch)
636 if action is None:
637 e_usage('got invalid flag %r' % ('-' + ch),
638 arg_r.Location())
639
640 attached_arg = char0 if ch in spec.plus_flags else None
641 quit = action.OnMatch(attached_arg, arg_r, out)
642 arg_r.Next() # process the next flag
643
644 if quit:
645 break
646 else:
647 continue
648
649 break # it's a regular arg
650
651 return out