1 #!/usr/bin/env python2
2 # Copyright 2016 Andy Chu. All rights reserved.
3 # Licensed under the Apache License, Version 2.0 (the "License");
4 # you may not use this file except in compliance with the License.
5 # You may obtain a copy of the License at
6 #
7 # http://www.apache.org/licenses/LICENSE-2.0
8 """
9 consts_gen.py - Code generation for consts.py, id_kind_def.py, etc.
10 """
11 from __future__ import print_function
12
13 import collections
14 import os
15 import sys
16
17 from asdl import gen_cpp
18 from mycpp.mylib import log
19 from frontend import id_kind_def
20 from frontend import builtin_def
21 from frontend import option_def
22
23
24 def _CreateModule(id_spec, ids):
25 """Create a SYNTHETIC ASDL module to generate code from."""
26 from asdl import ast
27
28 id_variants = [ast.Constructor(name) for name, _ in ids]
29 id_sum = ast.SimpleSum(id_variants,
30 generate=['integers', 'no_namespace_suffix'])
31
32 kind_variants = [ast.Constructor(name) for name in id_spec.kind_name_list]
33 kind_sum = ast.SimpleSum(kind_variants, generate=['no_namespace_suffix'])
34
35 id_ = ast.TypeDecl('Id', id_sum)
36 kind_ = ast.TypeDecl('Kind', kind_sum)
37
38 schema_ast = ast.Module('id_kind', [], [id_, kind_])
39 return schema_ast
40
41
42 _BUILTINS = builtin_def.All()
43
44
45 def GenBuiltinLookup(func_name, kind, f):
46 #log('%r %r', func_name, kind)
47
48 pairs = [(b.name, b.index) for b in _BUILTINS if b.kind == kind]
49
50 GenStringLookup('builtin_t', func_name, pairs, f)
51
52
53 def GenStringLookup(type_name, func_name, pairs, f):
54 #log('%s', pairs)
55
56 groups = collections.defaultdict(list)
57 for name, index in pairs:
58 first_char = name[0]
59 groups[first_char].append((name, index))
60
61 if 0:
62 for first_char, pairs in groups.iteritems():
63 log('%s %d', first_char, len(pairs))
64 log('%s', pairs)
65
66 # Note: we could optimize the length check, e.g. have a second level
67 # switch. But we would need to measure the difference. Caching the id on
68 # AST nodes is probably a bigger win, e.g. for loops.
69 #
70 # Size optimization: don't repeat constants literally?
71
72 f.write("""\
73 %s %s(Str* s) {
74 int length = len(s);
75 if (length == 0) return 0; // consts.NO_INDEX
76
77 const char* data = s->data_;
78 switch (data[0]) {
79 """ % (type_name, func_name))
80
81 for first_char in sorted(groups):
82 pairs = groups[first_char]
83 f.write(" case '%s':\n" % first_char)
84 for name, index in pairs:
85 # NOTE: we have to check the length because they're not NUL-terminated
86 f.write('''\
87 if (length == %d && memcmp("%s", data, %d) == 0) return %d;
88 ''' % (len(name), name, len(name), index))
89 f.write(' break;\n')
90
91 f.write("""\
92 }
93
94 return 0; // consts.NO_INDEX
95 }
96
97 """)
98
99
100 def GenStringMembership(func_name, strs, f):
101 groups = collections.defaultdict(list)
102 for s in strs:
103 first_char = s[0]
104 groups[first_char].append(s)
105
106 f.write("""\
107 bool %s(Str* s) {
108 int length = len(s);
109 if (length == 0) return false;
110
111 const char* data = s->data_;
112 switch (data[0]) {
113 """ % func_name)
114
115 for first_char in sorted(groups):
116 strs = groups[first_char]
117 f.write(" case '%s':\n" % first_char)
118 for s in strs:
119 # NOTE: we have to check the length because they're not NUL-terminated
120 f.write('''\
121 if (length == %d && memcmp("%s", data, %d) == 0) return true;
122 ''' % (len(s), s, len(s)))
123 f.write(' break;\n')
124
125 f.write("""\
126 }
127
128 return false;
129 }
130
131 """)
132
133
134 C_CHAR = {
135 # '\'' is a single quote in C
136 "'": "\\'",
137 '"': '\\"',
138 '\\': "\\\\",
139 '\t': '\\t',
140 '\r': '\\r',
141 '\n': '\\n',
142 '\v': '\\v',
143 '\0': '\\0',
144 '\a': '\\a',
145 '\b': '\\b',
146 '\f': '\\f',
147 '\x1b': '\\x1b',
148 }
149
150
151 def CChar(c):
152 return C_CHAR.get(c, c)
153
154
155 def GenCharLookup(func_name, lookup, f, required=False):
156 f.write("""\
157 Str* %s(Str* c) {
158 assert(len(c) == 1);
159
160 char ch = c->data_[0];
161
162 // TODO-intern: return value
163 switch (ch) {
164 """ % func_name)
165
166 for char_code in sorted(lookup):
167 f.write(" case '%s':\n" % CChar(char_code))
168 f.write(' return StrFromC("%s", 1);\n' % CChar(lookup[char_code]))
169 f.write(" break;\n")
170
171 f.write(" default:\n")
172 if required:
173 f.write(" assert(0);\n")
174 else:
175 f.write(" return nullptr;\n")
176
177 f.write("""
178 }
179 }
180 """)
181
182
183 def GenStrList(l, name, out):
184 element_globals = []
185 for i, elem in enumerate(l):
186 global_name = "k%s_%d" % (name, i)
187 out('GLOBAL_STR(%s, "%s");', global_name, elem)
188 element_globals.append(global_name)
189
190 lit = ' COMMA '.join(element_globals)
191 out('GLOBAL_LIST(%s, Str*, %d, {%s});\n', name, len(l), lit)
192
193
194 def main(argv):
195 try:
196 action = argv[1]
197 except IndexError:
198 raise RuntimeError('Action required')
199
200 # TODO: Remove duplication in core/meta.py
201 ID_TO_KIND = {}
202 BOOL_ARG_TYPES = {}
203 TEST_UNARY_LOOKUP = {}
204 TEST_BINARY_LOOKUP = {}
205 TEST_OTHER_LOOKUP = {}
206
207 ID_SPEC = id_kind_def.IdSpec(ID_TO_KIND, BOOL_ARG_TYPES)
208
209 id_kind_def.AddKinds(ID_SPEC)
210 id_kind_def.AddBoolKinds(ID_SPEC) # must come second
211
212 id_kind_def.SetupTestBuiltin(ID_SPEC, TEST_UNARY_LOOKUP, TEST_BINARY_LOOKUP,
213 TEST_OTHER_LOOKUP)
214
215 ids = ID_SPEC.id_str2int.items()
216 ids.sort(key=lambda pair: pair[1]) # Sort by ID
217
218 if action == 'c':
219 for name, id_int in ids:
220 print('#define id__%s %s' % (name, id_int))
221
222 elif action == 'cpp':
223 schema_ast = _CreateModule(ID_SPEC, ids)
224
225 out_prefix = argv[2]
226
227 with open(out_prefix + '.h', 'w') as f:
228 f.write("""\
229 #ifndef ID_KIND_ASDL_H
230 #define ID_KIND_ASDL_H
231
232 namespace id_kind_asdl {
233
234 #define ASDL_NAMES struct
235 """)
236
237 v = gen_cpp.ClassDefVisitor(f)
238 v.VisitModule(schema_ast)
239
240 f.write("""
241 } // namespace id_kind_asdl
242
243 #endif // ID_KIND_ASDL_H
244 """)
245
246 with open(out_prefix + '.cc', 'w') as f:
247 f.write("""\
248 #include <assert.h>
249 #include "_gen/frontend/id_kind.asdl.h"
250
251 namespace id_kind_asdl {
252
253 """)
254
255 v = gen_cpp.MethodDefVisitor(f)
256
257 v.VisitModule(schema_ast)
258
259 f.write('} // namespace id_kind_asdl\n')
260
261 elif action == 'mypy':
262 from asdl import gen_python
263
264 schema_ast = _CreateModule(ID_SPEC, ids)
265 #print(schema_ast)
266
267 f = sys.stdout
268
269 f.write("""\
270 from asdl import pybase
271
272 """)
273 # Minor style issue: we want Id and Kind, not Id_e and Kind_e
274 v = gen_python.GenMyPyVisitor(f)
275 v.VisitModule(schema_ast)
276
277 elif action == 'cpp-consts':
278
279 # Break circular deps
280
281 from core import pyutil
282 from frontend import consts
283 from _devbuild.gen.id_kind_asdl import Id_str, Kind_str
284 from _devbuild.gen.types_asdl import redir_arg_type_str, bool_arg_type_str
285
286 LIST_INT = [
287 'STRICT_ALL',
288 'YSH_UPGRADE',
289 'YSH_ALL',
290 'DEFAULT_TRUE',
291 'PARSE_OPTION_NUMS',
292 'SHOPT_OPTION_NUMS',
293 'SET_OPTION_NUMS',
294 'VISIBLE_SHOPT_NUMS',
295 ]
296
297 prefix = argv[2]
298
299 with open(prefix + '.h', 'w') as f:
300
301 def out(fmt, *args):
302 print(fmt % args, file=f)
303
304 out("""\
305 #ifndef CONSTS_H
306 #define CONSTS_H
307
308 #include "mycpp/runtime.h"
309
310 #include "_gen/frontend/id_kind.asdl.h"
311 #include "_gen/frontend/option.asdl.h"
312 #include "_gen/core/runtime.asdl.h"
313 #include "_gen/frontend/types.asdl.h"
314
315 namespace consts {
316 """)
317
318 for name in LIST_INT:
319 out('extern List<int>* %s;', name)
320
321 out('extern List<Str*>* BUILTIN_NAMES;')
322 out('extern List<Str*>* OSH_KEYWORD_NAMES;')
323 out('extern List<Str*>* SET_OPTION_NAMES;')
324 out('extern List<Str*>* SHOPT_OPTION_NAMES;')
325
326 out("""\
327
328 extern int NO_INDEX;
329
330 extern Str* gVersion;
331
332 int RedirDefaultFd(id_kind_asdl::Id_t id);
333 types_asdl::redir_arg_type_t RedirArgType(id_kind_asdl::Id_t id);
334 types_asdl::bool_arg_type_t BoolArgType(id_kind_asdl::Id_t id);
335 id_kind_asdl::Kind GetKind(id_kind_asdl::Id_t id);
336
337 types_asdl::opt_group_t OptionGroupNum(Str* s);
338 option_asdl::option_t OptionNum(Str* s);
339 option_asdl::builtin_t LookupNormalBuiltin(Str* s);
340 option_asdl::builtin_t LookupAssignBuiltin(Str* s);
341 option_asdl::builtin_t LookupSpecialBuiltin(Str* s);
342 bool IsControlFlow(Str* s);
343 bool IsKeyword(Str* s);
344 Str* LookupCharC(Str* c);
345 Str* LookupCharPrompt(Str* c);
346
347 Str* OptionName(option_asdl::option_t opt_num);
348
349 Tuple2<runtime_asdl::state_t, runtime_asdl::emit_t> IfsEdge(runtime_asdl::state_t state, runtime_asdl::char_kind_t ch);
350
351 } // namespace consts
352
353 #endif // CONSTS_H
354 """)
355
356 with open(prefix + '.cc', 'w') as f:
357
358 def out(fmt, *args):
359 print(fmt % args, file=f)
360
361 out("""\
362 #include "_gen/frontend/consts.h"
363
364 using id_kind_asdl::Id;
365 using id_kind_asdl::Kind;
366 using types_asdl::redir_arg_type_e;
367 using types_asdl::bool_arg_type_e;
368 using option_asdl::builtin_t;
369
370 namespace consts {
371
372 int NO_INDEX = 0; // duplicated from frontend/consts.py
373 """)
374
375 # Generate gVersion, which is read by pyutil::GetVersion()
376 this_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
377 root_dir = os.path.join(this_dir, '..') # ~/git/oilshell/oil
378 loader = pyutil._FileResourceLoader(root_dir)
379
380 version_str = pyutil.GetVersion(loader)
381 out('GLOBAL_STR(gVersion, "%s");' % version_str)
382 out('')
383
384 # Note: could use opt_num:: instead of raw ints
385 for name in LIST_INT:
386 val = getattr(consts, name)
387 val_str = ' COMMA '.join(str(i) for i in val)
388 out('GLOBAL_LIST(%s, int, %d, {%s});', name, len(val), val_str)
389
390 out("""\
391
392 int RedirDefaultFd(id_kind_asdl::Id_t id) {
393 // relies on "switch lowering"
394 switch (id) {
395 """)
396 for id_ in sorted(consts.REDIR_DEFAULT_FD):
397 a = Id_str(id_).replace('.', '::')
398 b = consts.REDIR_DEFAULT_FD[id_]
399 out(' case %s: return %s;' % (a, b))
400 out("""\
401 }
402 FAIL(kShouldNotGetHere);
403 }
404 """)
405
406 out("""\
407 types_asdl::redir_arg_type_t RedirArgType(id_kind_asdl::Id_t id) {
408 // relies on "switch lowering"
409 switch (id) {
410 """)
411 for id_ in sorted(consts.REDIR_ARG_TYPES):
412 a = Id_str(id_).replace('.', '::')
413 # redir_arg_type_e::Path, etc.
414 b = redir_arg_type_str(consts.REDIR_ARG_TYPES[id_]).replace(
415 '.', '_e::')
416 out(' case %s: return %s;' % (a, b))
417 out("""\
418 }
419 FAIL(kShouldNotGetHere);
420 }
421 """)
422
423 out("""\
424 types_asdl::bool_arg_type_t BoolArgType(id_kind_asdl::Id_t id) {
425 // relies on "switch lowering"
426 switch (id) {
427 """)
428 for id_ in sorted(BOOL_ARG_TYPES):
429 a = Id_str(id_).replace('.', '::')
430 # bool_arg_type_e::Str, etc.
431 b = bool_arg_type_str(BOOL_ARG_TYPES[id_]).replace('.', '_e::')
432 out(' case %s: return %s;' % (a, b))
433 out("""\
434 }
435 FAIL(kShouldNotGetHere);
436 }
437 """)
438
439 out("""\
440 Kind GetKind(id_kind_asdl::Id_t id) {
441 // relies on "switch lowering"
442 switch (id) {
443 """)
444 for id_ in sorted(ID_TO_KIND):
445 a = Id_str(id_).replace('.', '::')
446 b = Kind_str(ID_TO_KIND[id_]).replace('.', '::')
447 out(' case %s: return %s;' % (a, b))
448 out("""\
449 }
450 FAIL(kShouldNotGetHere);
451 }
452 """)
453
454 pairs = consts.OPTION_GROUPS.items()
455 GenStringLookup('types_asdl::opt_group_t', 'OptionGroupNum', pairs,
456 f)
457
458 pairs = [(opt.name, opt.index) for opt in option_def.All()]
459 GenStringLookup('option_asdl::option_t', 'OptionNum', pairs, f)
460
461 GenBuiltinLookup('LookupNormalBuiltin', 'normal', f)
462 GenBuiltinLookup('LookupAssignBuiltin', 'assign', f)
463 GenBuiltinLookup('LookupSpecialBuiltin', 'special', f)
464
465 from frontend import lexer_def # break circular dep
466 GenStringMembership('IsControlFlow', lexer_def.CONTROL_FLOW_NAMES,
467 f)
468 GenStringMembership('IsKeyword', consts.OSH_KEYWORD_NAMES, f)
469
470 GenCharLookup('LookupCharC', consts._ONE_CHAR_C, f, required=True)
471 GenCharLookup('LookupCharPrompt', consts._ONE_CHAR_PROMPT, f)
472
473 # OptionName() is a bit redundant with ADSL's debug print option_str(),
474 # but the latter should get stripped from the binary
475 out("""\
476 Str* OptionName(option_asdl::option_t opt_num) {
477 const char* s;
478 switch (opt_num) {
479 """)
480
481 for opt in option_def.All():
482 out(' case %s:' % opt.index)
483 out(' s = "%s";' % opt.name)
484 out(' break;')
485
486 out("""\
487 default:
488 FAIL(kShouldNotGetHere);
489 }
490 return StrFromC(s); // TODO-intern
491 }
492 """)
493
494 #
495 # Generate a tightly packed 2D array for C, from a Python dict.
496 #
497
498 edges = consts._IFS_EDGES
499 max_state = max(edge[0] for edge in edges)
500 max_char_kind = max(edge[1] for edge in edges)
501
502 edge_array = []
503 for i in xrange(max_state + 1):
504 # unused cells get -1
505 edge_array.append(['-1'] * (max_char_kind + 1))
506
507 for i in xrange(max_state + 1):
508 for j in xrange(max_char_kind + 1):
509 entry = edges.get((i, j))
510 if entry is not None:
511 # pack (new_state, action) into 32 bits
512 edge_array[i][j] = '(%d<<16)|%d' % entry
513
514 parts = []
515 for i in xrange(max_state + 1):
516 parts.append(' {')
517 parts.append(', '.join('%10s' % cell for cell in edge_array[i]))
518 parts.append(' },\n')
519
520 out("""\
521 int _IFS_EDGE[%d][%d] = {
522 %s
523 };
524 """ % (max_state + 1, max_char_kind + 1, ''.join(parts)))
525
526 out("""\
527 // Note: all of these are integers, e.g. state_i, emit_i, char_kind_i
528 using runtime_asdl::state_t;
529 using runtime_asdl::emit_t;
530 using runtime_asdl::char_kind_t;
531
532 Tuple2<state_t, emit_t> IfsEdge(state_t state, runtime_asdl::char_kind_t ch) {
533 int cell = _IFS_EDGE[state][ch];
534 state_t new_state = cell >> 16;
535 emit_t emit = cell & 0xFFFF;
536 return Tuple2<state_t, emit_t>(new_state, emit);
537 }
538 """)
539
540 GenStrList(consts.BUILTIN_NAMES, 'BUILTIN_NAMES', out)
541 GenStrList(consts.OSH_KEYWORD_NAMES, 'OSH_KEYWORD_NAMES', out)
542 GenStrList(consts.SET_OPTION_NAMES, 'SET_OPTION_NAMES', out)
543 GenStrList(consts.SHOPT_OPTION_NAMES, 'SHOPT_OPTION_NAMES', out)
544
545 out("""\
546 } // namespace consts
547 """)
548
549 elif action == 'py-consts':
550 # It's kind of weird to use the generated code to generate more code.
551 # Can we do this instead with the parsed module for "id" and "types.asdl"?
552
553 from frontend import consts
554 from _devbuild.gen.id_kind_asdl import Id_str, Kind_str
555 from _devbuild.gen.types_asdl import redir_arg_type_str, bool_arg_type_str
556
557 print("""
558 from _devbuild.gen.id_kind_asdl import Id, Kind
559 from _devbuild.gen.types_asdl import redir_arg_type_e, bool_arg_type_e
560 """)
561
562 print('')
563 print('BOOL_ARG_TYPES = {')
564 for id_ in sorted(BOOL_ARG_TYPES):
565 v = BOOL_ARG_TYPES[id_]
566 # HACK
567 v = bool_arg_type_str(v).replace('.', '_e.')
568 print(' %s: %s,' % (Id_str(id_), v))
569 print('}')
570
571 print('')
572 print('TEST_UNARY_LOOKUP = {')
573 for op_str in sorted(TEST_UNARY_LOOKUP):
574 v = Id_str(TEST_UNARY_LOOKUP[op_str])
575 print(' %r: %s,' % (op_str, v))
576 print('}')
577
578 print('')
579 print('TEST_BINARY_LOOKUP = {')
580 for op_str in sorted(TEST_BINARY_LOOKUP):
581 v = Id_str(TEST_BINARY_LOOKUP[op_str])
582 print(' %r: %s,' % (op_str, v))
583 print('}')
584
585 print('')
586 print('TEST_OTHER_LOOKUP = {')
587 for op_str in sorted(TEST_OTHER_LOOKUP):
588 v = Id_str(TEST_OTHER_LOOKUP[op_str])
589 print(' %r: %s,' % (op_str, v))
590 print('}')
591
592 print('')
593 print('ID_TO_KIND = {')
594 for id_ in sorted(ID_TO_KIND):
595 v = Kind_str(ID_TO_KIND[id_])
596 print(' %s: %s,' % (Id_str(id_), v))
597 print('}')
598
599 else:
600 raise RuntimeError('Invalid action %r' % action)
601
602
603 if __name__ == '__main__':
604 try:
605 main(sys.argv)
606 except RuntimeError as e:
607 print('FATAL: %s' % e, file=sys.stderr)
608 sys.exit(1)