1 #!/usr/bin/env bash
2 #
3 # Compile OVM tarball.
4 #
5 # Usage:
6 # build/ovm-compile.sh <function name>
7
8 set -o nounset
9 set -o pipefail
10 set -o errexit
11 shopt -s strict:all 2>/dev/null || true # dogfood for OSH
12
13 REPO_ROOT=$(cd $(dirname $0)/..; pwd)
14 readonly REPO_ROOT
15
16 source build/common.sh
17
18 source-detected-config-or-die() {
19 if ! source _build/detected-config.sh; then
20 # Make this error stand out.
21 echo
22 echo "FATAL: can't find _build/detected-config.h. Run './configure'"
23 echo
24 exit 1
25 fi
26 }
27
28 # NOTES on trying to delete certain modules:
29 #
30 # _warnings.c: There weren't that many; it probably could be deleted.
31 # bufferobject.c: the types.py module uses it.
32 # Python-ast.h: pythonrun.c uses it in several places (mod_ty), and a lot of
33 # stuff uses pythonrun.c.
34 # pythonrun.c: lots interpreter flags and interpreter initialization caused
35 # link errors.
36 # pyctype.c: Tables needed for many string operations.
37
38 # getargs.c: needed for Python-C API, e.g. PyArg_ParseTuple.
39 # dtoa.c: not tried, but I assume that %.3f for 'time' uses it.
40
41
42 readonly OVM_PYTHON_OBJS='
43 Python/_warnings.c
44 Python/bltinmodule.c
45 Python/ceval.c
46 Python/errors.c
47 Python/getargs.c
48 Python/getcompiler.c
49 Python/getplatform.c
50 Python/getversion.c
51 Python/import.c
52 Python/marshal.c
53 Python/modsupport.c
54 Python/mystrtoul.c
55 Python/mysnprintf.c
56 Python/pyarena.c
57 Python/pyctype.c
58 Python/pyfpe.c
59 Python/pystate.c
60 Python/pythonrun.c
61 Python/random.c
62 Python/structmember.c
63 Python/sysmodule.c
64 Python/traceback.c
65 Python/pystrtod.c
66 Python/dtoa.c
67 Python/pymath.c
68 '
69 # NOTE: pystrtod.c needs some floating point functions in pymath.c
70
71 OBJECT_OBJS='
72 Objects/abstract.c
73 Objects/boolobject.c
74 Objects/bufferobject.c
75 Objects/bytes_methods.c
76 Objects/capsule.c
77 Objects/cellobject.c
78 Objects/classobject.c
79 Objects/cobject.c
80 Objects/codeobject.c
81 Objects/descrobject.c
82 Objects/enumobject.c
83 Objects/exceptions.c
84 Objects/genobject.c
85 Objects/fileobject.c
86 Objects/floatobject.c
87 Objects/frameobject.c
88 Objects/funcobject.c
89 Objects/intobject.c
90 Objects/iterobject.c
91 Objects/listobject.c
92 Objects/longobject.c
93 Objects/dictobject.c
94 Objects/methodobject.c
95 Objects/moduleobject.c
96 Objects/object.c
97 Objects/obmalloc.c
98 Objects/rangeobject.c
99 Objects/setobject.c
100 Objects/sliceobject.c
101 Objects/stringobject.c
102 Objects/structseq.c
103 Objects/tupleobject.c
104 Objects/typeobject.c
105 Objects/weakrefobject.c
106 '
107
108 # Non-standard lib stuff.
109 MODULE_OBJS='
110 Modules/main.c
111 Modules/gcmodule.c
112 '
113
114 # The stuff in Modules/Setup.dist, signalmodule.c. NOTE: In Python,
115 # signalmodule.c is specified in Modules/Setup.config, which comes from
116 # 'configure' output.
117 MODOBJS='
118 Modules/errnomodule.c
119 Modules/pwdmodule.c
120 Modules/_weakref.c
121 Modules/zipimport.c
122 Modules/signalmodule.c
123 '
124
125 # Parser/myreadline.c is needed for raw_input() to work. There is a dependency
126 # from Python/bltinmodule.c to it.
127 OVM_LIBRARY_OBJS="
128 Modules/getbuildinfo.c
129 Parser/myreadline.c
130 $OBJECT_OBJS
131 $OVM_PYTHON_OBJS
132 $MODULE_OBJS
133 $MODOBJS
134 "
135
136 readonly EMPTY_STR='""'
137
138 # Stub out a few variables
139 readonly PREPROC_FLAGS=(
140 -D OVM_MAIN \
141 -D PYTHONPATH="$EMPTY_STR" \
142 -D VERSION="$EMPTY_STR" \
143 -D VPATH="$EMPTY_STR" \
144 -D Py_BUILD_CORE \
145 # Python already has support for disabling complex numbers!
146 -D WITHOUT_COMPLEX
147 )
148
149 # NOTE: build/oil-defs is hard-coded to the oil.ovm app. We're abandoning
150 # hello.ovm and opy.ovm for now, but those can easily be added later. We
151 # haven't mangled the CPython source!
152 readonly INCLUDE_PATHS=(
153 -I . # for pyconfig.h
154 -I .. # for _gen/frontend/id_kind_asdl_c.h etc.
155 -I Include
156 -I ../build/oil-defs
157 -I ../py-yajl
158 # Note: This depends on build/py.sh yajl-release
159 -I ../py-yajl/yajl/yajl-2.1.1/include
160 )
161 readonly CC=${CC:-cc} # cc should be on POSIX systems
162
163 # BASE_CFLAGS is copied by observation from what configure.ac does on my Ubuntu
164 # 16.04 system. Then we check if it works on Alpine Linux too.
165
166 # "Python violates C99 rules, by casting between incompatible pointer types.
167 # GCC may generate bad code as a result of that, so use -fno-strict-aliasing if
168 # supported."
169 # - gcc 4.x and Clang need -fwrapv
170
171 # TODO:
172 # - -DNDEBUG is also passed. That turns off asserts. Do we want that?
173 # - We should auto-detect the flags in configure, or simplify the source so it
174 # isn't necessary. Python's configure.ac sometimes does it by compiling a test
175 # file; at other times it does it by grepping $CC --help.
176
177 # pyext/fanos.c needs -std=c99
178 BASE_CFLAGS='-fno-strict-aliasing -fwrapv -Wall -Wstrict-prototypes -std=c99'
179
180 # These flags are disabled for OS X. I would have thought it would work in
181 # Clang? It works with both GCC and Clang on Linux.
182 # https://stackoverflow.com/questions/6687630/how-to-remove-unused-c-c-symbols-with-gcc-and-ld
183 #BASE_CFLAGS="$BASE_CFLAGS -fdata-sections -ffunction-sections"
184
185 # Needed after cpython-defs filtering.
186 BASE_CFLAGS="$BASE_CFLAGS -Wno-unused-variable -Wno-unused-function"
187 readonly BASE_CFLAGS
188
189 BASE_LDFLAGS=''
190 # Disabled for OS X
191 # BASE_LDFLAGS='-Wl,--gc-sections'
192
193 # The user should be able to customize CFLAGS, but it shouldn't disable what's
194 # in BASE_CFLAGS.
195 readonly CFLAGS=${CFLAGS:-}
196 readonly LDFLAGS=${LDFLAGS:-}
197
198 build() {
199 local out=${1:-$PY27/ovm2}
200 local module_init=${2:-$PY27/Modules/config.c}
201 local main_name=${3:-_tmp/hello/main_name.c}
202 local c_module_srcs=${4:-_tmp/hello/c-module-srcs.txt}
203 shift 4
204
205 local abs_out=$PWD/$out
206 local abs_module_init=$PWD/$module_init
207 local abs_main_name=$PWD/$main_name
208 local abs_c_module_srcs=$PWD/$c_module_srcs
209
210 #echo $OVM_LIBRARY_OBJS
211
212 # HAVE_READLINE defined in detected-config.sh.
213 source-detected-config-or-die
214
215 pushd $PY27
216
217 local readline_flags=''
218 if [[ "$HAVE_READLINE" -eq 1 ]]; then
219 # Readline interface for tokenizer.c and [raw_]input() in bltinmodule.c.
220 # For now, we are using raw_input() for the REPL. TODO: Parameterize this!
221 # We should create a special no_readline_raw_input().
222
223 c_module_src_list=$(cat $abs_c_module_srcs)
224
225 if [[ -n "$READLINE_DIR" ]]; then
226 readline_flags+="-L $READLINE_DIR/lib -I $READLINE_DIR/include "
227 fi
228
229 # NOTE: pyconfig.h has HAVE_LIBREADLINE but doesn't appear to use it?
230 readline_flags+="-l readline -D HAVE_READLINE"
231 else
232 # don't fail
233 c_module_src_list=$(grep -E -v '/readline.c|/line_input.c' $abs_c_module_srcs || true)
234 fi
235
236 # $PREFIX comes from ./configure and defaults to /usr/local.
237 # $EXEC_PREFIX is a GNU thing and used in getpath.c. Could probably get rid
238 # of it.
239
240 time $CC \
241 ${BASE_CFLAGS} \
242 ${CFLAGS} \
243 "${INCLUDE_PATHS[@]}" \
244 "${PREPROC_FLAGS[@]}" \
245 -D PREFIX="\"$PREFIX\"" \
246 -D EXEC_PREFIX="\"$PREFIX\"" \
247 -o $abs_out \
248 $OVM_LIBRARY_OBJS \
249 $abs_module_init \
250 $abs_main_name \
251 $c_module_src_list \
252 Modules/ovm.c \
253 -l m \
254 ${BASE_LDFLAGS} \
255 ${LDFLAGS} \
256 $readline_flags \
257 "$@"
258
259 # NOTE:
260 # -l readline -l termcap -- for Python readline. Hm it builds without -l
261 # termcap.
262 # -l z WOULD be needed for zlibmodule.c, but we don't need it because our zip
263 # file has no compression -- see build/make_zip.py with ZIP_STORED.
264 # zipimport works fine without this.
265 }
266
267 # build the optimized one. Makefile uses -O3.
268
269 # Clang -O2 is 1.37 MB. 18 seconds to compile.
270 # -m32 is 1.12 MB. But I probably have to redefine a few things because
271 # there are more warnings.
272 # -O3 is 1.40 MB.
273
274 # GCC -O2 is 1.35 MB. 21 seconds to compile.
275
276 build-dbg() {
277 build "$@" -O0 -g -D OVM_DEBUG
278 }
279
280 # This will be stripped later.
281 build-opt() {
282 # frame pointer for perf. Otherwise stack traces are messed up!
283 # http://www.brendangregg.com/FlameGraphs/cpuflamegraphs.html#C But why
284 # isn't debuginfo enough? Because it's a recursive function?
285 # Does this make things slower? Do I need a "perf" build?
286 build "$@" -O3 -fno-omit-frame-pointer
287 }
288
289 #
290 # Source Release (uses same files
291 #
292
293 add-py27() {
294 xargs -I {} -- echo $PY27/{}
295 }
296
297 python-sources() {
298 echo "$OVM_LIBRARY_OBJS" | add-py27
299 }
300
301 _headers() {
302 local c_module_srcs=${1:-_tmp/hello/c-module-srcs.txt}
303 local abs_c_module_srcs=$PWD/$c_module_srcs
304
305 cd $PY27
306
307 # -MM: no system headers
308 gcc \
309 "${INCLUDE_PATHS[@]}" \
310 "${PREPROC_FLAGS[@]}" \
311 -MM $OVM_LIBRARY_OBJS \
312 Modules/ovm.c \
313 $(cat $abs_c_module_srcs)
314 }
315
316 # NOTE: 91 headers in Include, but only 81 referenced here. So it's worth it.
317 # These are probably for the parser.
318 #
319 # NOTE: We also should get rid of asdl.h and so forth.
320
321 python-headers() {
322 local c_module_srcs=$1
323
324 # 1. -MM outputs Makefile fragments, so egrep turns those into proper lines.
325 #
326 # 2. The user should generated detected-config.h, so remove it.
327 #
328 # 3. # gcc outputs paths like
329 # Python-2.7.13/Python/../Objects/stringlib/stringdefs.h
330 # but 'Python/..' causes problems for tar.
331 #
332
333 # NOTE: need .def for build/oil-defs.
334 _headers $c_module_srcs \
335 | egrep --only-matching '[^ ]+\.(h|def)' \
336 | grep -v '_build/detected-config.h' \
337 | sed 's|^Python/../||' \
338 | sort | uniq | add-py27
339 }
340
341 make-tar() {
342 local app_name=${1:-hello}
343 local bytecode_zip=${2:-bytecode-cpython.zip}
344 local out=${3:-_release/hello.tar}
345
346 local version_file
347 case $app_name in
348 oil)
349 version_file=oil-version.txt
350 ;;
351 hello)
352 version_file=build/testdata/hello-version.txt
353 ;;
354 *)
355 die "Unknown app $app_name"
356 exit 1
357 ;;
358 esac
359 local version=$(head -n 1 $version_file)
360
361 echo "Creating $app_name version $version"
362
363 local c_module_srcs=_build/$app_name/c-module-srcs.txt
364
365 # Add oil-0.0.0/ to the beginning of every path.
366 local sed_expr="s,^,${app_name}-${version}/,"
367
368 # Differences between tarball and repo:
369 #
370 # - build/portable-rules.mk is intentionally not included in the release tarball.
371 # The Makefile can and should operate without it.
372 #
373 # - We include intermediate files like c-module-srcs.txt, so we don't have to
374 # ship tools dynamic_deps.py. The end-user build shouldn't depend on Python.
375
376 # Note: python-headers runs gcc -M, including pyconfig.h and
377 # _build/detected-config.h.
378
379 tar --create --transform "$sed_expr" --file $out \
380 LICENSE.txt \
381 INSTALL.txt \
382 configure \
383 install \
384 uninstall \
385 Makefile \
386 doc/osh.1 \
387 build/ovm-compile.sh \
388 build/ovm-actions.sh \
389 build/clean.sh \
390 build/common.sh \
391 build/detect-*.c \
392 _build/$app_name/$bytecode_zip \
393 _build/$app_name/*.c \
394 py-yajl/yajl/COPYING \
395 $PY27/LICENSE \
396 $PY27/Modules/ovm.c \
397 $c_module_srcs \
398 $(cat $c_module_srcs | add-py27) \
399 $(python-headers $c_module_srcs) \
400 $(python-sources)
401
402 ls -l $out
403 }
404
405 # 123K lines.
406 # Excluding MODOBJS, it's 104K lines.
407 #
408 # Biggest: posixmodule,unicodeobject,typeobject,ceval.
409 #
410 # Remove tmpnam from posixmodule, other cruft.
411 #
412 # Big ones to rid of: unicodeobject.c, import.c
413 # codecs and codecsmodule? There is some non-unicode stuff there though.
414 #
415 # Probably need unicode for compatibility with modules and web frameworks
416 # especially.
417
418 count-c-lines() {
419 pushd $PY27
420 wc -l $OVM_LIBRARY_OBJS | sort -n
421
422 # 90 files.
423 # NOTE: To count headers, use the tar file.
424 echo
425 echo 'Files:'
426 { for i in $OVM_LIBRARY_OBJS; do
427 echo $i
428 done
429 } | wc -l
430
431 popd
432 }
433
434 "$@"