OILS
/
build
/
ovm-compile.sh
1 |
#!/usr/bin/env bash
|
2 |
#
|
3 |
# Compile OVM tarball.
|
4 |
#
|
5 |
# Usage:
|
6 |
# build/ovm-compile.sh <function name>
|
7 |
|
8 |
set -o nounset
|
9 |
set -o pipefail
|
10 |
set -o errexit
|
11 |
shopt -s strict:all 2>/dev/null || true # dogfood for OSH
|
12 |
|
13 |
REPO_ROOT=$(cd $(dirname $0)/..; pwd)
|
14 |
readonly REPO_ROOT
|
15 |
|
16 |
source build/common.sh
|
17 |
|
18 |
source-detected-config-or-die() {
|
19 |
if ! source _build/detected-config.sh; then
|
20 |
# Make this error stand out.
|
21 |
echo
|
22 |
echo "FATAL: can't find _build/detected-config.h. Run './configure'"
|
23 |
echo
|
24 |
exit 1
|
25 |
fi
|
26 |
}
|
27 |
|
28 |
# NOTES on trying to delete certain modules:
|
29 |
#
|
30 |
# _warnings.c: There weren't that many; it probably could be deleted.
|
31 |
# bufferobject.c: the types.py module uses it.
|
32 |
# Python-ast.h: pythonrun.c uses it in several places (mod_ty), and a lot of
|
33 |
# stuff uses pythonrun.c.
|
34 |
# pythonrun.c: lots interpreter flags and interpreter initialization caused
|
35 |
# link errors.
|
36 |
# pyctype.c: Tables needed for many string operations.
|
37 |
|
38 |
# getargs.c: needed for Python-C API, e.g. PyArg_ParseTuple.
|
39 |
# dtoa.c: not tried, but I assume that %.3f for 'time' uses it.
|
40 |
|
41 |
|
42 |
readonly OVM_PYTHON_OBJS='
|
43 |
Python/_warnings.c
|
44 |
Python/bltinmodule.c
|
45 |
Python/ceval.c
|
46 |
Python/errors.c
|
47 |
Python/getargs.c
|
48 |
Python/getcompiler.c
|
49 |
Python/getplatform.c
|
50 |
Python/getversion.c
|
51 |
Python/import.c
|
52 |
Python/marshal.c
|
53 |
Python/modsupport.c
|
54 |
Python/mystrtoul.c
|
55 |
Python/mysnprintf.c
|
56 |
Python/pyarena.c
|
57 |
Python/pyctype.c
|
58 |
Python/pyfpe.c
|
59 |
Python/pystate.c
|
60 |
Python/pythonrun.c
|
61 |
Python/random.c
|
62 |
Python/structmember.c
|
63 |
Python/sysmodule.c
|
64 |
Python/traceback.c
|
65 |
Python/pystrtod.c
|
66 |
Python/dtoa.c
|
67 |
Python/pymath.c
|
68 |
'
|
69 |
# NOTE: pystrtod.c needs some floating point functions in pymath.c
|
70 |
|
71 |
OBJECT_OBJS='
|
72 |
Objects/abstract.c
|
73 |
Objects/boolobject.c
|
74 |
Objects/bufferobject.c
|
75 |
Objects/bytes_methods.c
|
76 |
Objects/capsule.c
|
77 |
Objects/cellobject.c
|
78 |
Objects/classobject.c
|
79 |
Objects/cobject.c
|
80 |
Objects/codeobject.c
|
81 |
Objects/descrobject.c
|
82 |
Objects/enumobject.c
|
83 |
Objects/exceptions.c
|
84 |
Objects/genobject.c
|
85 |
Objects/fileobject.c
|
86 |
Objects/floatobject.c
|
87 |
Objects/frameobject.c
|
88 |
Objects/funcobject.c
|
89 |
Objects/intobject.c
|
90 |
Objects/iterobject.c
|
91 |
Objects/listobject.c
|
92 |
Objects/longobject.c
|
93 |
Objects/dictobject.c
|
94 |
Objects/methodobject.c
|
95 |
Objects/moduleobject.c
|
96 |
Objects/object.c
|
97 |
Objects/obmalloc.c
|
98 |
Objects/rangeobject.c
|
99 |
Objects/setobject.c
|
100 |
Objects/sliceobject.c
|
101 |
Objects/stringobject.c
|
102 |
Objects/structseq.c
|
103 |
Objects/tupleobject.c
|
104 |
Objects/typeobject.c
|
105 |
Objects/weakrefobject.c
|
106 |
'
|
107 |
|
108 |
# Non-standard lib stuff.
|
109 |
MODULE_OBJS='
|
110 |
Modules/main.c
|
111 |
Modules/gcmodule.c
|
112 |
'
|
113 |
|
114 |
# The stuff in Modules/Setup.dist, signalmodule.c. NOTE: In Python,
|
115 |
# signalmodule.c is specified in Modules/Setup.config, which comes from
|
116 |
# 'configure' output.
|
117 |
MODOBJS='
|
118 |
Modules/errnomodule.c
|
119 |
Modules/pwdmodule.c
|
120 |
Modules/_weakref.c
|
121 |
Modules/zipimport.c
|
122 |
Modules/signalmodule.c
|
123 |
'
|
124 |
|
125 |
# Parser/myreadline.c is needed for raw_input() to work. There is a dependency
|
126 |
# from Python/bltinmodule.c to it.
|
127 |
OVM_LIBRARY_OBJS="
|
128 |
Modules/getbuildinfo.c
|
129 |
Parser/myreadline.c
|
130 |
$OBJECT_OBJS
|
131 |
$OVM_PYTHON_OBJS
|
132 |
$MODULE_OBJS
|
133 |
$MODOBJS
|
134 |
"
|
135 |
|
136 |
readonly EMPTY_STR='""'
|
137 |
|
138 |
# Stub out a few variables
|
139 |
readonly PREPROC_FLAGS=(
|
140 |
-D OVM_MAIN \
|
141 |
-D PYTHONPATH="$EMPTY_STR" \
|
142 |
-D VERSION="$EMPTY_STR" \
|
143 |
-D VPATH="$EMPTY_STR" \
|
144 |
-D Py_BUILD_CORE \
|
145 |
# Python already has support for disabling complex numbers!
|
146 |
-D WITHOUT_COMPLEX
|
147 |
)
|
148 |
|
149 |
# NOTE: build/oil-defs is hard-coded to the oil.ovm app. We're abandoning
|
150 |
# hello.ovm and opy.ovm for now, but those can easily be added later. We
|
151 |
# haven't mangled the CPython source!
|
152 |
readonly INCLUDE_PATHS=(
|
153 |
-I . # for pyconfig.h
|
154 |
-I .. # for _gen/frontend/id_kind_asdl_c.h etc.
|
155 |
-I Include
|
156 |
-I ../build/oil-defs
|
157 |
-I ../py-yajl
|
158 |
# Note: This depends on build/py.sh yajl-release
|
159 |
-I ../py-yajl/yajl/yajl-2.1.1/include
|
160 |
)
|
161 |
readonly CC=${CC:-cc} # cc should be on POSIX systems
|
162 |
|
163 |
# BASE_CFLAGS is copied by observation from what configure.ac does on my Ubuntu
|
164 |
# 16.04 system. Then we check if it works on Alpine Linux too.
|
165 |
|
166 |
# "Python violates C99 rules, by casting between incompatible pointer types.
|
167 |
# GCC may generate bad code as a result of that, so use -fno-strict-aliasing if
|
168 |
# supported."
|
169 |
# - gcc 4.x and Clang need -fwrapv
|
170 |
|
171 |
# TODO:
|
172 |
# - -DNDEBUG is also passed. That turns off asserts. Do we want that?
|
173 |
# - We should auto-detect the flags in configure, or simplify the source so it
|
174 |
# isn't necessary. Python's configure.ac sometimes does it by compiling a test
|
175 |
# file; at other times it does it by grepping $CC --help.
|
176 |
|
177 |
# pyext/fanos.c needs -std=c99
|
178 |
BASE_CFLAGS='-fno-strict-aliasing -fwrapv -Wall -Wstrict-prototypes -std=c99'
|
179 |
|
180 |
# These flags are disabled for OS X. I would have thought it would work in
|
181 |
# Clang? It works with both GCC and Clang on Linux.
|
182 |
# https://stackoverflow.com/questions/6687630/how-to-remove-unused-c-c-symbols-with-gcc-and-ld
|
183 |
#BASE_CFLAGS="$BASE_CFLAGS -fdata-sections -ffunction-sections"
|
184 |
|
185 |
# Needed after cpython-defs filtering.
|
186 |
BASE_CFLAGS="$BASE_CFLAGS -Wno-unused-variable -Wno-unused-function"
|
187 |
readonly BASE_CFLAGS
|
188 |
|
189 |
BASE_LDFLAGS=''
|
190 |
# Disabled for OS X
|
191 |
# BASE_LDFLAGS='-Wl,--gc-sections'
|
192 |
|
193 |
# The user should be able to customize CFLAGS, but it shouldn't disable what's
|
194 |
# in BASE_CFLAGS.
|
195 |
readonly CFLAGS=${CFLAGS:-}
|
196 |
readonly LDFLAGS=${LDFLAGS:-}
|
197 |
|
198 |
build() {
|
199 |
local out=${1:-$PY27/ovm2}
|
200 |
local module_init=${2:-$PY27/Modules/config.c}
|
201 |
local main_name=${3:-_tmp/hello/main_name.c}
|
202 |
local c_module_srcs=${4:-_tmp/hello/c-module-srcs.txt}
|
203 |
shift 4
|
204 |
|
205 |
local abs_out=$PWD/$out
|
206 |
local abs_module_init=$PWD/$module_init
|
207 |
local abs_main_name=$PWD/$main_name
|
208 |
local abs_c_module_srcs=$PWD/$c_module_srcs
|
209 |
|
210 |
#echo $OVM_LIBRARY_OBJS
|
211 |
|
212 |
# HAVE_READLINE defined in detected-config.sh.
|
213 |
source-detected-config-or-die
|
214 |
|
215 |
pushd $PY27
|
216 |
|
217 |
local readline_flags=''
|
218 |
if [[ "$HAVE_READLINE" -eq 1 ]]; then
|
219 |
# Readline interface for tokenizer.c and [raw_]input() in bltinmodule.c.
|
220 |
# For now, we are using raw_input() for the REPL. TODO: Parameterize this!
|
221 |
# We should create a special no_readline_raw_input().
|
222 |
|
223 |
c_module_src_list=$(cat $abs_c_module_srcs)
|
224 |
|
225 |
if [[ -n "$READLINE_DIR" ]]; then
|
226 |
readline_flags+="-L $READLINE_DIR/lib -I $READLINE_DIR/include "
|
227 |
fi
|
228 |
|
229 |
# NOTE: pyconfig.h has HAVE_LIBREADLINE but doesn't appear to use it?
|
230 |
readline_flags+="-l readline -D HAVE_READLINE"
|
231 |
else
|
232 |
# don't fail
|
233 |
c_module_src_list=$(grep -E -v '/readline.c|/line_input.c' $abs_c_module_srcs || true)
|
234 |
fi
|
235 |
|
236 |
# $PREFIX comes from ./configure and defaults to /usr/local.
|
237 |
# $EXEC_PREFIX is a GNU thing and used in getpath.c. Could probably get rid
|
238 |
# of it.
|
239 |
|
240 |
time $CC \
|
241 |
${BASE_CFLAGS} \
|
242 |
${CFLAGS} \
|
243 |
"${INCLUDE_PATHS[@]}" \
|
244 |
"${PREPROC_FLAGS[@]}" \
|
245 |
-D PREFIX="\"$PREFIX\"" \
|
246 |
-D EXEC_PREFIX="\"$PREFIX\"" \
|
247 |
-o $abs_out \
|
248 |
$OVM_LIBRARY_OBJS \
|
249 |
$abs_module_init \
|
250 |
$abs_main_name \
|
251 |
$c_module_src_list \
|
252 |
Modules/ovm.c \
|
253 |
-l m \
|
254 |
${BASE_LDFLAGS} \
|
255 |
${LDFLAGS} \
|
256 |
$readline_flags \
|
257 |
"$@"
|
258 |
|
259 |
# NOTE:
|
260 |
# -l readline -l termcap -- for Python readline. Hm it builds without -l
|
261 |
# termcap.
|
262 |
# -l z WOULD be needed for zlibmodule.c, but we don't need it because our zip
|
263 |
# file has no compression -- see build/make_zip.py with ZIP_STORED.
|
264 |
# zipimport works fine without this.
|
265 |
}
|
266 |
|
267 |
# build the optimized one. Makefile uses -O3.
|
268 |
|
269 |
# Clang -O2 is 1.37 MB. 18 seconds to compile.
|
270 |
# -m32 is 1.12 MB. But I probably have to redefine a few things because
|
271 |
# there are more warnings.
|
272 |
# -O3 is 1.40 MB.
|
273 |
|
274 |
# GCC -O2 is 1.35 MB. 21 seconds to compile.
|
275 |
|
276 |
build-dbg() {
|
277 |
build "$@" -O0 -g -D OVM_DEBUG
|
278 |
}
|
279 |
|
280 |
# This will be stripped later.
|
281 |
build-opt() {
|
282 |
# frame pointer for perf. Otherwise stack traces are messed up!
|
283 |
# http://www.brendangregg.com/FlameGraphs/cpuflamegraphs.html#C But why
|
284 |
# isn't debuginfo enough? Because it's a recursive function?
|
285 |
# Does this make things slower? Do I need a "perf" build?
|
286 |
build "$@" -O3 -fno-omit-frame-pointer
|
287 |
}
|
288 |
|
289 |
#
|
290 |
# Source Release (uses same files
|
291 |
#
|
292 |
|
293 |
add-py27() {
|
294 |
xargs -I {} -- echo $PY27/{}
|
295 |
}
|
296 |
|
297 |
python-sources() {
|
298 |
echo "$OVM_LIBRARY_OBJS" | add-py27
|
299 |
}
|
300 |
|
301 |
_headers() {
|
302 |
local c_module_srcs=${1:-_tmp/hello/c-module-srcs.txt}
|
303 |
local abs_c_module_srcs=$PWD/$c_module_srcs
|
304 |
|
305 |
cd $PY27
|
306 |
|
307 |
# -MM: no system headers
|
308 |
gcc \
|
309 |
"${INCLUDE_PATHS[@]}" \
|
310 |
"${PREPROC_FLAGS[@]}" \
|
311 |
-MM $OVM_LIBRARY_OBJS \
|
312 |
Modules/ovm.c \
|
313 |
$(cat $abs_c_module_srcs)
|
314 |
}
|
315 |
|
316 |
# NOTE: 91 headers in Include, but only 81 referenced here. So it's worth it.
|
317 |
# These are probably for the parser.
|
318 |
#
|
319 |
# NOTE: We also should get rid of asdl.h and so forth.
|
320 |
|
321 |
python-headers() {
|
322 |
local c_module_srcs=$1
|
323 |
|
324 |
# 1. -MM outputs Makefile fragments, so egrep turns those into proper lines.
|
325 |
#
|
326 |
# 2. The user should generated detected-config.h, so remove it.
|
327 |
#
|
328 |
# 3. # gcc outputs paths like
|
329 |
# Python-2.7.13/Python/../Objects/stringlib/stringdefs.h
|
330 |
# but 'Python/..' causes problems for tar.
|
331 |
#
|
332 |
|
333 |
# NOTE: need .def for build/oil-defs.
|
334 |
_headers $c_module_srcs \
|
335 |
| egrep --only-matching '[^ ]+\.(h|def)' \
|
336 |
| grep -v '_build/detected-config.h' \
|
337 |
| sed 's|^Python/../||' \
|
338 |
| sort | uniq | add-py27
|
339 |
}
|
340 |
|
341 |
make-tar() {
|
342 |
local app_name=${1:-hello}
|
343 |
local bytecode_zip=${2:-bytecode-cpython.zip}
|
344 |
local out=${3:-_release/hello.tar}
|
345 |
|
346 |
local version_file
|
347 |
case $app_name in
|
348 |
oil)
|
349 |
version_file=oil-version.txt
|
350 |
;;
|
351 |
hello)
|
352 |
version_file=build/testdata/hello-version.txt
|
353 |
;;
|
354 |
*)
|
355 |
die "Unknown app $app_name"
|
356 |
exit 1
|
357 |
;;
|
358 |
esac
|
359 |
local version=$(head -n 1 $version_file)
|
360 |
|
361 |
echo "Creating $app_name version $version"
|
362 |
|
363 |
local c_module_srcs=_build/$app_name/c-module-srcs.txt
|
364 |
|
365 |
# Add oil-0.0.0/ to the beginning of every path.
|
366 |
local sed_expr="s,^,${app_name}-${version}/,"
|
367 |
|
368 |
# Differences between tarball and repo:
|
369 |
#
|
370 |
# - build/portable-rules.mk is intentionally not included in the release tarball.
|
371 |
# The Makefile can and should operate without it.
|
372 |
#
|
373 |
# - We include intermediate files like c-module-srcs.txt, so we don't have to
|
374 |
# ship tools dynamic_deps.py. The end-user build shouldn't depend on Python.
|
375 |
|
376 |
# Note: python-headers runs gcc -M, including pyconfig.h and
|
377 |
# _build/detected-config.h.
|
378 |
|
379 |
tar --create --transform "$sed_expr" --file $out \
|
380 |
LICENSE.txt \
|
381 |
INSTALL.txt \
|
382 |
configure \
|
383 |
install \
|
384 |
uninstall \
|
385 |
Makefile \
|
386 |
doc/osh.1 \
|
387 |
build/ovm-compile.sh \
|
388 |
build/ovm-actions.sh \
|
389 |
build/clean.sh \
|
390 |
build/common.sh \
|
391 |
build/detect-*.c \
|
392 |
_build/$app_name/$bytecode_zip \
|
393 |
_build/$app_name/*.c \
|
394 |
py-yajl/yajl/COPYING \
|
395 |
$PY27/LICENSE \
|
396 |
$PY27/Modules/ovm.c \
|
397 |
$c_module_srcs \
|
398 |
$(cat $c_module_srcs | add-py27) \
|
399 |
$(python-headers $c_module_srcs) \
|
400 |
$(python-sources)
|
401 |
|
402 |
ls -l $out
|
403 |
}
|
404 |
|
405 |
# 123K lines.
|
406 |
# Excluding MODOBJS, it's 104K lines.
|
407 |
#
|
408 |
# Biggest: posixmodule,unicodeobject,typeobject,ceval.
|
409 |
#
|
410 |
# Remove tmpnam from posixmodule, other cruft.
|
411 |
#
|
412 |
# Big ones to rid of: unicodeobject.c, import.c
|
413 |
# codecs and codecsmodule? There is some non-unicode stuff there though.
|
414 |
#
|
415 |
# Probably need unicode for compatibility with modules and web frameworks
|
416 |
# especially.
|
417 |
|
418 |
count-c-lines() {
|
419 |
pushd $PY27
|
420 |
wc -l $OVM_LIBRARY_OBJS | sort -n
|
421 |
|
422 |
# 90 files.
|
423 |
# NOTE: To count headers, use the tar file.
|
424 |
echo
|
425 |
echo 'Files:'
|
426 |
{ for i in $OVM_LIBRARY_OBJS; do
|
427 |
echo $i
|
428 |
done
|
429 |
} | wc -l
|
430 |
|
431 |
popd
|
432 |
}
|
433 |
|
434 |
"$@"
|