1 #!/usr/bin/env bash
2 #
3 # Script for contributors to quickly set up core packages
4 #
5 # Usage:
6 # build/deps.sh <function name>
7 #
8 # Examples:
9 # build/deps.sh fetch
10 # build/deps.sh install-wedges
11 # build/deps.sh rm-oils-crap # rm /wedge ~/wedge to start over
12 #
13 # - re2c
14 # - cmark
15 # - python3
16 # - mypy and deps, so mycpp can import htem
17
18 # TODO:
19 # - remove cmark dependency for help. It's still used for docs and benchmarks.
20 # - remove re2c from dev build? Are there any bugs? I think it's just slow.
21 # - add spec-bin so people can always run the tests
22 #
23 # - change Contributing page
24 # - build/deps.sh fetch-py
25 # - build/deps.sh install-wedges-py
26 #
27 # mycpp/README.md:
28 #
29 # - build/deps.sh fetch
30 # - build/deps.sh install-wedges
31 #
32 # Can we make most of them non-root deps?
33
34 set -o nounset
35 set -o pipefail
36 set -o errexit
37
38 source build/dev-shell.sh # python3 in PATH, PY3_LIBS_VERSION
39 source deps/from-apt.sh # PY3_BUILD_DEPS
40 #source deps/podman.sh
41 source devtools/run-task.sh # run-task
42
43 # Also in build/dev-shell.sh
44 USER_WEDGE_DIR=~/wedge/oils-for-unix.org
45
46 readonly DEPS_SOURCE_DIR=_build/deps-source
47
48 readonly RE2C_VERSION=3.0
49 readonly RE2C_URL="https://github.com/skvadrik/re2c/releases/download/$RE2C_VERSION/re2c-$RE2C_VERSION.tar.xz"
50
51 readonly CMARK_VERSION=0.29.0
52 readonly CMARK_URL="https://github.com/commonmark/cmark/archive/$CMARK_VERSION.tar.gz"
53
54 readonly PY2_VERSION=2.7.18
55 readonly PY2_URL="https://www.python.org/ftp/python/2.7.18/Python-$PY2_VERSION.tar.xz"
56
57 readonly PY3_VERSION=3.10.4
58 readonly PY3_URL="https://www.python.org/ftp/python/3.10.4/Python-$PY3_VERSION.tar.xz"
59
60 readonly MYPY_GIT_URL=https://github.com/python/mypy
61 readonly MYPY_VERSION=0.780
62
63 readonly PY3_LIBS=~/wedge/oils-for-unix.org/pkg/py3-libs/$MYPY_VERSION
64
65 # Version 2.4.0 from 2021-10-06 was the last version that supported Python 2
66 # https://github.com/PyCQA/pyflakes/blob/main/NEWS.rst
67 readonly PYFLAKES_VERSION=2.4.0
68 #readonly PYFLAKES_URL='https://files.pythonhosted.org/packages/15/60/c577e54518086e98470e9088278247f4af1d39cb43bcbd731e2c307acd6a/pyflakes-2.4.0.tar.gz'
69 # 2023-07: Mirrored to avoid network problem on broome during release
70 readonly PYFLAKES_URL='https://www.oilshell.org/blob/pyflakes-2.4.0.tar.gz'
71
72 readonly BLOATY_VERSION=1.1
73 readonly BLOATY_URL='https://github.com/google/bloaty/releases/download/v1.1/bloaty-1.1.tar.bz2'
74
75 readonly UFTRACE_VERSION=0.13
76 readonly UFTRACE_URL='https://github.com/namhyung/uftrace/archive/refs/tags/v0.13.tar.gz'
77
78 log() {
79 echo "$0: $@" >& 2
80 }
81
82 die() {
83 log "$@"
84 exit 1
85 }
86
87 rm-oils-crap() {
88 ### When you want to start over
89
90 rm -r -f -v ~/wedge
91 sudo rm -r -f -v /wedge
92 }
93
94
95 install-ubuntu-packages() {
96 ### Packages for build/py.sh all, building wedges, etc.
97
98 # python2-dev is no longer available on Debian 12
99 # python-dev also seems gone
100 #
101 # g++: essential
102 # libreadline-dev: needed for the build/prepare.sh Python build.
103 # gawk: used by spec-runner.sh for the special match() function.
104 # cmake: for build/py.sh yajl-release (TODO: remove eventually)
105
106 set -x # show what needs sudo
107
108 # pass -y for say gitpod
109 sudo apt "$@" install \
110 g++ gawk libreadline-dev ninja-build cmake \
111 "${PY3_BUILD_DEPS[@]}"
112 set +x
113
114 test/spec-bin.sh install-shells-with-apt
115 }
116
117 download-to() {
118 local dir=$1
119 local url=$2
120 wget --no-clobber --directory "$dir" "$url"
121 }
122
123 maybe-extract() {
124 local wedge_dir=$1
125 local tar_name=$2
126 local out_dir=$3
127
128 if test -d "$wedge_dir/$out_dir"; then
129 log "Not extracting because $wedge_dir/$out_dir exists"
130 return
131 fi
132
133 local tar=$wedge_dir/$tar_name
134 case $tar_name in
135 *.gz)
136 flag='--gzip'
137 ;;
138 *.bz2)
139 flag='--bzip2'
140 ;;
141 *.xz)
142 flag='--xz'
143 ;;
144 *)
145 die "tar with unknown extension: $tar_name"
146 ;;
147 esac
148
149 tar --extract $flag --file $tar --directory $wedge_dir
150 }
151
152 clone-mypy() {
153 ### replaces deps/from-git
154 local dest_dir=$1
155 local version=${2:-$MYPY_VERSION}
156
157 local dest=$dest_dir/mypy-$version
158 if test -d $dest; then
159 log "Not cloning because $dest exists"
160 return
161 fi
162
163 # TODO: verify commit checksum
164
165 # v$VERSION is a tag, not a branch
166 git clone --recursive --depth=50 --branch v$version \
167 $MYPY_GIT_URL $dest
168 }
169
170 fetch() {
171 local py_only=${1:-}
172
173 # For now, simulate what 'medo expand deps/source.medo _build/deps-source'
174 # would do: fetch compressed tarballs designated by .treeptr files, and
175 # expand them.
176
177 # _build/deps-source/
178 # re2c/
179 # WEDGE
180 # re2c-3.0/ # expanded .tar.xz file
181
182 mkdir -p $DEPS_SOURCE_DIR
183
184 # Copy the whole tree, including the .treeptr files
185 cp --verbose --recursive --no-target-directory \
186 deps/source.medo/ $DEPS_SOURCE_DIR/
187
188 download-to $DEPS_SOURCE_DIR/re2c "$RE2C_URL"
189 download-to $DEPS_SOURCE_DIR/cmark "$CMARK_URL"
190 maybe-extract $DEPS_SOURCE_DIR/re2c "$(basename $RE2C_URL)" re2c-$RE2C_VERSION
191 maybe-extract $DEPS_SOURCE_DIR/cmark "$(basename $CMARK_URL)" cmark-$CMARK_VERSION
192
193 if test -n "$py_only"; then
194 log "Fetched dependencies for 'build/py.sh'"
195 return
196 fi
197
198 download-to $DEPS_SOURCE_DIR/pyflakes "$PYFLAKES_URL"
199 maybe-extract $DEPS_SOURCE_DIR/pyflakes "$(basename $PYFLAKES_URL)" \
200 pyflakes-$PYFLAKES_VERSION
201
202 download-to $DEPS_SOURCE_DIR/python2 "$PY2_URL"
203 download-to $DEPS_SOURCE_DIR/python3 "$PY3_URL"
204 maybe-extract $DEPS_SOURCE_DIR/python2 "$(basename $PY2_URL)" Python-$PY2_VERSION
205 maybe-extract $DEPS_SOURCE_DIR/python3 "$(basename $PY3_URL)" Python-$PY3_VERSION
206
207 # bloaty and uftrace are for benchmarks, in containers
208 download-to $DEPS_SOURCE_DIR/bloaty "$BLOATY_URL"
209 download-to $DEPS_SOURCE_DIR/uftrace "$UFTRACE_URL"
210 maybe-extract $DEPS_SOURCE_DIR/bloaty "$(basename $BLOATY_URL)" uftrace-$BLOATY_VERSION
211 maybe-extract $DEPS_SOURCE_DIR/uftrace "$(basename $UFTRACE_URL)" bloaty-$UFTRACE_VERSION
212
213 # This is in $DEPS_SOURCE_DIR to COPY into containers, which mycpp will directly import.
214 # It's also copied into a wedge in install-wedges.
215 clone-mypy $DEPS_SOURCE_DIR/mypy
216
217 if command -v tree > /dev/null; then
218 tree -L 2 $DEPS_SOURCE_DIR
219 tree -L 2 $USER_WEDGE_DIR
220 fi
221 }
222
223 mirror-pyflakes() {
224 ### Workaround for network error during release
225 scp \
226 $DEPS_SOURCE_DIR/pyflakes/"$(basename $PYFLAKES_URL)" \
227 oilshell.org:oilshell.org/blob/
228 }
229
230 fetch-py() {
231 fetch py_only
232 }
233
234 mypy-new() {
235 local version=0.971
236 # Do the latest version for Python 2
237 clone-mypy $DEPS_SOURCE_DIR/mypy $version
238
239 local dest_dir=$USER_WEDGE_DIR/pkg/mypy/$version
240 mkdir -p $dest_dir
241
242 cp --verbose --recursive --no-target-directory \
243 $DEPS_SOURCE_DIR/mypy/mypy-$version $dest_dir
244 }
245
246 wedge-exists() {
247 # TODO: Doesn't take into account ~/wedge/ vs. /wedge
248 local installed=/wedge/oils-for-unix.org/pkg/$1/$2
249
250 if test -d $installed; then
251 log "$installed already exists"
252 return 0
253 else
254 return 1
255 fi
256 }
257
258 # TODO: py3-libs needs to be a WEDGE, so that that you can run
259 # 'wedge build deps/source.medo/py3-libs/' and then get it in
260 #
261 # _build/wedge/{absolute,relative} # which one?
262 #
263 # It needs a BUILD DEPENDENCY on:
264
265 # - the python3 wedge, so you can do python3 -m pip install.
266 # - the mypy repo, which has test-requirements.txt
267
268 install-py3-libs-in-venv() {
269 local venv_dir=$1
270 local mypy_dir=$2 # This is a param for host build vs. container build
271
272 source $venv_dir/bin/activate # enter virtualenv
273
274 # 2023-07 bug fix: have to install MyPy deps FIRST, THEN yapf. Otherwise
275 # we get an error from pip:
276 # "ERROR: pip's dependency resolver does not currently take into account all
277 # the packages that are installed."
278
279 # for mycpp/
280 time python3 -m pip install -r $mypy_dir/test-requirements.txt
281
282 # pexpect: for spec/stateful/*.py
283 python3 -m pip install pexpect
284
285 # TODO:
286 # - Do something like this 'pip download' in build/deps.sh fetch
287 # - Then create a WEDGE which installs it
288 # - However note that this is NOT source code; there is binary code, e.g.
289 # in lxml-*.whl
290 if false; then
291 local pip_dir=_tmp/pip
292 mkdir -p $pip_dir
293 python3 -m pip download -d $pip_dir -r $mypy_dir/test-requirements.txt
294 python3 -m pip download -d $pip_dir pexpect
295 fi
296 }
297
298 install-py3-libs() {
299 local mypy_dir=${1:-$DEPS_SOURCE_DIR/mypy/mypy-$MYPY_VERSION}
300
301 local py3
302 py3=$(command -v python3)
303 case $py3 in
304 *wedge/oils-for-unix.org/*)
305 ;;
306 *)
307 die "python3 is '$py3', but expected it to be in a wedge"
308 ;;
309 esac
310
311 log "Ensuring pip is installed (interpreter $(command -v python3)"
312 python3 -m ensurepip
313
314 local venv_dir=$USER_WEDGE_DIR/pkg/py3-libs/$PY3_LIBS_VERSION
315 log "Creating venv in $venv_dir"
316
317 # Note: the bin/python3 in this venv is a symlink to python3 in $PATH, i.e.
318 # the /wedge we just built
319 python3 -m venv $venv_dir
320
321 log "Installing MyPy deps in venv"
322
323 # Run in a subshell because it mutates shell state
324 $0 install-py3-libs-in-venv $venv_dir $mypy_dir
325 }
326
327 install-wedges() {
328 local py_only=${1:-}
329
330 # TODO:
331 # - Make all of these RELATIVE wedges
332 # - Add
333 # - unboxed-rel-smoke-test -- move it inside container
334 # - rel-smoke-test -- mount it in a different location
335 # - Should have a CI task that does all of this!
336
337 if ! wedge-exists cmark 0.29.0; then
338 deps/wedge.sh unboxed-build _build/deps-source/cmark/
339 fi
340
341 if ! wedge-exists re2c 3.0; then
342 deps/wedge.sh unboxed-build _build/deps-source/re2c/
343 fi
344
345 if ! wedge-exists python2 $PY2_VERSION; then
346 deps/wedge.sh unboxed-build _build/deps-source/python2/
347 fi
348
349 if test -n "$py_only"; then
350 log "Installed dependencies for 'build/py.sh'"
351 return
352 fi
353
354 # Just copy this source tarball
355 if ! wedge-exists pyflakes $PYFLAKES_VERSION; then
356 local dest_dir=$USER_WEDGE_DIR/pkg/pyflakes/$PYFLAKES_VERSION
357 mkdir -p $dest_dir
358
359 cp --verbose --recursive --no-target-directory \
360 $DEPS_SOURCE_DIR/pyflakes/pyflakes-$PYFLAKES_VERSION $dest_dir
361 fi
362
363 # TODO: make the Python build faster by using all your cores?
364 if ! wedge-exists python3 $PY3_VERSION; then
365 deps/wedge.sh unboxed-build _build/deps-source/python3/
366 fi
367
368 # Copy all the contents, except for .git folder.
369 if ! wedge-exists mypy $MYPY_VERSION; then
370
371 # NOTE: We have to also copy the .git dir, because it has
372 # .git/modules/typeshed
373
374 local dest_dir=$USER_WEDGE_DIR/pkg/mypy/$MYPY_VERSION
375 mkdir -p $dest_dir
376
377 cp --verbose --recursive --no-target-directory \
378 $DEPS_SOURCE_DIR/mypy/mypy-$MYPY_VERSION $dest_dir
379 fi
380
381 install-py3-libs
382 }
383
384 # Host wedges end up in ~/wedge
385 uftrace-host() {
386 ### built on demand; run $0 first
387
388 # BUG: doesn't detect python3
389 # WEDGE tells me that it depends on pkg-config
390 # 'apt-get install pkgconf' gets it
391 # TODO: Should use python3 WEDGE instead of SYSTEM python3?
392
393 deps/wedge.sh unboxed-build _build/deps-source/uftrace
394 }
395
396 R-libs-host() {
397 deps/wedge.sh unboxed-build _build/deps-source/R-libs
398 }
399
400 bloaty-host() {
401 deps/wedge.sh unboxed-build _build/deps-source/bloaty
402 }
403
404 install-wedges-py() {
405 install-wedges py_only
406 }
407
408 container-wedges() {
409 ### Build wedges that are copied into containers, not run on host
410
411 # These end up in _build/wedge/binary
412
413 #export-podman
414
415 if true; then
416 deps/wedge.sh build deps/source.medo/time-helper
417 deps/wedge.sh build deps/source.medo/cmark/
418 deps/wedge.sh build deps/source.medo/re2c/
419 deps/wedge.sh build deps/source.medo/python3/
420 fi
421
422 if false; then
423 deps/wedge.sh build deps/source.medo/bloaty/
424 deps/wedge.sh build deps/source.medo/uftrace/
425 fi
426
427 if false; then
428 # For soil-benchmarks/ images
429 deps/wedge.sh build deps/source.medo/R-libs/
430 fi
431
432 }
433
434 run-task "$@"