OILS
/
build
/
deps.sh
1 |
#!/usr/bin/env bash
|
2 |
#
|
3 |
# Script for contributors to quickly set up core packages
|
4 |
#
|
5 |
# Usage:
|
6 |
# build/deps.sh <function name>
|
7 |
#
|
8 |
# Examples:
|
9 |
# build/deps.sh fetch
|
10 |
# build/deps.sh install-wedges
|
11 |
# build/deps.sh rm-oils-crap # rm /wedge ~/wedge to start over
|
12 |
#
|
13 |
# - re2c
|
14 |
# - cmark
|
15 |
# - python3
|
16 |
# - mypy and deps, so mycpp can import htem
|
17 |
|
18 |
# TODO:
|
19 |
# - remove cmark dependency for help. It's still used for docs and benchmarks.
|
20 |
# - remove re2c from dev build? Are there any bugs? I think it's just slow.
|
21 |
# - add spec-bin so people can always run the tests
|
22 |
#
|
23 |
# - change Contributing page
|
24 |
# - build/deps.sh fetch-py
|
25 |
# - build/deps.sh install-wedges-py
|
26 |
#
|
27 |
# mycpp/README.md:
|
28 |
#
|
29 |
# - build/deps.sh fetch
|
30 |
# - build/deps.sh install-wedges
|
31 |
#
|
32 |
# Can we make most of them non-root deps?
|
33 |
|
34 |
set -o nounset
|
35 |
set -o pipefail
|
36 |
set -o errexit
|
37 |
|
38 |
source build/dev-shell.sh # python3 in PATH, PY3_LIBS_VERSION
|
39 |
source deps/from-apt.sh # PY3_BUILD_DEPS
|
40 |
#source deps/podman.sh
|
41 |
source devtools/run-task.sh # run-task
|
42 |
|
43 |
# Also in build/dev-shell.sh
|
44 |
USER_WEDGE_DIR=~/wedge/oils-for-unix.org
|
45 |
|
46 |
readonly DEPS_SOURCE_DIR=_build/deps-source
|
47 |
|
48 |
readonly RE2C_VERSION=3.0
|
49 |
readonly RE2C_URL="https://github.com/skvadrik/re2c/releases/download/$RE2C_VERSION/re2c-$RE2C_VERSION.tar.xz"
|
50 |
|
51 |
readonly CMARK_VERSION=0.29.0
|
52 |
readonly CMARK_URL="https://github.com/commonmark/cmark/archive/$CMARK_VERSION.tar.gz"
|
53 |
|
54 |
readonly PY2_VERSION=2.7.18
|
55 |
readonly PY2_URL="https://www.python.org/ftp/python/2.7.18/Python-$PY2_VERSION.tar.xz"
|
56 |
|
57 |
readonly PY3_VERSION=3.10.4
|
58 |
readonly PY3_URL="https://www.python.org/ftp/python/3.10.4/Python-$PY3_VERSION.tar.xz"
|
59 |
|
60 |
readonly MYPY_GIT_URL=https://github.com/python/mypy
|
61 |
readonly MYPY_VERSION=0.780
|
62 |
|
63 |
readonly PY3_LIBS=~/wedge/oils-for-unix.org/pkg/py3-libs/$MYPY_VERSION
|
64 |
|
65 |
# Version 2.4.0 from 2021-10-06 was the last version that supported Python 2
|
66 |
# https://github.com/PyCQA/pyflakes/blob/main/NEWS.rst
|
67 |
readonly PYFLAKES_VERSION=2.4.0
|
68 |
#readonly PYFLAKES_URL='https://files.pythonhosted.org/packages/15/60/c577e54518086e98470e9088278247f4af1d39cb43bcbd731e2c307acd6a/pyflakes-2.4.0.tar.gz'
|
69 |
# 2023-07: Mirrored to avoid network problem on broome during release
|
70 |
readonly PYFLAKES_URL='https://www.oilshell.org/blob/pyflakes-2.4.0.tar.gz'
|
71 |
|
72 |
readonly BLOATY_VERSION=1.1
|
73 |
readonly BLOATY_URL='https://github.com/google/bloaty/releases/download/v1.1/bloaty-1.1.tar.bz2'
|
74 |
|
75 |
readonly UFTRACE_VERSION=0.13
|
76 |
readonly UFTRACE_URL='https://github.com/namhyung/uftrace/archive/refs/tags/v0.13.tar.gz'
|
77 |
|
78 |
log() {
|
79 |
echo "$0: $@" >& 2
|
80 |
}
|
81 |
|
82 |
die() {
|
83 |
log "$@"
|
84 |
exit 1
|
85 |
}
|
86 |
|
87 |
rm-oils-crap() {
|
88 |
### When you want to start over
|
89 |
|
90 |
rm -r -f -v ~/wedge
|
91 |
sudo rm -r -f -v /wedge
|
92 |
}
|
93 |
|
94 |
|
95 |
install-ubuntu-packages() {
|
96 |
### Packages for build/py.sh all, building wedges, etc.
|
97 |
|
98 |
# python2-dev is no longer available on Debian 12
|
99 |
# python-dev also seems gone
|
100 |
#
|
101 |
# g++: essential
|
102 |
# libreadline-dev: needed for the build/prepare.sh Python build.
|
103 |
# gawk: used by spec-runner.sh for the special match() function.
|
104 |
# cmake: for build/py.sh yajl-release (TODO: remove eventually)
|
105 |
|
106 |
set -x # show what needs sudo
|
107 |
|
108 |
# pass -y for say gitpod
|
109 |
sudo apt "$@" install \
|
110 |
g++ gawk libreadline-dev ninja-build cmake \
|
111 |
"${PY3_BUILD_DEPS[@]}"
|
112 |
set +x
|
113 |
|
114 |
test/spec-bin.sh install-shells-with-apt
|
115 |
}
|
116 |
|
117 |
download-to() {
|
118 |
local dir=$1
|
119 |
local url=$2
|
120 |
wget --no-clobber --directory "$dir" "$url"
|
121 |
}
|
122 |
|
123 |
maybe-extract() {
|
124 |
local wedge_dir=$1
|
125 |
local tar_name=$2
|
126 |
local out_dir=$3
|
127 |
|
128 |
if test -d "$wedge_dir/$out_dir"; then
|
129 |
log "Not extracting because $wedge_dir/$out_dir exists"
|
130 |
return
|
131 |
fi
|
132 |
|
133 |
local tar=$wedge_dir/$tar_name
|
134 |
case $tar_name in
|
135 |
*.gz)
|
136 |
flag='--gzip'
|
137 |
;;
|
138 |
*.bz2)
|
139 |
flag='--bzip2'
|
140 |
;;
|
141 |
*.xz)
|
142 |
flag='--xz'
|
143 |
;;
|
144 |
*)
|
145 |
die "tar with unknown extension: $tar_name"
|
146 |
;;
|
147 |
esac
|
148 |
|
149 |
tar --extract $flag --file $tar --directory $wedge_dir
|
150 |
}
|
151 |
|
152 |
clone-mypy() {
|
153 |
### replaces deps/from-git
|
154 |
local dest_dir=$1
|
155 |
local version=${2:-$MYPY_VERSION}
|
156 |
|
157 |
local dest=$dest_dir/mypy-$version
|
158 |
if test -d $dest; then
|
159 |
log "Not cloning because $dest exists"
|
160 |
return
|
161 |
fi
|
162 |
|
163 |
# TODO: verify commit checksum
|
164 |
|
165 |
# v$VERSION is a tag, not a branch
|
166 |
git clone --recursive --depth=50 --branch v$version \
|
167 |
$MYPY_GIT_URL $dest
|
168 |
}
|
169 |
|
170 |
fetch() {
|
171 |
local py_only=${1:-}
|
172 |
|
173 |
# For now, simulate what 'medo expand deps/source.medo _build/deps-source'
|
174 |
# would do: fetch compressed tarballs designated by .treeptr files, and
|
175 |
# expand them.
|
176 |
|
177 |
# _build/deps-source/
|
178 |
# re2c/
|
179 |
# WEDGE
|
180 |
# re2c-3.0/ # expanded .tar.xz file
|
181 |
|
182 |
mkdir -p $DEPS_SOURCE_DIR
|
183 |
|
184 |
# Copy the whole tree, including the .treeptr files
|
185 |
cp --verbose --recursive --no-target-directory \
|
186 |
deps/source.medo/ $DEPS_SOURCE_DIR/
|
187 |
|
188 |
download-to $DEPS_SOURCE_DIR/re2c "$RE2C_URL"
|
189 |
download-to $DEPS_SOURCE_DIR/cmark "$CMARK_URL"
|
190 |
maybe-extract $DEPS_SOURCE_DIR/re2c "$(basename $RE2C_URL)" re2c-$RE2C_VERSION
|
191 |
maybe-extract $DEPS_SOURCE_DIR/cmark "$(basename $CMARK_URL)" cmark-$CMARK_VERSION
|
192 |
|
193 |
if test -n "$py_only"; then
|
194 |
log "Fetched dependencies for 'build/py.sh'"
|
195 |
return
|
196 |
fi
|
197 |
|
198 |
download-to $DEPS_SOURCE_DIR/pyflakes "$PYFLAKES_URL"
|
199 |
maybe-extract $DEPS_SOURCE_DIR/pyflakes "$(basename $PYFLAKES_URL)" \
|
200 |
pyflakes-$PYFLAKES_VERSION
|
201 |
|
202 |
download-to $DEPS_SOURCE_DIR/python2 "$PY2_URL"
|
203 |
download-to $DEPS_SOURCE_DIR/python3 "$PY3_URL"
|
204 |
maybe-extract $DEPS_SOURCE_DIR/python2 "$(basename $PY2_URL)" Python-$PY2_VERSION
|
205 |
maybe-extract $DEPS_SOURCE_DIR/python3 "$(basename $PY3_URL)" Python-$PY3_VERSION
|
206 |
|
207 |
# bloaty and uftrace are for benchmarks, in containers
|
208 |
download-to $DEPS_SOURCE_DIR/bloaty "$BLOATY_URL"
|
209 |
download-to $DEPS_SOURCE_DIR/uftrace "$UFTRACE_URL"
|
210 |
maybe-extract $DEPS_SOURCE_DIR/bloaty "$(basename $BLOATY_URL)" uftrace-$BLOATY_VERSION
|
211 |
maybe-extract $DEPS_SOURCE_DIR/uftrace "$(basename $UFTRACE_URL)" bloaty-$UFTRACE_VERSION
|
212 |
|
213 |
# This is in $DEPS_SOURCE_DIR to COPY into containers, which mycpp will directly import.
|
214 |
# It's also copied into a wedge in install-wedges.
|
215 |
clone-mypy $DEPS_SOURCE_DIR/mypy
|
216 |
|
217 |
if command -v tree > /dev/null; then
|
218 |
tree -L 2 $DEPS_SOURCE_DIR
|
219 |
tree -L 2 $USER_WEDGE_DIR
|
220 |
fi
|
221 |
}
|
222 |
|
223 |
mirror-pyflakes() {
|
224 |
### Workaround for network error during release
|
225 |
scp \
|
226 |
$DEPS_SOURCE_DIR/pyflakes/"$(basename $PYFLAKES_URL)" \
|
227 |
oilshell.org:oilshell.org/blob/
|
228 |
}
|
229 |
|
230 |
fetch-py() {
|
231 |
fetch py_only
|
232 |
}
|
233 |
|
234 |
mypy-new() {
|
235 |
local version=0.971
|
236 |
# Do the latest version for Python 2
|
237 |
clone-mypy $DEPS_SOURCE_DIR/mypy $version
|
238 |
|
239 |
local dest_dir=$USER_WEDGE_DIR/pkg/mypy/$version
|
240 |
mkdir -p $dest_dir
|
241 |
|
242 |
cp --verbose --recursive --no-target-directory \
|
243 |
$DEPS_SOURCE_DIR/mypy/mypy-$version $dest_dir
|
244 |
}
|
245 |
|
246 |
wedge-exists() {
|
247 |
# TODO: Doesn't take into account ~/wedge/ vs. /wedge
|
248 |
local installed=/wedge/oils-for-unix.org/pkg/$1/$2
|
249 |
|
250 |
if test -d $installed; then
|
251 |
log "$installed already exists"
|
252 |
return 0
|
253 |
else
|
254 |
return 1
|
255 |
fi
|
256 |
}
|
257 |
|
258 |
# TODO: py3-libs needs to be a WEDGE, so that that you can run
|
259 |
# 'wedge build deps/source.medo/py3-libs/' and then get it in
|
260 |
#
|
261 |
# _build/wedge/{absolute,relative} # which one?
|
262 |
#
|
263 |
# It needs a BUILD DEPENDENCY on:
|
264 |
|
265 |
# - the python3 wedge, so you can do python3 -m pip install.
|
266 |
# - the mypy repo, which has test-requirements.txt
|
267 |
|
268 |
install-py3-libs-in-venv() {
|
269 |
local venv_dir=$1
|
270 |
local mypy_dir=$2 # This is a param for host build vs. container build
|
271 |
|
272 |
source $venv_dir/bin/activate # enter virtualenv
|
273 |
|
274 |
# 2023-07 bug fix: have to install MyPy deps FIRST, THEN yapf. Otherwise
|
275 |
# we get an error from pip:
|
276 |
# "ERROR: pip's dependency resolver does not currently take into account all
|
277 |
# the packages that are installed."
|
278 |
|
279 |
# for mycpp/
|
280 |
time python3 -m pip install -r $mypy_dir/test-requirements.txt
|
281 |
|
282 |
# pexpect: for spec/stateful/*.py
|
283 |
python3 -m pip install pexpect
|
284 |
|
285 |
# TODO:
|
286 |
# - Do something like this 'pip download' in build/deps.sh fetch
|
287 |
# - Then create a WEDGE which installs it
|
288 |
# - However note that this is NOT source code; there is binary code, e.g.
|
289 |
# in lxml-*.whl
|
290 |
if false; then
|
291 |
local pip_dir=_tmp/pip
|
292 |
mkdir -p $pip_dir
|
293 |
python3 -m pip download -d $pip_dir -r $mypy_dir/test-requirements.txt
|
294 |
python3 -m pip download -d $pip_dir pexpect
|
295 |
fi
|
296 |
}
|
297 |
|
298 |
install-py3-libs() {
|
299 |
local mypy_dir=${1:-$DEPS_SOURCE_DIR/mypy/mypy-$MYPY_VERSION}
|
300 |
|
301 |
local py3
|
302 |
py3=$(command -v python3)
|
303 |
case $py3 in
|
304 |
*wedge/oils-for-unix.org/*)
|
305 |
;;
|
306 |
*)
|
307 |
die "python3 is '$py3', but expected it to be in a wedge"
|
308 |
;;
|
309 |
esac
|
310 |
|
311 |
log "Ensuring pip is installed (interpreter $(command -v python3)"
|
312 |
python3 -m ensurepip
|
313 |
|
314 |
local venv_dir=$USER_WEDGE_DIR/pkg/py3-libs/$PY3_LIBS_VERSION
|
315 |
log "Creating venv in $venv_dir"
|
316 |
|
317 |
# Note: the bin/python3 in this venv is a symlink to python3 in $PATH, i.e.
|
318 |
# the /wedge we just built
|
319 |
python3 -m venv $venv_dir
|
320 |
|
321 |
log "Installing MyPy deps in venv"
|
322 |
|
323 |
# Run in a subshell because it mutates shell state
|
324 |
$0 install-py3-libs-in-venv $venv_dir $mypy_dir
|
325 |
}
|
326 |
|
327 |
install-wedges() {
|
328 |
local py_only=${1:-}
|
329 |
|
330 |
# TODO:
|
331 |
# - Make all of these RELATIVE wedges
|
332 |
# - Add
|
333 |
# - unboxed-rel-smoke-test -- move it inside container
|
334 |
# - rel-smoke-test -- mount it in a different location
|
335 |
# - Should have a CI task that does all of this!
|
336 |
|
337 |
if ! wedge-exists cmark 0.29.0; then
|
338 |
deps/wedge.sh unboxed-build _build/deps-source/cmark/
|
339 |
fi
|
340 |
|
341 |
if ! wedge-exists re2c 3.0; then
|
342 |
deps/wedge.sh unboxed-build _build/deps-source/re2c/
|
343 |
fi
|
344 |
|
345 |
if ! wedge-exists python2 $PY2_VERSION; then
|
346 |
deps/wedge.sh unboxed-build _build/deps-source/python2/
|
347 |
fi
|
348 |
|
349 |
if test -n "$py_only"; then
|
350 |
log "Installed dependencies for 'build/py.sh'"
|
351 |
return
|
352 |
fi
|
353 |
|
354 |
# Just copy this source tarball
|
355 |
if ! wedge-exists pyflakes $PYFLAKES_VERSION; then
|
356 |
local dest_dir=$USER_WEDGE_DIR/pkg/pyflakes/$PYFLAKES_VERSION
|
357 |
mkdir -p $dest_dir
|
358 |
|
359 |
cp --verbose --recursive --no-target-directory \
|
360 |
$DEPS_SOURCE_DIR/pyflakes/pyflakes-$PYFLAKES_VERSION $dest_dir
|
361 |
fi
|
362 |
|
363 |
# TODO: make the Python build faster by using all your cores?
|
364 |
if ! wedge-exists python3 $PY3_VERSION; then
|
365 |
deps/wedge.sh unboxed-build _build/deps-source/python3/
|
366 |
fi
|
367 |
|
368 |
# Copy all the contents, except for .git folder.
|
369 |
if ! wedge-exists mypy $MYPY_VERSION; then
|
370 |
|
371 |
# NOTE: We have to also copy the .git dir, because it has
|
372 |
# .git/modules/typeshed
|
373 |
|
374 |
local dest_dir=$USER_WEDGE_DIR/pkg/mypy/$MYPY_VERSION
|
375 |
mkdir -p $dest_dir
|
376 |
|
377 |
cp --verbose --recursive --no-target-directory \
|
378 |
$DEPS_SOURCE_DIR/mypy/mypy-$MYPY_VERSION $dest_dir
|
379 |
fi
|
380 |
|
381 |
install-py3-libs
|
382 |
}
|
383 |
|
384 |
# Host wedges end up in ~/wedge
|
385 |
uftrace-host() {
|
386 |
### built on demand; run $0 first
|
387 |
|
388 |
# BUG: doesn't detect python3
|
389 |
# WEDGE tells me that it depends on pkg-config
|
390 |
# 'apt-get install pkgconf' gets it
|
391 |
# TODO: Should use python3 WEDGE instead of SYSTEM python3?
|
392 |
|
393 |
deps/wedge.sh unboxed-build _build/deps-source/uftrace
|
394 |
}
|
395 |
|
396 |
R-libs-host() {
|
397 |
deps/wedge.sh unboxed-build _build/deps-source/R-libs
|
398 |
}
|
399 |
|
400 |
bloaty-host() {
|
401 |
deps/wedge.sh unboxed-build _build/deps-source/bloaty
|
402 |
}
|
403 |
|
404 |
install-wedges-py() {
|
405 |
install-wedges py_only
|
406 |
}
|
407 |
|
408 |
container-wedges() {
|
409 |
### Build wedges that are copied into containers, not run on host
|
410 |
|
411 |
# These end up in _build/wedge/binary
|
412 |
|
413 |
#export-podman
|
414 |
|
415 |
if true; then
|
416 |
deps/wedge.sh build deps/source.medo/time-helper
|
417 |
deps/wedge.sh build deps/source.medo/cmark/
|
418 |
deps/wedge.sh build deps/source.medo/re2c/
|
419 |
deps/wedge.sh build deps/source.medo/python3/
|
420 |
fi
|
421 |
|
422 |
if false; then
|
423 |
deps/wedge.sh build deps/source.medo/bloaty/
|
424 |
deps/wedge.sh build deps/source.medo/uftrace/
|
425 |
fi
|
426 |
|
427 |
if false; then
|
428 |
# For soil-benchmarks/ images
|
429 |
deps/wedge.sh build deps/source.medo/R-libs/
|
430 |
fi
|
431 |
|
432 |
}
|
433 |
|
434 |
run-task "$@"
|