diff --git a/.dockerignore b/.dockerignore index a3d0f33..04830a9 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,6 @@ cpython .git temp -/src/out \ No newline at end of file +/src/out +/builds/ +/ssl/ diff --git a/.gitignore b/.gitignore index ca59704..573e2b0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ -/temp -/src/out/ \ No newline at end of file +/src/out/ +/builds/ diff --git a/README.md b/README.md index 2ddea94..6f83748 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,50 @@ extpython ========= -Isolated python installations for Ubuntu +Isolated python .deb builder Extpython is a collection of tools for building installable packages containing various versions of Python. The installations are isolated from the operating system's python such that modules may be added, removed, or upgraded without risk of breaking. The builds are executed in Docker. -Example invocation: +*Example invocation:* ``` -./images.sh -./build.sh trusty-3.7.3.sh +./images.sh # pulls / creates base images with dependencies +./build.sh trusty-3.9.5.sh # build using the recipe in the named script file ``` -The vars file - `trusty-3.7.3.sh` above - contains information about what python version will be built. See -`src/bionic-3.7.3.sh` for an annotated example. +The vars file - `trusty-3.9.5.sh` above - contains information about what python version will be built. See +`src/bionic-3.9.5.sh` for an annotated example. -Tips: +Version support matrix +---------------------- -- Python >=3.8 won't build on trusty because Openssl is too old +| Ubuntu | P 2.7 | Python 3.6 | P 3.7 | Python 3.8 | P 3.9 | +|--------|--------|------------------|-------|----------------------|--------| +| 14.04 | | 3.6.7
3.6.8 | | | 3.9.5* | +| 16.04 | | 3.6.7 | 3.7.3 | | 3.9.5 | +| 18.04 | 2.7.16 | 3.6.7 | 3.7.3 | 3.8.0a4
3.8.0b1 | 3.9.5 | +| 19.04 | | | 3.7.3 | | | +| 20.04 | | | | | 3.9.5 | + +\* with extra extpython-openssl-1-1 package - see below -TODO: +Openssl +------- -- Parameter for pre-loaded pip modules +Python 3.8 and 3.9 require a version of openssl that is newer than what ships on Ubuntu 14.04. This will likely happen +with other os/version combinations in the future, but this is the only known case now. + +To satisfy this requirement for Python 3.9, extpython also builds an Openssl package that installs in a similar +isolated manner. + +See `./ssl/` for details. + + +TODO +---- + +* Parameter for pre-loaded pip modules diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh index 77c3b19..3efe7f6 100755 --- a/scripts/bootstrap.sh +++ b/scripts/bootstrap.sh @@ -5,6 +5,12 @@ ln -s /src/$VARSFILE /tmp/vars.sh . /tmp/vars.sh . /etc/lsb-release #for DISTRIB_CODENAME +if [ -v INSIDE_SCRIPT ] ; then + echo "$INSIDE_SCRIPT" > /tmp/inside.sh + cat /tmp/inside.sh + bash -x /tmp/inside.sh +fi + # run the build sudo -Hu builder /build.sh diff --git a/scripts/build.sh b/scripts/build.sh index 83ecd84..5ab4e60 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -21,10 +21,15 @@ export BUILD_DATE=$(date -R) template /src/debian/changelog | tee debian/changelog template /src/debian/control | tee debian/control cp /src/debian/compat debian/ -cp /src/debian/rules debian/ cp /src/Makefile ./ sed -i -E "s/_OPT_DIR_/${PYTHON_MAJOR}/" Makefile +if [ -v RULESFILE ]; then + cp /src/debian/$RULESFILE debian/rules +else + cp /src/debian/rules debian/ +fi + # build the deb time dpkg-buildpackage -us -uc -b diff --git a/src/Makefile b/src/Makefile index eb38b3d..aa5ab51 100644 --- a/src/Makefile +++ b/src/Makefile @@ -1,7 +1,7 @@ #!/usr/bin/make -f PY3_SRC=./cpython/ -PY3_CONFFLAGS=--prefix=$(DESTDIR)/opt/extpython/_OPT_DIR_ --with-ensurepip=install +PY3_CONFFLAGS=--prefix=$(DESTDIR)/opt/extpython/_OPT_DIR_ --with-ensurepip=install ${EXTRA_PY_CONFIGURE_FLAGS} # --enable-optimizations all: @@ -11,7 +11,7 @@ all: install: cd $(PY3_SRC) && \ - make install + make install -j8 # $(DESTDIR)/opt/extpython/3.7.0/bin/python3 $(DESTDIR)/opt/extpython/3.7/bin/pip3 install -r requirements.txt clean: diff --git a/src/bionic-3.9.5.sh b/src/bionic-3.9.5.sh new file mode 100644 index 0000000..76bd7cf --- /dev/null +++ b/src/bionic-3.9.5.sh @@ -0,0 +1,10 @@ +#!/bin/bash -eux + +export EXTPYTHON_DIST=bionic +export PYTHON_RELEASE=3.9.5 +export PYTHON_MAJOR=3.9 +export PYTHON_TARBALL="http://artifact.scc.net.davepedu.com/repo/tar/cpython/cpython/cpython-${PYTHON_RELEASE}.tar.gz" +export CPYTHON_DEPS="libssl-dev zlib1g-dev libncurses5-dev libncursesw5-dev \ + libreadline-dev libsqlite3-dev libgdbm-dev libdb5.3-dev libbz2-dev libexpat1-dev liblzma-dev tk-dev libffi-dev uuid-dev" + +export EXTRA_PY_CONFIGURE_FLAGS=--enable-optimizations diff --git a/src/debian/rules.trusty-3.9.5 b/src/debian/rules.trusty-3.9.5 new file mode 100644 index 0000000..7a0452a --- /dev/null +++ b/src/debian/rules.trusty-3.9.5 @@ -0,0 +1,13 @@ +#!/usr/bin/make -f +# -*- makefile -*- + +export DH_VERBOSE=1 + +override_dh_usrlocal: +override_dh_strip: + +override_dh_shlibdeps: + dh_shlibdeps -l /opt/extpython/ssl/1.1.1k/lib + +%: + dh $@ diff --git a/src/focal-3.9.5.sh b/src/focal-3.9.5.sh new file mode 100644 index 0000000..9c8cd2e --- /dev/null +++ b/src/focal-3.9.5.sh @@ -0,0 +1,10 @@ +#!/bin/bash -eux + +export EXTPYTHON_DIST=focal +export PYTHON_RELEASE=3.9.5 +export PYTHON_MAJOR=3.9 +export PYTHON_TARBALL="http://artifact.scc.net.davepedu.com/repo/tar/cpython/cpython/cpython-${PYTHON_RELEASE}.tar.gz" +export CPYTHON_DEPS="libssl-dev zlib1g-dev libncurses5-dev libncursesw5-dev \ + libreadline-dev libsqlite3-dev libgdbm-dev libdb5.3-dev libbz2-dev libexpat1-dev liblzma-dev tk-dev libffi-dev uuid-dev" + +export EXTRA_PY_CONFIGURE_FLAGS=--enable-optimizations diff --git a/src/trusty-3.6.8.sh b/src/trusty-3.6.8.sh index 8e2322f..e6c5f10 100644 --- a/src/trusty-3.6.8.sh +++ b/src/trusty-3.6.8.sh @@ -1,12 +1,8 @@ #!/bin/bash -eux export EXTPYTHON_DIST=trusty -# Release is used to select which tarball to download export PYTHON_RELEASE=3.6.8 -# Major version no is used in paths and labels (extpython-python${}, /opt/extpython/${}/...) export PYTHON_MAJOR=3.6 -# Tarball download URL export PYTHON_TARBALL="http://artifact.scc.net.davepedu.com/repo/tar/cpython/cpython/cpython-${PYTHON_RELEASE}.tar.gz" -# Deps needed to build this package export CPYTHON_DEPS="libssl-dev zlib1g-dev libncurses5-dev libncursesw5-dev \ libreadline-dev libsqlite3-dev libgdbm-dev libdb5.3-dev libbz2-dev libexpat1-dev liblzma-dev tk-dev libffi-dev" diff --git a/src/trusty-3.9.5.sh b/src/trusty-3.9.5.sh new file mode 100644 index 0000000..0af62dc --- /dev/null +++ b/src/trusty-3.9.5.sh @@ -0,0 +1,34 @@ +#!/bin/bash -eux + +# Dist chooses which ubuntu base image we build in +export EXTPYTHON_DIST=trusty + +# Release is used to select which tarball to download +export PYTHON_RELEASE=3.9.5 + +# Major version no is used in paths and labels (extpython-python${}, /opt/extpython/${}/...) +export PYTHON_MAJOR=3.9 + +# Tarball download URL +export PYTHON_TARBALL="http://artifact.scc.net.davepedu.com/repo/tar/cpython/cpython/cpython-${PYTHON_RELEASE}.tar.gz" + +# Dependencies needed to build cpython +export CPYTHON_DEPS="libssl-dev zlib1g-dev libncurses5-dev libncursesw5-dev \ + libreadline-dev libsqlite3-dev libgdbm-dev libdb5.3-dev libbz2-dev libexpat1-dev liblzma-dev tk-dev libffi-dev uuid-dev" + +# Optional script to run in the build container early in the build process +export INSIDE_SCRIPT='#!/bin/bash -eux + wget -q -O- http://artifact.scc.net.davepedu.com/repo/apt/extpython/dists/trusty/install | bash -x /dev/stdin + apt-get update -qq + apt-get install -y extpython-openssl-1-1 +' + +# Extra args for python's configure script +export EXTRA_PY_CONFIGURE_FLAGS=--with-openssl=/opt/extpython/ssl/1.1.1k + +# Name of the rules makefile used to build the deb package +export RULESFILE=rules.trusty-3.9.5 + +# Other arbitrary environment variables +# trusty's native openssl is too old, so we ship a newer version. Embed the library path +export LDFLAGS=-Wl,-rpath=/opt/extpython/ssl/1.1.1k/lib diff --git a/src/xenial-3.9.5.sh b/src/xenial-3.9.5.sh new file mode 100644 index 0000000..2b6026d --- /dev/null +++ b/src/xenial-3.9.5.sh @@ -0,0 +1,10 @@ +#!/bin/bash -eux + +export EXTPYTHON_DIST=xenial +export PYTHON_RELEASE=3.9.5 +export PYTHON_MAJOR=3.9 +export PYTHON_TARBALL="http://artifact.scc.net.davepedu.com/repo/tar/cpython/cpython/cpython-${PYTHON_RELEASE}.tar.gz" +export CPYTHON_DEPS="libssl-dev zlib1g-dev libncurses5-dev libncursesw5-dev \ + libreadline-dev libsqlite3-dev libgdbm-dev libdb5.3-dev libbz2-dev libexpat1-dev liblzma-dev tk-dev libffi-dev uuid-dev" + +export EXTRA_PY_CONFIGURE_FLAGS=--enable-optimizations