diff --git a/bin/compile b/bin/compile index f173fdee5f907e6126bf4b3f0d6f299afea8e2cb..d97af680d5428c3abb0d5f72444608a72df2248e 100755 --- a/bin/compile +++ b/bin/compile @@ -160,6 +160,8 @@ cp -R "$CACHE_DIR/.heroku/python" .heroku/ &> /dev/null || true cp -R "$CACHE_DIR/.heroku/python-stack" .heroku/ &> /dev/null || true # A plain text file which contains the current python version being used (used for cache busting). cp -R "$CACHE_DIR/.heroku/python-version" .heroku/ &> /dev/null || true +# A plain text file which contains the current sqlite3 version being used (used for cache busting). +cp -R "$CACHE_DIR/.heroku/python-sqlite3-version" .heroku/ &> /dev/null || true # Any pre-compiled binaries, provided by the buildpack. cp -R "$CACHE_DIR/.heroku/vendor" .heroku/ &> /dev/null || true # "editable" installations of code repositories, via pip or pipenv. @@ -274,6 +276,16 @@ sub_env "$BIN_DIR/steps/geo-libs" # shellcheck source=bin/steps/gdal source "$BIN_DIR/steps/gdal" +# SQLite3 support. +# This sets up and installs sqlite3 dev headers and the sqlite3 binary but not the +# libsqlite3-0 library since that exists on the stack image. +# Note: This only applies to Python 2.7.15+ and Python 3.6.6+ +(( start=$(nowms) )) +# shellcheck source=bin/steps/sqlite3 +source "$BIN_DIR/steps/sqlite3" +buildpack_sqlite3_install +mtime "sqlite3.install.time" "${start}" + # pip install # ----------- diff --git a/bin/steps/python b/bin/steps/python index 58d60436cadd212d86c8d3ccf7cfce2a7557b87a..e66a05dd7b48524ba0a6f6e7258cf0bb73aabfb9 100755 --- a/bin/steps/python +++ b/bin/steps/python @@ -29,7 +29,15 @@ fi if [[ "$STACK" != "$CACHED_PYTHON_STACK" ]]; then puts-step "Stack has changed from $CACHED_PYTHON_STACK to $STACK, clearing cache" - rm -fr .heroku/python-stack .heroku/python-version .heroku/python .heroku/vendor + rm -fr .heroku/python-stack .heroku/python-version .heroku/python .heroku/vendor .heroku/python .heroku/python-sqlite3-version +fi + +# need to clear the cache for first time installing SQLite3, +# since the version is changing and could lead to runtime errors +# with compiled extensions. +if [ -d .heroku/python ] && [ ! -f .heroku/python-sqlite3-version ] && python_sqlite3_check "$PYTHON_VERSION"; then + puts-step "Need to update SQLite3, clearing cache" + rm -fr .heroku/python-stack .heroku/python-version .heroku/python .heroku/vendor fi if [ -f .heroku/python-version ]; then diff --git a/bin/steps/sqlite3 b/bin/steps/sqlite3 new file mode 100644 index 0000000000000000000000000000000000000000..8855f30a19b09baa612df8f34ea0311e249484be --- /dev/null +++ b/bin/steps/sqlite3 @@ -0,0 +1,87 @@ +#!/usr/bin/env bash + +# shellcheck source=bin/utils +source "$BIN_DIR/utils" + +sqlite3_version() { + SQLITE3_VERSION=${SQLITE3_VERSION:-$(dpkg -s libsqlite3-0 | grep Version | sed 's/Version: //')} + + export SQLITE3_VERSION +} + +sqlite3_install() { + HEROKU_PYTHON_DIR="$1" + SQLITE3_VERSION="$2" + HEADERS_ONLY="$3" + + mkdir -p "$HEROKU_PYTHON_DIR" + + APT_CACHE_DIR="$HEROKU_PYTHON_DIR/apt/cache" + APT_STATE_DIR="$HEROKU_PYTHON_DIR/apt/state" + + mkdir -p "$APT_CACHE_DIR/archives/partial" + mkdir -p "$APT_STATE_DIR/lists/partial" + + APT_OPTIONS="-o debug::nolocking=true" + APT_OPTIONS="$APT_OPTIONS -o dir::cache=$APT_CACHE_DIR" + APT_OPTIONS="$APT_OPTIONS -o dir::state=$APT_STATE_DIR" + APT_OPTIONS="$APT_OPTIONS -o dir::etc::sourcelist=/etc/apt/sources.list" + + apt-get $APT_OPTIONS update > /dev/null 2>&1 + if [ -z "$HEADERS_ONLY" ]; then + apt-get $APT_OPTIONS -y -d --reinstall install libsqlite3-dev="$SQLITE3_VERSION" sqlite3="$SQLITE3_VERSION" > /dev/null 2>&1 + else + apt-get $APT_OPTIONS -y -d --reinstall install libsqlite3-dev="$SQLITE3_VERSION" + fi + + find "$APT_CACHE_DIR/archives/" -name "*.deb" -exec dpkg -x {} "$HEROKU_PYTHON_DIR/sqlite3/" \; + + mkdir -p "$HEROKU_PYTHON_DIR/include" + mkdir -p "$HEROKU_PYTHON_DIR/lib" + + # remove old sqlite3 libraries/binaries + find "$HEROKU_PYTHON_DIR/include/" -name "sqlite3*.h" -exec rm -f {} \; + find "$HEROKU_PYTHON_DIR/lib/" -name "libsqlite3.*" -exec rm -f {} \; + rm -f "$HEROKU_PYTHON_DIR/lib/pkgconfig/sqlite3.pc" + rm -f "$HEROKU_PYTHON_DIR/bin/sqlite3" + + # copy over sqlite3 headers & bins and setup linking against the stack image library + mv "$HEROKU_PYTHON_DIR/sqlite3/usr/include/"* "$HEROKU_PYTHON_DIR/include/" + mv "$HEROKU_PYTHON_DIR/sqlite3/usr/lib/x86_64-linux-gnu"/libsqlite3.*a "$HEROKU_PYTHON_DIR/lib/" + mkdir -p "$HEROKU_PYTHON_DIR/lib/pkgconfig" + # set the right prefix/lib directories + sed -e 's/prefix=\/usr/prefix=\/app\/.heroku\/python/' -e 's/\/x86_64-linux-gnu//' "$HEROKU_PYTHON_DIR/sqlite3/usr/lib/x86_64-linux-gnu/pkgconfig/sqlite3.pc" > "$HEROKU_PYTHON_DIR/lib/pkgconfig/sqlite3.pc" + # need to point the libsqlite3.so to the stack image library for /usr/bin/ld -lsqlite3 + SQLITE3_LIBFILE="/usr/lib/x86_64-linux-gnu/$(readlink -n "$HEROKU_PYTHON_DIR/sqlite3/usr/lib/x86_64-linux-gnu/libsqlite3.so")" + ln -s "$SQLITE3_LIBFILE" "$HEROKU_PYTHON_DIR/lib/libsqlite3.so" + if [ -z "$HEADERS_ONLY" ]; then + mv "$HEROKU_PYTHON_DIR/sqlite3/usr/bin"/* "$HEROKU_PYTHON_DIR/bin/" + fi + + # cleanup + rm -rf "$HEROKU_PYTHON_DIR/sqlite3/" + rm -rf "$HEROKU_PYTHON_DIR/apt/" +} + +buildpack_sqlite3_install() { + sqlite3_version + HEROKU_PYTHON_DIR="$BUILD_DIR/.heroku/python" + + SQLITE3_VERSION_FILE="$BUILD_DIR/.heroku/python-sqlite3-version" + if [ -f "$SQLITE3_VERSION_FILE" ]; then + INSTALLED_SQLITE3_VERSION=$(cat "$SQLITE3_VERSION_FILE") + fi + + # python version check + if python_sqlite3_check "$PYTHON_VERSION"; then + # only install if the sqlite3 version has changed + if [ "$INSTALLED_SQLITE3_VERSION" != "$SQLITE3_VERSION" ]; then + puts-step "Installing SQLite3" + sqlite3_install "$BUILD_DIR/.heroku/python" "$SQLITE3_VERSION" + + # save version installed + mkdir -p "$CACHE_DIR/.heroku/" + echo "$SQLITE3_VERSION" > "$CACHE_DIR/.heroku/python-sqlite3-version" + fi + fi +} diff --git a/bin/utils b/bin/utils index b748bbea9822a6303e6c86cf217a82551c1029f4..2bbf82a32e59e33c84bc5be925e72f6437cb4e68 100755 --- a/bin/utils +++ b/bin/utils @@ -58,3 +58,41 @@ measure-size() { echo "$(du -s .heroku/python 2>/dev/null || echo 0) | awk '{print $1}')" } +# Python version operator > +version_gt() { + test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1"; +} + +# Python verison operator >= +version_gte() { + if [ "$1" == "$2" ]; then + return 0 + fi + + version_gt "$1" "$2" +} + +# Check if Python 2 +python2_check() { + VERSION="$1" + + version_gte "$VERSION" "python-2.7.0" && version_gt "python-3.0.0" "$VERSION" +} + +# Check if Python 3 +python3_check() { + VERSION="$1" + + version_gte "$VERSION" "python-3.0.0" && version_gt "python-4.0.0" "$VERSION" +} + +# Check if Python version needs to install SQLite3 +python_sqlite3_check() { + VERSION="$1" + MIN_PYTHON_3="python-3.6.6" + MIN_PYTHON_2="python-2.7.15" + + ( python2_check "$VERSION" && version_gte "$VERSION" "$MIN_PYTHON_2" ) \ + || ( python3_check "$VERSION" && version_gte "$VERSION" "$MIN_PYTHON_3" ) \ + || ( version_gte "$VERSION" "3.7.0" ) +} diff --git a/builds/runtimes/python-2.7.15 b/builds/runtimes/python-2.7.15 index 8177f5ea9ea3b55497efe425a319575ef5fbc214..a2363fbe5bde1a9c911b413e0c462f0ac42ebfc2 100755 --- a/builds/runtimes/python-2.7.15 +++ b/builds/runtimes/python-2.7.15 @@ -1,8 +1,16 @@ #!/usr/bin/env bash # Build Path: /app/.heroku/python/ -# Build Deps: libraries/sqlite OUT_PREFIX=$1 +BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin" +export BIN_DIR + +# shellcheck source=bin/utils +source "$BIN_DIR/steps/sqlite3" + +sqlite3_version +echo "Setting up SQLite3 Headers for $SQLITE3_VERSION" +sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1 echo "Building Python…" SOURCE_TARBALL='https://python.org/ftp/python/2.7.15/Python-2.7.15.tgz' diff --git a/builds/runtimes/python-3.6.6 b/builds/runtimes/python-3.6.6 index 6da9e250e2a81c35f66cea151aa08be089be40a2..a3bf0f8a6febcf2f4414643dfcc0970cff048c10 100755 --- a/builds/runtimes/python-3.6.6 +++ b/builds/runtimes/python-3.6.6 @@ -1,8 +1,16 @@ #!/usr/bin/env bash # Build Path: /app/.heroku/python/ -# Build Deps: libraries/sqlite OUT_PREFIX=$1 +BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin" +export BIN_DIR + +# shellcheck source=bin/utils +source "$BIN_DIR/steps/sqlite3" + +sqlite3_version +echo "Setting up SQLite3 Headers for $SQLITE3_VERSION" +sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1 echo "Building Python…" SOURCE_TARBALL='https://python.org/ftp/python/3.6.6/Python-3.6.6.tgz' diff --git a/builds/runtimes/python-3.7.0 b/builds/runtimes/python-3.7.0 index 2046c4555e7927b574ca48ed0e8434e7901000a7..74dccc590403b20c557ee29048d7395ccf6e6f5c 100755 --- a/builds/runtimes/python-3.7.0 +++ b/builds/runtimes/python-3.7.0 @@ -1,8 +1,16 @@ #!/usr/bin/env bash # Build Path: /app/.heroku/python/ -# Build Deps: libraries/sqlite OUT_PREFIX=$1 +BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin" +export BIN_DIR + +# shellcheck source=bin/utils +source "$BIN_DIR/steps/sqlite3" + +sqlite3_version +echo "Setting up SQLite3 Headers for $SQLITE3_VERSION" +sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1 echo "Building Python…" SOURCE_TARBALL='https://python.org/ftp/python/3.7.0/Python-3.7.0.tgz'