From 221722fb27908dcef63e05c023deb0f2525ca713 Mon Sep 17 00:00:00 2001
From: Terence Lee <hone02@gmail.com>
Date: Tue, 19 Jun 2018 17:48:06 -0500
Subject: [PATCH] setup libsqlite3-dev and sqlite3 binary to match stack's
 libsqlite3-0

With inspiration from @KevinBrolly, this patch uses the stack image
SQLite3 package but also still providing the dev headers and binary that
users may still be using today. The benefit is that we won't need to
rebuild all the python binaries for this to take affect. We can just
stop shipping SQLite3 from future binaries. In addition, we don't need
to worry about what version and when to update SQLite3 and maintaining
the packages ourselves.

This also includes updates to Python 2.7.15 and Python 3.6.6 so they can
rebuilt with the stack image dev headers instead of building our own
vendored SQLite3.
---
 bin/compile                   | 12 +++++
 bin/steps/python              | 10 +++-
 bin/steps/sqlite3             | 87 +++++++++++++++++++++++++++++++++++
 bin/utils                     | 11 +++++
 builds/runtimes/python-2.7.15 | 10 +++-
 builds/runtimes/python-3.6.6  | 10 +++-
 builds/runtimes/python-3.7.0  | 10 +++-
 7 files changed, 146 insertions(+), 4 deletions(-)
 create mode 100644 bin/steps/sqlite3

diff --git a/bin/compile b/bin/compile
index f173fdee..d97af680 100755
--- a/bin/compile
+++ b/bin/compile
@@ -160,6 +160,8 @@ cp -R "$CACHE_DIR/.heroku/python" .heroku/ &> /dev/null || true
 cp -R "$CACHE_DIR/.heroku/python-stack" .heroku/ &> /dev/null || true
 # A plain text file which contains the current python version being used (used for cache busting).
 cp -R "$CACHE_DIR/.heroku/python-version" .heroku/ &> /dev/null || true
+# A plain text file which contains the current sqlite3 version being used (used for cache busting).
+cp -R "$CACHE_DIR/.heroku/python-sqlite3-version" .heroku/ &> /dev/null || true
 # Any pre-compiled binaries, provided by the buildpack.
 cp -R "$CACHE_DIR/.heroku/vendor" .heroku/ &> /dev/null || true
 # "editable" installations of code repositories, via pip or pipenv.
@@ -274,6 +276,16 @@ sub_env "$BIN_DIR/steps/geo-libs"
 # shellcheck source=bin/steps/gdal
 source "$BIN_DIR/steps/gdal"
 
+# SQLite3 support.
+# This sets up and installs sqlite3 dev headers and the sqlite3 binary but not the
+# libsqlite3-0 library since that exists on the stack image.
+# Note: This only applies to Python 2.7.15+ and Python 3.6.6+
+(( start=$(nowms) ))
+# shellcheck source=bin/steps/sqlite3
+source "$BIN_DIR/steps/sqlite3"
+buildpack_sqlite3_install
+mtime "sqlite3.install.time" "${start}"
+
 # pip install
 # -----------
 
diff --git a/bin/steps/python b/bin/steps/python
index 58d60436..e66a05dd 100755
--- a/bin/steps/python
+++ b/bin/steps/python
@@ -29,7 +29,15 @@ fi
 
 if [[ "$STACK" != "$CACHED_PYTHON_STACK" ]]; then
     puts-step "Stack has changed from $CACHED_PYTHON_STACK to $STACK, clearing cache"
-    rm -fr .heroku/python-stack .heroku/python-version .heroku/python .heroku/vendor
+    rm -fr .heroku/python-stack .heroku/python-version .heroku/python .heroku/vendor .heroku/python .heroku/python-sqlite3-version
+fi
+
+# need to clear the cache for first time installing SQLite3,
+# since the version is changing and could lead to runtime errors
+# with compiled extensions.
+if [ -d .heroku/python ] && [ ! -f .heroku/python-sqlite3-version ] && python_sqlite3_check "$PYTHON_VERSION"; then
+  puts-step "Need to update SQLite3, clearing cache"
+  rm -fr .heroku/python-stack .heroku/python-version .heroku/python .heroku/vendor
 fi
 
 if [ -f .heroku/python-version ]; then
diff --git a/bin/steps/sqlite3 b/bin/steps/sqlite3
new file mode 100644
index 00000000..8855f30a
--- /dev/null
+++ b/bin/steps/sqlite3
@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+
+# shellcheck source=bin/utils
+source "$BIN_DIR/utils"
+
+sqlite3_version() {
+  SQLITE3_VERSION=${SQLITE3_VERSION:-$(dpkg -s libsqlite3-0 | grep Version | sed 's/Version: //')}
+
+  export SQLITE3_VERSION
+}
+
+sqlite3_install() {
+  HEROKU_PYTHON_DIR="$1"
+  SQLITE3_VERSION="$2"
+  HEADERS_ONLY="$3"
+
+  mkdir -p "$HEROKU_PYTHON_DIR"
+
+  APT_CACHE_DIR="$HEROKU_PYTHON_DIR/apt/cache"
+  APT_STATE_DIR="$HEROKU_PYTHON_DIR/apt/state"
+
+  mkdir -p "$APT_CACHE_DIR/archives/partial"
+  mkdir -p "$APT_STATE_DIR/lists/partial"
+
+  APT_OPTIONS="-o debug::nolocking=true"
+  APT_OPTIONS="$APT_OPTIONS -o dir::cache=$APT_CACHE_DIR"
+  APT_OPTIONS="$APT_OPTIONS -o dir::state=$APT_STATE_DIR"
+  APT_OPTIONS="$APT_OPTIONS -o dir::etc::sourcelist=/etc/apt/sources.list"
+
+  apt-get $APT_OPTIONS update > /dev/null 2>&1
+  if [ -z "$HEADERS_ONLY" ]; then
+    apt-get $APT_OPTIONS -y -d --reinstall install libsqlite3-dev="$SQLITE3_VERSION" sqlite3="$SQLITE3_VERSION" > /dev/null 2>&1
+  else
+    apt-get $APT_OPTIONS -y -d --reinstall install libsqlite3-dev="$SQLITE3_VERSION"
+  fi
+
+  find "$APT_CACHE_DIR/archives/" -name "*.deb" -exec dpkg -x {} "$HEROKU_PYTHON_DIR/sqlite3/" \;
+
+  mkdir -p "$HEROKU_PYTHON_DIR/include"
+  mkdir -p "$HEROKU_PYTHON_DIR/lib"
+
+  # remove old sqlite3 libraries/binaries
+  find "$HEROKU_PYTHON_DIR/include/" -name "sqlite3*.h" -exec rm -f {} \;
+  find "$HEROKU_PYTHON_DIR/lib/" -name "libsqlite3.*" -exec rm -f {} \;
+  rm -f "$HEROKU_PYTHON_DIR/lib/pkgconfig/sqlite3.pc"
+  rm -f "$HEROKU_PYTHON_DIR/bin/sqlite3"
+
+  # copy over sqlite3 headers & bins and setup linking against the stack image library
+  mv "$HEROKU_PYTHON_DIR/sqlite3/usr/include/"* "$HEROKU_PYTHON_DIR/include/"
+  mv "$HEROKU_PYTHON_DIR/sqlite3/usr/lib/x86_64-linux-gnu"/libsqlite3.*a "$HEROKU_PYTHON_DIR/lib/"
+  mkdir -p "$HEROKU_PYTHON_DIR/lib/pkgconfig"
+  # set the right prefix/lib directories
+  sed -e 's/prefix=\/usr/prefix=\/app\/.heroku\/python/' -e 's/\/x86_64-linux-gnu//' "$HEROKU_PYTHON_DIR/sqlite3/usr/lib/x86_64-linux-gnu/pkgconfig/sqlite3.pc" > "$HEROKU_PYTHON_DIR/lib/pkgconfig/sqlite3.pc"
+  # need to point the libsqlite3.so to the stack image library for /usr/bin/ld -lsqlite3
+  SQLITE3_LIBFILE="/usr/lib/x86_64-linux-gnu/$(readlink -n "$HEROKU_PYTHON_DIR/sqlite3/usr/lib/x86_64-linux-gnu/libsqlite3.so")"
+  ln -s "$SQLITE3_LIBFILE" "$HEROKU_PYTHON_DIR/lib/libsqlite3.so"
+  if [ -z "$HEADERS_ONLY" ]; then
+    mv "$HEROKU_PYTHON_DIR/sqlite3/usr/bin"/* "$HEROKU_PYTHON_DIR/bin/"
+  fi
+
+  # cleanup
+  rm -rf "$HEROKU_PYTHON_DIR/sqlite3/"
+  rm -rf "$HEROKU_PYTHON_DIR/apt/"
+}
+
+buildpack_sqlite3_install() {
+  sqlite3_version
+  HEROKU_PYTHON_DIR="$BUILD_DIR/.heroku/python"
+
+  SQLITE3_VERSION_FILE="$BUILD_DIR/.heroku/python-sqlite3-version"
+  if [ -f "$SQLITE3_VERSION_FILE" ]; then
+    INSTALLED_SQLITE3_VERSION=$(cat "$SQLITE3_VERSION_FILE")
+  fi
+
+  # python version check
+  if python_sqlite3_check "$PYTHON_VERSION"; then
+    # only install if the sqlite3 version has changed
+    if [ "$INSTALLED_SQLITE3_VERSION" != "$SQLITE3_VERSION" ]; then
+      puts-step "Installing SQLite3"
+      sqlite3_install "$BUILD_DIR/.heroku/python" "$SQLITE3_VERSION"
+
+      # save version installed
+      mkdir -p "$CACHE_DIR/.heroku/"
+      echo "$SQLITE3_VERSION" > "$CACHE_DIR/.heroku/python-sqlite3-version"
+    fi
+  fi
+}
diff --git a/bin/utils b/bin/utils
index 33fd2b30..2bbf82a3 100755
--- a/bin/utils
+++ b/bin/utils
@@ -85,3 +85,14 @@ python3_check() {
 
   version_gte "$VERSION" "python-3.0.0" && version_gt "python-4.0.0" "$VERSION"
 }
+
+# Check if Python version needs to install SQLite3
+python_sqlite3_check() {
+  VERSION="$1"
+  MIN_PYTHON_3="python-3.6.6"
+  MIN_PYTHON_2="python-2.7.15"
+
+  ( python2_check "$VERSION" && version_gte "$VERSION" "$MIN_PYTHON_2" ) \
+    || ( python3_check "$VERSION" && version_gte "$VERSION" "$MIN_PYTHON_3" ) \
+    || ( version_gte "$VERSION" "3.7.0" )
+}
diff --git a/builds/runtimes/python-2.7.15 b/builds/runtimes/python-2.7.15
index 8177f5ea..a2363fbe 100755
--- a/builds/runtimes/python-2.7.15
+++ b/builds/runtimes/python-2.7.15
@@ -1,8 +1,16 @@
 #!/usr/bin/env bash
 # Build Path: /app/.heroku/python/
-# Build Deps: libraries/sqlite
 
 OUT_PREFIX=$1
+BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
+export BIN_DIR
+
+# shellcheck source=bin/utils
+source "$BIN_DIR/steps/sqlite3"
+
+sqlite3_version
+echo "Setting up SQLite3 Headers for $SQLITE3_VERSION"
+sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1
 
 echo "Building Python…"
 SOURCE_TARBALL='https://python.org/ftp/python/2.7.15/Python-2.7.15.tgz'
diff --git a/builds/runtimes/python-3.6.6 b/builds/runtimes/python-3.6.6
index 6da9e250..a3bf0f8a 100755
--- a/builds/runtimes/python-3.6.6
+++ b/builds/runtimes/python-3.6.6
@@ -1,8 +1,16 @@
 #!/usr/bin/env bash
 # Build Path: /app/.heroku/python/
-# Build Deps: libraries/sqlite
 
 OUT_PREFIX=$1
+BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
+export BIN_DIR
+
+# shellcheck source=bin/utils
+source "$BIN_DIR/steps/sqlite3"
+
+sqlite3_version
+echo "Setting up SQLite3 Headers for $SQLITE3_VERSION"
+sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1
 
 echo "Building Python…"
 SOURCE_TARBALL='https://python.org/ftp/python/3.6.6/Python-3.6.6.tgz'
diff --git a/builds/runtimes/python-3.7.0 b/builds/runtimes/python-3.7.0
index 2046c455..74dccc59 100755
--- a/builds/runtimes/python-3.7.0
+++ b/builds/runtimes/python-3.7.0
@@ -1,8 +1,16 @@
 #!/usr/bin/env bash
 # Build Path: /app/.heroku/python/
-# Build Deps: libraries/sqlite
 
 OUT_PREFIX=$1
+BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
+export BIN_DIR
+
+# shellcheck source=bin/utils
+source "$BIN_DIR/steps/sqlite3"
+
+sqlite3_version
+echo "Setting up SQLite3 Headers for $SQLITE3_VERSION"
+sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1
 
 echo "Building Python…"
 SOURCE_TARBALL='https://python.org/ftp/python/3.7.0/Python-3.7.0.tgz'
-- 
GitLab