diff --git a/bin/compile b/bin/compile index 688e96a3c9d88a0a9815f61b07860a48f0ea808c..affe18f0484e14a836212e5fb0bc7072d581c5ac 100755 --- a/bin/compile +++ b/bin/compile @@ -15,70 +15,96 @@ # Fail fast and fail hard. set -eo pipefail -# Standard Library. +# Boostrap the Buildpack Standard Library. export BPLOG_PREFIX="buildpack.python" export BUILDPACK_LOG_FILE=${BUILDPACK_LOG_FILE:-/dev/null} [ "$BUILDPACK_XTRACE" ] && set -o xtrace -# Prepend proper path for virtualenv hackery. This will be deprecated soon. +# Prepend proper path for old-school virtualenv hackery. +# This may not be neccessary. export PATH=:/usr/local/bin:$PATH -# Paths. +# Setup Path variables, for later use in the Buildpack. BIN_DIR=$(cd "$(dirname "$0")"; pwd) # absolute path ROOT_DIR=$(dirname "$BIN_DIR") BUILD_DIR=$1 CACHE_DIR=$2 ENV_DIR=$3 +# Export Path variables, for use in sub-scripts. export BUILD_DIR CACHE_DIR ENV_DIR +# Set the Buildpack's internet target for downloading Python distributions. +# The user can provide BUILDPACK_VENDOR_URL to specify a custom target. +# Note: this is designed for non-Heroku use, as it does not use the user-provided +# environment variable mechanism (the ENV_DIR). VENDOR_URL="https://lang-python.s3.amazonaws.com/$STACK" if [[ -n ${BUILDPACK_VENDOR_URL:-} ]]; then VENDOR_URL="$BUILDPACK_VENDOR_URL" fi export VENDOR_URL -# Python defaults -DEFAULT_PYTHON_VERSION="python-3.6.5" -LATEST_3="python-3.6.5" +# Which versions of Python are we using? +# These variables are used to specify which versions of Python to install by default, +# as well as prompt the user to upgrade if they are using an un–supported version. +# Note: When 3.7 lands, I recommend switing to LATEST_36 and LATEST_37. +DEFAULT_PYTHON_VERSION="python-3.6.4" +LATEST_3="python-3.6.4" LATEST_2="python-2.7.15" +# Which stack is used (for binary downloading), if none is provided (e.g. outside of Heroku)? DEFAULT_PYTHON_STACK="cedar-14" +# If pip doesn't match this version (the version we install), run the installer. PIP_UPDATE="9.0.2" export DEFAULT_PYTHON_VERSION DEFAULT_PYTHON_STACK PIP_UPDATE LATEST_2 LATEST_3 -# Common Problem Warnings +# Common Problem Warnings: +# This section creates a temporary file in which to stick the output of `pip install`. +# The `warnings` subscript then greps through this for common problems and guides +# the user towards resolution of known issues. WARNINGS_LOG=$(mktemp) export WARNINGS_LOG export RECOMMENDED_PYTHON_VERSION=$DEFAULT_PYTHON_VERSION -# Setup vendored tools and pip-pop (pip-diff) +# The buildpack ships with a few executable tools (e.g. pip-grep, etc). +# This installs them into the path, so we can execute them directly. export PATH=$PATH:$ROOT_DIR/vendor/:$ROOT_DIR/vendor/pip-pop -# Support Anvil Build_IDs +# Set environment variables if they weren't set by the platform. +# Note: this is legacy, for a deprecated build system known as Anvil. +# This can likely be removed, with caution. [ ! "$SLUG_ID" ] && SLUG_ID="defaultslug" [ ! "$REQUEST_ID" ] && REQUEST_ID=$SLUG_ID [ ! "$STACK" ] && STACK=$DEFAULT_PYTHON_STACK -# Sanitizing environment variables. +# Sanitize externally-provided environment variables: +# The following environment variables are either problematic or simply unneccessary +# for the buildpack to have knowledge of, so we unset them, to keep the environment +# as clean and pristine as possible. unset GIT_DIR PYTHONHOME PYTHONPATH unset RECEIVE_DATA RUN_KEY BUILD_INFO DEPLOY LOG_TOKEN unset CYTOKINE_LOG_FILE GEM_PATH -# Syntax sugar. +# Import the utils script, which contains helper functions used throughout the buildpack. # shellcheck source=bin/utils source "$BIN_DIR/utils" -# Import collection of warnings. +# Import the warnings script, which contains the `pip install` user warning mechanisms +# (mentioned and explained above) # shellcheck source=bin/warnings source "$BIN_DIR/warnings" -# we need to put a bunch of symlinks in there later +# Make the directory in which we will create symlinks from the temporary build directory +# to `/app`. +# Symlinks are required, since Python is not a portable installation. +# More on this topic later. mkdir -p /app/.heroku -# Set up outputs under new context +# This buildpack programatically generates (or simply copies) a number of files for +# buildpack machinery: an export script, and a number of `.profile.d` scripts. This +# section declares the locations of those files and targets. PROFILE_PATH="$BUILD_DIR/.profile.d/python.sh" EXPORT_PATH="$BIN_DIR/../export" GUNICORN_PROFILE_PATH="$BUILD_DIR/.profile.d/python.gunicorn.sh" @@ -87,43 +113,75 @@ WEB_CONCURRENCY_PROFILE_PATH="$BUILD_DIR/.profile.d/WEB_CONCURRENCY.sh" # We'll need to send these statics to other scripts we `source`. export BUILD_DIR CACHE_DIR BIN_DIR PROFILE_PATH EXPORT_PATH -# Prepend proper environment variables for Python use. +# Python Environment Variables +# Set Python-specific environment variables, for running Python within the buildpack. +# Notes on each variable included. + +# PATH is relatively obvious, we need to be able to execute 'python'. export PATH=/app/.heroku/python/bin:/app/.heroku/vendor/bin:$PATH +# Tell Python to not buffer it's stdin/stdout. export PYTHONUNBUFFERED=1 +# Set the locale to a well-known and expected standard. export LANG=en_US.UTF-8 +# `~/.heroku/vendor` is an place where the buildpack may stick pre-build binaries for known +# C dependencies (e.g. libmemcached on cedar-14). This section configures Python (GCC, more specifically) +# and pip to automatically include these paths when building binaries. export C_INCLUDE_PATH=/app/.heroku/vendor/include:/app/.heroku/python/include:$C_INCLUDE_PATH export CPLUS_INCLUDE_PATH=/app/.heroku/vendor/include:/app/.heroku/python/include:$CPLUS_INCLUDE_PATH export LIBRARY_PATH=/app/.heroku/vendor/lib:/app/.heroku/python/lib:$LIBRARY_PATH export LD_LIBRARY_PATH=/app/.heroku/vendor/lib:/app/.heroku/python/lib:$LD_LIBRARY_PATH export PKG_CONFIG_PATH=/app/.heroku/vendor/lib/pkg-config:/app/.heroku/python/lib/pkg-config:$PKG_CONFIG_PATH +# The Application Code +# -------------------- + # Switch to the repo's context. cd "$BUILD_DIR" -# Prepare the cache. +# The Cache +# --------- + +# The workflow for the Python Buildpack's cache is as follows: +# +# - `~/.heroku/{known-paths}` are copied from the cache into the slug. +# - The build is executed, modifying `~/.heroku/{known-paths}`. +# - Once the build is complete, `~/.heroku/{known-paths}` is copied back into the cache. + +# Create the cache directory, if it doesn't exist. mkdir -p "$CACHE_DIR" # Restore old artifacts from the cache. mkdir -p .heroku +# The Python installation. cp -R "$CACHE_DIR/.heroku/python" .heroku/ &> /dev/null || true +# A plain text file which contains the current stack being used (used for cache busting). cp -R "$CACHE_DIR/.heroku/python-stack" .heroku/ &> /dev/null || true +# A plain text file which contains the current python version being used (used for cache busting). cp -R "$CACHE_DIR/.heroku/python-version" .heroku/ &> /dev/null || true +# Any pre-compiled binaries, provided by the buildpack. cp -R "$CACHE_DIR/.heroku/vendor" .heroku/ &> /dev/null || true +# "editable" installations of code repositories, via pip or pipenv. if [[ -d "$CACHE_DIR/.heroku/src" ]]; then cp -R "$CACHE_DIR/.heroku/src" .heroku/ &> /dev/null || true fi -# Experimental pre_compile hook. +# The pre_compile hook. Customers rely on this. Don't remove it. +# This part of the code is used to allow users to customize their build experience +# without forking the buildpack by providing a `bin/pre_compile` script, which gets +# run inline with the buildpack automatically. + # shellcheck source=bin/steps/hooks/pre_compile source "$BIN_DIR/steps/hooks/pre_compile" -# Sticky runtimes. +# Sticky runtimes. If there was a previous build, and it used a given version of Python, +# continue to use that version of Python in perpituity (warnings will be raised if +# they are out–of–date). if [ -f "$CACHE_DIR/.heroku/python-version" ]; then DEFAULT_PYTHON_VERSION=$(cat "$CACHE_DIR/.heroku/python-version") fi -# Stack fallback for non-declared caches. +# We didn't always record the stack version. This code is in place because of that. if [ -f "$CACHE_DIR/.heroku/python-stack" ]; then CACHED_PYTHON_STACK=$(cat "$CACHE_DIR/.heroku/python-stack") else @@ -133,17 +191,26 @@ fi export CACHED_PYTHON_STACK # Pipenv Python version support. +# Detect the version of Python requested from a Pipfile (e.g. python_version or python_full_version). +# Convert it to a runtime.txt file. + # shellcheck source=bin/steps/pipenv-python-version source "$BIN_DIR/steps/pipenv-python-version" -# If no runtime given, assume default version. +# If no runtime was provided by the user, assume the default Python runtime version. if [ ! -f runtime.txt ]; then echo "$DEFAULT_PYTHON_VERSION" > runtime.txt fi +# Create the directory for .profile.d, if it doesn't exist. mkdir -p "$(dirname "$PROFILE_PATH")" +# Create the directory for editable source code installation, if it doesn't exist. mkdir -p /app/.heroku/src +# On Heroku CI, builds happen in `/app`. Otherwise, on the Heroku platform, +# they occur in a temp directory. Beacuse Python is not portable, we must create +# symlinks to emulate that we are operating in `/app` during the build process. +# This is (hopefully obviously) because apps end up running from `/app` in production. if [[ $BUILD_DIR != '/app' ]]; then # python expects to reside in /app, so set up symlinks # we will not remove these later so subsequent buildpacks can still invoke it @@ -152,60 +219,79 @@ if [[ $BUILD_DIR != '/app' ]]; then # Note: .heroku/src is copied in later. fi -# Install Python. +# Download / Install Python, from pre-build binaries available on Amazon S3. +# This step also bootstraps pip / setuptools. let start=$(nowms) # shellcheck source=bin/steps/python source "$BIN_DIR/steps/python" mtime "python.install.time" "${start}" -# Pipenv support. +# Install Pipenv dependencies, if a Pipfile was provided. # shellcheck source=bin/steps/pipenv source "$BIN_DIR/steps/pipenv" # Uninstall removed dependencies with Pip. +# The buildpack will automatically remove any declared dependencies (in requirements.txt) +# that were explicitly removed. This machinery is a bit complex, but it is not complicated. let start=$(nowms) # shellcheck source=bin/steps/pip-uninstall source "$BIN_DIR/steps/pip-uninstall" mtime "pip.uninstall.time" "${start}" - # If no requirements.txt file given, assume `setup.py develop` is intended. +# This allows for people to ship a setup.py application to Heroku +# (which is rare, but I vouch that it should work!) + if [ ! -f requirements.txt ] && [ ! -f Pipfile ]; then echo "-e ." > requirements.txt fi # Fix egg-links. +# Because we're installing things into a different path than we're running them (temp dir vs app dir), +# We must re-write all of Python's eggpath links to target the proper directory. # shellcheck source=bin/steps/eggpath-fix source "$BIN_DIR/steps/eggpath-fix" # Mercurial support. +# If a customer appears to be using mercurial for dependency resolution, we install it first. +# Note: this only applies to pip, not pipenv. This can likely be removed, over time. Measure it first. # shellcheck source=bin/steps/mercurial source "$BIN_DIR/steps/mercurial" # Pylibmc support. +# On cedar-14, libmemcached was not available. The buildpack provides its own version, instead. # shellcheck source=bin/steps/pylibmc source "$BIN_DIR/steps/pylibmc" -# Support for Geo libraries. +# Support for Geo libraries. This is deprecated functionality, only functional on cedar-14. +# It is undocumented. # shellcheck source=bin/steps/geo-libs sub_env "$BIN_DIR/steps/geo-libs" # GDAL support. +# This is part of the Geo support. # shellcheck source=bin/steps/gdal source "$BIN_DIR/steps/gdal" -# Install dependencies with Pip (where the magic happens). +# pip install +# ----------- + +# Install dependencies with pip (where the magic happens). let start=$(nowms) # shellcheck source=bin/steps/pip-install source "$BIN_DIR/steps/pip-install" mtime "pip.install.time" "${start}" # Support for NLTK corpora. +# Note: this may only work on Python 2.7. I don't think many customers use this functionality, +# and it should probably be undocumented. +# (there's an import error on 3.6 that should hopefully be fixed upstream at some point) let start=$(nowms) sub_env "$BIN_DIR/steps/nltk" mtime "nltk.download.time" "${start}" -# Support for pip install -e. +# Support for editable installations. Here, we are copying pip–created src directory, +# and copying it into the proper place (the logical place to do this was early, but it must be done here). # In CI, $BUILD_DIR is /app. if [[ ! "$BUILD_DIR" == "/app" ]]; then rm -fr "$BUILD_DIR/.heroku/src" @@ -214,24 +300,35 @@ fi # Django collectstatic support. +# The buildpack automatically runs collectstatic for Django applications. +# This is the cause for the majority of build failures on the Python platform. +# These failures are intentional — if collectstatic (which can be tricky, at times) fails, +# your build fails. let start=$(nowms) sub_env "$BIN_DIR/steps/collectstatic" mtime "collectstatic.time" "${start}" -# Create .profile script for application runtime environment variables. + +# Progamatically create .profile.d script for application runtime environment variables. + +# Set the PATH to include Python / pip / pipenv / etc. set_env PATH "\$HOME/.heroku/python/bin:\$PATH" +# Tell Python to run in unbuffered mode. set_env PYTHONUNBUFFERED true +# Tell Python where it lives. set_env PYTHONHOME "\$HOME/.heroku/python" - +# Set variables for C libraries. set_env LIBRARY_PATH "\$HOME/.heroku/vendor/lib:\$HOME/.heroku/python/lib:\$LIBRARY_PATH" set_env LD_LIBRARY_PATH "\$HOME/.heroku/vendor/lib:\$HOME/.heroku/python/lib:\$LD_LIBRARY_PATH" - +# Locale. set_default_env LANG en_US.UTF-8 +# The Python hash seed is set to random. set_default_env PYTHONHASHSEED random +# Tell Python to look for Python modules in the /app dir. Don't change this. set_default_env PYTHONPATH "\$HOME" -# python expects to be in /app, if at runtime, it is not, set -# up symlinks... this can occur when the subdir buildpack is used +# Python expects to be in /app, if at runtime, it is not, set +# up symlinks… this can occur when the subdir buildpack is used. cat <<EOT >> "$PROFILE_PATH" if [[ \$HOME != "/app" ]]; then mkdir -p /app/.heroku @@ -244,8 +341,7 @@ EOT cp "$ROOT_DIR/vendor/WEB_CONCURRENCY.sh" "$WEB_CONCURRENCY_PROFILE_PATH" cp "$ROOT_DIR/vendor/python.gunicorn.sh" "$GUNICORN_PROFILE_PATH" - -# Experimental post_compile hook. +# Experimental post_compile hook. Don't remove this. # shellcheck source=bin/steps/hooks/post_compile source "$BIN_DIR/steps/hooks/post_compile" @@ -253,8 +349,7 @@ source "$BIN_DIR/steps/hooks/post_compile" # shellcheck source=bin/steps/eggpath-fix2 source "$BIN_DIR/steps/eggpath-fix2" -# Store new artifacts in cache. - +# Store new artifacts in the cache. rm -rf "$CACHE_DIR/.heroku/python" rm -rf "$CACHE_DIR/.heroku/python-version" rm -rf "$CACHE_DIR/.heroku/python-stack"