diff --git a/bin/compile b/bin/compile index 858e7065ee7fa22545613d29ce7cf4eddd393baa..89e30019183d32a260b465fdd8a0bb6edb5b2c94 100755 --- a/bin/compile +++ b/bin/compile @@ -15,6 +15,10 @@ # Fail fast and fail hard. set -eo pipefail +# Standard Library. +export BPLOG_PREFIX="buildpack.python" +export BUILDPACK_LOG_FILE=${BUILDPACK_LOG_FILE:-/dev/null} + [ "$BUILDPACK_XTRACE" ] && set -o xtrace # Prepend proper path for virtualenv hackery. This will be deprecated soon. @@ -137,10 +141,14 @@ if [[ $BUILD_DIR != '/app' ]]; then fi # Install Python. +let start=$(nowms) source $BIN_DIR/steps/python +mtime "python.install.time" "${start}" # Sanity check for setuptools/distribute. +let start=$(nowms) source $BIN_DIR/steps/setuptools +mtime "setuptools.install.time" "${start}" # Pipenv support. source $BIN_DIR/steps/pipenv @@ -169,20 +177,28 @@ sub-env $BIN_DIR/steps/geo-libs source $BIN_DIR/steps/gdal # Install dependencies with Pip (where the magic happens). +let start=$(nowms) source $BIN_DIR/steps/pip-install +mtime "pip.install.time" "${start}" # Uninstall removed dependencies with Pip. +let start=$(nowms) source $BIN_DIR/steps/pip-uninstall +mtime "pip.uninstall.time" "${start}" # Support for NLTK corpora. +let start=$(nowms) sub-env $BIN_DIR/steps/nltk +mtime "nltk.download.time" "${start}" # Support for pip install -e. rm -fr $BUILD_DIR/.heroku/src deep-cp /app/.heroku/src $BUILD_DIR/.heroku/src # Django collectstatic support. +let start=$(nowms) sub-env $BIN_DIR/steps/collectstatic +mtime "collectstatic.time" "${start}" # Create .profile script for application runtime environment variables. set-env PATH '$HOME/.heroku/python/bin:$PATH' @@ -219,3 +235,6 @@ cp -R .heroku/vendor $CACHE_DIR/.heroku/ &> /dev/null || true if [[ -d .heroku/src ]]; then cp -R .heroku/src $CACHE_DIR/.heroku/ &> /dev/null || true fi + +# Measure the size of the Python installation. +mmeasure 'python.size' "$(measure-size)" diff --git a/bin/steps/collectstatic b/bin/steps/collectstatic index 6189a5fd486789143a1c379f54da435c2552e624..6e36c7aabceb3e890e96f3b8839f65479611b5bc 100755 --- a/bin/steps/collectstatic +++ b/bin/steps/collectstatic @@ -26,7 +26,7 @@ pip-grep -s requirements.txt django Django && DJANGO_INSTALLED=1 if [ ! "$DISABLE_COLLECTSTATIC" ] && [ -f "$MANAGE_FILE" ] && [ "$DJANGO_INSTALLED" ]; then set +e - puts-cmd "python $MANAGE_FILE collectstatic --noinput" + puts-step "$ python $MANAGE_FILE collectstatic --noinput" # Run collectstatic, cleanup some of the noisy output. python $MANAGE_FILE collectstatic --noinput --traceback 2>&1 | sed '/^Post-processed/d;/^Copying/d;/^$/d' | indent diff --git a/bin/steps/python b/bin/steps/python index 9d539be44bb551e0f761e6e7b4539fab1ae95bab..30aa2c8cafc6eeeb571100f3014b4786a0387462 100755 --- a/bin/steps/python +++ b/bin/steps/python @@ -25,6 +25,8 @@ if [ ! "$SKIP_INSTALL" ]; then mkdir -p .heroku/python curl https://lang-python.s3.amazonaws.com/$STACK/runtimes/$PYTHON_VERSION.tar.gz -s | tar zxv -C .heroku/python &> /dev/null + mcount "version.python.$PYTHON_VERSION" + if [[ $? != 0 ]] ; then puts-warn "Requested runtime ($PYTHON_VERSION) is not available for this stack ($STACK)." puts-warn "Aborting. More info: https://devcenter.heroku.com/articles/python-support" diff --git a/bin/utils b/bin/utils index 8d037fe32f27dfaba9aea78145654e8029282caf..6d16aac6da5855033fcce160d361d01b515d34a4 100755 --- a/bin/utils +++ b/bin/utils @@ -1,5 +1,12 @@ +#!/usr/bin/env bash shopt -s extglob +# The standard library. +if [[ ! -f /tmp/stdlib.sh ]]; then + curl -s https://raw.githubusercontent.com/heroku/buildpack-stdlib/v2/stdlib.sh > /tmp/stdlib.sh +fi +source /tmp/stdlib.sh + if [ $(uname) == Darwin ]; then sed() { command sed -l "$@"; } else @@ -11,16 +18,12 @@ indent() { sed "s/^/ /" } + # Clean up pip output cleanup() { sed -e 's/\.\.\.\+/.../g' | sed -e '/already satisfied/Id' | sed -e '/No files were found to uninstall/Id' | sed -e '/Overwriting/Id' | sed -e '/python executable/Id' | sed -e '/no previously-included files/Id' } -# Buildpack Indented line. -puts-line() { - echo " $@" -} - # Buildpack Steps. puts-step() { echo "-----> $@" @@ -31,28 +34,6 @@ puts-warn() { echo " ! $@" } -# Buildpack Commands. -puts-cmd() { - echo " $ $@" -} - -# Usage: $ set-env key value -set-env() { - echo "export $1=$2" >> $PROFILE_PATH - echo "export $1=$2" >> $EXPORT_PATH -} - -# Usage: $ set-default-env key value -set-default-env() { - echo "export $1=\${$1:-$2}" >> $PROFILE_PATH - echo "export $1=\${$1:-$2}" >> $EXPORT_PATH -} - -# Usage: $ un-set-env key -un-set-env() { - echo "unset $1" >> $PROFILE_PATH -} - # Does some serious copying. deep-cp() { declare source="$1" target="$2" @@ -69,21 +50,6 @@ deep-cp() { ) } -# Does some serious moving. -deep-mv() { - deep-cp "$1" "$2" - deep-rm "$1" -} - -# Does some serious deleting. -deep-rm() { - # subshell to avoid surprising caller with shopts. - ( - shopt -s dotglob - rm -rf "$1"/!(.curlrc|.netrc|tmp|.|..) - ) -} - sub-env() { @@ -108,3 +74,8 @@ sub-env() { ) } +# Measure the size of the Python installation. +measure-size() { + echo "$((du -s .heroku/python 2>/dev/null || echo 0) | awk '{print $1}')" +} + diff --git a/bin/warnings b/bin/warnings index 3ec061bba522853f0e0a949274fcb1c71875f9c7..01e43d58f097b6e00355dfe1721950ce920b523d 100755 --- a/bin/warnings +++ b/bin/warnings @@ -7,6 +7,7 @@ old-platform() { puts-warn "This caused the security warning you saw above during the 'pip install' step." puts-warn "We recommend '$RECOMMENDED_PYTHON_VERSION', which you can specify in a 'runtime.txt' file." puts-warn " -- Much Love, Heroku." + mcount 'warnings.python.old' fi } @@ -16,6 +17,7 @@ pylibmc-missing() { puts-warn "Hello! There was a problem with your build related to libmemcache." puts-warn "The Python library 'pylibmc' must be explicitly specified in 'requirements.txt' in order to build correctly." puts-warn "Once you do that, everything should work as expected. -- Much Love, Heroku." + mcount 'warnings.libmemcache' fi } @@ -27,6 +29,7 @@ scipy-included() { puts-warn "There is, however, a buildpack available that makes it possible to use it on Heroku." puts-warn "You can learn more here: https://devcenter.heroku.com/articles/python-c-deps" puts-warn "Sorry for the inconvenience. -- Much Love, Heroku." + mcount 'warnings.scipy' fi } @@ -37,6 +40,18 @@ distribute-included() { puts-warn "This library is automatically installed by Heroku and shouldn't be in" puts-warn "Your requirements.txt file. This can cause unexpected behavior." puts-warn " -- Much Love, Heroku." + mcount 'warnings.distribute' + fi +} + +six-included() { + if grep -qi 'Running setup.py install for six' "$WARNINGS_LOG"; then + echo + puts-warn "Hello! Your requirements.txt file contains the six package." + puts-warn "This library is automatically installed by Heroku and shouldn't be in" + puts-warn "Your requirements.txt file. This can cause unexpected behavior." + puts-warn " -- Much Love, Heroku." + mcount 'warnings.six' fi } @@ -45,5 +60,6 @@ show-warnings() { pylibmc-missing scipy-included distribute-included + six-included }