diff --git a/.travis.yml b/.travis.yml index 7f5f026790a7d36caef33063435bd9e38acaf672..c6c1b47b354bc4269300e2780bfa386b2a378be8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,25 @@ language: bash -sudo: required +# sudo: required +addons: + apt: + sources: + - debian-sid # Grab shellcheck from the Debian repo (o_O) + packages: + - shellcheck services: - docker # install: docker pull heroku/cedar:14 -script: ./tests.sh -env: - - STACK=heroku-16 - - STACK=cedar-14 \ No newline at end of file +jobs: + include: + - stage: "Bash linting (shellcheck)" + script: make check + + - stage: "Heroku-16 Stack Tests" + script: ./tests.sh + env: + - STACK=heroku-16 + + - stage: "Cedar-14 Stack Tests" + script: ./tests.sh + env: + - STACK=cedar-14 \ No newline at end of file diff --git a/Makefile b/Makefile index 7d4ae900bbfb1eafcf6f70ae8cc861b57d822735..81d0df8fa90092a6735d12e8c23f057847584f9a 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,11 @@ test: test-heroku-16 +check: + @shellcheck -x bin/compile bin/detect bin/release bin/test-compile bin/utils bin/warnings + @shellcheck -x bin/steps/collectstatic bin/steps/cryptography bin/steps/eggpath-fix bin/steps/eggpath-fix2 bin/steps/gdal bin/steps/geo-libs bin/steps/mercurial bin/steps/nltk bin/steps/pip-install bin/steps/pip-uninstall bin/steps/pipenv bin/steps/pipenv-python-version bin/steps/pylibmc bin/steps/python + @shellcheck -x bin/steps/hooks/* + test-cedar-14: @echo "Running tests in docker (cedar-14)..." @docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=cedar-14" heroku/cedar:14 bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run;' diff --git a/bin/compile b/bin/compile index b5c3108345032be3a2bdd8611b768c40fa476aa3..ad0ac09f2d1ea17cb684db17a85af1f4cf9f4d61 100755 --- a/bin/compile +++ b/bin/compile @@ -25,20 +25,24 @@ export BUILDPACK_LOG_FILE=${BUILDPACK_LOG_FILE:-/dev/null} export PATH=:/usr/local/bin:$PATH # Paths. -BIN_DIR=$(cd $(dirname $0); pwd) # absolute path -ROOT_DIR=$(dirname $BIN_DIR) +BIN_DIR=$(cd "$(dirname "$0")"; pwd) # absolute path +ROOT_DIR=$(dirname "$BIN_DIR") BUILD_DIR=$1 CACHE_DIR=$2 ENV_DIR=$3 +export BUILD_DIR CACHE_DIR ENV_DIR + # Python defaults DEFAULT_PYTHON_VERSION="python-3.6.2" DEFAULT_PYTHON_STACK="cedar-14" -PYTHON_EXE="/app/.heroku/python/bin/python" -PIP_VERSION="9.0.1" +PIP_UPDATE="9.0.1" + +export DEFAULT_PYTHON_VERSION DEFAULT_PYTHON_STACK PIP_UPDATE # Common Problem Warnings -export WARNINGS_LOG=$(mktemp) +WARNINGS_LOG=$(mktemp) +export WARNINGS_LOG export RECOMMENDED_PYTHON_VERSION=$DEFAULT_PYTHON_VERSION # Setup vendored tools and pip-pop (pip-diff) @@ -55,10 +59,12 @@ unset RECEIVE_DATA RUN_KEY BUILD_INFO DEPLOY LOG_TOKEN unset CYTOKINE_LOG_FILE GEM_PATH # Syntax sugar. -source $BIN_DIR/utils +# shellcheck source=bin/utils +source "$BIN_DIR/utils" # Import collection of warnings. -source $BIN_DIR/warnings +# shellcheck source=bin/warnings +source "$BIN_DIR/warnings" # we need to put a bunch of symlinks in there later mkdir -p /app/.heroku @@ -83,7 +89,7 @@ export LD_LIBRARY_PATH=/app/.heroku/vendor/lib:/app/.heroku/python/lib:$LD_LIBRA export PKG_CONFIG_PATH=/app/.heroku/vendor/lib/pkg-config:/app/.heroku/python/lib/pkg-config:$PKG_CONFIG_PATH # Switch to the repo's context. -cd $BUILD_DIR +cd "$BUILD_DIR" # Warn for lack of Procfile. if [[ ! -f Procfile ]]; then @@ -92,60 +98,66 @@ if [[ ! -f Procfile ]]; then fi # Prepare the cache. -mkdir -p $CACHE_DIR +mkdir -p "$CACHE_DIR" # Restore old artifacts from the cache. mkdir -p .heroku -cp -R $CACHE_DIR/.heroku/python .heroku/ &> /dev/null || true -cp -R $CACHE_DIR/.heroku/python-stack .heroku/ &> /dev/null || true -cp -R $CACHE_DIR/.heroku/python-version .heroku/ &> /dev/null || true -cp -R $CACHE_DIR/.heroku/vendor .heroku/ &> /dev/null || true -if [[ -d $CACHE_DIR/.heroku/src ]]; then - cp -R $CACHE_DIR/.heroku/src .heroku/ &> /dev/null || true +cp -R "$CACHE_DIR/.heroku/python" .heroku/ &> /dev/null || true +cp -R "$CACHE_DIR/.heroku/python-stack" .heroku/ &> /dev/null || true +cp -R "$CACHE_DIR/.heroku/python-version" .heroku/ &> /dev/null || true +cp -R "$CACHE_DIR/.heroku/vendor" .heroku/ &> /dev/null || true +if [[ -d "$CACHE_DIR/.heroku/src" ]]; then + cp -R "$CACHE_DIR/.heroku/src" .heroku/ &> /dev/null || true fi # Experimental pre_compile hook. -source $BIN_DIR/steps/hooks/pre_compile +# shellcheck source=bin/steps/hooks/pre_compile +source "$BIN_DIR/steps/hooks/pre_compile" # Sticky runtimes. -if [ -f $CACHE_DIR/.heroku/python-version ]; then - DEFAULT_PYTHON_VERSION=$(cat $CACHE_DIR/.heroku/python-version) +if [ -f "$CACHE_DIR/.heroku/python-version" ]; then + DEFAULT_PYTHON_VERSION=$(cat "$CACHE_DIR/.heroku/python-version") fi # Stack fallback for non-declared caches. -if [ -f $CACHE_DIR/.heroku/python-stack ]; then - CACHED_PYTHON_STACK=$(cat $CACHE_DIR/.heroku/python-stack) +if [ -f "$CACHE_DIR/.heroku/python-stack" ]; then + CACHED_PYTHON_STACK=$(cat "$CACHE_DIR/.heroku/python-stack") else CACHED_PYTHON_STACK=$STACK fi +export CACHED_PYTHON_STACK + # Pipenv Python version support. -source $BIN_DIR/steps/pipenv-python-version +# shellcheck source=bin/steps/pipenv-python-version +source "$BIN_DIR/steps/pipenv-python-version" # If no runtime given, assume default version. if [ ! -f runtime.txt ]; then - echo $DEFAULT_PYTHON_VERSION > runtime.txt + echo "$DEFAULT_PYTHON_VERSION" > runtime.txt fi -mkdir -p $(dirname $PROFILE_PATH) +mkdir -p "$(dirname "$PROFILE_PATH")" mkdir -p /app/.heroku/src if [[ $BUILD_DIR != '/app' ]]; then # python expects to reside in /app, so set up symlinks # we will not remove these later so subsequent buildpacks can still invoke it - ln -nsf $BUILD_DIR/.heroku/python /app/.heroku/python - ln -nsf $BUILD_DIR/.heroku/vendor /app/.heroku/vendor + ln -nsf "$BUILD_DIR/.heroku/python" /app/.heroku/python + ln -nsf "$BUILD_DIR/.heroku/vendor" /app/.heroku/vendor # Note: .heroku/src is copied in later. fi # Install Python. let start=$(nowms) -source $BIN_DIR/steps/python +# shellcheck source=bin/steps/python +source "$BIN_DIR/steps/python" mtime "python.install.time" "${start}" # Pipenv support. -source $BIN_DIR/steps/pipenv +# shellcheck source=bin/steps/pipenv +source "$BIN_DIR/steps/pipenv" # If no requirements.txt file given, assume `setup.py develop` is intended. if [ ! -f requirements.txt ] && [ ! -f Pipfile ]; then @@ -153,88 +165,101 @@ if [ ! -f requirements.txt ] && [ ! -f Pipfile ]; then fi # Fix egg-links. -source $BIN_DIR/steps/eggpath-fix +# shellcheck source=bin/steps/eggpath-fix +source "$BIN_DIR/steps/eggpath-fix" # Mercurial support. -source $BIN_DIR/steps/mercurial +# shellcheck source=bin/steps/mercurial +source "$BIN_DIR/steps/mercurial" # Pylibmc support. -source $BIN_DIR/steps/pylibmc +# shellcheck source=bin/steps/pylibmc +source "$BIN_DIR/steps/pylibmc" # Libffi support. -source $BIN_DIR/steps/cryptography +# shellcheck source=bin/steps/cryptography +source "$BIN_DIR/steps/cryptography" # Support for Geo libraries. -sub-env $BIN_DIR/steps/geo-libs +# shellcheck source=bin/steps/geo-libs +sub-env "$BIN_DIR/steps/geo-libs" # GDAL support. -source $BIN_DIR/steps/gdal +# shellcheck source=bin/steps/gdal +source "$BIN_DIR/steps/gdal" # Uninstall removed dependencies with Pip. let start=$(nowms) -source $BIN_DIR/steps/pip-uninstall +# shellcheck source=bin/steps/pip-uninstall +source "$BIN_DIR/steps/pip-uninstall" mtime "pip.uninstall.time" "${start}" # Install dependencies with Pip (where the magic happens). let start=$(nowms) -source $BIN_DIR/steps/pip-install +# shellcheck source=bin/steps/pip-install +source "$BIN_DIR/steps/pip-install" mtime "pip.install.time" "${start}" # Support for NLTK corpora. let start=$(nowms) -sub-env $BIN_DIR/steps/nltk +sub-env "$BIN_DIR/steps/nltk" mtime "nltk.download.time" "${start}" # Support for pip install -e. # In CI, $BUILD_DIR is /app. if [[ ! "$BUILD_DIR" == "/app" ]]; then - rm -fr $BUILD_DIR/.heroku/src - deep-cp /app/.heroku/src $BUILD_DIR/.heroku/src + rm -fr "$BUILD_DIR/.heroku/src" + deep-cp /app/.heroku/src "$BUILD_DIR/.heroku/src" fi # Django collectstatic support. let start=$(nowms) -sub-env $BIN_DIR/steps/collectstatic +sub-env "$BIN_DIR/steps/collectstatic" mtime "collectstatic.time" "${start}" # Create .profile script for application runtime environment variables. -set-env PATH '$HOME/.heroku/python/bin:$PATH' +set-env PATH "\$HOME/.heroku/python/bin:\$PATH" set-env PYTHONUNBUFFERED true set-env PYTHONHOME /app/.heroku/python -set-env LIBRARY_PATH '/app/.heroku/vendor/lib:/app/.heroku/python/lib:$LIBRARY_PATH' -set-env LD_LIBRARY_PATH '/app/.heroku/vendor/lib:/app/.heroku/python/lib:$LD_LIBRARY_PATH' + +set-env LIBRARY_PATH "/app/.heroku/vendor/lib:/app/.heroku/python/lib:\$LIBRARY_PATH" +set-env LD_LIBRARY_PATH "/app/.heroku/vendor/lib:/app/.heroku/python/lib:\$LD_LIBRARY_PATH" + set-default-env LANG en_US.UTF-8 set-default-env PYTHONHASHSEED random set-default-env PYTHONPATH /app/ # Install sane-default script for $WEB_CONCURRENCY and $FORWARDED_ALLOW_IPS. -cp $ROOT_DIR/vendor/WEB_CONCURRENCY.sh $WEB_CONCURRENCY_PROFILE_PATH -cp $ROOT_DIR/vendor/python.gunicorn.sh $GUNICORN_PROFILE_PATH +cp "$ROOT_DIR/vendor/WEB_CONCURRENCY.sh" "$WEB_CONCURRENCY_PROFILE_PATH" +cp "$ROOT_DIR/vendor/python.gunicorn.sh" "$GUNICORN_PROFILE_PATH" # Experimental post_compile hook. -source $BIN_DIR/steps/hooks/post_compile +# shellcheck source=bin/steps/hooks/post_compile +source "$BIN_DIR/steps/hooks/post_compile" # Fix egg-links, again. -source $BIN_DIR/steps/eggpath-fix2 +# shellcheck source=bin/steps/eggpath-fix2 +source "$BIN_DIR/steps/eggpath-fix2" # Store new artifacts in cache. -rm -rf $CACHE_DIR/.heroku/python -rm -rf $CACHE_DIR/.heroku/python-version -rm -rf $CACHE_DIR/.heroku/python-stack -rm -rf $CACHE_DIR/.heroku/vendor -rm -rf $CACHE_DIR/.heroku/src - -mkdir -p $CACHE_DIR/.heroku -cp -R .heroku/python $CACHE_DIR/.heroku/ -cp -R .heroku/python-version $CACHE_DIR/.heroku/ -cp -R .heroku/python-stack $CACHE_DIR/.heroku/ &> /dev/null || true -cp -R .heroku/vendor $CACHE_DIR/.heroku/ &> /dev/null || true +rm -rf "$CACHE_DIR/.heroku/python" +rm -rf "$CACHE_DIR/.heroku/python-version" +rm -rf "$CACHE_DIR/.heroku/python-stack" +rm -rf "$CACHE_DIR/.heroku/vendor" +rm -rf "$CACHE_DIR/.heroku/src" + +mkdir -p "$CACHE_DIR/.heroku" +cp -R .heroku/python "$CACHE_DIR/.heroku/" +cp -R .heroku/python-version "$CACHE_DIR/.heroku/" +cp -R .heroku/python-stack "$CACHE_DIR/.heroku/" &> /dev/null || true +cp -R .heroku/vendor "$CACHE_DIR/.heroku/" &> /dev/null || true if [[ -d .heroku/src ]]; then - cp -R .heroku/src $CACHE_DIR/.heroku/ &> /dev/null || true + cp -R .heroku/src "$CACHE_DIR/.heroku/" &> /dev/null || true fi # Measure the size of the Python installation. +# shellcheck disable=SC2119 mmeasure 'python.size' "$(measure-size)" diff --git a/bin/detect b/bin/detect index cee85b1982865c722a6fba5da4d01264bbcc9435..eeb965b0f26db4c006af830484494ff5e9dd93d0 100755 --- a/bin/detect +++ b/bin/detect @@ -15,7 +15,7 @@ BUILD_DIR=$1 # Exit early if app is clearly not Python. -if [ ! -f $BUILD_DIR/requirements.txt ] && [ ! -f $BUILD_DIR/setup.py ] && [ ! -f $BUILD_DIR/Pipfile ]; then +if [ ! -f "$BUILD_DIR/requirements.txt" ] && [ ! -f "$BUILD_DIR/setup.py" ] && [ ! -f "$BUILD_DIR/Pipfile" ]; then exit 1 fi diff --git a/bin/release b/bin/release index 8e342ca2f7ce73f3a994992aacfe8458718d82ec..86fca78c0c377d0614db2fd3a8b67593445456aa 100755 --- a/bin/release +++ b/bin/release @@ -1,10 +1,9 @@ #!/usr/bin/env bash # bin/release <build-dir> -BIN_DIR=$(cd $(dirname $0); pwd) # absolute path BUILD_DIR=$1 -MANAGE_FILE=$(cd $BUILD_DIR && find . -maxdepth 3 -type f -name 'manage.py' | head -1) +MANAGE_FILE=$(cd "$BUILD_DIR" && find . -maxdepth 3 -type f -name 'manage.py' | head -1) MANAGE_FILE=${MANAGE_FILE:2} cat <<EOF diff --git a/bin/steps/collectstatic b/bin/steps/collectstatic index 51a4f432bbaca1a98560016ced4852aae7e2b6f9..647ab92d5a03ec001e2b3a1db9bf92b90831391f 100755 --- a/bin/steps/collectstatic +++ b/bin/steps/collectstatic @@ -10,6 +10,7 @@ # - $DISABLE_COLLECTSTATIC: disables this functionality. # - $DEBUG_COLLECTSTATIC: upon failure, print out environment variables. +# shellcheck source=bin/utils source $BIN_DIR/utils # Location of 'manage.py', if it exists. @@ -29,13 +30,13 @@ if [ ! "$DISABLE_COLLECTSTATIC" ] && [ -f "$MANAGE_FILE" ] && [ "$DJANGO_INSTALL puts-step "$ python $MANAGE_FILE collectstatic --noinput" # Run collectstatic, cleanup some of the noisy output. - python $MANAGE_FILE collectstatic --noinput --traceback 2>&1 | sed '/^Post-processed/d;/^Copying/d;/^$/d' | indent + python "$MANAGE_FILE" collectstatic --noinput --traceback 2>&1 | sed '/^Post-processed/d;/^Copying/d;/^$/d' | indent COLLECTSTATIC_STATUS="${PIPESTATUS[0]}" set -e # Display a warning if collectstatic failed. - [ $COLLECTSTATIC_STATUS -ne 0 ] && { + [ "$COLLECTSTATIC_STATUS" -ne 0 ] && { echo echo " ! Error while running '$ python $MANAGE_FILE collectstatic --noinput'." diff --git a/bin/steps/cryptography b/bin/steps/cryptography index 8a507999d05256b7dad9a95ea553b7b2ab7fc3c9..06afa5b9de5e75f80855988a309dfd9db744f529 100755 --- a/bin/steps/cryptography +++ b/bin/steps/cryptography @@ -15,7 +15,8 @@ VENDORED_LIBFFI="https://lang-python.s3.amazonaws.com/$STACK/libraries/vendor/li PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH" # Syntax sugar. -source $BIN_DIR/utils +# shellcheck source=bin/utils +source "$BIN_DIR/utils" # If a package using cffi exists within requirements, use vendored libffi. if (pip-grep -s requirements.txt argon2-cffi bcrypt cffi cryptography django[argon2] Django[argon2] django[bcrypt] Django[bcrypt] PyNaCl pyOpenSSL PyOpenSSL requests[security] misaka &> /dev/null) then @@ -24,8 +25,9 @@ if (pip-grep -s requirements.txt argon2-cffi bcrypt cffi cryptography django[arg echo "-----> Noticed cffi. Bootstrapping libffi." mkdir -p .heroku/vendor # Download and extract libffi into target vendor directory. - curl $VENDORED_LIBFFI -s | tar zxv -C .heroku/vendor &> /dev/null + curl "$VENDORED_LIBFFI" -s | tar zxv -C .heroku/vendor &> /dev/null fi - export LIBFFI=$(pwd)/vendor + LIBFFI=$(pwd)/vendor + export LIBFFI fi diff --git a/bin/steps/eggpath-fix b/bin/steps/eggpath-fix index 65d16d2b322ab359c853ce5d21882666457fae34..f1e3f38c024cd2686ba67fa082870da297766738 100644 --- a/bin/steps/eggpath-fix +++ b/bin/steps/eggpath-fix @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + set +e # delete any existing egg links, to uninstall exisisting installations. find .heroku/python/lib/python*/site-packages/ -name "*.egg-link" -delete 2> /dev/null diff --git a/bin/steps/eggpath-fix2 b/bin/steps/eggpath-fix2 index f119a2914396d58cc0197c53787d812d8b27db42..303c12a9f0efd4af00932d4d4b916e3f0e4438d3 100644 --- a/bin/steps/eggpath-fix2 +++ b/bin/steps/eggpath-fix2 @@ -1,3 +1,4 @@ +#!/usr/bin/env bash set +e # rewrite build dir in egg links to /app so things are found at runtime find .heroku/python/lib/python*/site-packages/ -name "*.pth" -print0 2> /dev/null | xargs -r -0 -n 1 sed -i -e "s#$(pwd)#/app#" &> /dev/null diff --git a/bin/steps/gdal b/bin/steps/gdal index 50a09ac331aad10a53e797b49e0d5c01250e2d8f..b060ff5678db0491dfa021d28291d4bee0b08ee2 100755 --- a/bin/steps/gdal +++ b/bin/steps/gdal @@ -15,7 +15,8 @@ VENDORED_GDAL="https://lang-python.s3.amazonaws.com/$STACK/libraries/vendor/gdal PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH" # Syntax sugar. -source $BIN_DIR/utils +# shellcheck source=bin/utils +source "$BIN_DIR/utils" # If GDAL exists within requirements, use vendored gdal. if (pip-grep -s requirements.txt GDAL gdal pygdal &> /dev/null) then @@ -24,9 +25,10 @@ if (pip-grep -s requirements.txt GDAL gdal pygdal &> /dev/null) then echo "-----> Noticed GDAL. Bootstrapping gdal." mkdir -p .heroku/vendor # Download and extract cryptography into target vendor directory. - curl $VENDORED_GDAL -s | tar zxv -C .heroku/vendor &> /dev/null + curl "$VENDORED_GDAL" -s | tar zxv -C .heroku/vendor &> /dev/null fi - export GDAL=$(pwd)/vendor + GDAL=$(pwd)/vendor + export GDAL fi diff --git a/bin/steps/geo-libs b/bin/steps/geo-libs index 3240fed0cdea322863d0e181192e43ac4d8a8a61..c41efb407b8a9a6dc4e13bc40d90aaa56e85c0eb 100755 --- a/bin/steps/geo-libs +++ b/bin/steps/geo-libs @@ -17,7 +17,8 @@ VENDORED_PROJ="https://lang-python.s3.amazonaws.com/$STACK/libraries/vendor/proj PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH" # Syntax sugar. -source $BIN_DIR/utils +# shellcheck source=bin/utils +source "$BIN_DIR/utils" # If GDAL exists within requirements, use vendored gdal. if [[ "$BUILD_WITH_GEO_LIBRARIES" ]]; then @@ -26,11 +27,12 @@ if [[ "$BUILD_WITH_GEO_LIBRARIES" ]]; then echo "-----> Bootstrapping gdal, geos, proj." mkdir -p .heroku/vendor # Download and extract cryptography into target vendor directory. - curl $VENDORED_GDAL -s | tar zxv -C .heroku/vendor &> /dev/null - curl $VENDORED_GEOS -s | tar zxv -C .heroku/vendor &> /dev/null - curl $VENDORED_PROJ -s | tar zxv -C .heroku/vendor &> /dev/null + curl "$VENDORED_GDAL" -s | tar zxv -C .heroku/vendor &> /dev/null + curl "$VENDORED_GEOS" -s | tar zxv -C .heroku/vendor &> /dev/null + curl "$VENDORED_PROJ" -s | tar zxv -C .heroku/vendor &> /dev/null fi - export GDAL=$(pwd)/vendor + GDAL=$(pwd)/vendor + export GDAL fi diff --git a/bin/steps/mercurial b/bin/steps/mercurial index cd4ad707b6cd0d4acbceb2df161ab6e98e457df7..0eaba33101d7365781fa5a474ec0d79519eb1123 100755 --- a/bin/steps/mercurial +++ b/bin/steps/mercurial @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + # Install Mercurial if it appears to be required. if (grep -Fiq "hg+" requirements.txt) then /app/.heroku/python/bin/pip install mercurial | cleanup | indent diff --git a/bin/steps/nltk b/bin/steps/nltk index ea07799bc7e07d018849c65a1d23873f3dae1e11..f1557fdca39db8df3f2e456f3bde1506944490ad 100755 --- a/bin/steps/nltk +++ b/bin/steps/nltk @@ -10,18 +10,23 @@ # This script is invoked by [`bin/compile`](/). # Syntax sugar. -source $BIN_DIR/utils +# shellcheck source=bin/utils +source "$BIN_DIR/utils" # Check that nltk was installed by pip, otherwise obviously not needed -python -m nltk.downloader -h >/dev/null 2>&1 -if [ $? -eq 0 ]; then +if sp-grep -s nltk; then puts-step "Downloading NLTK corpora..." + nltk_packages_definition="$BUILD_DIR/nltk.txt" + if [ -f "$nltk_packages_definition" ]; then + nltk_packages=$(tr "\n" " " < "$nltk_packages_definition") puts-step "Downloading NLTK packages: $nltk_packages" - python -m nltk.downloader -d $BUILD_DIR/.heroku/python/nltk_data $nltk_packages | indent + + python -m nltk.downloader -d "$BUILD_DIR/.heroku/python/nltk_data" "$nltk_packages" | indent set-env NLTK_DATA "/app/.heroku/python/nltk_data" + else puts-warn "'nltk.txt' not found, not downloading any corpora" puts-warn "Learn more: https://devcenter.heroku.com/articles/python-nltk" diff --git a/bin/steps/pip-install b/bin/steps/pip-install index 017f95138b484fe3ba795d6d647a6e5c107af977..ee7e3589d673f651bcfb10f937e70972df8f6ef4 100755 --- a/bin/steps/pip-install +++ b/bin/steps/pip-install @@ -1,10 +1,12 @@ +#!/usr/bin/env bash + if [ ! "$SKIP_PIP_INSTALL" ]; then # Install dependencies with Pip. puts-step "Installing requirements with pip" set +e - /app/.heroku/python/bin/pip install -r $BUILD_DIR/requirements.txt --exists-action=w --src=/app/.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | tee $WARNINGS_LOG | cleanup | indent + /app/.heroku/python/bin/pip install -r "$BUILD_DIR/requirements.txt" --exists-action=w --src=/app/.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | tee "$WARNINGS_LOG" | cleanup | indent PIP_STATUS="${PIPESTATUS[0]}" set -e diff --git a/bin/steps/pip-uninstall b/bin/steps/pip-uninstall index 64950c2382ca398f88f6ccbf79ccccd7e2753123..6cf469f1dcb631fb25a8bf16a0ea4d49db435c87 100755 --- a/bin/steps/pip-uninstall +++ b/bin/steps/pip-uninstall @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + set +e # Install dependencies with Pip. diff --git a/bin/steps/pipenv b/bin/steps/pipenv index cdb9d18c4bd3381f58295e38d72203691913aab8..23032ade06a85340f7a30c99ba1e847d027f9a4d 100644 --- a/bin/steps/pipenv +++ b/bin/steps/pipenv @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + # Pipenv support (Generate requriements.txt with pipenv). if [[ -f Pipfile ]]; then if [[ ! -f requirements.txt ]]; then diff --git a/bin/steps/pipenv-python-version b/bin/steps/pipenv-python-version index ed462657cc4bf9c867733ae78ff009f9c38bf688..e69ef31ced7954f38ec16f354a28949eccb7ce9f 100755 --- a/bin/steps/pipenv-python-version +++ b/bin/steps/pipenv-python-version @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + # Detect Python-version with Pipenv. if [[ -f $BUILD_DIR/Pipfile.lock ]]; then @@ -8,14 +10,14 @@ if [[ -f $BUILD_DIR/Pipfile.lock ]]; then fi if [[ -f $BUILD_DIR/Pipfile.lock ]]; then set +e - PYTHON=$(cat $BUILD_DIR/Pipfile.lock | jq '._meta.requires.python_version' -r) + PYTHON=$(jq -r '._meta.requires.python_version' "$BUILD_DIR/Pipfile.lock") set -e if [ "$PYTHON" = 2.7 ]; then - echo "python-2.7.13" > $BUILD_DIR/runtime.txt + echo "python-2.7.13" > "$BUILD_DIR/runtime.txt" fi if [ "$PYTHON" = 3.6 ]; then - echo "python-3.6.0" > $BUILD_DIR/runtime.txt + echo "python-3.6.2" > "$BUILD_DIR/runtime.txt" fi fi fi diff --git a/bin/steps/pylibmc b/bin/steps/pylibmc index f574e5350b8adabc4571578767cc3d11924b89f7..2ebba64ed1d592db455b1fb2b9bfb65aab7a5649 100755 --- a/bin/steps/pylibmc +++ b/bin/steps/pylibmc @@ -13,7 +13,8 @@ VENDORED_MEMCACHED="https://lang-python.s3.amazonaws.com/$STACK/libraries/vendor/libmemcache.tar.gz" # Syntax sugar. -source $BIN_DIR/utils +# shellcheck source=bin/utils +source "$BIN_DIR/utils" # If pylibmc exists within requirements, use vendored libmemcached. @@ -23,8 +24,9 @@ if (pip-grep -s requirements.txt pylibmc &> /dev/null) then echo "-----> Noticed pylibmc. Bootstrapping libmemcached." mkdir -p .heroku/vendor # Download and extract libmemcached into target vendor directory. - curl $VENDORED_MEMCACHED -s | tar zxv -C .heroku/vendor &> /dev/null + curl "$VENDORED_MEMCACHED" -s | tar zxv -C .heroku/vendor &> /dev/null fi - export LIBMEMCACHED=$(pwd)/vendor + LIBMEMCACHED=$(pwd)/vendor + export LIBMEMCACHED fi diff --git a/bin/steps/python b/bin/steps/python index 30aa2c8cafc6eeeb571100f3014b4786a0387462..32a25903da0262cbda9914a8bb643cc015b8c5d8 100755 --- a/bin/steps/python +++ b/bin/steps/python @@ -1,10 +1,12 @@ +#!/usr/bin/env bash + set +e runtime-fixer runtime.txt PYTHON_VERSION=$(cat runtime.txt) # Install Python. if [ -f .heroku/python-version ]; then - if [ ! $(cat .heroku/python-version) = $PYTHON_VERSION ]; then + if [ ! "$(cat .heroku/python-version)" = "$PYTHON_VERSION" ]; then puts-step "Found $(cat .heroku/python-version), removing" rm -fr .heroku/python else @@ -12,7 +14,7 @@ if [ -f .heroku/python-version ]; then fi fi -if [ ! $STACK = $CACHED_PYTHON_STACK ]; then +if [ ! "$STACK" = "$CACHED_PYTHON_STACK" ]; then rm -fr .heroku/python .heroku/python-stack .heroku/vendor unset SKIP_INSTALL fi @@ -24,29 +26,24 @@ if [ ! "$SKIP_INSTALL" ]; then # Prepare destination directory. mkdir -p .heroku/python - curl https://lang-python.s3.amazonaws.com/$STACK/runtimes/$PYTHON_VERSION.tar.gz -s | tar zxv -C .heroku/python &> /dev/null mcount "version.python.$PYTHON_VERSION" - if [[ $? != 0 ]] ; then + if ! curl "https://lang-python.s3.amazonaws.com/$STACK/runtimes/$PYTHON_VERSION.tar.gz" -s | tar zxv -C .heroku/python &> /dev/null; then puts-warn "Requested runtime ($PYTHON_VERSION) is not available for this stack ($STACK)." puts-warn "Aborting. More info: https://devcenter.heroku.com/articles/python-support" exit 1 fi # Record for future reference. - echo $PYTHON_VERSION > .heroku/python-version - echo $STACK > .heroku/python-stack + echo "$PYTHON_VERSION" > .heroku/python-version + echo "$STACK" > .heroku/python-stack FRESH_PYTHON=true hash -r fi # If Pip isn't up to date: -if [ "$FRESH_PYTHON" ] || [[ ! $(pip --version) == *$PIP_VERSION* ]]; then - WORKING_DIR=$(pwd) - - TMPTARDIR=$(mktemp -d) - trap "rm -rf $TMPTARDIR" RETURN +if [ "$FRESH_PYTHON" ] || [[ ! $(pip --version) == *$PIP_UPDATE* ]]; then puts-step "Installing pip" @@ -54,7 +51,7 @@ if [ "$FRESH_PYTHON" ] || [[ ! $(pip --version) == *$PIP_VERSION* ]]; then rm -fr /app/.heroku/python/lib/python2.7/site-packages/pip-* rm -fr /app/.heroku/python/lib/python2.7/site-packages/setuptools-* - /app/.heroku/python/bin/python $ROOT_DIR/vendor/get-pip.py &> /dev/null + /app/.heroku/python/bin/python "$ROOT_DIR/vendor/get-pip.py" &> /dev/null fi diff --git a/bin/test-compile b/bin/test-compile index 51b162e635220c05a61f75d737ae577b42adbba0..d19814dbbe6bd55413a972fd0ed711ed89bc2c57 100755 --- a/bin/test-compile +++ b/bin/test-compile @@ -1,10 +1,12 @@ #!/usr/bin/env bash # Syntax sugar. -BIN_DIR=$(cd $(dirname $0); pwd) # absolute path -source $BIN_DIR/utils +BIN_DIR=$(cd "$(dirname "$0")" || return; pwd) # absolute path -DISABLE_COLLECTSTATIC=1 "$(dirname ${0:-})/compile" "$1" "$2" "$3" +# shellcheck source=bin/utils +source "$BIN_DIR/utils" + +DISABLE_COLLECTSTATIC=1 "$(dirname "${0:-}")/compile" "$1" "$2" "$3" if [[ -f "$1/requirements-test.txt" ]]; then /app/.heroku/python/bin/pip install -r "$1/requirements-test.txt" --exists-action=w --src=./.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | cleanup | indent diff --git a/bin/utils b/bin/utils index dc2a69d778ac148af16256ec7505f18796b678da..59273a7c49124c231dd9fdbbc2a3c55b25062199 100755 --- a/bin/utils +++ b/bin/utils @@ -1,13 +1,15 @@ #!/usr/bin/env bash shopt -s extglob +shopt -s nullglob # The standard library. if [[ ! -f /tmp/stdlib.sh ]]; then curl --retry 3 -s https://lang-common.s3.amazonaws.com/buildpack-stdlib/v2/stdlib.sh > /tmp/stdlib.sh fi +# shellcheck source=/dev/null source /tmp/stdlib.sh -if [ $(uname) == Darwin ]; then +if [ "$(uname)" == Darwin ]; then sed() { command sed -l "$@"; } else sed() { command sed -u "$@"; } @@ -26,12 +28,12 @@ cleanup() { # Buildpack Steps. puts-step() { - echo "-----> $@" + echo "-----> $*" } # Buildpack Warnings. puts-warn() { - echo " ! $@" + echo " ! $*" } # Does some serious copying. @@ -62,9 +64,10 @@ sub-env() { ( if [ -d "$ENV_DIR" ]; then - for e in $(ls $ENV_DIR); do + # shellcheck disable=SC2045 + for e in $(ls "$ENV_DIR"); do echo "$e" | grep -E "$WHITELIST" | grep -qvE "$BLACKLIST" && - export "$e=$(cat $ENV_DIR/$e)" + export "$e=$(cat "$ENV_DIR/$e")" : done fi @@ -76,6 +79,6 @@ sub-env() { # Measure the size of the Python installation. measure-size() { - echo "$((du -s .heroku/python 2>/dev/null || echo 0) | awk '{print $1}')" + echo "$(du -s .heroku/python 2>/dev/null || echo 0) | awk '{print $1}')" } diff --git a/bin/warnings b/bin/warnings index 01e43d58f097b6e00355dfe1721950ce920b523d..b43748b42799e0cc663883dc5572d078c773d3f3 100755 --- a/bin/warnings +++ b/bin/warnings @@ -1,3 +1,4 @@ +#!/usr/bin/env bash shopt -s extglob old-platform() { diff --git a/test/fixtures/python3/runtime.txt b/test/fixtures/python3/runtime.txt deleted file mode 100644 index 80aea67443c78b91acc7fd40849e1c28daf6ae7c..0000000000000000000000000000000000000000 --- a/test/fixtures/python3/runtime.txt +++ /dev/null @@ -1 +0,0 @@ -python-3.6.0 \ No newline at end of file diff --git a/test/run b/test/run index 7c042a3fcb5a3ac3ed3aebf5cb13f2d511385f1c..e125c13f4d4fab1fc05cd218bf03aecb4d2465c8 100755 --- a/test/run +++ b/test/run @@ -7,7 +7,7 @@ testPipenv() { testPipenvVersion() { compile "pipenv-version" - assertCaptured "3.6.0" + assertCaptured "3.6.2" assertCapturedSuccess } @@ -61,7 +61,7 @@ testPython2() { testPython3() { compile "python3" - assertCaptured "python-3.6.0" + assertCaptured "python-3.6.2" assertCapturedSuccess }