From f74d4db8c5953b39ef69852157ad8f78d678bf8d Mon Sep 17 00:00:00 2001 From: Kenneth Reitz <me@kennethreitz.org> Date: Thu, 3 Aug 2017 17:36:23 -0400 Subject: [PATCH] Shellcheck Compliance (#438) * fixed the bug for pypy-5.8.0 * enable shellcheck in makefile Signed-off-by: Kenneth Reitz <me@kennethreitz.org> * make compile script shellcheck compatible Signed-off-by: Kenneth Reitz <me@kennethreitz.org> * let's see what this says * fix pip installation Signed-off-by: Kenneth Reitz <me@kennethreitz.org> * shellcheck detect script * shellcheck release Signed-off-by: Kenneth Reitz <me@kennethreitz.org> * shellcheck test-compile Signed-off-by: Kenneth Reitz <me@kennethreitz.org> * shellcheck warnings/utils * updated makefile Signed-off-by: Kenneth Reitz <me@kennethreitz.org> * shellcheck collectstatic Signed-off-by: Kenneth Reitz <me@kennethreitz.org> * shellcheck cryptography Signed-off-by: Kenneth Reitz <me@kennethreitz.org> * shellcheck eggpath fixes Signed-off-by: Kenneth Reitz <me@kennethreitz.org> * shellcheck gdal * shellcheck goes-libs Signed-off-by: Kenneth Reitz <me@kennethreitz.org> * shellcheck mercurial Signed-off-by: Kenneth Reitz <me@kennethreitz.org> * nltk shellcheck * shellcheck pip-install Signed-off-by: Kenneth Reitz <me@kennethreitz.org> * shellcheck pip-uninstall Signed-off-by: Kenneth Reitz <me@kennethreitz.org> * shellcheck pipenv Signed-off-by: Kenneth Reitz <me@kennethreitz.org> * shellcheck pylibmc Signed-off-by: Kenneth Reitz <me@kennethreitz.org> * shellcheck python Signed-off-by: Kenneth Reitz <me@kennethreitz.org> * update makefile Signed-off-by: Kenneth Reitz <me@kennethreitz.org> * final update to makefile Signed-off-by: Kenneth Reitz <me@kennethreitz.org> * fix tests Signed-off-by: Kenneth Reitz <me@kennethreitz.org> * actually fix the tests Signed-off-by: Kenneth Reitz <me@kennethreitz.org> * update pipenv-python-version Signed-off-by: Kenneth Reitz <me@kennethreitz.org> * requested fixes Signed-off-by: Kenneth Reitz <me@kennethreitz.org> * update spellcheck sources * attempted fix for shell globbing Signed-off-by: Kenneth Reitz <me@kennethreitz.org> * further attempted fix Signed-off-by: Kenneth Reitz <me@kennethreitz.org> * switch nltk styling Signed-off-by: Kenneth Reitz <me@kennethreitz.org> * no more need for temp app for installing pip Signed-off-by: Kenneth Reitz <me@kennethreitz.org> * attempt at staged build for shellcheck Signed-off-by: Kenneth Reitz <me@kennethreitz.org> * attempted fix of travis file Signed-off-by: Kenneth Reitz <me@kennethreitz.org> * reorder travis file * add shellcheck to install * make things clearer in travis file Signed-off-by: Kenneth Reitz <me@kennethreitz.org> * better travis file Signed-off-by: Kenneth Reitz <me@kennethreitz.org> * attempt at better travis file Signed-off-by: Kenneth Reitz <me@kennethreitz.org> * better travis file Signed-off-by: Kenneth Reitz <me@kennethreitz.org> --- .travis.yml | 26 ++++-- Makefile | 5 ++ bin/compile | 141 ++++++++++++++++++------------ bin/detect | 2 +- bin/release | 3 +- bin/steps/collectstatic | 5 +- bin/steps/cryptography | 8 +- bin/steps/eggpath-fix | 2 + bin/steps/eggpath-fix2 | 1 + bin/steps/gdal | 8 +- bin/steps/geo-libs | 12 +-- bin/steps/mercurial | 2 + bin/steps/nltk | 13 ++- bin/steps/pip-install | 4 +- bin/steps/pip-uninstall | 2 + bin/steps/pipenv | 2 + bin/steps/pipenv-python-version | 8 +- bin/steps/pylibmc | 8 +- bin/steps/python | 21 ++--- bin/test-compile | 8 +- bin/utils | 15 ++-- bin/warnings | 1 + test/fixtures/python3/runtime.txt | 1 - test/run | 4 +- 24 files changed, 188 insertions(+), 114 deletions(-) delete mode 100644 test/fixtures/python3/runtime.txt diff --git a/.travis.yml b/.travis.yml index 7f5f0267..c6c1b47b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,25 @@ language: bash -sudo: required +# sudo: required +addons: + apt: + sources: + - debian-sid # Grab shellcheck from the Debian repo (o_O) + packages: + - shellcheck services: - docker # install: docker pull heroku/cedar:14 -script: ./tests.sh -env: - - STACK=heroku-16 - - STACK=cedar-14 \ No newline at end of file +jobs: + include: + - stage: "Bash linting (shellcheck)" + script: make check + + - stage: "Heroku-16 Stack Tests" + script: ./tests.sh + env: + - STACK=heroku-16 + + - stage: "Cedar-14 Stack Tests" + script: ./tests.sh + env: + - STACK=cedar-14 \ No newline at end of file diff --git a/Makefile b/Makefile index 7d4ae900..81d0df8f 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,11 @@ test: test-heroku-16 +check: + @shellcheck -x bin/compile bin/detect bin/release bin/test-compile bin/utils bin/warnings + @shellcheck -x bin/steps/collectstatic bin/steps/cryptography bin/steps/eggpath-fix bin/steps/eggpath-fix2 bin/steps/gdal bin/steps/geo-libs bin/steps/mercurial bin/steps/nltk bin/steps/pip-install bin/steps/pip-uninstall bin/steps/pipenv bin/steps/pipenv-python-version bin/steps/pylibmc bin/steps/python + @shellcheck -x bin/steps/hooks/* + test-cedar-14: @echo "Running tests in docker (cedar-14)..." @docker run -v $(shell pwd):/buildpack:ro --rm -it -e "STACK=cedar-14" heroku/cedar:14 bash -c 'cp -r /buildpack /buildpack_test; cd /buildpack_test/; test/run;' diff --git a/bin/compile b/bin/compile index b5c31083..ad0ac09f 100755 --- a/bin/compile +++ b/bin/compile @@ -25,20 +25,24 @@ export BUILDPACK_LOG_FILE=${BUILDPACK_LOG_FILE:-/dev/null} export PATH=:/usr/local/bin:$PATH # Paths. -BIN_DIR=$(cd $(dirname $0); pwd) # absolute path -ROOT_DIR=$(dirname $BIN_DIR) +BIN_DIR=$(cd "$(dirname "$0")"; pwd) # absolute path +ROOT_DIR=$(dirname "$BIN_DIR") BUILD_DIR=$1 CACHE_DIR=$2 ENV_DIR=$3 +export BUILD_DIR CACHE_DIR ENV_DIR + # Python defaults DEFAULT_PYTHON_VERSION="python-3.6.2" DEFAULT_PYTHON_STACK="cedar-14" -PYTHON_EXE="/app/.heroku/python/bin/python" -PIP_VERSION="9.0.1" +PIP_UPDATE="9.0.1" + +export DEFAULT_PYTHON_VERSION DEFAULT_PYTHON_STACK PIP_UPDATE # Common Problem Warnings -export WARNINGS_LOG=$(mktemp) +WARNINGS_LOG=$(mktemp) +export WARNINGS_LOG export RECOMMENDED_PYTHON_VERSION=$DEFAULT_PYTHON_VERSION # Setup vendored tools and pip-pop (pip-diff) @@ -55,10 +59,12 @@ unset RECEIVE_DATA RUN_KEY BUILD_INFO DEPLOY LOG_TOKEN unset CYTOKINE_LOG_FILE GEM_PATH # Syntax sugar. -source $BIN_DIR/utils +# shellcheck source=bin/utils +source "$BIN_DIR/utils" # Import collection of warnings. -source $BIN_DIR/warnings +# shellcheck source=bin/warnings +source "$BIN_DIR/warnings" # we need to put a bunch of symlinks in there later mkdir -p /app/.heroku @@ -83,7 +89,7 @@ export LD_LIBRARY_PATH=/app/.heroku/vendor/lib:/app/.heroku/python/lib:$LD_LIBRA export PKG_CONFIG_PATH=/app/.heroku/vendor/lib/pkg-config:/app/.heroku/python/lib/pkg-config:$PKG_CONFIG_PATH # Switch to the repo's context. -cd $BUILD_DIR +cd "$BUILD_DIR" # Warn for lack of Procfile. if [[ ! -f Procfile ]]; then @@ -92,60 +98,66 @@ if [[ ! -f Procfile ]]; then fi # Prepare the cache. -mkdir -p $CACHE_DIR +mkdir -p "$CACHE_DIR" # Restore old artifacts from the cache. mkdir -p .heroku -cp -R $CACHE_DIR/.heroku/python .heroku/ &> /dev/null || true -cp -R $CACHE_DIR/.heroku/python-stack .heroku/ &> /dev/null || true -cp -R $CACHE_DIR/.heroku/python-version .heroku/ &> /dev/null || true -cp -R $CACHE_DIR/.heroku/vendor .heroku/ &> /dev/null || true -if [[ -d $CACHE_DIR/.heroku/src ]]; then - cp -R $CACHE_DIR/.heroku/src .heroku/ &> /dev/null || true +cp -R "$CACHE_DIR/.heroku/python" .heroku/ &> /dev/null || true +cp -R "$CACHE_DIR/.heroku/python-stack" .heroku/ &> /dev/null || true +cp -R "$CACHE_DIR/.heroku/python-version" .heroku/ &> /dev/null || true +cp -R "$CACHE_DIR/.heroku/vendor" .heroku/ &> /dev/null || true +if [[ -d "$CACHE_DIR/.heroku/src" ]]; then + cp -R "$CACHE_DIR/.heroku/src" .heroku/ &> /dev/null || true fi # Experimental pre_compile hook. -source $BIN_DIR/steps/hooks/pre_compile +# shellcheck source=bin/steps/hooks/pre_compile +source "$BIN_DIR/steps/hooks/pre_compile" # Sticky runtimes. -if [ -f $CACHE_DIR/.heroku/python-version ]; then - DEFAULT_PYTHON_VERSION=$(cat $CACHE_DIR/.heroku/python-version) +if [ -f "$CACHE_DIR/.heroku/python-version" ]; then + DEFAULT_PYTHON_VERSION=$(cat "$CACHE_DIR/.heroku/python-version") fi # Stack fallback for non-declared caches. -if [ -f $CACHE_DIR/.heroku/python-stack ]; then - CACHED_PYTHON_STACK=$(cat $CACHE_DIR/.heroku/python-stack) +if [ -f "$CACHE_DIR/.heroku/python-stack" ]; then + CACHED_PYTHON_STACK=$(cat "$CACHE_DIR/.heroku/python-stack") else CACHED_PYTHON_STACK=$STACK fi +export CACHED_PYTHON_STACK + # Pipenv Python version support. -source $BIN_DIR/steps/pipenv-python-version +# shellcheck source=bin/steps/pipenv-python-version +source "$BIN_DIR/steps/pipenv-python-version" # If no runtime given, assume default version. if [ ! -f runtime.txt ]; then - echo $DEFAULT_PYTHON_VERSION > runtime.txt + echo "$DEFAULT_PYTHON_VERSION" > runtime.txt fi -mkdir -p $(dirname $PROFILE_PATH) +mkdir -p "$(dirname "$PROFILE_PATH")" mkdir -p /app/.heroku/src if [[ $BUILD_DIR != '/app' ]]; then # python expects to reside in /app, so set up symlinks # we will not remove these later so subsequent buildpacks can still invoke it - ln -nsf $BUILD_DIR/.heroku/python /app/.heroku/python - ln -nsf $BUILD_DIR/.heroku/vendor /app/.heroku/vendor + ln -nsf "$BUILD_DIR/.heroku/python" /app/.heroku/python + ln -nsf "$BUILD_DIR/.heroku/vendor" /app/.heroku/vendor # Note: .heroku/src is copied in later. fi # Install Python. let start=$(nowms) -source $BIN_DIR/steps/python +# shellcheck source=bin/steps/python +source "$BIN_DIR/steps/python" mtime "python.install.time" "${start}" # Pipenv support. -source $BIN_DIR/steps/pipenv +# shellcheck source=bin/steps/pipenv +source "$BIN_DIR/steps/pipenv" # If no requirements.txt file given, assume `setup.py develop` is intended. if [ ! -f requirements.txt ] && [ ! -f Pipfile ]; then @@ -153,88 +165,101 @@ if [ ! -f requirements.txt ] && [ ! -f Pipfile ]; then fi # Fix egg-links. -source $BIN_DIR/steps/eggpath-fix +# shellcheck source=bin/steps/eggpath-fix +source "$BIN_DIR/steps/eggpath-fix" # Mercurial support. -source $BIN_DIR/steps/mercurial +# shellcheck source=bin/steps/mercurial +source "$BIN_DIR/steps/mercurial" # Pylibmc support. -source $BIN_DIR/steps/pylibmc +# shellcheck source=bin/steps/pylibmc +source "$BIN_DIR/steps/pylibmc" # Libffi support. -source $BIN_DIR/steps/cryptography +# shellcheck source=bin/steps/cryptography +source "$BIN_DIR/steps/cryptography" # Support for Geo libraries. -sub-env $BIN_DIR/steps/geo-libs +# shellcheck source=bin/steps/geo-libs +sub-env "$BIN_DIR/steps/geo-libs" # GDAL support. -source $BIN_DIR/steps/gdal +# shellcheck source=bin/steps/gdal +source "$BIN_DIR/steps/gdal" # Uninstall removed dependencies with Pip. let start=$(nowms) -source $BIN_DIR/steps/pip-uninstall +# shellcheck source=bin/steps/pip-uninstall +source "$BIN_DIR/steps/pip-uninstall" mtime "pip.uninstall.time" "${start}" # Install dependencies with Pip (where the magic happens). let start=$(nowms) -source $BIN_DIR/steps/pip-install +# shellcheck source=bin/steps/pip-install +source "$BIN_DIR/steps/pip-install" mtime "pip.install.time" "${start}" # Support for NLTK corpora. let start=$(nowms) -sub-env $BIN_DIR/steps/nltk +sub-env "$BIN_DIR/steps/nltk" mtime "nltk.download.time" "${start}" # Support for pip install -e. # In CI, $BUILD_DIR is /app. if [[ ! "$BUILD_DIR" == "/app" ]]; then - rm -fr $BUILD_DIR/.heroku/src - deep-cp /app/.heroku/src $BUILD_DIR/.heroku/src + rm -fr "$BUILD_DIR/.heroku/src" + deep-cp /app/.heroku/src "$BUILD_DIR/.heroku/src" fi # Django collectstatic support. let start=$(nowms) -sub-env $BIN_DIR/steps/collectstatic +sub-env "$BIN_DIR/steps/collectstatic" mtime "collectstatic.time" "${start}" # Create .profile script for application runtime environment variables. -set-env PATH '$HOME/.heroku/python/bin:$PATH' +set-env PATH "\$HOME/.heroku/python/bin:\$PATH" set-env PYTHONUNBUFFERED true set-env PYTHONHOME /app/.heroku/python -set-env LIBRARY_PATH '/app/.heroku/vendor/lib:/app/.heroku/python/lib:$LIBRARY_PATH' -set-env LD_LIBRARY_PATH '/app/.heroku/vendor/lib:/app/.heroku/python/lib:$LD_LIBRARY_PATH' + +set-env LIBRARY_PATH "/app/.heroku/vendor/lib:/app/.heroku/python/lib:\$LIBRARY_PATH" +set-env LD_LIBRARY_PATH "/app/.heroku/vendor/lib:/app/.heroku/python/lib:\$LD_LIBRARY_PATH" + set-default-env LANG en_US.UTF-8 set-default-env PYTHONHASHSEED random set-default-env PYTHONPATH /app/ # Install sane-default script for $WEB_CONCURRENCY and $FORWARDED_ALLOW_IPS. -cp $ROOT_DIR/vendor/WEB_CONCURRENCY.sh $WEB_CONCURRENCY_PROFILE_PATH -cp $ROOT_DIR/vendor/python.gunicorn.sh $GUNICORN_PROFILE_PATH +cp "$ROOT_DIR/vendor/WEB_CONCURRENCY.sh" "$WEB_CONCURRENCY_PROFILE_PATH" +cp "$ROOT_DIR/vendor/python.gunicorn.sh" "$GUNICORN_PROFILE_PATH" # Experimental post_compile hook. -source $BIN_DIR/steps/hooks/post_compile +# shellcheck source=bin/steps/hooks/post_compile +source "$BIN_DIR/steps/hooks/post_compile" # Fix egg-links, again. -source $BIN_DIR/steps/eggpath-fix2 +# shellcheck source=bin/steps/eggpath-fix2 +source "$BIN_DIR/steps/eggpath-fix2" # Store new artifacts in cache. -rm -rf $CACHE_DIR/.heroku/python -rm -rf $CACHE_DIR/.heroku/python-version -rm -rf $CACHE_DIR/.heroku/python-stack -rm -rf $CACHE_DIR/.heroku/vendor -rm -rf $CACHE_DIR/.heroku/src - -mkdir -p $CACHE_DIR/.heroku -cp -R .heroku/python $CACHE_DIR/.heroku/ -cp -R .heroku/python-version $CACHE_DIR/.heroku/ -cp -R .heroku/python-stack $CACHE_DIR/.heroku/ &> /dev/null || true -cp -R .heroku/vendor $CACHE_DIR/.heroku/ &> /dev/null || true +rm -rf "$CACHE_DIR/.heroku/python" +rm -rf "$CACHE_DIR/.heroku/python-version" +rm -rf "$CACHE_DIR/.heroku/python-stack" +rm -rf "$CACHE_DIR/.heroku/vendor" +rm -rf "$CACHE_DIR/.heroku/src" + +mkdir -p "$CACHE_DIR/.heroku" +cp -R .heroku/python "$CACHE_DIR/.heroku/" +cp -R .heroku/python-version "$CACHE_DIR/.heroku/" +cp -R .heroku/python-stack "$CACHE_DIR/.heroku/" &> /dev/null || true +cp -R .heroku/vendor "$CACHE_DIR/.heroku/" &> /dev/null || true if [[ -d .heroku/src ]]; then - cp -R .heroku/src $CACHE_DIR/.heroku/ &> /dev/null || true + cp -R .heroku/src "$CACHE_DIR/.heroku/" &> /dev/null || true fi # Measure the size of the Python installation. +# shellcheck disable=SC2119 mmeasure 'python.size' "$(measure-size)" diff --git a/bin/detect b/bin/detect index cee85b19..eeb965b0 100755 --- a/bin/detect +++ b/bin/detect @@ -15,7 +15,7 @@ BUILD_DIR=$1 # Exit early if app is clearly not Python. -if [ ! -f $BUILD_DIR/requirements.txt ] && [ ! -f $BUILD_DIR/setup.py ] && [ ! -f $BUILD_DIR/Pipfile ]; then +if [ ! -f "$BUILD_DIR/requirements.txt" ] && [ ! -f "$BUILD_DIR/setup.py" ] && [ ! -f "$BUILD_DIR/Pipfile" ]; then exit 1 fi diff --git a/bin/release b/bin/release index 8e342ca2..86fca78c 100755 --- a/bin/release +++ b/bin/release @@ -1,10 +1,9 @@ #!/usr/bin/env bash # bin/release <build-dir> -BIN_DIR=$(cd $(dirname $0); pwd) # absolute path BUILD_DIR=$1 -MANAGE_FILE=$(cd $BUILD_DIR && find . -maxdepth 3 -type f -name 'manage.py' | head -1) +MANAGE_FILE=$(cd "$BUILD_DIR" && find . -maxdepth 3 -type f -name 'manage.py' | head -1) MANAGE_FILE=${MANAGE_FILE:2} cat <<EOF diff --git a/bin/steps/collectstatic b/bin/steps/collectstatic index 51a4f432..647ab92d 100755 --- a/bin/steps/collectstatic +++ b/bin/steps/collectstatic @@ -10,6 +10,7 @@ # - $DISABLE_COLLECTSTATIC: disables this functionality. # - $DEBUG_COLLECTSTATIC: upon failure, print out environment variables. +# shellcheck source=bin/utils source $BIN_DIR/utils # Location of 'manage.py', if it exists. @@ -29,13 +30,13 @@ if [ ! "$DISABLE_COLLECTSTATIC" ] && [ -f "$MANAGE_FILE" ] && [ "$DJANGO_INSTALL puts-step "$ python $MANAGE_FILE collectstatic --noinput" # Run collectstatic, cleanup some of the noisy output. - python $MANAGE_FILE collectstatic --noinput --traceback 2>&1 | sed '/^Post-processed/d;/^Copying/d;/^$/d' | indent + python "$MANAGE_FILE" collectstatic --noinput --traceback 2>&1 | sed '/^Post-processed/d;/^Copying/d;/^$/d' | indent COLLECTSTATIC_STATUS="${PIPESTATUS[0]}" set -e # Display a warning if collectstatic failed. - [ $COLLECTSTATIC_STATUS -ne 0 ] && { + [ "$COLLECTSTATIC_STATUS" -ne 0 ] && { echo echo " ! Error while running '$ python $MANAGE_FILE collectstatic --noinput'." diff --git a/bin/steps/cryptography b/bin/steps/cryptography index 8a507999..06afa5b9 100755 --- a/bin/steps/cryptography +++ b/bin/steps/cryptography @@ -15,7 +15,8 @@ VENDORED_LIBFFI="https://lang-python.s3.amazonaws.com/$STACK/libraries/vendor/li PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH" # Syntax sugar. -source $BIN_DIR/utils +# shellcheck source=bin/utils +source "$BIN_DIR/utils" # If a package using cffi exists within requirements, use vendored libffi. if (pip-grep -s requirements.txt argon2-cffi bcrypt cffi cryptography django[argon2] Django[argon2] django[bcrypt] Django[bcrypt] PyNaCl pyOpenSSL PyOpenSSL requests[security] misaka &> /dev/null) then @@ -24,8 +25,9 @@ if (pip-grep -s requirements.txt argon2-cffi bcrypt cffi cryptography django[arg echo "-----> Noticed cffi. Bootstrapping libffi." mkdir -p .heroku/vendor # Download and extract libffi into target vendor directory. - curl $VENDORED_LIBFFI -s | tar zxv -C .heroku/vendor &> /dev/null + curl "$VENDORED_LIBFFI" -s | tar zxv -C .heroku/vendor &> /dev/null fi - export LIBFFI=$(pwd)/vendor + LIBFFI=$(pwd)/vendor + export LIBFFI fi diff --git a/bin/steps/eggpath-fix b/bin/steps/eggpath-fix index 65d16d2b..f1e3f38c 100644 --- a/bin/steps/eggpath-fix +++ b/bin/steps/eggpath-fix @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + set +e # delete any existing egg links, to uninstall exisisting installations. find .heroku/python/lib/python*/site-packages/ -name "*.egg-link" -delete 2> /dev/null diff --git a/bin/steps/eggpath-fix2 b/bin/steps/eggpath-fix2 index f119a291..303c12a9 100644 --- a/bin/steps/eggpath-fix2 +++ b/bin/steps/eggpath-fix2 @@ -1,3 +1,4 @@ +#!/usr/bin/env bash set +e # rewrite build dir in egg links to /app so things are found at runtime find .heroku/python/lib/python*/site-packages/ -name "*.pth" -print0 2> /dev/null | xargs -r -0 -n 1 sed -i -e "s#$(pwd)#/app#" &> /dev/null diff --git a/bin/steps/gdal b/bin/steps/gdal index 50a09ac3..b060ff56 100755 --- a/bin/steps/gdal +++ b/bin/steps/gdal @@ -15,7 +15,8 @@ VENDORED_GDAL="https://lang-python.s3.amazonaws.com/$STACK/libraries/vendor/gdal PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH" # Syntax sugar. -source $BIN_DIR/utils +# shellcheck source=bin/utils +source "$BIN_DIR/utils" # If GDAL exists within requirements, use vendored gdal. if (pip-grep -s requirements.txt GDAL gdal pygdal &> /dev/null) then @@ -24,9 +25,10 @@ if (pip-grep -s requirements.txt GDAL gdal pygdal &> /dev/null) then echo "-----> Noticed GDAL. Bootstrapping gdal." mkdir -p .heroku/vendor # Download and extract cryptography into target vendor directory. - curl $VENDORED_GDAL -s | tar zxv -C .heroku/vendor &> /dev/null + curl "$VENDORED_GDAL" -s | tar zxv -C .heroku/vendor &> /dev/null fi - export GDAL=$(pwd)/vendor + GDAL=$(pwd)/vendor + export GDAL fi diff --git a/bin/steps/geo-libs b/bin/steps/geo-libs index 3240fed0..c41efb40 100755 --- a/bin/steps/geo-libs +++ b/bin/steps/geo-libs @@ -17,7 +17,8 @@ VENDORED_PROJ="https://lang-python.s3.amazonaws.com/$STACK/libraries/vendor/proj PKG_CONFIG_PATH="/app/.heroku/vendor/lib/pkgconfig:$PKG_CONFIG_PATH" # Syntax sugar. -source $BIN_DIR/utils +# shellcheck source=bin/utils +source "$BIN_DIR/utils" # If GDAL exists within requirements, use vendored gdal. if [[ "$BUILD_WITH_GEO_LIBRARIES" ]]; then @@ -26,11 +27,12 @@ if [[ "$BUILD_WITH_GEO_LIBRARIES" ]]; then echo "-----> Bootstrapping gdal, geos, proj." mkdir -p .heroku/vendor # Download and extract cryptography into target vendor directory. - curl $VENDORED_GDAL -s | tar zxv -C .heroku/vendor &> /dev/null - curl $VENDORED_GEOS -s | tar zxv -C .heroku/vendor &> /dev/null - curl $VENDORED_PROJ -s | tar zxv -C .heroku/vendor &> /dev/null + curl "$VENDORED_GDAL" -s | tar zxv -C .heroku/vendor &> /dev/null + curl "$VENDORED_GEOS" -s | tar zxv -C .heroku/vendor &> /dev/null + curl "$VENDORED_PROJ" -s | tar zxv -C .heroku/vendor &> /dev/null fi - export GDAL=$(pwd)/vendor + GDAL=$(pwd)/vendor + export GDAL fi diff --git a/bin/steps/mercurial b/bin/steps/mercurial index cd4ad707..0eaba331 100755 --- a/bin/steps/mercurial +++ b/bin/steps/mercurial @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + # Install Mercurial if it appears to be required. if (grep -Fiq "hg+" requirements.txt) then /app/.heroku/python/bin/pip install mercurial | cleanup | indent diff --git a/bin/steps/nltk b/bin/steps/nltk index ea07799b..f1557fdc 100755 --- a/bin/steps/nltk +++ b/bin/steps/nltk @@ -10,18 +10,23 @@ # This script is invoked by [`bin/compile`](/). # Syntax sugar. -source $BIN_DIR/utils +# shellcheck source=bin/utils +source "$BIN_DIR/utils" # Check that nltk was installed by pip, otherwise obviously not needed -python -m nltk.downloader -h >/dev/null 2>&1 -if [ $? -eq 0 ]; then +if sp-grep -s nltk; then puts-step "Downloading NLTK corpora..." + nltk_packages_definition="$BUILD_DIR/nltk.txt" + if [ -f "$nltk_packages_definition" ]; then + nltk_packages=$(tr "\n" " " < "$nltk_packages_definition") puts-step "Downloading NLTK packages: $nltk_packages" - python -m nltk.downloader -d $BUILD_DIR/.heroku/python/nltk_data $nltk_packages | indent + + python -m nltk.downloader -d "$BUILD_DIR/.heroku/python/nltk_data" "$nltk_packages" | indent set-env NLTK_DATA "/app/.heroku/python/nltk_data" + else puts-warn "'nltk.txt' not found, not downloading any corpora" puts-warn "Learn more: https://devcenter.heroku.com/articles/python-nltk" diff --git a/bin/steps/pip-install b/bin/steps/pip-install index 017f9513..ee7e3589 100755 --- a/bin/steps/pip-install +++ b/bin/steps/pip-install @@ -1,10 +1,12 @@ +#!/usr/bin/env bash + if [ ! "$SKIP_PIP_INSTALL" ]; then # Install dependencies with Pip. puts-step "Installing requirements with pip" set +e - /app/.heroku/python/bin/pip install -r $BUILD_DIR/requirements.txt --exists-action=w --src=/app/.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | tee $WARNINGS_LOG | cleanup | indent + /app/.heroku/python/bin/pip install -r "$BUILD_DIR/requirements.txt" --exists-action=w --src=/app/.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | tee "$WARNINGS_LOG" | cleanup | indent PIP_STATUS="${PIPESTATUS[0]}" set -e diff --git a/bin/steps/pip-uninstall b/bin/steps/pip-uninstall index 64950c23..6cf469f1 100755 --- a/bin/steps/pip-uninstall +++ b/bin/steps/pip-uninstall @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + set +e # Install dependencies with Pip. diff --git a/bin/steps/pipenv b/bin/steps/pipenv index cdb9d18c..23032ade 100644 --- a/bin/steps/pipenv +++ b/bin/steps/pipenv @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + # Pipenv support (Generate requriements.txt with pipenv). if [[ -f Pipfile ]]; then if [[ ! -f requirements.txt ]]; then diff --git a/bin/steps/pipenv-python-version b/bin/steps/pipenv-python-version index ed462657..e69ef31c 100755 --- a/bin/steps/pipenv-python-version +++ b/bin/steps/pipenv-python-version @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + # Detect Python-version with Pipenv. if [[ -f $BUILD_DIR/Pipfile.lock ]]; then @@ -8,14 +10,14 @@ if [[ -f $BUILD_DIR/Pipfile.lock ]]; then fi if [[ -f $BUILD_DIR/Pipfile.lock ]]; then set +e - PYTHON=$(cat $BUILD_DIR/Pipfile.lock | jq '._meta.requires.python_version' -r) + PYTHON=$(jq -r '._meta.requires.python_version' "$BUILD_DIR/Pipfile.lock") set -e if [ "$PYTHON" = 2.7 ]; then - echo "python-2.7.13" > $BUILD_DIR/runtime.txt + echo "python-2.7.13" > "$BUILD_DIR/runtime.txt" fi if [ "$PYTHON" = 3.6 ]; then - echo "python-3.6.0" > $BUILD_DIR/runtime.txt + echo "python-3.6.2" > "$BUILD_DIR/runtime.txt" fi fi fi diff --git a/bin/steps/pylibmc b/bin/steps/pylibmc index f574e535..2ebba64e 100755 --- a/bin/steps/pylibmc +++ b/bin/steps/pylibmc @@ -13,7 +13,8 @@ VENDORED_MEMCACHED="https://lang-python.s3.amazonaws.com/$STACK/libraries/vendor/libmemcache.tar.gz" # Syntax sugar. -source $BIN_DIR/utils +# shellcheck source=bin/utils +source "$BIN_DIR/utils" # If pylibmc exists within requirements, use vendored libmemcached. @@ -23,8 +24,9 @@ if (pip-grep -s requirements.txt pylibmc &> /dev/null) then echo "-----> Noticed pylibmc. Bootstrapping libmemcached." mkdir -p .heroku/vendor # Download and extract libmemcached into target vendor directory. - curl $VENDORED_MEMCACHED -s | tar zxv -C .heroku/vendor &> /dev/null + curl "$VENDORED_MEMCACHED" -s | tar zxv -C .heroku/vendor &> /dev/null fi - export LIBMEMCACHED=$(pwd)/vendor + LIBMEMCACHED=$(pwd)/vendor + export LIBMEMCACHED fi diff --git a/bin/steps/python b/bin/steps/python index 30aa2c8c..32a25903 100755 --- a/bin/steps/python +++ b/bin/steps/python @@ -1,10 +1,12 @@ +#!/usr/bin/env bash + set +e runtime-fixer runtime.txt PYTHON_VERSION=$(cat runtime.txt) # Install Python. if [ -f .heroku/python-version ]; then - if [ ! $(cat .heroku/python-version) = $PYTHON_VERSION ]; then + if [ ! "$(cat .heroku/python-version)" = "$PYTHON_VERSION" ]; then puts-step "Found $(cat .heroku/python-version), removing" rm -fr .heroku/python else @@ -12,7 +14,7 @@ if [ -f .heroku/python-version ]; then fi fi -if [ ! $STACK = $CACHED_PYTHON_STACK ]; then +if [ ! "$STACK" = "$CACHED_PYTHON_STACK" ]; then rm -fr .heroku/python .heroku/python-stack .heroku/vendor unset SKIP_INSTALL fi @@ -24,29 +26,24 @@ if [ ! "$SKIP_INSTALL" ]; then # Prepare destination directory. mkdir -p .heroku/python - curl https://lang-python.s3.amazonaws.com/$STACK/runtimes/$PYTHON_VERSION.tar.gz -s | tar zxv -C .heroku/python &> /dev/null mcount "version.python.$PYTHON_VERSION" - if [[ $? != 0 ]] ; then + if ! curl "https://lang-python.s3.amazonaws.com/$STACK/runtimes/$PYTHON_VERSION.tar.gz" -s | tar zxv -C .heroku/python &> /dev/null; then puts-warn "Requested runtime ($PYTHON_VERSION) is not available for this stack ($STACK)." puts-warn "Aborting. More info: https://devcenter.heroku.com/articles/python-support" exit 1 fi # Record for future reference. - echo $PYTHON_VERSION > .heroku/python-version - echo $STACK > .heroku/python-stack + echo "$PYTHON_VERSION" > .heroku/python-version + echo "$STACK" > .heroku/python-stack FRESH_PYTHON=true hash -r fi # If Pip isn't up to date: -if [ "$FRESH_PYTHON" ] || [[ ! $(pip --version) == *$PIP_VERSION* ]]; then - WORKING_DIR=$(pwd) - - TMPTARDIR=$(mktemp -d) - trap "rm -rf $TMPTARDIR" RETURN +if [ "$FRESH_PYTHON" ] || [[ ! $(pip --version) == *$PIP_UPDATE* ]]; then puts-step "Installing pip" @@ -54,7 +51,7 @@ if [ "$FRESH_PYTHON" ] || [[ ! $(pip --version) == *$PIP_VERSION* ]]; then rm -fr /app/.heroku/python/lib/python2.7/site-packages/pip-* rm -fr /app/.heroku/python/lib/python2.7/site-packages/setuptools-* - /app/.heroku/python/bin/python $ROOT_DIR/vendor/get-pip.py &> /dev/null + /app/.heroku/python/bin/python "$ROOT_DIR/vendor/get-pip.py" &> /dev/null fi diff --git a/bin/test-compile b/bin/test-compile index 51b162e6..d19814db 100755 --- a/bin/test-compile +++ b/bin/test-compile @@ -1,10 +1,12 @@ #!/usr/bin/env bash # Syntax sugar. -BIN_DIR=$(cd $(dirname $0); pwd) # absolute path -source $BIN_DIR/utils +BIN_DIR=$(cd "$(dirname "$0")" || return; pwd) # absolute path -DISABLE_COLLECTSTATIC=1 "$(dirname ${0:-})/compile" "$1" "$2" "$3" +# shellcheck source=bin/utils +source "$BIN_DIR/utils" + +DISABLE_COLLECTSTATIC=1 "$(dirname "${0:-}")/compile" "$1" "$2" "$3" if [[ -f "$1/requirements-test.txt" ]]; then /app/.heroku/python/bin/pip install -r "$1/requirements-test.txt" --exists-action=w --src=./.heroku/src --disable-pip-version-check --no-cache-dir 2>&1 | cleanup | indent diff --git a/bin/utils b/bin/utils index dc2a69d7..59273a7c 100755 --- a/bin/utils +++ b/bin/utils @@ -1,13 +1,15 @@ #!/usr/bin/env bash shopt -s extglob +shopt -s nullglob # The standard library. if [[ ! -f /tmp/stdlib.sh ]]; then curl --retry 3 -s https://lang-common.s3.amazonaws.com/buildpack-stdlib/v2/stdlib.sh > /tmp/stdlib.sh fi +# shellcheck source=/dev/null source /tmp/stdlib.sh -if [ $(uname) == Darwin ]; then +if [ "$(uname)" == Darwin ]; then sed() { command sed -l "$@"; } else sed() { command sed -u "$@"; } @@ -26,12 +28,12 @@ cleanup() { # Buildpack Steps. puts-step() { - echo "-----> $@" + echo "-----> $*" } # Buildpack Warnings. puts-warn() { - echo " ! $@" + echo " ! $*" } # Does some serious copying. @@ -62,9 +64,10 @@ sub-env() { ( if [ -d "$ENV_DIR" ]; then - for e in $(ls $ENV_DIR); do + # shellcheck disable=SC2045 + for e in $(ls "$ENV_DIR"); do echo "$e" | grep -E "$WHITELIST" | grep -qvE "$BLACKLIST" && - export "$e=$(cat $ENV_DIR/$e)" + export "$e=$(cat "$ENV_DIR/$e")" : done fi @@ -76,6 +79,6 @@ sub-env() { # Measure the size of the Python installation. measure-size() { - echo "$((du -s .heroku/python 2>/dev/null || echo 0) | awk '{print $1}')" + echo "$(du -s .heroku/python 2>/dev/null || echo 0) | awk '{print $1}')" } diff --git a/bin/warnings b/bin/warnings index 01e43d58..b43748b4 100755 --- a/bin/warnings +++ b/bin/warnings @@ -1,3 +1,4 @@ +#!/usr/bin/env bash shopt -s extglob old-platform() { diff --git a/test/fixtures/python3/runtime.txt b/test/fixtures/python3/runtime.txt deleted file mode 100644 index 80aea674..00000000 --- a/test/fixtures/python3/runtime.txt +++ /dev/null @@ -1 +0,0 @@ -python-3.6.0 \ No newline at end of file diff --git a/test/run b/test/run index 7c042a3f..e125c13f 100755 --- a/test/run +++ b/test/run @@ -7,7 +7,7 @@ testPipenv() { testPipenvVersion() { compile "pipenv-version" - assertCaptured "3.6.0" + assertCaptured "3.6.2" assertCapturedSuccess } @@ -61,7 +61,7 @@ testPython2() { testPython3() { compile "python3" - assertCaptured "python-3.6.0" + assertCaptured "python-3.6.2" assertCapturedSuccess } -- GitLab