diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c19386392fbfd9c7d56a6be41c8bf8a55f28854c --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,5 @@ +repos: +- repo: git://github.com/detailyang/pre-commit-shell + rev: 1.0.4 + hooks: + - id: shell-lint diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a5e3a0e032e019be3728732e9a62ca9431ed009..fcfb1c9188c4649e86181347ea5ce9ba00784f63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,19 +1,27 @@ # Python Buildpack Changelog -# 141 (2018-10-10) +# 145 (2018-11-08) + +Python 3.7.1, 3.6.7, 3.5.6 and 3.4.9 now available on all Heroku stacks. + +# 144 (2018-10-10) Switch to cautious upgrade for Pipenv install to ensure the pinned pip version is used with Pipenv -# 140 (2018-10-09) +# 143 (2018-10-09) Add support for detecting SLUGIFY_USES_TEXT_UNIDECODE, which is required to install Apache Airflow version 1.10 or higher. -# 139 (2018-10-08) +# 142 (2018-10-08) Improvements to Python install messaging +# 139, 140, 141 + +No user-facing changes, documenting for version clarity + # 138 (2018-08-01) Use stack image SQLite3 instead of vendoring diff --git a/bin/compile b/bin/compile index d97af680d5428c3abb0d5f72444608a72df2248e..a092a28d762b1d57a0008bdcbb7c0db34bd183e5 100755 --- a/bin/compile +++ b/bin/compile @@ -52,14 +52,25 @@ export VENDOR_URL DEFAULT_PYTHON_VERSION="python-3.6.6" LATEST_36="python-3.6.6" LATEST_37="python-3.7.0" -LATEST_2="python-2.7.15" +LATEST_35="python-3.5.6" +LATEST_34="python-3.4.9" +LATEST_27="python-2.7.15" + +# Supported Python Branches +PY37="python-3.7" +PY36="python-3.6" +PY35="python-3.5" +PY34="python-3.4" +PY27="python-2.7" # Which stack is used (for binary downloading), if none is provided (e.g. outside of Heroku)? DEFAULT_PYTHON_STACK="cedar-14" # If pip doesn't match this version (the version we install), run the installer. PIP_UPDATE="9.0.2" -export DEFAULT_PYTHON_VERSION DEFAULT_PYTHON_STACK PIP_UPDATE LATEST_2 LATEST_36 LATEST_37 +export DEFAULT_PYTHON_VERSION DEFAULT_PYTHON_STACK PIP_UPDATE +export LATEST_27 LATEST_36 LATEST_37 LATEST_35 LATEST_34 +export PY37 PY36 PY35 PY27 PY34 # Common Problem Warnings: # This section creates a temporary file in which to stick the output of `pip install`. diff --git a/bin/steps/python b/bin/steps/python index ea623b1bfdf72daef78dcb7c12bfa766f44e352a..d10eccf23463b56bcbe0d7c0252d9d2db43509ec 100755 --- a/bin/steps/python +++ b/bin/steps/python @@ -7,41 +7,49 @@ PYTHON_VERSION=$(cat runtime.txt) # The location of the pre-compiled python binary. VENDORED_PYTHON="${VENDOR_URL}/runtimes/$PYTHON_VERSION.tar.gz" -if [[ $PYTHON_VERSION =~ ^python-2 ]]; then - if [[ "$PYTHON_VERSION" != "$LATEST_2" ]]; then - puts-warn "The latest version of Python 2 is $LATEST_2 (you are using $PYTHON_VERSION, which is unsupported)." - puts-warn "We recommend upgrading by specifying the latest version ($LATEST_2)." - echo " Learn More: https://devcenter.heroku.com/articles/python-runtimes" - else - echo " Using supported version of Python 2 ($PYTHON_VERSION)" +SECURITY_UPDATE="Python has released a security update! Please consider upgrading to" + +# check if runtime exists +if curl --output /dev/null --silent --head --fail "$VENDORED_PYTHON"; then + if [[ "$PYTHON_VERSION" == $PY37* ]]; then + # do things to alert the user of security release available + if [ "$PYTHON_VERSION" != "$LATEST_37" ]; then + puts-warn "$SECURITY_UPDATE" "$LATEST_37" + echo " Learn More: https://devcenter.heroku.com/articles/python-runtimes" + fi fi -else - if [[ $PYTHON_VERSION =~ ^python-3 ]]; then - if [[ $PYTHON_VERSION =~ ^python-3.7 ]]; then - if [[ "$PYTHON_VERSION" != "$LATEST_37" ]]; then - puts-warn "The latest version of Python 3.7 is $LATEST_37 (you are using $PYTHON_VERSION, which is unsupported)." - puts-warn "We recommend upgrading by specifying the latest version ($LATEST_37)." + if [[ "$PYTHON_VERSION" == $PY36* ]]; then + # security update note + if [ "$PYTHON_VERSION" != "$LATEST_36" ]; then + puts-warn "$SECURITY_UPDATE" "$LATEST_36" + echo " Learn More: https://devcenter.heroku.com/articles/python-runtimes" + fi + fi + if [[ "$PYTHON_VERSION" == $PY35* ]]; then + # security update note + if [ "$PYTHON_VERSION" != "$LATEST_35" ]; then + puts-warn "$SECURITY_UPDATE" "$LATEST_35" echo " Learn More: https://devcenter.heroku.com/articles/python-runtimes" - else - echo " Using supported version of Python 3.7 ($PYTHON_VERSION)" - fi - else - if [[ $PYTHON_VERSION =~ ^python-3.6 ]]; then - if [[ "$PYTHON_VERSION" != "$LATEST_36" ]]; then - puts-warn "The latest version of Python 3.6 is $LATEST_36 (you are using $PYTHON_VERSION, which is unsupported)." - puts-warn "We recommend upgrading by specifying the latest version ($LATEST_36)." - echo " Learn More: https://devcenter.heroku.com/articles/python-runtimes" - else - echo " Using supported version of Python 3.6 ($PYTHON_VERSION)" - fi - else - puts-warn "Heroku supports runtime versions $LATEST_37, $LATEST_36 and $LATEST_2." - puts-warn "You are using $PYTHON_VERSION, which is unsupported." - puts-warn "We recommend upgrading by specifying the default supported version ($LATEST_36)." - echo " Learn More: https://devcenter.heroku.com/articles/python-runtimes" - fi fi fi + if [[ "$PYTHON_VERSION" == $PY34* ]]; then + # security update note + if [ "$PYTHON_VERSION" != "$LATEST_34" ]; then + puts-warn "$SECURITY_UPDATE" "$LATEST_34" + echo " Learn More: https://devcenter.heroku.com/articles/python-runtimes" + fi + fi + if [[ "$PYTHON_VERSION" == $PY27* ]]; then + # security update note + if [ "$PYTHON_VERSION" != "$LATEST_27" ]; then + puts-warn "$SECURITY_UPDATE" "$LATEST_27" + echo " Learn More: https://devcenter.heroku.com/articles/python-runtimes" + fi + fi +else + puts-warn "Requested runtime ($PYTHON_VERSION) is not available for this stack ($STACK)." + puts-warn "Aborting. More info: https://devcenter.heroku.com/articles/python-support" + exit 1 fi if [[ "$STACK" != "$CACHED_PYTHON_STACK" ]]; then diff --git a/bin/utils b/bin/utils index 2bbf82a32e59e33c84bc5be925e72f6437cb4e68..f625645dff0ddaf8b21e86e9cfe6e4b45e183fc8 100755 --- a/bin/utils +++ b/bin/utils @@ -89,10 +89,9 @@ python3_check() { # Check if Python version needs to install SQLite3 python_sqlite3_check() { VERSION="$1" - MIN_PYTHON_3="python-3.6.6" + MIN_PYTHON_3="python-3.5.6" MIN_PYTHON_2="python-2.7.15" ( python2_check "$VERSION" && version_gte "$VERSION" "$MIN_PYTHON_2" ) \ - || ( python3_check "$VERSION" && version_gte "$VERSION" "$MIN_PYTHON_3" ) \ - || ( version_gte "$VERSION" "3.7.0" ) + || ( python3_check "$VERSION" && version_gte "$VERSION" "$MIN_PYTHON_3" ) } diff --git a/builds/runtimes/python-3.4.9 b/builds/runtimes/python-3.4.9 new file mode 100755 index 0000000000000000000000000000000000000000..00cf5078458461fed5ca810e06d864419674dd09 --- /dev/null +++ b/builds/runtimes/python-3.4.9 @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Build Path: /app/.heroku/python/ + +OUT_PREFIX=$1 +BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin" +export BIN_DIR + +# shellcheck source=bin/utils +source "$BIN_DIR/steps/sqlite3" + +sqlite3_version +echo "Setting up SQLite3 Headers for $SQLITE3_VERSION" +sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1 + +echo "Building Python…" +SOURCE_TARBALL='https://python.org/ftp/python/3.4.9/Python-3.4.9.tgz' +curl -L $SOURCE_TARBALL | tar xz +mv Python-3.4.9 src +cd src + +./configure --prefix=$OUT_PREFIX --with-ensurepip=no +make +make install + +# Remove unneeded test directories, similar to the official Docker Python images: +# https://github.com/docker-library/python +find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' + + +# Remove spare / +LOCATION=${OUT_PREFIX%?} + +ln $LOCATION/bin/python3 $LOCATION/bin/python diff --git a/builds/runtimes/python-3.5.6 b/builds/runtimes/python-3.5.6 new file mode 100755 index 0000000000000000000000000000000000000000..0d179352b1f3c996b1003d0ba9d5999a5bbd8c7c --- /dev/null +++ b/builds/runtimes/python-3.5.6 @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Build Path: /app/.heroku/python/ + +OUT_PREFIX=$1 +BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin" +export BIN_DIR + +# shellcheck source=bin/utils +source "$BIN_DIR/steps/sqlite3" + +sqlite3_version +echo "Setting up SQLite3 Headers for $SQLITE3_VERSION" +sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1 + +echo "Building Python…" +SOURCE_TARBALL='https://python.org/ftp/python/3.5.6/Python-3.5.6.tgz' +curl -L $SOURCE_TARBALL | tar xz +mv Python-3.5.6 src +cd src + +./configure --prefix=$OUT_PREFIX --with-ensurepip=no +make +make install + +# Remove unneeded test directories, similar to the official Docker Python images: +# https://github.com/docker-library/python +find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' + + +# Remove spare / +LOCATION=${OUT_PREFIX%?} + +ln $LOCATION/bin/python3 $LOCATION/bin/python diff --git a/builds/runtimes/python-3.6.7 b/builds/runtimes/python-3.6.7 new file mode 100755 index 0000000000000000000000000000000000000000..052ee1e2fb0275ed9f5cf164f00951c61eda4e1e --- /dev/null +++ b/builds/runtimes/python-3.6.7 @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Build Path: /app/.heroku/python/ + +OUT_PREFIX=$1 +BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin" +export BIN_DIR + +# shellcheck source=bin/utils +source "$BIN_DIR/steps/sqlite3" + +sqlite3_version +echo "Setting up SQLite3 Headers for $SQLITE3_VERSION" +sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1 + +echo "Building Python…" +SOURCE_TARBALL='https://python.org/ftp/python/3.6.7/Python-3.6.7.tgz' +curl -L $SOURCE_TARBALL | tar xz +mv Python-3.6.7 src +cd src + +./configure --prefix=$OUT_PREFIX --with-ensurepip=no +make +make install + +# Remove unneeded test directories, similar to the official Docker Python images: +# https://github.com/docker-library/python +find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' + + +# Remove spare / +LOCATION=${OUT_PREFIX%?} + +ln $LOCATION/bin/python3 $LOCATION/bin/python diff --git a/builds/runtimes/python-3.7.1 b/builds/runtimes/python-3.7.1 new file mode 100755 index 0000000000000000000000000000000000000000..b8051393f22aaf95ac504c64ea082eb025c17ca0 --- /dev/null +++ b/builds/runtimes/python-3.7.1 @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Build Path: /app/.heroku/python/ + +OUT_PREFIX=$1 +BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin" +export BIN_DIR + +# shellcheck source=bin/utils +source "$BIN_DIR/steps/sqlite3" + +sqlite3_version +echo "Setting up SQLite3 Headers for $SQLITE3_VERSION" +sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1 + +echo "Building Python…" +SOURCE_TARBALL='https://python.org/ftp/python/3.7.1/Python-3.7.1.tgz' +curl -L $SOURCE_TARBALL | tar xz +mv Python-3.7.1 src +cd src + +./configure --prefix=$OUT_PREFIX --with-ensurepip=no +make +make install + +# Remove unneeded test directories, similar to the official Docker Python images: +# https://github.com/docker-library/python +find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' + + +# Remove spare / +LOCATION=${OUT_PREFIX%?} + +ln $LOCATION/bin/python3 $LOCATION/bin/python diff --git a/test/fixtures/python3/requirements.txt b/test/fixtures/python2_warn/requirements.txt similarity index 100% rename from test/fixtures/python3/requirements.txt rename to test/fixtures/python2_warn/requirements.txt diff --git a/test/fixtures/python2_warn/runtime.txt b/test/fixtures/python2_warn/runtime.txt new file mode 100644 index 0000000000000000000000000000000000000000..538568204a78b4e4ec20255b64fa25533b4edae3 --- /dev/null +++ b/test/fixtures/python2_warn/runtime.txt @@ -0,0 +1 @@ +python-2.7.14 diff --git a/test/fixtures/python3_4/requirements.txt b/test/fixtures/python3_4/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..7e1060246fd6746a14204539a72e199a25469a05 --- /dev/null +++ b/test/fixtures/python3_4/requirements.txt @@ -0,0 +1 @@ +flask diff --git a/test/fixtures/python3_4/runtime.txt b/test/fixtures/python3_4/runtime.txt new file mode 100644 index 0000000000000000000000000000000000000000..cc8325f75753b807e16f0c2a1d05ac2d24b269b0 --- /dev/null +++ b/test/fixtures/python3_4/runtime.txt @@ -0,0 +1 @@ +python-3.4.9 diff --git a/test/fixtures/python3_4_warn/requirements.txt b/test/fixtures/python3_4_warn/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..7e1060246fd6746a14204539a72e199a25469a05 --- /dev/null +++ b/test/fixtures/python3_4_warn/requirements.txt @@ -0,0 +1 @@ +flask diff --git a/test/fixtures/python3_4_warn/runtime.txt b/test/fixtures/python3_4_warn/runtime.txt new file mode 100644 index 0000000000000000000000000000000000000000..fe5dcf7a3aeb68d0bb349ddd6f7d61ba731a1057 --- /dev/null +++ b/test/fixtures/python3_4_warn/runtime.txt @@ -0,0 +1 @@ +python-3.4.0 diff --git a/test/fixtures/python3_5/requirements.txt b/test/fixtures/python3_5/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..663bd1f6a2ae02f29df59fb4963c17934034f731 --- /dev/null +++ b/test/fixtures/python3_5/requirements.txt @@ -0,0 +1 @@ +requests \ No newline at end of file diff --git a/test/fixtures/python3_5/runtime.txt b/test/fixtures/python3_5/runtime.txt new file mode 100644 index 0000000000000000000000000000000000000000..e6391f4a68edd6f81269b05227977459a9fbd6a8 --- /dev/null +++ b/test/fixtures/python3_5/runtime.txt @@ -0,0 +1 @@ +python-3.5.6 diff --git a/test/fixtures/python3_5_warn/requirements.txt b/test/fixtures/python3_5_warn/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..7e1060246fd6746a14204539a72e199a25469a05 --- /dev/null +++ b/test/fixtures/python3_5_warn/requirements.txt @@ -0,0 +1 @@ +flask diff --git a/test/fixtures/python3_5_warn/runtime.txt b/test/fixtures/python3_5_warn/runtime.txt new file mode 100644 index 0000000000000000000000000000000000000000..5486d7adb9a188bcfc1d0d8844b4091b1c369ec9 --- /dev/null +++ b/test/fixtures/python3_5_warn/runtime.txt @@ -0,0 +1 @@ +python-3.5.3 diff --git a/test/fixtures/python3_6/requirements.txt b/test/fixtures/python3_6/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..663bd1f6a2ae02f29df59fb4963c17934034f731 --- /dev/null +++ b/test/fixtures/python3_6/requirements.txt @@ -0,0 +1 @@ +requests \ No newline at end of file diff --git a/test/fixtures/python3_6/runtime.txt b/test/fixtures/python3_6/runtime.txt new file mode 100644 index 0000000000000000000000000000000000000000..1935e9778db66275403e99759f51768cc7599918 --- /dev/null +++ b/test/fixtures/python3_6/runtime.txt @@ -0,0 +1 @@ +python-3.6.6 diff --git a/test/fixtures/python3_6_7/requirements.txt b/test/fixtures/python3_6_7/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..7e1060246fd6746a14204539a72e199a25469a05 --- /dev/null +++ b/test/fixtures/python3_6_7/requirements.txt @@ -0,0 +1 @@ +flask diff --git a/test/fixtures/python3_6_7/runtime.txt b/test/fixtures/python3_6_7/runtime.txt new file mode 100644 index 0000000000000000000000000000000000000000..34b35b713f56868abc889d5798cab36939bec38f --- /dev/null +++ b/test/fixtures/python3_6_7/runtime.txt @@ -0,0 +1 @@ +python-3.6.7 diff --git a/test/fixtures/python3_6_warn/requirements.txt b/test/fixtures/python3_6_warn/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..663bd1f6a2ae02f29df59fb4963c17934034f731 --- /dev/null +++ b/test/fixtures/python3_6_warn/requirements.txt @@ -0,0 +1 @@ +requests \ No newline at end of file diff --git a/test/fixtures/python3_6_warn/runtime.txt b/test/fixtures/python3_6_warn/runtime.txt new file mode 100644 index 0000000000000000000000000000000000000000..486fcce127f00eba90a4096290b67c31adaef0bd --- /dev/null +++ b/test/fixtures/python3_6_warn/runtime.txt @@ -0,0 +1 @@ +python-3.6.5 diff --git a/test/fixtures/python3_7/requirements.txt b/test/fixtures/python3_7/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..663bd1f6a2ae02f29df59fb4963c17934034f731 --- /dev/null +++ b/test/fixtures/python3_7/requirements.txt @@ -0,0 +1 @@ +requests \ No newline at end of file diff --git a/test/fixtures/python3_7/runtime.txt b/test/fixtures/python3_7/runtime.txt new file mode 100644 index 0000000000000000000000000000000000000000..881a2db1abd7719381d1925771e0d4208fb46fb5 --- /dev/null +++ b/test/fixtures/python3_7/runtime.txt @@ -0,0 +1 @@ +python-3.7.0 diff --git a/test/fixtures/python3_7_1/requirements.txt b/test/fixtures/python3_7_1/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..7e1060246fd6746a14204539a72e199a25469a05 --- /dev/null +++ b/test/fixtures/python3_7_1/requirements.txt @@ -0,0 +1 @@ +flask diff --git a/test/fixtures/python3_7_1/runtime.txt b/test/fixtures/python3_7_1/runtime.txt new file mode 100644 index 0000000000000000000000000000000000000000..4255f73ae6037a2a4a5ddd6a1093c841fc339845 --- /dev/null +++ b/test/fixtures/python3_7_1/runtime.txt @@ -0,0 +1 @@ +python-3.7.1 diff --git a/test/run b/test/run index f6368161bae2719f3f75e8fa4d18febfe1fef5bd..67d97c8adf10ba9bf61a187dd9e5919616e6fd26 100755 --- a/test/run +++ b/test/run @@ -87,18 +87,98 @@ testPylibmc() { assertCapturedSuccess } +testPython2_warn() { + compile "python2_warn" + if [[ $STACK = "heroku-18" ]]; then + assertCapturedError + else + assertCaptured "python-2.7.14" + assertCaptured "security update!" + assertCapturedSuccess + fi +} + testPython2() { compile "python2" assertCaptured "python-2.7.15" assertCapturedSuccess } -testPython3() { - compile "python3" +# This fail +testPython3_4_warn() { + compile "python3_4_warn" + if [[ $STACK = "cedar-14" ]]; then + assertCaptured "python-3.4.0" + assertCapturedSuccess + else + assertCapturedError + fi +} + +# This fail +testPython3_4() { + compile "python3_4" + assertCaptured "python-3.4.9" + assertCapturedError +} + +# This fail +testPython3_5_warn() { + compile "python3_5_warn" + if [[ $STACK = "cedar-14" ]]; then + assertCaptured "python-3.5.3" + assertCaptured "security update!" + assertCapturedError + else + assertCapturedError + fi +} + +# This will fail +testPython3_5() { + compile "python3_5" + assertCaptured "python-3.5.6" + assertCapturedError +} + +# This will warn +testPython3_6_warn() { + compile "python3_6_warn" + assertCaptured "python-3.6.5" + assertCaptured "security update!" + assertCapturedSuccess +} + +testPython3_6() { + compile "python3_6" assertCaptured "python-3.6.6" assertCapturedSuccess } +# This will fail +testPython3_6_7() { + compile "python3_6_7" + assertCaptured "python-3.6.7" + assertCapturedError +} + +testPython3_7() { + compile "python3_7" + if [[ $STACK = "cedar-14" ]]; then + assertCapturedError + else + assertCaptured "python-3.7.0" + assertCapturedSuccess + fi +} + +# This will fail +testPython3_7_1() { + compile "python3_7_1" + assertCaptured "python-3.7.1" + assertCapturedError +} + testGitEgg() { compile "git-egg" assertCaptured "requests"