Merge remote-tracking branch 'github/master'

.pre-commit-config.yaml

+repos:
+-   repo: git://github.com/detailyang/pre-commit-shell
+    rev: 1.0.4
+    hooks:
+    - id: shell-lint

.travis.yml

@@ -5,9 +5,7 @@ rvm:
 - 2.4.4
 before_script:
   - gem install bundler -v 1.16.2
-  - bundle exec rake hatchet:setup_travis
-before_install:
-  - sudo bash etc/ci-setup.sh
+  - bundle exec hatchet ci:setup
 jobs:
   include:
   - stage: Bash linting (shellcheck)
@@ -35,5 +33,6 @@ env:
   - HATCHET_RETRIES=3
   - IS_RUNNING_ON_CI=true
   - HATCHET_APP_LIMIT=5
+  - HATCHET_DEPLOY_STRATEGY=git
   - secure: yjtlPE5FbVxTKnjUy/tZUBgSEf4qADD3QOxtgziuid73S0U/1IEXlMGFULsQzIjtlHKmHeywZqpVVEpthIH4RuT7uoX1Pb7SSM/g0T8fT3VoEFbFK1uYl0oZQbUS4Klxv9tPiumj8if3m6ULEGIz1X0wZcMOC0tMLwVCnwmap0E=
   - secure: ZeFTHWwnpIKE9nAqs88ocmiQh7bKce84lilGm5J23nf3N6V4wNyLwqlkvsM008WGBCaOg9AUx7ZunasT0ANsR5gLP3eV2UUg7ILdRgV2Gy13eNRFheC4PHdN92RqQ3aKoqlIv2K999xlhVjod0NzhkQQXB6PddfQINbuU7ks6As=

CHANGELOG.md

 # Python Buildpack Changelog
 
-# 141 (2018-10-10)
+# 146 (2018-15-08)
+
+Python 3.7.1, 3.6.7, 3.5.6 and 3.4.9 now available on all Heroku stacks.
+
+# 145 (2018-11-08)
+
+Testing and tooling expanded to better support new runtimes
+
+# 144 (2018-10-10)
 
 Switch to cautious upgrade for Pipenv install to ensure the pinned pip version
 is used with Pipenv
 
-# 140 (2018-10-09)
+# 143 (2018-10-09)
 
 Add support for detecting SLUGIFY_USES_TEXT_UNIDECODE, which is required to
 install Apache Airflow version 1.10 or higher.
 
-# 139 (2018-10-08)
+# 142 (2018-10-08)
 
 Improvements to Python install messaging
 
+# 139, 140, 141
+
+No user-facing changes, documenting for version clarity
+
 # 138 (2018-08-01)
 
 Use stack image SQLite3 instead of vendoring

Gemfile.lock

@@ -6,16 +6,16 @@ GEM
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
-    concurrent-ruby (1.0.5)
+    concurrent-ruby (1.1.3)
     diff-lcs (1.3)
     erubis (2.7.0)
     excon (0.62.0)
-    heroics (0.0.24)
+    heroics (0.0.25)
       erubis (~> 2.0)
       excon
       moneta
       multi_json (>= 1.9.2)
-    heroku_hatchet (4.0.2)
+    heroku_hatchet (4.0.6)
       excon (~> 0)
       minitest-retry (~> 0.1.9)
       platform-api (~> 2)
@@ -23,16 +23,16 @@ GEM
       rrrretry (~> 1)
       thor (~> 0)
       threaded (~> 0)
-    i18n (1.1.0)
+    i18n (1.1.1)
       concurrent-ruby (~> 1.0)
     minitest (5.11.3)
     minitest-retry (0.1.9)
       minitest (>= 5.0)
-    moneta (0.8.1)
+    moneta (1.0.0)
     multi_json (1.13.1)
-    platform-api (2.1.0)
-      heroics (~> 0.0.23)
-      moneta (~> 0.8.1)
+    platform-api (2.2.0)
+      heroics (~> 0.0.25)
+      moneta (~> 1.0.0)
     rake (12.3.1)
     repl_runner (0.0.3)
       activesupport
@@ -52,7 +52,7 @@ GEM
     rspec-retry (0.6.1)
       rspec-core (> 3.3)
     rspec-support (3.8.0)
-    thor (0.20.0)
+    thor (0.20.3)
     thread_safe (0.3.6)
     threaded (0.0.4)
     tzinfo (1.2.5)

Makefile

 # These targets are not files
 .PHONY: tests
 
-test: test-heroku-16
+test: test-heroku-18 test-heroku-16
 
 check:
 	@shellcheck -x bin/compile bin/detect bin/release bin/test-compile bin/utils bin/warnings

README.md

@@ -4,36 +4,54 @@
 
 [![Build Status](https://travis-ci.org/heroku/heroku-buildpack-python.svg?branch=master)](https://travis-ci.org/heroku/heroku-buildpack-python)
 
-This is the official [Heroku buildpack](https://devcenter.heroku.com/articles/buildpacks) for Python apps, powered by [Pipenv](http://docs.pipenv.org/), [pip](https://pip.pypa.io/) and other excellent software.
+This is the official [Heroku buildpack](https://devcenter.heroku.com/articles/buildpacks) for Python apps.
 
-Recommended web frameworks include **Django** and **Flask**. The recommended webserver is **Gunicorn**. There are no restrictions around what software can be used (as long as it's pip-installable). Web processes must bind to `$PORT`, and only the HTTP protocol is permitted for incoming connections.
+Recommended web frameworks include **Django** and **Flask**, among others. The recommended webserver is **Gunicorn**. There are no restrictions around what software can be used (as long as it's pip-installable). Web processes must bind to `$PORT`, and only the HTTP protocol is permitted for incoming connections.
 
-Python packages with C dependencies that are not [available on the stack image](https://devcenter.heroku.com/articles/stack-packages) are generally not supported, unless `manylinux` wheels are provided by the package maintainers (common). For recommended solutions, check out [this article](https://devcenter.heroku.com/articles/python-c-deps) for more information. 
+Python packages with C dependencies that are not [available on the stack image](https://devcenter.heroku.com/articles/stack-packages) are generally not supported, unless `manylinux` wheels are provided by the package maintainers (common). For recommended solutions, check out [this article](https://devcenter.heroku.com/articles/python-c-deps) for more information.
 
 See it in Action
 ----------------
-
-Deploying a Python application couldn't be easier:
-
-    $ ls
-    Pipfile		Pipfile.lock	Procfile	web.py
-
-    $ heroku create --buildpack heroku/python
-
-    $ git push heroku master
-    …
-    -----> Python app detected
-    -----> Installing python-3.6.6
-    -----> Installing pip
-    -----> Installing requirements with Pipenv 2018.5.18…
-           ...
-           Installing dependencies from Pipfile…
-    -----> Discovering process types
-           Procfile declares types -> (none)
-
-A `Pipfile` or `requirements.txt` must be present at the root of your application's repository.
-
-You can also specify the latest production release of this buildpack for upcoming builds of an existing application:
+```
+$ ls
+my-application		requirements.txt	runtime.txt
+
+$ git push heroku master
+Counting objects: 4, done.
+Delta compression using up to 8 threads.
+Compressing objects: 100% (2/2), done.
+Writing objects: 100% (4/4), 276 bytes | 276.00 KiB/s, done.
+Total 4 (delta 0), reused 0 (delta 0)
+remote: Compressing source files... done.
+remote: Building source:
+remote:
+remote: -----> Python app detected
+remote: -----> Installing python-3.7.1
+remote: -----> Installing pip
+remote: -----> Installing SQLite3
+remote: -----> Installing requirements with pip
+remote:        Collecting flask (from -r /tmp/build_c2c067ef79ff14c9bf1aed6796f9ed1f/requirements.txt (line 1))
+remote:          Downloading ...
+remote:        Installing collected packages: Werkzeug, click, MarkupSafe, Jinja2, itsdangerous, flask
+remote:        Successfully installed Jinja2-2.10 MarkupSafe-1.1.0 Werkzeug-0.14.1 click-7.0 flask-1.0.2 itsdangerous-1.1.0
+remote:
+remote: -----> Discovering process types
+remote:        Procfile declares types -> (none)
+remote:
+```
+
+A `requirements.txt` must be present at the root of your application's repository to deploy.
+
+To specify your python version, you also need a `runtime.txt` file - unless you are using the default Python runtime version.
+
+Current default Python Runtime: Python 3.6.7
+
+Alternatively, you can provide a `setup.py` file, or a `Pipfile`. Using `Pipenv` will generate `runtime.txt` based on `python-version` at build time.
+
+Specify a Buildpack Version
+---------------------------
+
+You can specify the latest production release of this buildpack for upcoming builds of an existing application:
 
     $ heroku buildpacks:set heroku/python
 
@@ -41,23 +59,30 @@ You can also specify the latest production release of this buildpack for upcomin
 Specify a Python Runtime
 ------------------------
 
-Specific versions of the Python runtime can be specified in your `Pipfile`:
+Supported runtime options include:
+
+- `python-3.7.1`
+- `python-3.6.7`
+- `python-2.7.15`
 
-    [requires]
-    python_version = "2.7"
+## Tests
 
-Or, more specifically:
+The buildpack tests use [Docker](https://www.docker.com/) to simulate
+Heroku's [stack images.](https://devcenter.heroku.com/articles/stack)
 
-    [requires]
-    python_full_version = "2.7.15"
+To run the test suite:
 
-Or, with a `runtime.txt` file:
+```
+make test
+```
 
-    $ cat runtime.txt
-    python-2.7.15
+Or to test in a particular stack:
 
-Runtime options include:
+```
+make test-heroku-18
+make test-heroku-16
+```
 
-- `python-3.7.0`
-- `python-3.6.6`
-- `python-2.7.15`
+The tests are run via the vendored
+[shunit2](https://github.com/kward/shunit2)
+test framework.

bin/compile

@@ -49,17 +49,28 @@ export VENDOR_URL
 # These variables are used to specify which versions of Python to install by default,
 # as well as prompt the user to upgrade if they are using an un–supported version.
 # Note: When 3.7 lands, I recommend switching to LATEST_36 and LATEST_37.
-DEFAULT_PYTHON_VERSION="python-3.6.6"
-LATEST_36="python-3.6.6"
-LATEST_37="python-3.7.0"
-LATEST_2="python-2.7.15"
+DEFAULT_PYTHON_VERSION="python-3.6.7"
+LATEST_36="python-3.6.7"
+LATEST_37="python-3.7.1"
+LATEST_35="python-3.5.6"
+LATEST_34="python-3.4.9"
+LATEST_27="python-2.7.15"
+
+# Supported Python Branches
+PY37="python-3.7"
+PY36="python-3.6"
+PY35="python-3.5"
+PY34="python-3.4"
+PY27="python-2.7"
 
 # Which stack is used (for binary downloading), if none is provided (e.g. outside of Heroku)?
 DEFAULT_PYTHON_STACK="cedar-14"
 # If pip doesn't match this version (the version we install), run the installer.
 PIP_UPDATE="9.0.2"
 
-export DEFAULT_PYTHON_VERSION DEFAULT_PYTHON_STACK PIP_UPDATE LATEST_2 LATEST_36 LATEST_37
+export DEFAULT_PYTHON_VERSION DEFAULT_PYTHON_STACK PIP_UPDATE
+export LATEST_27 LATEST_36 LATEST_37 LATEST_35 LATEST_34
+export PY37 PY36 PY35 PY27 PY34
 
 # Common Problem Warnings:
 # This section creates a temporary file in which to stick the output of `pip install`.

bin/steps/pipenv-python-version

@@ -19,7 +19,7 @@ if [[ -f $BUILD_DIR/Pipfile ]]; then
             if [[ "$PYTHON" == "null" ]]; then
                 PYTHON=$(jq -r '._meta.requires.python_version' "$BUILD_DIR/Pipfile.lock")
                 if [ "$PYTHON" = 2.7 ]; then
-                    echo "$LATEST_2" > "$BUILD_DIR/runtime.txt"
+                    echo "$LATEST_27" > "$BUILD_DIR/runtime.txt"
                 fi
                 if [ "$PYTHON" = 3.6 ]; then
                     echo "$LATEST_36" > "$BUILD_DIR/runtime.txt"
@@ -33,4 +33,3 @@ if [[ -f $BUILD_DIR/Pipfile ]]; then
         fi
     fi
 fi
-

bin/steps/python

@@ -7,41 +7,49 @@ PYTHON_VERSION=$(cat runtime.txt)
 # The location of the pre-compiled python binary.
 VENDORED_PYTHON="${VENDOR_URL}/runtimes/$PYTHON_VERSION.tar.gz"
 
-if [[ $PYTHON_VERSION =~ ^python-2 ]]; then
-  if [[ "$PYTHON_VERSION" != "$LATEST_2" ]]; then
-    puts-warn "The latest version of Python 2 is $LATEST_2 (you are using $PYTHON_VERSION, which is unsupported)."
-    puts-warn "We recommend upgrading by specifying the latest version ($LATEST_2)."
-    echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
-  else
-    echo "       Using supported version of Python 2 ($PYTHON_VERSION)"
+SECURITY_UPDATE="Python has released a security update! Please consider upgrading to"
+
+# check if runtime exists
+if curl --output /dev/null --silent --head --fail "$VENDORED_PYTHON"; then
+  if [[ "$PYTHON_VERSION" == $PY37* ]]; then
+    # do things to alert the user of security release available
+    if [ "$PYTHON_VERSION" != "$LATEST_37" ]; then
+      puts-warn "$SECURITY_UPDATE" "$LATEST_37"
+      echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
+    fi
   fi
-else
-  if [[ $PYTHON_VERSION =~ ^python-3 ]]; then
-    if [[ $PYTHON_VERSION =~ ^python-3.7 ]]; then
-      if [[ "$PYTHON_VERSION" != "$LATEST_37" ]]; then
-      puts-warn "The latest version of Python 3.7 is $LATEST_37 (you are using $PYTHON_VERSION, which is unsupported)."
-      puts-warn "We recommend upgrading by specifying the latest version ($LATEST_37)."
+  if [[ "$PYTHON_VERSION" == $PY36* ]]; then
+    # security update note
+    if [ "$PYTHON_VERSION" != "$LATEST_36" ]; then
+      puts-warn "$SECURITY_UPDATE" "$LATEST_36"
+      echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
+    fi
+  fi
+  if [[ "$PYTHON_VERSION" == $PY35* ]]; then
+    # security update note
+    if [ "$PYTHON_VERSION" != "$LATEST_35" ]; then
+      puts-warn "$SECURITY_UPDATE" "$LATEST_35"
       echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
-      else
-        echo "       Using supported version of Python 3.7 ($PYTHON_VERSION)"
-      fi
-    else
-      if [[ $PYTHON_VERSION =~ ^python-3.6 ]]; then
-        if [[ "$PYTHON_VERSION" != "$LATEST_36" ]]; then
-          puts-warn "The latest version of Python 3.6 is $LATEST_36 (you are using $PYTHON_VERSION, which is unsupported)."
-          puts-warn "We recommend upgrading by specifying the latest version ($LATEST_36)."
-          echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
-        else
-          echo "       Using supported version of Python 3.6 ($PYTHON_VERSION)"
-        fi
-      else
-        puts-warn "Heroku supports runtime versions $LATEST_37, $LATEST_36 and $LATEST_2." 
-        puts-warn "You are using $PYTHON_VERSION, which is unsupported."
-        puts-warn "We recommend upgrading by specifying the default supported version ($LATEST_36)."
-        echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
-      fi
     fi
   fi
+  if [[ "$PYTHON_VERSION" == $PY34* ]]; then
+    # security update note
+    if [ "$PYTHON_VERSION" != "$LATEST_34" ]; then
+      puts-warn "$SECURITY_UPDATE" "$LATEST_34"
+      echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
+    fi
+  fi
+  if [[ "$PYTHON_VERSION" == $PY27* ]]; then
+    # security update note
+    if [ "$PYTHON_VERSION" != "$LATEST_27" ]; then
+      puts-warn "$SECURITY_UPDATE" "$LATEST_27"
+      echo "       Learn More: https://devcenter.heroku.com/articles/python-runtimes"
+    fi
+  fi
+else
+  puts-warn "Requested runtime ($PYTHON_VERSION) is not available for this stack ($STACK)."
+  puts-warn "Aborting.  More info: https://devcenter.heroku.com/articles/python-support"
+  exit 1
 fi
 
 if [[ "$STACK" != "$CACHED_PYTHON_STACK" ]]; then

bin/utils

@@ -89,10 +89,9 @@ python3_check() {
 # Check if Python version needs to install SQLite3
 python_sqlite3_check() {
   VERSION="$1"
-  MIN_PYTHON_3="python-3.6.6"
+  MIN_PYTHON_3="python-3.5.6"
   MIN_PYTHON_2="python-2.7.15"
 
   ( python2_check "$VERSION" && version_gte "$VERSION" "$MIN_PYTHON_2" ) \
-    || ( python3_check "$VERSION" && version_gte "$VERSION" "$MIN_PYTHON_3" ) \
-    || ( version_gte "$VERSION" "3.7.0" )
+    || ( python3_check "$VERSION" && version_gte "$VERSION" "$MIN_PYTHON_3" )
 }

builds/runtimes/python-3.4.9

+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+OUT_PREFIX=$1
+BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
+export BIN_DIR
+
+# shellcheck source=bin/utils
+source "$BIN_DIR/steps/sqlite3"
+
+sqlite3_version
+echo "Setting up SQLite3 Headers for $SQLITE3_VERSION"
+sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1
+
+echo "Building Python…"
+SOURCE_TARBALL='https://python.org/ftp/python/3.4.9/Python-3.4.9.tgz'
+curl -L $SOURCE_TARBALL | tar xz
+mv Python-3.4.9 src
+cd src
+
+./configure --prefix=$OUT_PREFIX --with-ensurepip=no
+make
+make install
+
+# Remove unneeded test directories, similar to the official Docker Python images:
+# https://github.com/docker-library/python
+find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' +
+
+# Remove spare /
+LOCATION=${OUT_PREFIX%?}
+
+ln $LOCATION/bin/python3 $LOCATION/bin/python

builds/runtimes/python-3.5.6

+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+OUT_PREFIX=$1
+BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
+export BIN_DIR
+
+# shellcheck source=bin/utils
+source "$BIN_DIR/steps/sqlite3"
+
+sqlite3_version
+echo "Setting up SQLite3 Headers for $SQLITE3_VERSION"
+sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1
+
+echo "Building Python…"
+SOURCE_TARBALL='https://python.org/ftp/python/3.5.6/Python-3.5.6.tgz'
+curl -L $SOURCE_TARBALL | tar xz
+mv Python-3.5.6 src
+cd src
+
+./configure --prefix=$OUT_PREFIX --with-ensurepip=no
+make
+make install
+
+# Remove unneeded test directories, similar to the official Docker Python images:
+# https://github.com/docker-library/python
+find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' +
+
+# Remove spare /
+LOCATION=${OUT_PREFIX%?}
+
+ln $LOCATION/bin/python3 $LOCATION/bin/python

builds/runtimes/python-3.6.7

+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+OUT_PREFIX=$1
+BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
+export BIN_DIR
+
+# shellcheck source=bin/utils
+source "$BIN_DIR/steps/sqlite3"
+
+sqlite3_version
+echo "Setting up SQLite3 Headers for $SQLITE3_VERSION"
+sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1
+
+echo "Building Python…"
+SOURCE_TARBALL='https://python.org/ftp/python/3.6.7/Python-3.6.7.tgz'
+curl -L $SOURCE_TARBALL | tar xz
+mv Python-3.6.7 src
+cd src
+
+./configure --prefix=$OUT_PREFIX --with-ensurepip=no
+make
+make install
+
+# Remove unneeded test directories, similar to the official Docker Python images:
+# https://github.com/docker-library/python
+find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' +
+
+# Remove spare /
+LOCATION=${OUT_PREFIX%?}
+
+ln $LOCATION/bin/python3 $LOCATION/bin/python

builds/runtimes/python-3.7.1

+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+OUT_PREFIX=$1
+BIN_DIR="$(cd "$(dirname "$0")"/../.. || exit; pwd)/bin"
+export BIN_DIR
+
+# shellcheck source=bin/utils
+source "$BIN_DIR/steps/sqlite3"
+
+sqlite3_version
+echo "Setting up SQLite3 Headers for $SQLITE3_VERSION"
+sqlite3_install "$OUT_PREFIX" "$SQLITE3_VERSION" 1
+
+echo "Building Python…"
+SOURCE_TARBALL='https://python.org/ftp/python/3.7.1/Python-3.7.1.tgz'
+curl -L $SOURCE_TARBALL | tar xz
+mv Python-3.7.1 src
+cd src
+
+./configure --prefix=$OUT_PREFIX --with-ensurepip=no
+make
+make install
+
+# Remove unneeded test directories, similar to the official Docker Python images:
+# https://github.com/docker-library/python
+find "${OUT_PREFIX}" \( -type d -a \( -name test -o -name tests \) \) -exec rm -rf '{}' +
+
+# Remove spare /
+LOCATION=${OUT_PREFIX%?}
+
+ln $LOCATION/bin/python3 $LOCATION/bin/python

requirements.txt

 docopt==0.6.2
-bob-builder==0.0.13
+bob-builder
 boto==2.48.0

spec/hatchet/python_spec.rb

@@ -4,7 +4,7 @@ describe "Python!!!!!!!!!!!" do
   it "🐍" do
     Hatchet::Runner.new('python-getting-started', stack: DEFAULT_STACK).deploy do |app|
       expect(app.output).to           match(/Installing pip/)
-      expect(app.run('python -V')).to match(/3.6.6/)
+      expect(app.run('python -V')).to match(/3.6.7/)
     end
   end
 end

spec/spec_helper.rb

-ENV['HATCHET_BUILDPACK_BASE'] = 'https://github.com/heroku/heroku-buildpack-python.git'
+ENV['HATCHET_BUILDPACK_BASE'] = 'https://github.com/' + ENV['TRAVIS_REPO_SLUG'] + '.git'
 
 require 'rspec/core'
 require 'rspec/retry'

test/fixtures/pipenv-version/Pipfile

@@ -6,4 +6,4 @@ verify_ssl = true
 requests = "*"
 
 [requires]
-python_version = "3.6"
+python_version = "3.6"
\ No newline at end of file

test/fixtures/pipenv-version2/Pipfile

+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+
+[dev-packages]
+
+[requires]
+python_version = "2.7"

test/fixtures/pipenv-version2/Pipfile.lock

+{
+    "_meta": {
+        "hash": {
+            "sha256": "ae4bdd7d4157baab65ae9d0e8389a6011e6b640995372c45ec81fa5d1ddfae9f"
+        },
+        "pipfile-spec": 6,
+        "requires": {
+            "python_version": "2.7"
+        },
+        "sources": [
+            {
+                "name": "pypi",
+                "url": "https://pypi.org/simple",
+                "verify_ssl": true
+            }
+        ]
+    },
+    "default": {},
+    "develop": {}
+}