diff --git a/test/fixtures/python2_warn/requirements.txt b/test/fixtures/python2_warn/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..663bd1f6a2ae02f29df59fb4963c17934034f731
--- /dev/null
+++ b/test/fixtures/python2_warn/requirements.txt
@@ -0,0 +1 @@
+requests
\ No newline at end of file
diff --git a/test/fixtures/python2_warn/runtime.txt b/test/fixtures/python2_warn/runtime.txt
new file mode 100644
index 0000000000000000000000000000000000000000..538568204a78b4e4ec20255b64fa25533b4edae3
--- /dev/null
+++ b/test/fixtures/python2_warn/runtime.txt
@@ -0,0 +1 @@
+python-2.7.14
diff --git a/test/fixtures/python3_4_warn/requirements.txt b/test/fixtures/python3_4_warn/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..7e1060246fd6746a14204539a72e199a25469a05
--- /dev/null
+++ b/test/fixtures/python3_4_warn/requirements.txt
@@ -0,0 +1 @@
+flask
diff --git a/test/fixtures/python3_4_warn/runtime.txt b/test/fixtures/python3_4_warn/runtime.txt
new file mode 100644
index 0000000000000000000000000000000000000000..fe5dcf7a3aeb68d0bb349ddd6f7d61ba731a1057
--- /dev/null
+++ b/test/fixtures/python3_4_warn/runtime.txt
@@ -0,0 +1 @@
+python-3.4.0
diff --git a/test/fixtures/python3_5_warn/requirements.txt b/test/fixtures/python3_5_warn/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..7e1060246fd6746a14204539a72e199a25469a05
--- /dev/null
+++ b/test/fixtures/python3_5_warn/requirements.txt
@@ -0,0 +1 @@
+flask
diff --git a/test/fixtures/python3_5_warn/runtime.txt b/test/fixtures/python3_5_warn/runtime.txt
new file mode 100644
index 0000000000000000000000000000000000000000..5486d7adb9a188bcfc1d0d8844b4091b1c369ec9
--- /dev/null
+++ b/test/fixtures/python3_5_warn/runtime.txt
@@ -0,0 +1 @@
+python-3.5.3
diff --git a/test/fixtures/python3_6_warn/requirements.txt b/test/fixtures/python3_6_warn/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..663bd1f6a2ae02f29df59fb4963c17934034f731
--- /dev/null
+++ b/test/fixtures/python3_6_warn/requirements.txt
@@ -0,0 +1 @@
+requests
\ No newline at end of file
diff --git a/test/fixtures/python3_6_warn/runtime.txt b/test/fixtures/python3_6_warn/runtime.txt
new file mode 100644
index 0000000000000000000000000000000000000000..486fcce127f00eba90a4096290b67c31adaef0bd
--- /dev/null
+++ b/test/fixtures/python3_6_warn/runtime.txt
@@ -0,0 +1 @@
+python-3.6.5
diff --git a/test/run b/test/run
index 67d4113dbb6a0d1f8aed8bdca54d0f5ba45001a6..484f241d4f503af08c4bf56cc50425114ecf6cc1 100755
--- a/test/run
+++ b/test/run
@@ -87,19 +87,48 @@ testPylibmc() {
   assertCapturedSuccess
 }
 
+testPython2_warn() {
+    compile "python2_warn"
+    assertCaptured "python-2.7.14"
+    assertCaptured "security update!"
+    assertCapturedSuccess
+}
+
 testPython2() {
     compile "python2"
     assertCaptured "python-2.7.15"
     assertCapturedSuccess
 }
 
-# This will succeed on Cedar-14 and fail on 16 and 18
+# This fail
+testPython3_4_warn() {
+  compile "python3_4_warn"
+  if [[ $STACK = "cedar-14" ]]; then
+    assertCaptured "python-3.4.0"
+    assertCapturedSuccess
+  else
+    assertCapturedError
+  fi
+}
+
+# This fail
 testPython3_4() {
   compile "python3_4"
   assertCaptured "python-3.4.9"
   assertCapturedError
 }
 
+# This fail
+testPython3_5_warn() {
+  compile "python3_5_warn"
+  if [[ $STACK = "cedar-14" ]]; then
+    assertCaptured "python-3.5.3"
+    assertCapturedSuccess
+  else
+    assertCapturedError
+  fi
+}
+
 # This will fail
 testPython3_5() {
   compile "python3_5"
@@ -107,6 +136,14 @@ testPython3_5() {
   assertCapturedError
 }
 
+# This will warn
+testPython3_6_warn() {
+  compile "python3_6_warn"
+  assertCaptured "python-3.6.5"
+  assertCaptured "security update!"
+  assertCapturedSuccess
+}
+
 testPython3_6() {
   compile "python3_6"
   assertCaptured "python-3.6.6"
@@ -122,8 +159,12 @@ testPython3_6_7() {
 
 testPython3_7() {
   compile "python3_7"
-  assertCaptured "python-3.7.0"
-  assertCapturedSuccess
+  if [[ $STACK = "cedar-14" ]]; then
+    assertCapturedError
+  else
+    assertCaptured "python-3.7.0"
+    assertCapturedSuccess
+  fi
 }
 
 # This will fail