From 94fea04f17136e5a985f34df5472ecd68d8b97ad Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Tue, 21 Apr 2020 13:59:40 +0700 Subject: [PATCH 1/7] tox.ini: Use pytest-instafail --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 3f106bc..34b6e86 100644 --- a/tox.ini +++ b/tox.ini @@ -12,10 +12,11 @@ deps = !py27: git+https://github.com/google/sre_yield pytest pytest-cov + pytest-instafail unittest-expander !py34: lxml # Indirect deps via https_everywhere_checker tldextract regex commands = - pytest --cov=https_everywhere --cov-report=term-missing:skip-covered -rs --maxfail=15 {posargs:--cov-fail-under={env:COV_FAIL_UNDER:95}} + pytest --cov=https_everywhere --cov-report=term-missing:skip-covered -rs --instafail --maxfail=15 {posargs:--cov-fail-under={env:COV_FAIL_UNDER:95}} From 80e4f449904a34dd8c63fc6cf937fdd7545c7785 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Tue, 21 Apr 2020 13:58:55 +0700 Subject: [PATCH 2/7] test_upstream: Create Tests purely of requests HTTPAdapter, which can be used to prepare offline tests, and base all other tests on to show where other adapters differ from pure requests. Related to https://github.com/jayvdb/https-everywhere-py/issues/14 Related to https://github.com/jayvdb/https-everywhere-py/issues/25 --- tests/test_upstream.py | 172 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100644 tests/test_upstream.py diff --git a/tests/test_upstream.py b/tests/test_upstream.py new file mode 100644 index 0000000..5748e2e --- /dev/null +++ b/tests/test_upstream.py @@ -0,0 +1,172 @@ +from __future__ import unicode_literals + +import unittest + +import requests + + +class TestUpstreamAdapter(unittest.TestCase): + + cls = requests.adapters.HTTPAdapter + + def test_freerangekitten_com(self): + url = "http://freerangekitten.com/" + s = requests.Session() + s.mount("http://", self.cls()) + r = s.get(url) + r.raise_for_status() + self.assertEqual(r.url, url) + self.assertEqual(r.history, []) + + def test_example_com(self): + url = "http://example.com/" + s = requests.Session() + s.mount("http://", self.cls()) + r = s.get(url) + r.raise_for_status() + self.assertEqual(r.url, url) + self.assertEqual(r.history, []) + + def test_webhostinggeeks_com_science(self): + # https://github.com/EFForg/https-everywhere/issues/18867 + # https has wrong cert for a quick failure + url = "http://science.webhostinggeeks.com/" + s = requests.Session() + s.mount("http://", self.cls()) + r = s.get(url) + r.raise_for_status() + self.assertEqual(r.url, url) + self.assertEqual(r.history, []) + + def test_fedmsg_com(self): + # https://github.com/EFForg/https-everywhere/issues/18867 + # https redirects to http, so a manual exclusion is needed + url = "http://fedmsg.com/" + s = requests.Session() + s.mount("http://", self.cls()) + r = s.get(url) + r.raise_for_status() + self.assertEqual(r.url, url) + self.assertEqual(r.history, []) + + def test_shopzilla(self): + url = "http://www.shopzilla.com/" + s = requests.Session() + s.mount("http://", self.cls()) + r = s.get(url) + r.raise_for_status() + self.assertEqual(r.url, url) + self.assertEqual(r.history, []) + + def test_whisper_sh(self): + url = "http://whisper.sh/" + s = requests.Session() + s.mount("http://", self.cls()) + r = s.get(url) + r.raise_for_status() + self.assertEqual(r.url, url) + self.assertEqual(r.history, []) + + def test_thesyriacampaign(self): + url = "http://www.thesyriacampaign.org/" + s = requests.Session() + s.mount("http://", self.cls()) + r = s.get(url) + self.assertEqual(r.status_code, 403) + + def test_esncz_org(self): + url = "http://www.isc.vutbr.cz/" + s = requests.Session() + s.mount("http://", self.cls()) + r = s.get(url) + r.raise_for_status() + self.assertEqual(r.url, "https://www.esncz.org") + target = "https://www.esncz.org" + self.assertEqual(r.url, target) + self.assertEqual(len(r.history), 1) + original = r.history[0] + self.assertEqual(original.url, url) + self.assertEqual(original.status_code, 302) + self.assertEqual(original.reason, "Found") + + def test_01_org(self): + url = "http://01.org/" + s = requests.Session() + s.mount("http://", self.cls()) + r = s.get(url) + r.raise_for_status() + self.assertEqual(r.url, url.replace("http://", "https://")) + self.assertEqual(len(r.history), 1) + original = r.history[0] + self.assertEqual(original.url, url) + self.assertEqual(original.status_code, 301) + self.assertEqual(original.reason, "Moved Permanently") + + def test_01_org_www(self): + url = "http://www.01.org/" + s = requests.Session() + s.mount("http://", self.cls()) + r = s.get(url) + r.raise_for_status() + self.assertEqual(r.url, "https://01.org/") + self.assertEqual(len(r.history), 1) + original = r.history[0] + self.assertEqual(original.url, url) + self.assertEqual(original.status_code, 301) + self.assertEqual(original.reason, "Moved Permanently") + + def test_medbank_mt(self): + url = "http://business.medbank.com.mt/" + s = requests.Session() + s.mount("http://", self.cls()) + r = s.get(url) + r.raise_for_status() + self.assertEqual(r.url, "https://www.medirect.com.mt") + self.assertEqual(len(r.history), 1) + original = r.history[0] + self.assertEqual(original.url, url) + self.assertEqual(original.status_code, 301) + self.assertEqual(original.reason, "Moved Permanently") + + def test_my_vpnglobe(self): + url = "http://my.vpnglobe.com/" + s = requests.Session() + s.mount("http://", self.cls()) + with self.assertRaises(requests.exceptions.SSLError): + s.get(url) + + def _test_modwsgi_org(self): + # https://github.com/EFForg/https-everywhere/issues/18867 + # http has a redirect to readthedocs; https fails + url = "http://www.modwsgi.org/" + s = requests.Session() + s.mount("http://", self.cls()) + with self.assertRaises(requests.exceptions.Timeout): + s.get(url, timeout=5) + + def test_python_org_packages(self): + url = "http://packages.python.org/" + s = requests.Session() + s.mount("http://", self.cls()) + r = s.get(url) + r.raise_for_status() + self.assertEqual(r.url, "https://pythonhosted.org/") + self.assertEqual(len(r.history), 2) + original = r.history[0] + self.assertEqual(original.url, url) + self.assertEqual(original.status_code, 301) + self.assertEqual(original.reason, "Moved Permanently") + self.assertEqual(r.history[1].url, url.replace("http://", "https://")) + self.assertEqual(r.history[1].status_code, 301) + self.assertEqual(r.history[1].reason, "Moved Permanently") + + def test_ros_wiki(self): + # https://github.com/jayvdb/pypidb/issues/115 + # Short-lived problem + url = "http://wiki.ros.org/" + s = requests.Session() + s.mount("http://", self.cls()) + r = s.get(url) + r.raise_for_status() + self.assertEqual(r.url, url) + self.assertEqual(r.history, []) From ea34315fdb6a110a2a124654b12e4e41df57beec Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Tue, 21 Apr 2020 13:58:55 +0700 Subject: [PATCH 3/7] test_adapter: Remove TestUpgradeAdapter Removing this in a separate commit radically improves the readability of the next commit. Related to https://github.com/jayvdb/https-everywhere-py/issues/25 --- tests/test_adapter.py | 124 ------------------------------------------ 1 file changed, 124 deletions(-) diff --git a/tests/test_adapter.py b/tests/test_adapter.py index 4319327..057e6e3 100644 --- a/tests/test_adapter.py +++ b/tests/test_adapter.py @@ -540,130 +540,6 @@ def test_python_org_packages(self): self.assertEqual(r.history[1].status_code, 301) -class TestUpgradeAdapter(unittest.TestCase): - - cls = UpgradeHTTPSAdapter - - def test_freerangekitten_com(self): - url = "http://freerangekitten.com/" - s = requests.Session() - s.mount("http://", self.cls()) - r = s.get(url) - r.raise_for_status() - self.assertEqual(r.url, url.replace("http://", "https://")) - self.assertEqual(len(r.history), 1) - original = r.history[0] - self.assertEqual(original.url, url) - - def test_example_com(self): - url = "http://example.com/" - s = requests.Session() - s.mount("http://", self.cls()) - r = s.get(url) - r.raise_for_status() - self.assertEqual(r.url, url.replace("http://", "https://")) - self.assertEqual(len(r.history), 1) - original = r.history[0] - self.assertEqual(original.url, url) - - def test_webhostinggeeks_com_science(self): - # https://github.com/EFForg/https-everywhere/issues/18867 - # https has wrong cert for quick failure. - url = "http://science.webhostinggeeks.com/" - s = requests.Session() - s.mount("http://", self.cls()) - s.mount("https://", self.cls()) - r = s.get(url, timeout=5) - r.raise_for_status() - # There is an extra history item, with a redirect, but - # the final request has disregarded that redirect - self.assertEqual(r.url, url) - self.assertEqual(len(r.history), 1) - original = r.history[0] - self.assertEqual(original.url, url) - self.assertEqual( - original.headers["location"], url.replace("http://", "https://") - ) - - def test_shopzilla(self): - url = "http://www.shopzilla.com/" - s = requests.Session() - s.mount("http://", self.cls()) - with self.assertRaises(requests.exceptions.TooManyRedirects): - s.get(url) - - def test_whisper_sh(self): - url = "http://whisper.sh/" - s = requests.Session() - s.mount("http://", self.cls()) - with self.assertRaises(requests.exceptions.TooManyRedirects): - s.get(url) - - def test_thesyriacampaign(self): - url = "http://www.thesyriacampaign.org/" - s = requests.Session() - s.mount("http://", self.cls()) - with self.assertRaises(requests.exceptions.TooManyRedirects): - s.get(url) - - def test_esncz_org(self): - url = "http://www.isc.vutbr.cz/" - s = requests.Session() - s.mount("http://", self.cls()) - with self.assertRaises(requests.exceptions.SSLError): - s.get(url) - - def test_my_vpnglobe(self): - url = "http://my.vpnglobe.com/" - s = requests.Session() - s.mount("http://", self.cls()) - with self.assertRaises(requests.exceptions.SSLError): - s.get(url) - - def _test_modwsgi_org(self): - # https://github.com/EFForg/https-everywhere/issues/18867 - # http has a redirect to readthedocs; https fails - url = "http://www.modwsgi.org/" - s = requests.Session() - s.mount("http://", self.cls()) - s.mount("https://", self.cls()) - r = s.get(url, timeout=5) - r.raise_for_status() - self.assertEqual(r.url, "https://modwsgi.readthedocs.io/en/develop/") - self.assertEqual(len(r.history), 4) - original = r.history[0] - self.assertEqual(original.url, "http://www.modwsgi.org/") - injected_failed_upgrade = r.history[1] - self.assertEqual(injected_failed_upgrade.url, "http://www.modwsgi.org/") - source_redirect = r.history[2] - self.assertEqual(source_redirect.url, "http://modwsgi.readthedocs.io/") - - def test_fedmsg_com(self): - # https://github.com/EFForg/https-everywhere/issues/18867 - # https redirects to http, but there are no rules to upgrade it to https - url = "http://fedmsg.com/" - s = requests.Session() - s.mount("http://", self.cls()) - s.mount("https://", self.cls()) - with self.assertRaises(requests.exceptions.TooManyRedirects): - s.get(url) - - def test_python_org_packages(self): - url = "http://packages.python.org/" - s = requests.Session() - s.mount("http://", self.cls()) - r = s.get(url) - r.raise_for_status() - self.assertEqual(r.url, "https://pythonhosted.org/") - self.assertEqual(len(r.history), 2) - original = r.history[0] - self.assertEqual(original.url, url) - # This adapter provides the real response, with code of 301 - self.assertEqual(original.status_code, 302) - self.assertEqual(r.history[1].url, url.replace("http://", "https://")) - self.assertEqual(r.history[1].status_code, 301) - - class TestSafeUpgradeAdapter(unittest.TestCase): cls = SafeUpgradeHTTPSAdapter From 06ec55254107987798d25565809e26e42ad7f6f5 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Tue, 21 Apr 2020 13:47:26 +0700 Subject: [PATCH 4/7] test_adapter: Use inheritance Reduce duplication of expected results, and better show which adapters are most similar, and the remaining tests better demonstrate how they are different. Improved some tests to be more specific. Also add tests for wiki.ros.org. Closes https://github.com/jayvdb/https-everywhere-py/issues/25 --- tests/test_adapter.py | 349 ++++++++---------------------------------- 1 file changed, 63 insertions(+), 286 deletions(-) diff --git a/tests/test_adapter.py b/tests/test_adapter.py index 057e6e3..4ca867e 100644 --- a/tests/test_adapter.py +++ b/tests/test_adapter.py @@ -19,6 +19,8 @@ _HTTP_BLOCK_CODE, ) +from tests.test_upstream import TestUpstreamAdapter + # Throughout _test_modwsgi_org refers to a scenario where the website # has been fixed, an a similar scenario has not yet been found # It may need to be mocked. @@ -68,7 +70,7 @@ def test_whisper_sh(self): self.assertEqual(r.status_code, _HTTP_BLOCK_CODE) -class TestEverywhereOnlyAdapter(unittest.TestCase): +class TestEverywhereOnlyAdapter(TestUpstreamAdapter): cls = HTTPSEverywhereOnlyAdapter @@ -83,76 +85,11 @@ def test_freerangekitten_com(self): original = r.history[0] self.assertEqual(original.url, url) - def test_example_com(self): - url = "http://example.com/" - s = requests.Session() - s.mount("http://", self.cls()) - r = s.get(url) - r.raise_for_status() - self.assertEqual(r.url, url) - self.assertEqual(r.history, []) - - def test_webhostinggeeks_com_science(self): + def _test_webhostinggeeks_com_science(self): # https://github.com/EFForg/https-everywhere/issues/18867 # https has wrong cert, which is a quick failure, but an exception exists url = "http://science.webhostinggeeks.com/" - s = requests.Session() - s.mount("http://", self.cls()) - r = s.get(url) - r.raise_for_status() - self.assertEqual(r.url, url) - self.assertEqual(r.history, []) - - def test_fedmsg_com(self): - # https://github.com/EFForg/https-everywhere/issues/18867 - # https redirects to http, but there are no rules to upgrade it to https - url = "http://fedmsg.com/" - s = requests.Session() - s.mount("http://", self.cls()) - r = s.get(url) - r.raise_for_status() - self.assertEqual(r.url, url) - self.assertEqual(r.history, []) - - def test_shopzilla(self): - url = "http://www.shopzilla.com/" - s = requests.Session() - s.mount("http://", self.cls()) - r = s.get(url, timeout=5) - r.raise_for_status() - self.assertEqual(r.url, url) - self.assertEqual(r.history, []) - - def test_whisper_sh(self): - url = "http://whisper.sh/" - s = requests.Session() - s.mount("http://", self.cls()) - r = s.get(url, timeout=5) - r.raise_for_status() - self.assertEqual(r.url, url) - self.assertEqual(r.history, []) - - def test_thesyriacampaign(self): - url = "http://www.thesyriacampaign.org/" - s = requests.Session() - s.mount("http://", self.cls()) - r = s.get(url, timeout=5) - self.assertEqual(r.url, url) - self.assertEqual(r.status_code, 403) - - def test_esncz_org(self): - url = "http://www.isc.vutbr.cz/" - s = requests.Session() - s.mount("http://", self.cls()) - r = s.get(url, timeout=5) - r.raise_for_status() - target = "https://www.esncz.org" - self.assertEqual(r.url, target) - self.assertEqual(len(r.history), 1) - original = r.history[0] - self.assertEqual(original.url, url) - self.assertEqual(original.status_code, 302) - self.assertEqual(original.reason, "Found") + # See super method def test_01_org(self): url = "http://01.org/" @@ -181,23 +118,6 @@ def test_01_org_www(self): self.assertEqual(original.reason, _REASON) self.assertEqual(r.history[-1].url, "https://www.01.org/") - def test_medbank_mt(self): - url = "http://business.medbank.com.mt/" - s = requests.Session() - s.mount("http://", self.cls()) - r = s.get(url) - r.raise_for_status() - self.assertEqual(r.url, "https://www.medirect.com.mt") - # with self.assertRaises(requests.exceptions.ConnectionError): - # s.get(url) - - def test_my_vpnglobe(self): - url = "http://my.vpnglobe.com/" - s = requests.Session() - s.mount("http://", self.cls()) - with self.assertRaises(requests.exceptions.SSLError): - s.get(url) - def test_python_org_packages(self): url = "http://packages.python.org/" s = requests.Session() @@ -244,22 +164,7 @@ def test_medbank_mt(self): self.assertEqual(r.history[1].status_code, 301) self.assertEqual(r.history[1].reason, "Moved Permanently") - def test_python_org_packages(self): - url = "http://packages.python.org/" - s = requests.Session() - s.mount("http://", self.cls()) - r = s.get(url) - r.raise_for_status() - self.assertEqual(r.url, "https://pythonhosted.org/") - self.assertEqual(len(r.history), 2) - original = r.history[0] - self.assertEqual(original.url, url) - # 301 here vs 302 - self.assertEqual(original.status_code, 301) - self.assertEqual(original.reason, "Moved Permanently") - self.assertEqual(r.history[1].url, url.replace("http://", "https://")) - self.assertEqual(r.history[1].status_code, 301) - self.assertEqual(r.history[1].reason, "Moved Permanently") + test_python_org_packages = TestUpstreamAdapter.test_python_org_packages class TestMozillaPreloadAdapter(TestChromePreloadAdapter): @@ -277,12 +182,12 @@ class TestEverywhereAdapter(TestChromePreloadAdapter): test_python_org_packages = TestEverywhereOnlyAdapter.test_python_org_packages -class TestForceAdapter(unittest.TestCase): +class TestForceAdapter(TestEverywhereAdapter): cls = ForceHTTPSAdapter - def test_freerangekitten_com(self): - url = "http://freerangekitten.com/" + def test_ros_wiki(self): + url = "http://wiki.ros.org/" s = requests.Session() s.mount("http://", self.cls()) r = s.get(url) @@ -352,13 +257,6 @@ def test_esncz_org(self): with self.assertRaises(requests.exceptions.SSLError): s.get(url, timeout=5) - def test_my_vpnglobe(self): - url = "http://my.vpnglobe.com/" - s = requests.Session() - s.mount("http://", self.cls()) - with self.assertRaises(requests.exceptions.SSLError): - s.get(url) - def _test_modwsgi_org(self): # https://github.com/EFForg/https-everywhere/issues/18867 # http has a redirect to readthedocs; https fails @@ -392,76 +290,15 @@ def test_fedmsg_com(self): original = r.history[0] self.assertEqual(original.url, url.replace("http://", "https://")) - def test_python_org_packages(self): - url = "http://packages.python.org/" - s = requests.Session() - s.mount("http://", self.cls()) - r = s.get(url) - r.raise_for_status() - self.assertEqual(r.url, "https://pythonhosted.org/") - self.assertEqual(len(r.history), 2) - original = r.history[0] - self.assertEqual(original.url, url) - self.assertEqual(original.status_code, 302) - self.assertEqual(r.history[1].url, url.replace("http://", "https://")) - self.assertEqual(r.history[1].status_code, 301) - -class TestPreferAdapter(unittest.TestCase): +class TestPreferAdapter(TestForceAdapter): cls = PreferHTTPSAdapter - def test_freerangekitten_com(self): - url = "http://freerangekitten.com/" - s = requests.Session() - s.mount("http://", self.cls()) - r = s.get(url) - r.raise_for_status() - self.assertEqual(r.url, url.replace("http://", "https://")) - self.assertEqual(len(r.history), 1) - original = r.history[0] - self.assertEqual(original.url, url) - - def test_example_com(self): - url = "http://example.com/" - s = requests.Session() - s.mount("http://", self.cls()) - r = s.get(url) - r.raise_for_status() - self.assertEqual(r.url, url.replace("http://", "https://")) - self.assertEqual(len(r.history), 1) - original = r.history[0] - self.assertEqual(original.url, url) - - def test_webhostinggeeks_com_science(self): - # https://github.com/EFForg/https-everywhere/issues/18867 - # https has wrong cert for quick failure. - url = "http://science.webhostinggeeks.com/" - s = requests.Session() - s.mount("http://", self.cls()) - with self.assertRaises(requests.exceptions.SSLError): - s.get(url) - - def test_shopzilla(self): - url = "http://www.shopzilla.com/" - s = requests.Session() - s.mount("http://", self.cls()) - with self.assertRaises(requests.exceptions.TooManyRedirects): - s.get(url) - - def test_whisper_sh(self): - url = "http://whisper.sh/" - s = requests.Session() - s.mount("http://", self.cls()) - with self.assertRaises(requests.exceptions.TooManyRedirects): - s.get(url) - - def test_thesyriacampaign(self): - url = "http://www.thesyriacampaign.org/" - s = requests.Session() - s.mount("http://", self.cls()) - with self.assertRaises(requests.exceptions.TooManyRedirects): - s.get(url) + test_01_org = TestUpstreamAdapter.test_01_org + test_01_org_www = TestUpstreamAdapter.test_01_org + test_medbank_mt = TestUpstreamAdapter.test_medbank_mt + test_python_org_packages = TestUpstreamAdapter.test_python_org_packages def test_esncz_org(self): url = "http://www.isc.vutbr.cz/" @@ -476,13 +313,6 @@ def test_esncz_org(self): self.assertEqual(original.url, url) self.assertEqual(r.url, "https://www.esncz.org") - def test_my_vpnglobe(self): - url = "http://my.vpnglobe.com/" - s = requests.Session() - s.mount("http://", self.cls()) - with self.assertRaises(requests.exceptions.SSLError): - s.get(url) - def _test_modwsgi_org(self): # https://github.com/EFForg/https-everywhere/issues/18867 # http has a redirect to readthedocs; https fails @@ -498,49 +328,8 @@ def _test_modwsgi_org(self): self.assertEqual(len(r.history), 2) self.assertEqual(r.history[1].url, "https://modwsgi.readthedocs.io/") - def test_fedmsg_com(self): - # https://github.com/EFForg/https-everywhere/issues/18867 - # https redirects to http, but there are no rules to upgrade it to https - url = "http://fedmsg.com/" - s = requests.Session() - s.mount("http://", self.cls()) - with self.assertRaises(requests.exceptions.TooManyRedirects): - s.get(url) - # It can be solved with an explicit manual exclusion, but cant - # be managed in an adapter because the TooManyRedirects is raised - # in the session - s.mount("http://", self.cls(https_exclusions=["fedmsg.com"])) - r = s.get(url) - r.raise_for_status() - self.assertEqual(r.url, url) - self.assertEqual(r.history, []) - - s.mount("https://", self.cls(https_exclusions=["fedmsg.com"])) - r = s.get(url.replace("http://", "https://")) - r.raise_for_status() - self.assertEqual(r.url, url) - self.assertEqual(len(r.history), 1) - original = r.history[0] - self.assertEqual(original.url, url.replace("http://", "https://")) - - def test_python_org_packages(self): - url = "http://packages.python.org/" - s = requests.Session() - s.mount("http://", self.cls()) - r = s.get(url) - r.raise_for_status() - self.assertEqual(r.url, "https://pythonhosted.org/") - self.assertEqual(len(r.history), 2) - original = r.history[0] - self.assertEqual(original.url, url) - # This adapter provides the real response, with code of 301 - self.assertEqual(original.status_code, 301) - self.assertEqual(r.history[1].url, url.replace("http://", "https://")) - self.assertEqual(r.history[1].status_code, 301) - - -class TestSafeUpgradeAdapter(unittest.TestCase): +class TestSafeUpgradeAdapter(TestEverywhereAdapter): cls = SafeUpgradeHTTPSAdapter @@ -562,55 +351,7 @@ def test_example_com(self): self.assertEqual(r.url, url.replace("http://", "https://")) self.assertEqual(r.history, []) - def test_webhostinggeeks_com_science(self): - # https://github.com/EFForg/https-everywhere/issues/18867 - # https has wrong cert for quick failure. - url = "http://science.webhostinggeeks.com/" - s = requests.Session() - s.mount("http://", self.cls()) - r = s.get(url, timeout=5) - r.raise_for_status() - self.assertEqual(r.url, url) - self.assertEqual(r.history, []) - - def test_shopzilla(self): - url = "http://www.shopzilla.com/" - s = requests.Session() - s.mount("http://", self.cls()) - r = s.get(url, timeout=5) - r.raise_for_status() - self.assertEqual(r.url, url) - - def test_whisper_sh(self): - url = "http://whisper.sh/" - s = requests.Session() - s.mount("http://", self.cls()) - r = s.get(url, timeout=5) - r.raise_for_status() - self.assertEqual(r.url, url) - - def test_thesyriacampaign(self): - url = "http://www.thesyriacampaign.org/" - s = requests.Session() - s.mount("http://", self.cls()) - r = s.get(url, timeout=5) - self.assertEqual(r.url, url) - # FIXME: this should redirect to 'https://thesyriacampaign.org/' - self.assertEqual(r.status_code, 403) - - def test_esncz_org(self): - url = "http://www.isc.vutbr.cz/" - s = requests.Session() - s.mount("http://", self.cls()) - r = s.get(url, timeout=5) - r.raise_for_status() - target = "https://www.esncz.org" - self.assertEqual(r.url, target) - self.assertEqual(len(r.history), 1) - original = r.history[0] - self.assertEqual(original.url, "http://www.isc.vutbr.cz/") - - def test_my_vpnglobe(self): + def _test_my_vpnglobe(self): url = "http://my.vpnglobe.com/" s = requests.Session() s.mount("http://", self.cls()) @@ -633,34 +374,70 @@ def _test_modwsgi_org(self): source_redirect = r.history[1] self.assertEqual(source_redirect.url, "https://modwsgi.readthedocs.io/") - def test_fedmsg_com(self): - # https://github.com/EFForg/https-everywhere/issues/18867 - # https redirects to http, but there are no rules to upgrade it to https - url = "http://fedmsg.com/" + def test_python_org_packages(self): + url = "http://packages.python.org/" + s = requests.Session() + s.mount("http://", self.cls()) + r = s.get(url) + r.raise_for_status() + self.assertEqual(r.url, "https://pythonhosted.org/") + self.assertEqual(len(r.history), 1) + original = r.history[0] + self.assertEqual(original.url, url.replace("http://", "https://")) + # This adapter provides the real response, with code of 301 + self.assertEqual(original.status_code, 301) + + def test_ros_wiki(self): + # https://github.com/jayvdb/pypidb/issues/115 + # Short-lived problem + url = "http://wiki.ros.org/" s = requests.Session() s.mount("http://", self.cls()) + r = s.get(url) + r.raise_for_status() + self.assertEqual(r.url, url.replace("http://", "https://")) + self.assertEqual(r.history, []) + def test_01_org(self): + url = "http://01.org/" + s = requests.Session() + s.mount("http://", self.cls()) r = s.get(url) r.raise_for_status() - self.assertEqual(r.url, url) + self.assertEqual(r.url, url.replace("http://", "https://")) self.assertEqual(r.history, []) - r = s.get(url.replace("http://", "https://")) + def test_01_org_www(self): + url = "http://www.01.org/" + s = requests.Session() + s.mount("http://", self.cls()) + r = s.get(url) r.raise_for_status() - self.assertEqual(r.url, url) + self.assertEqual(r.url, "https://01.org/") self.assertEqual(len(r.history), 1) original = r.history[0] + # Here is where it differs from upstream, with an unusual + # original url which differs from the real original self.assertEqual(original.url, url.replace("http://", "https://")) + self.assertEqual(original.status_code, 301) + self.assertEqual(original.reason, "Moved Permanently") - def test_python_org_packages(self): - url = "http://packages.python.org/" + def test_medbank_mt(self): + url = "http://business.medbank.com.mt/" s = requests.Session() s.mount("http://", self.cls()) r = s.get(url) r.raise_for_status() - self.assertEqual(r.url, "https://pythonhosted.org/") + self.assertEqual(r.url, "https://www.medirect.com.mt") self.assertEqual(len(r.history), 1) original = r.history[0] + # Here is where it differs from upstream, with an unusual + # original url which differs from the real original self.assertEqual(original.url, url.replace("http://", "https://")) - # This adapter provides the real response, with code of 301 self.assertEqual(original.status_code, 301) + self.assertEqual(original.reason, "Moved Permanently") + + +class TestUpgradeAdapter(TestForceAdapter): + + cls = UpgradeHTTPSAdapter From 6f130a0def2f9ddfcf3934822dbe610335c6b503 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Tue, 21 Apr 2020 17:00:52 +0700 Subject: [PATCH 5/7] test_adapter: Use self.cls --- tests/test_adapter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_adapter.py b/tests/test_adapter.py index 4ca867e..4581197 100644 --- a/tests/test_adapter.py +++ b/tests/test_adapter.py @@ -275,8 +275,8 @@ def test_fedmsg_com(self): with self.assertRaises(requests.exceptions.TooManyRedirects): s.get(url) - s.mount("http://", ForceHTTPSAdapter(https_exclusions=["fedmsg.com"])) - s.mount("https://", ForceHTTPSAdapter(https_exclusions=["fedmsg.com"])) + s.mount("http://", self.cls(https_exclusions=["fedmsg.com"])) + s.mount("https://", self.cls(https_exclusions=["fedmsg.com"])) r = s.get(url) r.raise_for_status() From b49eb522755637491342038ffccd3de42dac6816 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Wed, 22 Apr 2020 07:04:55 +0700 Subject: [PATCH 6/7] tests: Increase timeout for fedmsg.com --- tests/test_adapter.py | 2 +- tests/test_upstream.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_adapter.py b/tests/test_adapter.py index 4581197..41d9f75 100644 --- a/tests/test_adapter.py +++ b/tests/test_adapter.py @@ -273,7 +273,7 @@ def test_fedmsg_com(self): s = requests.Session() s.mount("http://", self.cls()) with self.assertRaises(requests.exceptions.TooManyRedirects): - s.get(url) + s.get(url, timeout=60) s.mount("http://", self.cls(https_exclusions=["fedmsg.com"])) s.mount("https://", self.cls(https_exclusions=["fedmsg.com"])) diff --git a/tests/test_upstream.py b/tests/test_upstream.py index 5748e2e..b746fc0 100644 --- a/tests/test_upstream.py +++ b/tests/test_upstream.py @@ -44,7 +44,7 @@ def test_fedmsg_com(self): url = "http://fedmsg.com/" s = requests.Session() s.mount("http://", self.cls()) - r = s.get(url) + r = s.get(url, timeout=60) r.raise_for_status() self.assertEqual(r.url, url) self.assertEqual(r.history, []) From 260e6275e2cb49f5823d2f45aacaed99a9da00d5 Mon Sep 17 00:00:00 2001 From: John Vandenberg Date: Tue, 22 Sep 2020 12:11:02 +0700 Subject: [PATCH 7/7] Add .travis.yml --- .travis.yml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..17e0dfb --- /dev/null +++ b/.travis.yml @@ -0,0 +1,8 @@ +language: python +python: + - 2.7 + - 3.4 + - 3.6 + - 3.8 +install: pip install tox-travis +script: tox