From 0763108a68865de0ccbfc0964b08a77be2987ff3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Diego=20Elio=20Petten=C3=B2?= <flameeyes@flameeyes.com>
Date: Mon, 13 Apr 2020 23:15:38 +0100
Subject: [PATCH 01/11] Add some more debug logging for FileCache and for the
pass-through path.
This makes it easier to figure out _why_ something fails to look up altogether.
---
cachecontrol/caches/file_cache.py | 11 +++++++++--
cachecontrol/controller.py | 9 +++++++--
2 files changed, 16 insertions(+), 4 deletions(-)
diff --git a/cachecontrol/caches/file_cache.py b/cachecontrol/caches/file_cache.py
index 43393b8f..ed15f7bb 100644
--- a/cachecontrol/caches/file_cache.py
+++ b/cachecontrol/caches/file_cache.py
@@ -3,6 +3,7 @@
# SPDX-License-Identifier: Apache-2.0
import hashlib
+import logging
import os
from textwrap import dedent
@@ -16,6 +17,9 @@
FileNotFoundError = (IOError, OSError)
+logger = logging.getLogger(__name__)
+
+
def _secure_open_write(filename, fmode):
# We only want to write to this file, so open it in write only mode
flags = os.O_WRONLY
@@ -111,6 +115,7 @@ def _fn(self, name):
def get(self, key):
name = self._fn(key)
+ logger.debug("Looking up '%s' in '%s'", key, name)
try:
with open(name, "rb") as fh:
return fh.read()
@@ -120,12 +125,14 @@ def get(self, key):
def set(self, key, value):
name = self._fn(key)
+ logger.debug("Caching '%s' in '%s'", key, name)
# Make sure the directory exists
+ parentdir = os.path.dirname(name)
try:
- os.makedirs(os.path.dirname(name), self.dirmode)
+ os.makedirs(parentdir, self.dirmode)
except (IOError, OSError):
- pass
+ logging.debug("Error trying to create directory '%s'", parentdir, exc_info=True)
with self.lock_class(name) as lock:
# Write our actual file
diff --git a/cachecontrol/controller.py b/cachecontrol/controller.py
index 4d76877f..ce32e91f 100644
--- a/cachecontrol/controller.py
+++ b/cachecontrol/controller.py
@@ -284,7 +284,7 @@ def cache_response(self, request, response, body=None, status_codes=None):
cc = self.parse_cache_control(response_headers)
cache_url = self.cache_url(request.url)
- logger.debug('Updating cache with response from "%s"', cache_url)
+ logger.debug('Updating cache %r with response from "%s"', self.cache, cache_url)
# Delete it from the cache if we happen to have it stored there
no_store = False
@@ -325,7 +325,10 @@ def cache_response(self, request, response, body=None, status_codes=None):
# Add to the cache if the response headers demand it. If there
# is no date header then we can't do anything about expiring
# the cache.
- elif "date" in response_headers:
+ elif "date" not in response_headers:
+ logger.debug("No date header, expiration cannot be set.")
+ return
+ else:
# cache when there is a max-age > 0
if "max-age" in cc and cc["max-age"] > 0:
logger.debug("Caching b/c date exists and max-age > 0")
@@ -341,6 +344,8 @@ def cache_response(self, request, response, body=None, status_codes=None):
self.cache.set(
cache_url, self.serializer.dumps(request, response, body)
)
+ else:
+ logger.debug("No combination of headers to cache.")
def update_cached_response(self, request, response):
"""On a 304 we will get a new set of headers that we want to
From 5ac125137d1d26b06f8e33b83d514f1f76f2f286 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Diego=20Elio=20Petten=C3=B2?= <flameeyes@flameeyes.com>
Date: Mon, 13 Apr 2020 23:37:58 +0100
Subject: [PATCH 02/11] Make `Serializer.dumps()` require a `body` parameter.
When caching permanent redirects, if `body` is left to `None`, there's an infinite recursion that will lead to the caching to silently fail and not cache anything at all.
So instead, make `body` a required parameter, which can be empty (`''`) for cached redirects.
---
cachecontrol/controller.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/cachecontrol/controller.py b/cachecontrol/controller.py
index ce32e91f..8206b793 100644
--- a/cachecontrol/controller.py
+++ b/cachecontrol/controller.py
@@ -284,7 +284,7 @@ def cache_response(self, request, response, body=None, status_codes=None):
cc = self.parse_cache_control(response_headers)
cache_url = self.cache_url(request.url)
- logger.debug('Updating cache %r with response from "%s"', self.cache, cache_url)
+ logger.debug('Updating cache with response from "%s"', cache_url)
# Delete it from the cache if we happen to have it stored there
no_store = False
From 46b1f3c5bf9c4d2c23eb6d3ccf0784b61bc70aa4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Diego=20Elio=20Petten=C3=B2?= <flameeyes@flameeyes.com>
Date: Fri, 17 Apr 2020 14:23:48 +0100
Subject: [PATCH 03/11] Add an explicit encoding to test_chunked_response.
This is to workaround an isort bug that appears fixed in master, where the Transfer-Encoding: chunked line is interpreted as an encoding for the file.
---
tests/test_chunked_response.py | 2 ++
1 file changed, 2 insertions(+)
diff --git a/tests/test_chunked_response.py b/tests/test_chunked_response.py
index 46840870..9c505d00 100644
--- a/tests/test_chunked_response.py
+++ b/tests/test_chunked_response.py
@@ -1,3 +1,5 @@
+# encoding: utf-8
+
# SPDX-FileCopyrightText: 2015 Eric Larson
#
# SPDX-License-Identifier: Apache-2.0
From dea9cf5c8bc43968d3b46da8dc44eb6d6238742d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Diego=20Elio=20Petten=C3=B2?= <flameeyes@flameeyes.com>
Date: Fri, 17 Apr 2020 14:27:32 +0100
Subject: [PATCH 04/11] Use [isort](https://github.com/timothycrosley/isort)
throughout the source.
---
cachecontrol/__init__.py | 2 +-
cachecontrol/_cmd.py | 3 +--
cachecontrol/adapter.py | 4 ++--
cachecontrol/caches/redis_cache.py | 1 +
cachecontrol/controller.py | 3 +--
cachecontrol/heuristics.py | 4 +---
dev_requirements.txt | 17 +++++++++--------
docs/conf.py | 3 ++-
examples/benchmark.py | 9 +++++----
pyproject.toml | 3 +++
tests/conftest.py | 6 ++----
tests/test_adapter.py | 2 +-
tests/test_cache_control.py | 4 ++--
tests/test_etag.py | 4 +---
tests/test_expires_heuristics.py | 11 ++++-------
tests/test_max_age.py | 3 ++-
tests/test_regressions.py | 4 ++--
tests/test_serialization.py | 1 -
tests/test_storage_filecache.py | 6 +++---
tests/test_storage_redis.py | 1 +
tests/test_vary.py | 2 --
21 files changed, 44 insertions(+), 49 deletions(-)
create mode 100644 pyproject.toml
diff --git a/cachecontrol/__init__.py b/cachecontrol/__init__.py
index 002e3a05..23ab25ed 100644
--- a/cachecontrol/__init__.py
+++ b/cachecontrol/__init__.py
@@ -10,6 +10,6 @@
__email__ = "eric@ionrock.org"
__version__ = "0.12.6"
-from .wrapper import CacheControl
from .adapter import CacheControlAdapter
from .controller import CacheController
+from .wrapper import CacheControl
diff --git a/cachecontrol/_cmd.py b/cachecontrol/_cmd.py
index ccee0079..bf04b5db 100644
--- a/cachecontrol/_cmd.py
+++ b/cachecontrol/_cmd.py
@@ -3,6 +3,7 @@
# SPDX-License-Identifier: Apache-2.0
import logging
+from argparse import ArgumentParser
import requests
@@ -10,8 +11,6 @@
from cachecontrol.cache import DictCache
from cachecontrol.controller import logger
-from argparse import ArgumentParser
-
def setup_logging():
logger.setLevel(logging.DEBUG)
diff --git a/cachecontrol/adapter.py b/cachecontrol/adapter.py
index 22b49638..e3e4c512 100644
--- a/cachecontrol/adapter.py
+++ b/cachecontrol/adapter.py
@@ -2,14 +2,14 @@
#
# SPDX-License-Identifier: Apache-2.0
-import types
import functools
+import types
import zlib
from requests.adapters import HTTPAdapter
-from .controller import CacheController, PERMANENT_REDIRECT_STATUSES
from .cache import DictCache
+from .controller import PERMANENT_REDIRECT_STATUSES, CacheController
from .filewrapper import CallbackFileWrapper
diff --git a/cachecontrol/caches/redis_cache.py b/cachecontrol/caches/redis_cache.py
index 564c30e4..0e4b072d 100644
--- a/cachecontrol/caches/redis_cache.py
+++ b/cachecontrol/caches/redis_cache.py
@@ -5,6 +5,7 @@
from __future__ import division
from datetime import datetime
+
from cachecontrol.cache import BaseCache
diff --git a/cachecontrol/controller.py b/cachecontrol/controller.py
index 8206b793..beca7851 100644
--- a/cachecontrol/controller.py
+++ b/cachecontrol/controller.py
@@ -5,9 +5,9 @@
"""
The httplib2 algorithms ported for use with requests.
"""
+import calendar
import logging
import re
-import calendar
import time
from email.utils import parsedate_tz
@@ -16,7 +16,6 @@
from .cache import DictCache
from .serialize import Serializer
-
logger = logging.getLogger(__name__)
URI = re.compile(r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?")
diff --git a/cachecontrol/heuristics.py b/cachecontrol/heuristics.py
index ebe4a96f..3707bc68 100644
--- a/cachecontrol/heuristics.py
+++ b/cachecontrol/heuristics.py
@@ -4,10 +4,8 @@
import calendar
import time
-
-from email.utils import formatdate, parsedate, parsedate_tz
-
from datetime import datetime, timedelta
+from email.utils import formatdate, parsedate, parsedate_tz
TIME_FMT = "%a, %d %b %Y %H:%M:%S GMT"
diff --git a/dev_requirements.txt b/dev_requirements.txt
index ce7f9994..86b414a1 100644
--- a/dev_requirements.txt
+++ b/dev_requirements.txt
@@ -4,15 +4,16 @@
-e .
-tox
-pytest-cov
-pytest
-mock
+black
+bumpversion
cherrypy
-sphinx
-redis
+isort
lockfile
-bumpversion
+mock
+pytest
+pytest-cov
+redis
+sphinx
+tox
twine
-black
wheel
diff --git a/docs/conf.py b/docs/conf.py
index bd7cedb7..b4447a89 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -15,7 +15,8 @@
# All configuration values have a default; values that are commented out
# serve to show the default.
-import sys, os
+import os
+import sys
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
diff --git a/examples/benchmark.py b/examples/benchmark.py
index b036f788..2d4f59b8 100644
--- a/examples/benchmark.py
+++ b/examples/benchmark.py
@@ -2,13 +2,14 @@
#
# SPDX-License-Identifier: Apache-2.0
-import sys
-import requests
import argparse
-
-from multiprocessing import Process
+import sys
from datetime import datetime
+from multiprocessing import Process
from wsgiref.simple_server import make_server
+
+import requests
+
from cachecontrol import CacheControl
HOST = "localhost"
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 00000000..56a038e5
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,3 @@
+[tool.isort]
+line_length = 88
+known_first_party = ['cachecontrol']
diff --git a/tests/conftest.py b/tests/conftest.py
index e68b1548..9a645b00 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -2,14 +2,12 @@
#
# SPDX-License-Identifier: Apache-2.0
-from pprint import pformat
-
import os
import socket
-
-import pytest
+from pprint import pformat
import cherrypy
+import pytest
class SimpleApp(object):
diff --git a/tests/test_adapter.py b/tests/test_adapter.py
index a6820571..bd63978a 100644
--- a/tests/test_adapter.py
+++ b/tests/test_adapter.py
@@ -4,8 +4,8 @@
import mock
import pytest
-
from requests import Session
+
from cachecontrol.adapter import CacheControlAdapter
from cachecontrol.cache import DictCache
from cachecontrol.wrapper import CacheControl
diff --git a/tests/test_cache_control.py b/tests/test_cache_control.py
index 18c75624..0cce8244 100644
--- a/tests/test_cache_control.py
+++ b/tests/test_cache_control.py
@@ -5,14 +5,14 @@
"""
Unit tests that verify our caching methods work correctly.
"""
+import time
+
import pytest
from mock import ANY, Mock
-import time
from cachecontrol import CacheController
from cachecontrol.cache import DictCache
-
TIME_FMT = "%a, %d %b %Y %H:%M:%S GMT"
diff --git a/tests/test_etag.py b/tests/test_etag.py
index 700a0c54..f3e74bc2 100644
--- a/tests/test_etag.py
+++ b/tests/test_etag.py
@@ -3,10 +3,8 @@
# SPDX-License-Identifier: Apache-2.0
import pytest
-
-from mock import Mock, patch
-
import requests
+from mock import Mock, patch
from cachecontrol import CacheControl
from cachecontrol.cache import DictCache
diff --git a/tests/test_expires_heuristics.py b/tests/test_expires_heuristics.py
index 5d62f157..968be3cc 100644
--- a/tests/test_expires_heuristics.py
+++ b/tests/test_expires_heuristics.py
@@ -4,20 +4,17 @@
import calendar
import time
-
-from email.utils import formatdate, parsedate
from datetime import datetime
+from email.utils import formatdate, parsedate
+from pprint import pprint
from mock import Mock
from requests import Session, get
from requests.structures import CaseInsensitiveDict
from cachecontrol import CacheControl
-from cachecontrol.heuristics import LastModified, ExpiresAfter, OneDayCache
-from cachecontrol.heuristics import TIME_FMT
-from cachecontrol.heuristics import BaseHeuristic
-
-from pprint import pprint
+from cachecontrol.heuristics import (TIME_FMT, BaseHeuristic, ExpiresAfter,
+ LastModified, OneDayCache)
class TestHeuristicWithoutWarning(object):
diff --git a/tests/test_max_age.py b/tests/test_max_age.py
index 739f27e1..bdf4e867 100644
--- a/tests/test_max_age.py
+++ b/tests/test_max_age.py
@@ -3,9 +3,10 @@
# SPDX-License-Identifier: Apache-2.0
from __future__ import print_function
-import pytest
+import pytest
from requests import Session
+
from cachecontrol.adapter import CacheControlAdapter
from cachecontrol.cache import DictCache
diff --git a/tests/test_regressions.py b/tests/test_regressions.py
index 0806035a..9cc531a4 100644
--- a/tests/test_regressions.py
+++ b/tests/test_regressions.py
@@ -3,13 +3,13 @@
# SPDX-License-Identifier: Apache-2.0
import sys
-import pytest
+import pytest
+from requests import Session
from cachecontrol import CacheControl
from cachecontrol.caches import FileCache
from cachecontrol.filewrapper import CallbackFileWrapper
-from requests import Session
class Test39(object):
diff --git a/tests/test_serialization.py b/tests/test_serialization.py
index 59771c5a..5750b6b7 100644
--- a/tests/test_serialization.py
+++ b/tests/test_serialization.py
@@ -4,7 +4,6 @@
import msgpack
import requests
-
from mock import Mock
from cachecontrol.compat import pickle
diff --git a/tests/test_storage_filecache.py b/tests/test_storage_filecache.py
index 4ac8a4f0..67b94c06 100644
--- a/tests/test_storage_filecache.py
+++ b/tests/test_storage_filecache.py
@@ -7,16 +7,16 @@
"""
import os
import string
-
from random import randint, sample
import pytest
import requests
-from cachecontrol import CacheControl
-from cachecontrol.caches import FileCache
from lockfile import LockFile
from lockfile.mkdirlockfile import MkdirLockFile
+from cachecontrol import CacheControl
+from cachecontrol.caches import FileCache
+
def randomdata():
"""Plain random http data generator:"""
diff --git a/tests/test_storage_redis.py b/tests/test_storage_redis.py
index 4646be50..e2b6cf62 100644
--- a/tests/test_storage_redis.py
+++ b/tests/test_storage_redis.py
@@ -5,6 +5,7 @@
from datetime import datetime
from mock import Mock
+
from cachecontrol.caches import RedisCache
diff --git a/tests/test_vary.py b/tests/test_vary.py
index 543294b3..cb4b582c 100644
--- a/tests/test_vary.py
+++ b/tests/test_vary.py
@@ -9,8 +9,6 @@
from cachecontrol.cache import DictCache
from cachecontrol.compat import urljoin
-from pprint import pprint
-
class TestVary(object):
From 545c5e31cb20c0837b30156a3d192b42537d8cee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Diego=20Elio=20Petten=C3=B2?= <flameeyes@flameeyes.com>
Date: Fri, 17 Apr 2020 14:29:14 +0100
Subject: [PATCH 05/11] Refresh [black](https://github.com/psf/black)
formatting.
---
cachecontrol/cache.py | 2 --
cachecontrol/caches/file_cache.py | 5 +++--
cachecontrol/caches/redis_cache.py | 1 -
cachecontrol/controller.py | 8 +++-----
cachecontrol/heuristics.py | 14 ++++++++++++--
cachecontrol/serialize.py | 1 -
examples/benchmark.py | 4 ++--
pyproject.toml | 3 +++
tests/conftest.py | 1 -
tests/test_adapter.py | 1 -
tests/test_cache_control.py | 5 +++--
tests/test_chunked_response.py | 1 -
tests/test_etag.py | 1 -
tests/test_expires_heuristics.py | 20 +++++++-------------
tests/test_max_age.py | 2 --
tests/test_redirects.py | 2 --
tests/test_regressions.py | 1 -
tests/test_serialization.py | 5 +++--
tests/test_storage_filecache.py | 1 -
tests/test_storage_redis.py | 1 -
tests/test_vary.py | 1 -
21 files changed, 36 insertions(+), 44 deletions(-)
diff --git a/cachecontrol/cache.py b/cachecontrol/cache.py
index 8037e528..55786457 100644
--- a/cachecontrol/cache.py
+++ b/cachecontrol/cache.py
@@ -10,7 +10,6 @@
class BaseCache(object):
-
def get(self, key):
raise NotImplementedError()
@@ -25,7 +24,6 @@ def close(self):
class DictCache(BaseCache):
-
def __init__(self, init_dict=None):
self.lock = Lock()
self.data = init_dict or {}
diff --git a/cachecontrol/caches/file_cache.py b/cachecontrol/caches/file_cache.py
index ed15f7bb..de4e79bd 100644
--- a/cachecontrol/caches/file_cache.py
+++ b/cachecontrol/caches/file_cache.py
@@ -62,7 +62,6 @@ def _secure_open_write(filename, fmode):
class FileCache(BaseCache):
-
def __init__(
self,
directory,
@@ -132,7 +131,9 @@ def set(self, key, value):
try:
os.makedirs(parentdir, self.dirmode)
except (IOError, OSError):
- logging.debug("Error trying to create directory '%s'", parentdir, exc_info=True)
+ logging.debug(
+ "Error trying to create directory '%s'", parentdir, exc_info=True
+ )
with self.lock_class(name) as lock:
# Write our actual file
diff --git a/cachecontrol/caches/redis_cache.py b/cachecontrol/caches/redis_cache.py
index 0e4b072d..f0b146e0 100644
--- a/cachecontrol/caches/redis_cache.py
+++ b/cachecontrol/caches/redis_cache.py
@@ -10,7 +10,6 @@
class RedisCache(BaseCache):
-
def __init__(self, conn):
self.conn = conn
diff --git a/cachecontrol/controller.py b/cachecontrol/controller.py
index beca7851..8a2fee50 100644
--- a/cachecontrol/controller.py
+++ b/cachecontrol/controller.py
@@ -163,7 +163,7 @@ def cached_request(self, request):
# with cache busting headers as usual (ie no-cache).
if int(resp.status) in PERMANENT_REDIRECT_STATUSES:
msg = (
- 'Returning cached permanent redirect response '
+ "Returning cached permanent redirect response "
"(ignoring date and etag information)"
)
logger.debug(msg)
@@ -311,15 +311,13 @@ def cache_response(self, request, response, body=None, status_codes=None):
# If we've been given an etag, then keep the response
if self.cache_etags and "etag" in response_headers:
logger.debug("Caching due to etag")
- self.cache.set(
- cache_url, self.serializer.dumps(request, response, body)
- )
+ self.cache.set(cache_url, self.serializer.dumps(request, response, body))
# Add to the cache any permanent redirects. We do this before looking
# that the Date headers.
elif int(response.status) in PERMANENT_REDIRECT_STATUSES:
logger.debug("Caching permanent redirect")
- self.cache.set(cache_url, self.serializer.dumps(request, response, b''))
+ self.cache.set(cache_url, self.serializer.dumps(request, response, b""))
# Add to the cache if the response headers demand it. If there
# is no date header then we can't do anything about expiring
diff --git a/cachecontrol/heuristics.py b/cachecontrol/heuristics.py
index 3707bc68..27ef7dae 100644
--- a/cachecontrol/heuristics.py
+++ b/cachecontrol/heuristics.py
@@ -20,7 +20,6 @@ def datetime_to_header(dt):
class BaseHeuristic(object):
-
def warning(self, response):
"""
Return a valid 1xx warning header value describing the cache
@@ -99,8 +98,19 @@ class LastModified(BaseHeuristic):
http://lxr.mozilla.org/mozilla-release/source/netwerk/protocol/http/nsHttpResponseHead.cpp#397
Unlike mozilla we limit this to 24-hr.
"""
+
cacheable_by_default_statuses = {
- 200, 203, 204, 206, 300, 301, 404, 405, 410, 414, 501
+ 200,
+ 203,
+ 204,
+ 206,
+ 300,
+ 301,
+ 404,
+ 405,
+ 410,
+ 414,
+ 501,
}
def update_headers(self, resp):
diff --git a/cachecontrol/serialize.py b/cachecontrol/serialize.py
index 4e49a90e..5beb8ec6 100644
--- a/cachecontrol/serialize.py
+++ b/cachecontrol/serialize.py
@@ -25,7 +25,6 @@ def _b64_decode_str(s):
class Serializer(object):
-
def dumps(self, request, response, body):
response_headers = CaseInsensitiveDict(response.headers)
diff --git a/examples/benchmark.py b/examples/benchmark.py
index 2d4f59b8..2eac44b7 100644
--- a/examples/benchmark.py
+++ b/examples/benchmark.py
@@ -18,12 +18,12 @@
class Server(object):
-
def __call__(self, env, sr):
body = "Hello World!"
status = "200 OK"
headers = [
- ("Cache-Control", "max-age=%i" % (60 * 10)), ("Content-Type", "text/plain")
+ ("Cache-Control", "max-age=%i" % (60 * 10)),
+ ("Content-Type", "text/plain"),
]
sr(status, headers)
return body
diff --git a/pyproject.toml b/pyproject.toml
index 56a038e5..f35b9a04 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,3 +1,6 @@
[tool.isort]
line_length = 88
known_first_party = ['cachecontrol']
+# Set multi-line output to "Vertical Hanging indent" to avoid fighting with black.
+multi_line_output = 3
+include_trailing_comma = true
diff --git a/tests/conftest.py b/tests/conftest.py
index 9a645b00..2681ad43 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -11,7 +11,6 @@
class SimpleApp(object):
-
def __init__(self):
self.etag_count = 0
self.update_etag_string()
diff --git a/tests/test_adapter.py b/tests/test_adapter.py
index bd63978a..ce85928f 100644
--- a/tests/test_adapter.py
+++ b/tests/test_adapter.py
@@ -35,7 +35,6 @@ def sess(url, request):
class TestSessionActions(object):
-
def test_get_caches(self, url, sess):
r2 = sess.get(url)
assert r2.from_cache is True
diff --git a/tests/test_cache_control.py b/tests/test_cache_control.py
index 0cce8244..0b7c0f8f 100644
--- a/tests/test_cache_control.py
+++ b/tests/test_cache_control.py
@@ -17,7 +17,6 @@
class NullSerializer(object):
-
def dumps(self, request, response):
return response
@@ -156,7 +155,9 @@ def req(self, headers):
return self.c.cached_request(mock_request)
def test_cache_request_no_headers(self):
- cached_resp = Mock(headers={"ETag": "jfd9094r808", "Content-Length": 100}, status=200)
+ cached_resp = Mock(
+ headers={"ETag": "jfd9094r808", "Content-Length": 100}, status=200
+ )
self.c.cache = DictCache({self.url: cached_resp})
resp = self.req({})
assert not resp
diff --git a/tests/test_chunked_response.py b/tests/test_chunked_response.py
index 9c505d00..36d9da36 100644
--- a/tests/test_chunked_response.py
+++ b/tests/test_chunked_response.py
@@ -24,7 +24,6 @@ def sess():
class TestChunkedResponses(object):
-
def test_cache_chunked_response(self, url, sess):
"""
Verify that an otherwise cacheable response is cached when the
diff --git a/tests/test_etag.py b/tests/test_etag.py
index f3e74bc2..97b8d94e 100644
--- a/tests/test_etag.py
+++ b/tests/test_etag.py
@@ -12,7 +12,6 @@
class NullSerializer(object):
-
def dumps(self, request, response, body=None):
return response
diff --git a/tests/test_expires_heuristics.py b/tests/test_expires_heuristics.py
index 968be3cc..913704e7 100644
--- a/tests/test_expires_heuristics.py
+++ b/tests/test_expires_heuristics.py
@@ -13,14 +13,17 @@
from requests.structures import CaseInsensitiveDict
from cachecontrol import CacheControl
-from cachecontrol.heuristics import (TIME_FMT, BaseHeuristic, ExpiresAfter,
- LastModified, OneDayCache)
+from cachecontrol.heuristics import (
+ TIME_FMT,
+ BaseHeuristic,
+ ExpiresAfter,
+ LastModified,
+ OneDayCache,
+)
class TestHeuristicWithoutWarning(object):
-
def setup(self):
-
class NoopHeuristic(BaseHeuristic):
warning = Mock()
@@ -38,11 +41,8 @@ def test_no_header_change_means_no_warning_header(self, url):
class TestHeuristicWith3xxResponse(object):
-
def setup(self):
-
class DummyHeuristic(BaseHeuristic):
-
def update_headers(self, resp):
return {"x-dummy-header": "foobar"}
@@ -60,7 +60,6 @@ def test_heuristic_applies_to_304(self, url):
class TestUseExpiresHeuristic(object):
-
def test_expires_heuristic_arg(self):
sess = Session()
cached_sess = CacheControl(sess, heuristic=Mock())
@@ -68,7 +67,6 @@ def test_expires_heuristic_arg(self):
class TestOneDayCache(object):
-
def setup(self):
self.sess = Session()
self.cached_sess = CacheControl(self.sess, heuristic=OneDayCache())
@@ -88,7 +86,6 @@ def test_cache_for_one_day(self, url):
class TestExpiresAfter(object):
-
def setup(self):
self.sess = Session()
self.cache_sess = CacheControl(self.sess, heuristic=ExpiresAfter(days=1))
@@ -109,7 +106,6 @@ def test_expires_after_one_day(self, url):
class TestLastModified(object):
-
def setup(self):
self.sess = Session()
self.cached_sess = CacheControl(self.sess, heuristic=LastModified())
@@ -129,7 +125,6 @@ def test_last_modified(self, url):
class DummyResponse:
-
def __init__(self, status, headers):
self.status = status
self.headers = CaseInsensitiveDict(headers)
@@ -140,7 +135,6 @@ def datetime_to_header(dt):
class TestModifiedUnitTests(object):
-
def last_modified(self, period):
return time.strftime(TIME_FMT, time.gmtime(self.time_now - period))
diff --git a/tests/test_max_age.py b/tests/test_max_age.py
index bdf4e867..a04776cd 100644
--- a/tests/test_max_age.py
+++ b/tests/test_max_age.py
@@ -12,7 +12,6 @@
class NullSerializer(object):
-
def dumps(self, request, response, body=None):
return response
@@ -23,7 +22,6 @@ def loads(self, request, data):
class TestMaxAge(object):
-
@pytest.fixture()
def sess(self, url):
self.url = url
diff --git a/tests/test_redirects.py b/tests/test_redirects.py
index 56571f66..40db5f6e 100644
--- a/tests/test_redirects.py
+++ b/tests/test_redirects.py
@@ -11,7 +11,6 @@
class TestPermanentRedirects(object):
-
def setup(self):
self.sess = CacheControl(requests.Session())
@@ -33,7 +32,6 @@ def test_bust_cache_on_redirect(self, url):
class TestMultipleChoicesRedirects(object):
-
def setup(self):
self.sess = CacheControl(requests.Session())
diff --git a/tests/test_regressions.py b/tests/test_regressions.py
index 9cc531a4..eccd2797 100644
--- a/tests/test_regressions.py
+++ b/tests/test_regressions.py
@@ -13,7 +13,6 @@
class Test39(object):
-
@pytest.mark.skipif(
sys.version.startswith("2"), reason="Only run this for python 3.x"
)
diff --git a/tests/test_serialization.py b/tests/test_serialization.py
index 5750b6b7..465f0042 100644
--- a/tests/test_serialization.py
+++ b/tests/test_serialization.py
@@ -11,7 +11,6 @@
class TestSerializer(object):
-
def setup(self):
self.serializer = Serializer()
self.response_data = {
@@ -92,7 +91,9 @@ def test_read_latest_version_streamable(self, url):
original_resp = requests.get(url, stream=True)
req = original_resp.request
- resp = self.serializer.loads(req, self.serializer.dumps(req, original_resp.raw, original_resp.content))
+ resp = self.serializer.loads(
+ req, self.serializer.dumps(req, original_resp.raw, original_resp.content)
+ )
assert resp.read()
diff --git a/tests/test_storage_filecache.py b/tests/test_storage_filecache.py
index 67b94c06..38f178b0 100644
--- a/tests/test_storage_filecache.py
+++ b/tests/test_storage_filecache.py
@@ -26,7 +26,6 @@ def randomdata():
class TestStorageFileCache(object):
-
@pytest.fixture()
def sess(self, url, tmpdir):
self.url = url
diff --git a/tests/test_storage_redis.py b/tests/test_storage_redis.py
index e2b6cf62..3edfb8ac 100644
--- a/tests/test_storage_redis.py
+++ b/tests/test_storage_redis.py
@@ -10,7 +10,6 @@
class TestRedisCache(object):
-
def setup(self):
self.conn = Mock()
self.cache = RedisCache(self.conn)
diff --git a/tests/test_vary.py b/tests/test_vary.py
index cb4b582c..4b0e2d16 100644
--- a/tests/test_vary.py
+++ b/tests/test_vary.py
@@ -11,7 +11,6 @@
class TestVary(object):
-
@pytest.fixture()
def sess(self, url):
self.url = urljoin(url, "/vary_accept")
From 190d7de993131ff6dd718d66d2630eb6578f88b7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Diego=20Elio=20Petten=C3=B2?= <flameeyes@flameeyes.com>
Date: Fri, 17 Apr 2020 14:38:14 +0100
Subject: [PATCH 06/11] Set up [pre-commit](https://pre-commit.com/).
This includes isort, black and some basic hygiene on text files.
---
.bumpversion.cfg | 1 -
.gitignore | 2 +-
.pre-commit-config.yaml | 17 +++++++++++++++++
dev_requirements.txt | 1 +
pyproject.toml | 1 +
5 files changed, 20 insertions(+), 2 deletions(-)
create mode 100644 .pre-commit-config.yaml
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index 4c2faedb..aa028df5 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -7,4 +7,3 @@ current_version = 0.12.6
files = setup.py cachecontrol/__init__.py docs/conf.py
commit = True
tag = True
-
diff --git a/.gitignore b/.gitignore
index e2de968b..cb7b8bc0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,4 +14,4 @@ include
.Python
docs/_build
build/
-.tox
\ No newline at end of file
+.tox
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
new file mode 100644
index 00000000..acc64e9d
--- /dev/null
+++ b/.pre-commit-config.yaml
@@ -0,0 +1,17 @@
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+ rev: v2.3.0
+ hooks:
+ - id: check-yaml
+ - id: end-of-file-fixer
+ - id: trailing-whitespace
+- repo: https://github.com/timothycrosley/isort
+ rev: 4.3.21
+ hooks:
+ - id: isort
+ additional_dependencies:
+ - toml
+- repo: https://github.com/python/black
+ rev: 19.10b0
+ hooks:
+ - id: black
diff --git a/dev_requirements.txt b/dev_requirements.txt
index 86b414a1..e1896819 100644
--- a/dev_requirements.txt
+++ b/dev_requirements.txt
@@ -10,6 +10,7 @@ cherrypy
isort
lockfile
mock
+pre-commit
pytest
pytest-cov
redis
diff --git a/pyproject.toml b/pyproject.toml
index f35b9a04..12ffa088 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,7 @@
[tool.isort]
line_length = 88
known_first_party = ['cachecontrol']
+known_third_party = ['mock', 'lockfile', 'requests', 'pytest', 'msgpack', 'cherrypy']
# Set multi-line output to "Vertical Hanging indent" to avoid fighting with black.
multi_line_output = 3
include_trailing_comma = true
From 03d034fc2edddcdeb46260c7e5a44ffc5c053f10 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Diego=20Elio=20Petten=C3=B2?= <flameeyes@flameeyes.com>
Date: Fri, 17 Apr 2020 14:51:12 +0100
Subject: [PATCH 07/11] Add *~ to gitignore.
This is a fairly common unix extension for backup files used at least by Emacsen and vim.
---
.gitignore | 17 +++++++++--------
1 file changed, 9 insertions(+), 8 deletions(-)
diff --git a/.gitignore b/.gitignore
index cb7b8bc0..92826fa8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,16 +2,17 @@
#
# SPDX-License-Identifier: Apache-2.0
-.DS_Store
+*.egg-info/*
*.pyc
*.pyo
-*.egg-info/*
-dist
+*~
+.DS_Store
+.Python
+.tox
bin
+build/
+dist
+docs/_build
+include
lib
lib64
-include
-.Python
-docs/_build
-build/
-.tox
From 0fe4cd0b8b3107777bbb783642a6f70daa1848ed Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Diego=20Elio=20Petten=C3=B2?= <flameeyes@flameeyes.com>
Date: Fri, 17 Apr 2020 19:16:29 +0100
Subject: [PATCH 08/11] Use six instead of manually maintaining compatibility
with PY2.
This makes it easier to use constants for status codes as well.
---
cachecontrol/compat.py | 17 -----------------
cachecontrol/serialize.py | 5 ++++-
setup.py | 2 +-
tests/test_etag.py | 2 +-
tests/test_serialization.py | 2 +-
tests/test_vary.py | 2 +-
6 files changed, 8 insertions(+), 22 deletions(-)
diff --git a/cachecontrol/compat.py b/cachecontrol/compat.py
index 72c456cf..d602c4aa 100644
--- a/cachecontrol/compat.py
+++ b/cachecontrol/compat.py
@@ -2,17 +2,6 @@
#
# SPDX-License-Identifier: Apache-2.0
-try:
- from urllib.parse import urljoin
-except ImportError:
- from urlparse import urljoin
-
-
-try:
- import cPickle as pickle
-except ImportError:
- import pickle
-
# Handle the case where the requests module has been patched to not have
# urllib3 bundled as part of its source.
try:
@@ -24,9 +13,3 @@
from requests.packages.urllib3.util import is_fp_closed
except ImportError:
from urllib3.util import is_fp_closed
-
-# Replicate some six behaviour
-try:
- text_type = unicode
-except NameError:
- text_type = str
diff --git a/cachecontrol/serialize.py b/cachecontrol/serialize.py
index 5beb8ec6..0d40ca5a 100644
--- a/cachecontrol/serialize.py
+++ b/cachecontrol/serialize.py
@@ -10,7 +10,10 @@
import msgpack
from requests.structures import CaseInsensitiveDict
-from .compat import HTTPResponse, pickle, text_type
+from six import text_type
+from six.moves import cPickle as pickle
+
+from .compat import HTTPResponse
def _b64_decode_bytes(b):
diff --git a/setup.py b/setup.py
index 7cdae8c7..b571cc3f 100644
--- a/setup.py
+++ b/setup.py
@@ -21,7 +21,7 @@
include_package_data=True,
description="httplib2 caching for requests",
long_description=long_description,
- install_requires=["requests", "msgpack>=0.5.2"],
+ install_requires=["requests", "msgpack>=0.5.2", "six"],
extras_require={"filecache": ["lockfile>=0.9"], "redis": ["redis>=2.10.5"]},
entry_points={"console_scripts": ["doesitcache = cachecontrol._cmd:main"]},
python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*",
diff --git a/tests/test_etag.py b/tests/test_etag.py
index 97b8d94e..2b627763 100644
--- a/tests/test_etag.py
+++ b/tests/test_etag.py
@@ -8,7 +8,7 @@
from cachecontrol import CacheControl
from cachecontrol.cache import DictCache
-from cachecontrol.compat import urljoin
+from six.moves.urllib.parse import urljoin
class NullSerializer(object):
diff --git a/tests/test_serialization.py b/tests/test_serialization.py
index 465f0042..598ae289 100644
--- a/tests/test_serialization.py
+++ b/tests/test_serialization.py
@@ -6,8 +6,8 @@
import requests
from mock import Mock
-from cachecontrol.compat import pickle
from cachecontrol.serialize import Serializer
+from six.moves import cPickle as pickle
class TestSerializer(object):
diff --git a/tests/test_vary.py b/tests/test_vary.py
index 4b0e2d16..a1785895 100644
--- a/tests/test_vary.py
+++ b/tests/test_vary.py
@@ -7,7 +7,7 @@
from cachecontrol import CacheControl
from cachecontrol.cache import DictCache
-from cachecontrol.compat import urljoin
+from six.moves.urllib.parse import urljoin
class TestVary(object):
From f3d00981598cc4a379774eacb3104fded9891112 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Diego=20Elio=20Petten=C3=B2?= <flameeyes@flameeyes.com>
Date: Sun, 19 Apr 2020 15:18:09 +0100
Subject: [PATCH 09/11] Make the project compliant with the REUSE guidelines.
See https://reuse.software/ for details.
---
.pre-commit-config.yaml | 4 ++++
pyproject.toml | 4 ++++
tests/test_chunked_response.py | 4 ++++
tests/test_vary.py | 2 ++
4 files changed, 14 insertions(+)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index acc64e9d..c69b49d4 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2015 Eric Larson
+#
+# SPDX-License-Identifier: Apache-2.0
+
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v2.3.0
diff --git a/pyproject.toml b/pyproject.toml
index 12ffa088..1d48a866 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2015 Eric Larson
+#
+# SPDX-License-Identifier: Apache-2.0
+
[tool.isort]
line_length = 88
known_first_party = ['cachecontrol']
diff --git a/tests/test_chunked_response.py b/tests/test_chunked_response.py
index 36d9da36..bc390fbb 100644
--- a/tests/test_chunked_response.py
+++ b/tests/test_chunked_response.py
@@ -1,3 +1,7 @@
+# SPDX-FileCopyrightText: 2015 Eric Larson
+#
+# SPDX-License-Identifier: Apache-2.0
+
# encoding: utf-8
# SPDX-FileCopyrightText: 2015 Eric Larson
diff --git a/tests/test_vary.py b/tests/test_vary.py
index a1785895..a9d6fc96 100644
--- a/tests/test_vary.py
+++ b/tests/test_vary.py
@@ -2,6 +2,8 @@
#
# SPDX-License-Identifier: Apache-2.0
+from pprint import pprint
+
import pytest
import requests
From f75db54c91d9f9e51c9bd9e039b94c248c03ed9e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Diego=20Elio=20Petten=C3=B2?= <flameeyes@flameeyes.com>
Date: Mon, 20 Apr 2020 15:59:43 +0100
Subject: [PATCH 10/11] Make the BaseCache class an abstract class.
This is just a matter of cleanup, I can't think of any good reason for this
_not_ to be marked abstract.
---
cachecontrol/cache.py | 14 +++++++++++---
1 file changed, 11 insertions(+), 3 deletions(-)
diff --git a/cachecontrol/cache.py b/cachecontrol/cache.py
index 55786457..9500f5fb 100644
--- a/cachecontrol/cache.py
+++ b/cachecontrol/cache.py
@@ -6,18 +6,26 @@
The cache object API for implementing caches. The default is a thread
safe in-memory dictionary.
"""
+
+from abc import ABCMeta, abstractmethod
from threading import Lock
+from six import add_metaclass
+
+@add_metaclass(ABCMeta)
class BaseCache(object):
+ @abstractmethod
def get(self, key):
- raise NotImplementedError()
+ pass
+ @abstractmethod
def set(self, key, value):
- raise NotImplementedError()
+ pass
+ @abstractmethod
def delete(self, key):
- raise NotImplementedError()
+ pass
def close(self):
pass
From 15b70418de50048d8d6235da78e401bcecf03d5a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Diego=20Elio=20Petten=C3=B2?= <flameeyes@flameeyes.com>
Date: Mon, 20 Apr 2020 16:13:23 +0100
Subject: [PATCH 11/11] Suppress errors for `os.makedirs()`, again.
This is a bit more nuanced in Python 3, where only EEXIST errors are
suppressed, to match the `delete` codepath.
---
cachecontrol/caches/file_cache.py | 8 +++-----
1 file changed, 3 insertions(+), 5 deletions(-)
diff --git a/cachecontrol/caches/file_cache.py b/cachecontrol/caches/file_cache.py
index de4e79bd..523b07a5 100644
--- a/cachecontrol/caches/file_cache.py
+++ b/cachecontrol/caches/file_cache.py
@@ -15,7 +15,7 @@
except NameError:
# py2.X
FileNotFoundError = (IOError, OSError)
-
+ FileExistsError = (IOError, OSError)
logger = logging.getLogger(__name__)
@@ -130,10 +130,8 @@ def set(self, key, value):
parentdir = os.path.dirname(name)
try:
os.makedirs(parentdir, self.dirmode)
- except (IOError, OSError):
- logging.debug(
- "Error trying to create directory '%s'", parentdir, exc_info=True
- )
+ except FileExistsError:
+ pass
with self.lock_class(name) as lock:
# Write our actual file