incremental cache improvements

oduwsdl · May 24, 2019 · 74159bdc50fb82ca7a0085c06e5d427925edd175 · 74159bd
1 parent d8925e6
commit 74159bdc50fb82ca7a0085c06e5d427925edd175
Unified Split

Showing with 10 additions and 14 deletions.

+8 −13 mementoembed/memstock/uricache.py

+2 −1 setup.py
diff --git a/mementoembed/memstock/uricache.py b/mementoembed/memstock/uricache.py
@@ -35,7 +35,7 @@ def __init__(self, credentials, session, expiration_delta):
        module_logger.debug("setting up Redis cache at with credentials {}".format(credentials))
-  
-        self.conn = redis.Redis(
+  
+        self.conn = redis.StrictRedis(
            host=credentials['host'],
            port=credentials['port'],
            password=credentials['password'],
@@ -44,8 +44,6 @@ def __init__(self, credentials, session, expiration_delta):
        module_logger.debug("Redis cache set up with object {}".format(self.conn))
-  
-        self.conn.set("started", "yes")
-  
-
        self.session = session
        self.expiration_delta = expiration_delta
@@ -55,28 +53,25 @@ def purgeuri(self, uri):
    def saveuri(self, uri):
-  
-        self.conn.set("savingurl {}".format(uri), "yes")
-  
-
        module_logger.debug("saving URI to cache: {}".format(uri))
-  
-        # TODO: purge URI if expired
-  
-
        r = self.session.get(uri)
        observation_datetime = datetime.datetime.utcnow()
-  
-        ret = self.conn.hset(uri, "obdt", observation_datetime)
-  
-        module_logger.debug("hset {} obdt returned {}".format(uri, ret))
-  
-
        self.conn.hset(uri, "request_headers", json.dumps(dict(r.request.headers)))
        self.conn.hset(uri, "request_method", r.request.method)
        self.conn.hset(uri, "response_status", r.status_code)
        self.conn.hset(uri, "response_reason", r.reason)
        self.conn.hset(uri, "response_elapsed", r.elapsed.microseconds)
        self.conn.hset(uri, "response_headers", json.dumps(dict(r.headers)))
-  
-        self.conn.hset(uri, "response_encoding", r.encoding)
+  
+
+  
+        # sometimes there is no encoding
+  
+        if r.encoding is not None:
+  
+            self.conn.hset(uri, "response_encoding", r.encoding)
+  
+
        self.conn.hset(uri, "response_content", r.content)
-  
-        self.conn.hset(uri, "observation_datetime", observation_datetime)
+  
+        self.conn.hset(uri, "observation_datetime", observation_datetime.strftime("%Y-%m-%dT%H:%M:%S"))
        module_logger.debug("URI {} should now be written to the cache {}".format(
            uri, self.conn))
@@ -85,7 +80,7 @@ def saveuri(self, uri):
    def get(self, uri, headers={}, timeout=None):
-  
-        if self.conn.hget(uri, "status") is None:
+  
+        if self.conn.hget(uri, "response_status") is None:
            self.saveuri(uri)
        req_headers = CaseInsensitiveDict(json.loads(self.conn.hget(uri, "request_headers")))

diff --git a/setup.py b/setup.py
@@ -25,7 +25,8 @@
        'Pillow==5.2.0',
        'python-magic==0.4.15',
        'readability-lxml==0.7',
-  
-        'redis==2.10.6',
+  
+        'redis==3.0.1',
+  
+        'redis_namespace==3.0.1.1',
        'requests>=2.20.0',
        'requests_cache==0.4.13',
        'sphinx==1.8.4',