Fixed up cursor iteration more properly(?)
[automanga.git] / manga / htcache.py
CommitLineData
3cc7937c 1import os, hashlib, urllib.request, time
40277671 2from . import profile
f3ad0817
FT
3pj = os.path.join
4
90b3abc1
FT
5class notfound(Exception):
6 pass
7
f3ad0817
FT
8class cache(object):
9 def __init__(self, dir):
10 self.dir = dir
11
12 def mangle(self, url):
3cc7937c
FT
13 n = hashlib.md5()
14 n.update(url.encode("ascii"))
f3ad0817
FT
15 return n.hexdigest()
16
90b3abc1 17 def open(self, url):
531e4473 18 req = urllib.request.Request(url, headers={"User-Agent": "automanga/1"})
90b3abc1
FT
19 return urllib.request.urlopen(req)
20
21 def miss(self, url):
22 try:
23 s = self.open(url)
24 except urllib.error.HTTPError as exc:
25 if exc.code == 404:
26 raise notfound(url)
27 raise
28 with s:
75efe5be 29 if s.headers.get("content-encoding") == "gzip":
3cc7937c
FT
30 import gzip, io
31 return gzip.GzipFile(fileobj=io.BytesIO(s.read()), mode="r").read()
d6c0e189 32 return s.read()
d6c0e189 33
3cc7937c 34 def fetch(self, url, expire=3600):
f3ad0817
FT
35 path = pj(self.dir, self.mangle(url))
36 if os.path.exists(path):
37 if time.time() - os.stat(path).st_mtime < expire:
3cc7937c 38 with open(path, "rb") as f:
f3ad0817 39 return f.read()
d6c0e189 40 data = self.miss(url)
f3ad0817
FT
41 if not os.path.isdir(self.dir):
42 os.makedirs(self.dir)
3cc7937c 43 with open(path, "wb") as f:
f3ad0817
FT
44 f.write(data)
45 return data
46
40277671 47default = cache(pj(profile.confdir, "htcache"))
f3ad0817 48
3cc7937c 49def fetch(url, expire=3600):
f3ad0817 50 return default.fetch(url, expire)