Commit | Line | Data |
---|---|---|
3cc7937c | 1 | import os, hashlib, urllib.request, time |
f3ad0817 FT |
2 | pj = os.path.join |
3 | ||
4 | class cache(object): | |
5 | def __init__(self, dir): | |
6 | self.dir = dir | |
7 | ||
8 | def mangle(self, url): | |
3cc7937c FT |
9 | n = hashlib.md5() |
10 | n.update(url.encode("ascii")) | |
f3ad0817 FT |
11 | return n.hexdigest() |
12 | ||
d6c0e189 | 13 | def miss(self, url): |
531e4473 FT |
14 | req = urllib.request.Request(url, headers={"User-Agent": "automanga/1"}) |
15 | with urllib.request.urlopen(req) as s: | |
75efe5be | 16 | if s.headers.get("content-encoding") == "gzip": |
3cc7937c FT |
17 | import gzip, io |
18 | return gzip.GzipFile(fileobj=io.BytesIO(s.read()), mode="r").read() | |
d6c0e189 | 19 | return s.read() |
d6c0e189 | 20 | |
3cc7937c | 21 | def fetch(self, url, expire=3600): |
f3ad0817 FT |
22 | path = pj(self.dir, self.mangle(url)) |
23 | if os.path.exists(path): | |
24 | if time.time() - os.stat(path).st_mtime < expire: | |
3cc7937c | 25 | with open(path, "rb") as f: |
f3ad0817 | 26 | return f.read() |
d6c0e189 | 27 | data = self.miss(url) |
f3ad0817 FT |
28 | if not os.path.isdir(self.dir): |
29 | os.makedirs(self.dir) | |
3cc7937c | 30 | with open(path, "wb") as f: |
f3ad0817 FT |
31 | f.write(data) |
32 | return data | |
33 | ||
34 | home = os.getenv("HOME") | |
35 | if home is None or not os.path.isdir(home): | |
36 | raise Exception("Could not find home directory for HTTP caching") | |
37 | default = cache(pj(home, ".manga", "htcache")) | |
38 | ||
3cc7937c | 39 | def fetch(url, expire=3600): |
f3ad0817 | 40 | return default.fetch(url, expire) |