[vcfs.git] / filestore.c

#define _LARGEFILE64_SOURCE
#define _XOPEN_SOURCE 500
#include <stdlib.h>
#include <unistd.h>
#include <stdio.h>
#include <fcntl.h>
#include <errno.h>
#include <string.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <gcrypt.h>
#include <assert.h>

#include "utils.h"
#include "store.h"
#include "log.h"

#define LOGMAGIC "Dolda/Venti-1"
#define IDXMAGIC "Dolda/Index-1"
#define LOGENTMAGIC "\xca\xe5\x7a\x93"

typedef loff_t idx_t;

struct loghdr {
    char magic[sizeof(LOGMAGIC)];
};

struct idxhdr {
    char magic[sizeof(IDXMAGIC)];
    u_int64_t size;
};

struct idxent {
    struct addr addr;
    u_int64_t l, r;
    u_int64_t off;
};

struct logent {
    u_int8_t magic[4];
    struct addr name;
    u_int16_t len;
    u_int8_t fl;
};

struct fstore {
    int logfd;
    int idxfd;
    loff_t logsize;
    idx_t idxsize;
};

static int release(struct fstore *fst)
{
    if(fst->logfd >= 0) {
	fsync(fst->logfd);
	close(fst->logfd);
    }
    if(fst->idxfd >= 0) {
	fsync(fst->idxfd);
	close(fst->idxfd);
    }
    free(fst);
    return(0);
}

static int releaseg(struct store *st)
{
    return(release(st->pdata));
}

static void hash(const void *buf, size_t len, struct addr *a)
{
    gcry_md_hash_buffer(GCRY_MD_SHA256, a->hash, buf, len);
}

static int getidx(struct fstore *fst, idx_t i, struct idxent *ie)
{
    return(readall(fst->idxfd, ie, sizeof(*ie), sizeof(struct idxhdr) + i * sizeof(struct idxent)));
}

static int putidx(struct fstore *fst, idx_t i, struct idxent *ie)
{
    return(writeall(fst->idxfd, ie, sizeof(*ie), sizeof(struct idxhdr) + i * sizeof(struct idxent)));
}

static idx_t lookup(struct fstore *fst, struct addr *a, idx_t *parent)
{
    idx_t i;
    struct idxent ie;
    int c;
    
    if(fst->idxsize == 0) {
	if(parent != NULL)
	    *parent = -1;
	return(-1);
    }
    i = 0;
    while(1) {
	assert(!getidx(fst, i, &ie));
	c = addrcmp(a, &ie.addr);
	if(c < 0) {
	    if(ie.l == 0) {
		if(parent != NULL)
		    *parent = i;
		return(-1);
	    }
	    i = ie.l;
	} else if(c > 0) {
	    if(ie.r == 0) {
		if(parent != NULL)
		    *parent = i;
		return(-1);
	    }
	    i = ie.r;
	} else {
	    return(i);
	}
    }
}

static idx_t newindex(struct fstore *fst)
{
    size_t newsize;
    idx_t ni;
    struct idxent ne;
    struct idxhdr ih;
    
    /* XXX: Thread safety */
    ni = fst->idxsize++;
    newsize = sizeof(struct idxhdr) + fst->idxsize * sizeof(struct idxent);
    if(ftruncate(fst->idxfd, newsize))
	return(-1);
    ne.l = ne.r = 0;
    assert(!putidx(fst, ni, &ne));
    assert(!readall(fst->idxfd, &ih, sizeof(ih), 0));
    ih.size = fst->idxsize;
    assert(!writeall(fst->idxfd, &ih, sizeof(ih), 0));
    return(ni);
}

static int put(struct store *st, const void *buf, size_t len, struct addr *at)
{
    struct fstore *fst;
    struct addr pa;
    idx_t i, pi;
    struct idxent ie;
    loff_t leoff;
    int c;
    struct logent le;
    
    if(len > STORE_MAXBLSZ) {
	errno = E2BIG;
	return(-1);
    }

    fst = st->pdata;
    hash(buf, len, &pa);
    if(at != NULL)
	memcpy(at->hash, pa.hash, 32);
    
    if(lookup(fst, &pa, &pi) != -1)
	return(0);
    
    memcpy(le.magic, LOGENTMAGIC, 4);
    le.name = pa;
    le.len = len;
    le.fl = 0;
    /* XXX: Thread safety { */
    leoff = fst->logsize;
    fst->logsize += sizeof(le) + len;
    /* } */
    /* XXX: Handle data with embedded LOGENTMAGIC */
    writeall(fst->logfd, &le, sizeof(le), leoff);
    writeall(fst->logfd, buf, len, leoff + sizeof(le));

    i = newindex(fst);
    assert(!getidx(fst, i, &ie));
    ie.addr = pa;
    ie.off = leoff;
    assert(!putidx(fst, i, &ie));
    if(pi != -1) {
	assert(!getidx(fst, pi, &ie));
	c = addrcmp(&pa, &ie.addr);
	if(c < 0)
	    ie.l = i;
	else
	    ie.r = i;
	assert(!putidx(fst, pi, &ie));
    }
    
    return(0);
}

#define min(a, b) (((b) < (a))?(b):(a))

static ssize_t get(struct store *st, void *buf, size_t len, struct addr *at)
{
    idx_t i;
    struct idxent ie;
    struct fstore *fst;
    struct logent le;
    struct addr v;
    char tmpbuf[STORE_MAXBLSZ];
    
    fst = st->pdata;
    if((i = lookup(fst, at, NULL)) == -1) {
	errno = ENOENT;
	return(-1);
    }
    assert(!getidx(fst, i, &ie));
    
    if(readall(fst->logfd, &le, sizeof(le), ie.off)) {
	flog(LOG_CRIT, "could not read log entry: %s", strerror(errno));
	errno = EIO;
	return(-1);
    }
    if(memcmp(le.magic, LOGENTMAGIC, 4)) {
	flog(LOG_CRIT, "invalid magic in log");
	errno = EIO;
	return(-1);
    }
    if(addrcmp(&le.name, at)) {
	flog(LOG_CRIT, "did not receive correct block from log");
	errno = EIO;
	return(-1);
    }
    if(readall(fst->logfd, tmpbuf, le.len, ie.off + sizeof(le))) {
	flog(LOG_CRIT, "could not read log data: %s", strerror(errno));
	errno = EIO;
	return(-1);
    }
    hash(tmpbuf, le.len, &v);
    if(addrcmp(&v, &le.name)) {
	flog(LOG_CRIT, "log data did not verify against hash");
	errno = EIO;
	return(-1);
    }
    if(buf != NULL)
	memcpy(buf, tmpbuf, min(len, le.len));
    return(le.len);
}

static struct storeops fstops = {
    .release = releaseg,
    .put = put,
    .get = get,
};

struct store *newfstore(char *dir)
{
    struct store *st;
    struct fstore *fst;
    char tbuf[1024];
    struct loghdr lh;
    struct idxhdr ih;
    struct stat64 sb;
    
    fst = calloc(1, sizeof(*fst));
    fst->logfd = -1;
    fst->idxfd = -1;
    
    snprintf(tbuf, sizeof(tbuf), "%s/log", dir);
    if((fst->logfd = open(tbuf, O_RDWR | O_LARGEFILE)) < 0) {
	flog(LOG_ERR, "could not open log %s: %s", tbuf, strerror(errno));
	release(fst);
	return(NULL);
    }
    if(fstat64(fst->logfd, &sb)) {
	flog(LOG_ERR, "could not stat log: %s", strerror(errno));
	release(fst);
	return(NULL);
    }
    fst->logsize = sb.st_size;
    if(readall(fst->logfd, &lh, sizeof(lh), 0)) {
	flog(LOG_ERR, "could not read log header: %s", strerror(errno));
	release(fst);
	return(NULL);
    }
    if(memcmp(lh.magic, LOGMAGIC, sizeof(LOGMAGIC))) {
	flog(LOG_ERR, "invalid log magic");
	release(fst);
	return(NULL);
    }
    
    snprintf(tbuf, sizeof(tbuf), "%s/index", dir);
    if((fst->idxfd = open(tbuf, O_RDWR | O_LARGEFILE)) < 0) {
	flog(LOG_ERR, "could not open index %s: %s", tbuf, strerror(errno));
	release(fst);
	return(NULL);
    }
    if(fstat64(fst->idxfd, &sb)) {
	flog(LOG_ERR, "could not stat index: %s", strerror(errno));
	release(fst);
	return(NULL);
    }
    if(readall(fst->idxfd, &ih, sizeof(ih), 0)) {
	flog(LOG_ERR, "could not read index header: %s", strerror(errno));
	release(fst);
	return(NULL);
    }
    if(memcmp(ih.magic, IDXMAGIC, sizeof(IDXMAGIC))) {
	flog(LOG_ERR, "invalid index magic");
	release(fst);
	return(NULL);
    }
    if(sb.st_size != (sizeof(struct idxhdr) + ih.size * sizeof(struct idxent))) {
	flog(LOG_ERR, "invalid index size");
	release(fst);
	return(NULL);
    }
    fst->idxsize = ih.size;
    
    st = newstore(&fstops);
    st->pdata = fst;
    return(st);
}

int mkfstore(char *dir)
{
    char tbuf[1024];
    int fd;
    struct loghdr lh;
    struct idxhdr ih;
    
    if(access(dir, F_OK)) {
	if(mkdir(dir, 0700)) {
	    flog(LOG_ERR, "could not create %s: %s", dir, strerror(errno));
	    return(-1);
	}
    }
    
    snprintf(tbuf, sizeof(tbuf), "%s/log", dir);
    if((fd = open(tbuf, O_WRONLY | O_CREAT | O_EXCL, 0600)) < 0) {
	flog(LOG_ERR, "could not create log %s: %s", tbuf, strerror(errno));
	return(-1);
    }
    memcpy(lh.magic, LOGMAGIC, sizeof(LOGMAGIC));
    if(writeall(fd, &lh, sizeof(lh), 0)) {
	flog(LOG_ERR, "could not write log header: %s", strerror(errno));
	close(fd);
	return(-1);
    }
    close(fd);
    
    snprintf(tbuf, sizeof(tbuf), "%s/index", dir);
    if((fd = open(tbuf, O_WRONLY | O_CREAT | O_EXCL, 0600)) < 0) {
	flog(LOG_ERR, "could not create index %s: %s", tbuf, strerror(errno));
	return(-1);
    }
    memcpy(ih.magic, IDXMAGIC, sizeof(IDXMAGIC));
    ih.size = 0;
    if(writeall(fd, &ih, sizeof(ih), 0)) {
	flog(LOG_ERR, "could not write index header: %s", strerror(errno));
	close(fd);
	return(-1);
    }
    close(fd);
    return(0);
}
Commit	Line	Data
d5cf5351	1	#define _LARGEFILE64_SOURCE
	2	#define _XOPEN_SOURCE 500
	3	#include <stdlib.h>
	4	#include <unistd.h>
	5	#include <stdio.h>
	6	#include <fcntl.h>
	7	#include <errno.h>
	8	#include <string.h>
	9	#include <sys/types.h>
	10	#include <sys/stat.h>
	11	#include <gcrypt.h>
	12	#include <assert.h>
	13
	14	#include "utils.h"
	15	#include "store.h"
	16	#include "log.h"
	17
	18	#define LOGMAGIC "Dolda/Venti-1"
	19	#define IDXMAGIC "Dolda/Index-1"
	20	#define LOGENTMAGIC "\xca\xe5\x7a\x93"
	21
	22	typedef loff_t idx_t;
	23
	24	struct loghdr {
	25	char magic[sizeof(LOGMAGIC)];
	26	};
	27
	28	struct idxhdr {
	29	char magic[sizeof(IDXMAGIC)];
	30	u_int64_t size;
	31	};
	32
	33	struct idxent {
	34	struct addr addr;
	35	u_int64_t l, r;
	36	u_int64_t off;
	37	};
	38
	39	struct logent {
	40	u_int8_t magic[4];
	41	struct addr name;
	42	u_int16_t len;
	43	u_int8_t fl;
	44	};
	45
	46	struct fstore {
	47	int logfd;
	48	int idxfd;
	49	loff_t logsize;
	50	idx_t idxsize;
	51	};
	52
	53	static int release(struct fstore *fst)
	54	{
	55	if(fst->logfd >= 0) {
	56	fsync(fst->logfd);
	57	close(fst->logfd);
	58	}
	59	if(fst->idxfd >= 0) {
	60	fsync(fst->idxfd);
	61	close(fst->idxfd);
	62	}
	63	free(fst);
	64	return(0);
65	}
66
67	static int releaseg(struct store *st)
68	{
69	return(release(st->pdata));
70	}
71
72	static void hash(const void buf, size_t len, struct addr a)
73	{
74	gcry_md_hash_buffer(GCRY_MD_SHA256, a->hash, buf, len);
75	}
76
77	static int getidx(struct fstore fst, idx_t i, struct idxent ie)
78	{
79	return(readall(fst->idxfd, ie, sizeof(ie), sizeof(struct idxhdr) + i sizeof(struct idxent)));
80	}
81
82	static int putidx(struct fstore fst, idx_t i, struct idxent ie)
83	{
84	return(writeall(fst->idxfd, ie, sizeof(ie), sizeof(struct idxhdr) + i sizeof(struct idxent)));
85	}
86
87	static idx_t lookup(struct fstore fst, struct addr a, idx_t *parent)
88	{
89	idx_t i;
90	struct idxent ie;
91	int c;
92
93	if(fst->idxsize == 0) {
94	if(parent != NULL)
95	*parent = -1;
96	return(-1);
97	}
98	i = 0;
99	while(1) {
100	assert(!getidx(fst, i, &ie));
101	c = addrcmp(a, &ie.addr);
102	if(c < 0) {
103	if(ie.l == 0) {
104	if(parent != NULL)
105	*parent = i;
106	return(-1);
107	}
108	i = ie.l;
109	} else if(c > 0) {
110	if(ie.r == 0) {
111	if(parent != NULL)
112	*parent = i;
113	return(-1);
114	}
115	i = ie.r;
116	} else {
117	return(i);
118	}
119	}
120	}
121
122	static idx_t newindex(struct fstore *fst)
123	{
124	size_t newsize;
125	idx_t ni;
126	struct idxent ne;
127	struct idxhdr ih;
128
129	/* XXX: Thread safety */
130	ni = fst->idxsize++;
131	newsize = sizeof(struct idxhdr) + fst->idxsize * sizeof(struct idxent);
132	if(ftruncate(fst->idxfd, newsize))
133	return(-1);
134	ne.l = ne.r = 0;
135	assert(!putidx(fst, ni, &ne));
136	assert(!readall(fst->idxfd, &ih, sizeof(ih), 0));
137	ih.size = fst->idxsize;
138	assert(!writeall(fst->idxfd, &ih, sizeof(ih), 0));
139	return(ni);
140	}
141
142	static int put(struct store st, const void buf, size_t len, struct addr *at)
143	{
144	struct fstore *fst;
145	struct addr pa;
146	idx_t i, pi;
147	struct idxent ie;
148	loff_t leoff;
149	int c;
150	struct logent le;
151
152	if(len > STORE_MAXBLSZ) {
153	errno = E2BIG;
154	return(-1);
155	}
156
157	fst = st->pdata;
158	hash(buf, len, &pa);
159	if(at != NULL)
160	memcpy(at->hash, pa.hash, 32);
161
162	if(lookup(fst, &pa, &pi) != -1)
163	return(0);
164
165	memcpy(le.magic, LOGENTMAGIC, 4);
166	le.name = pa;
167	le.len = len;
168	le.fl = 0;
169	/* XXX: Thread safety { */
170	leoff = fst->logsize;
171	fst->logsize += sizeof(le) + len;
172	/* } */
173	/* XXX: Handle data with embedded LOGENTMAGIC */
174	writeall(fst->logfd, &le, sizeof(le), leoff);
175	writeall(fst->logfd, buf, len, leoff + sizeof(le));
176
177	i = newindex(fst);
178	assert(!getidx(fst, i, &ie));
179	ie.addr = pa;
180	ie.off = leoff;
181	assert(!putidx(fst, i, &ie));
182	if(pi != -1) {
183	assert(!getidx(fst, pi, &ie));
184	c = addrcmp(&pa, &ie.addr);
185	if(c < 0)
186	ie.l = i;
187	else
188	ie.r = i;
189	assert(!putidx(fst, pi, &ie));
190	}
191
192	return(0);
193	}
194
195	#define min(a, b) (((b) < (a))?(b):(a))
196
197	static ssize_t get(struct store st, void buf, size_t len, struct addr *at)
198	{
199	idx_t i;
200	struct idxent ie;
201	struct fstore *fst;
202	struct logent le;
203	struct addr v;
204	char tmpbuf[STORE_MAXBLSZ];
205
206	fst = st->pdata;
207	if((i = lookup(fst, at, NULL)) == -1) {
208	errno = ENOENT;
209	return(-1);
210	}
211	assert(!getidx(fst, i, &ie));
212
213	if(readall(fst->logfd, &le, sizeof(le), ie.off)) {
214	flog(LOG_CRIT, "could not read log entry: %s", strerror(errno));
215	errno = EIO;
216	return(-1);
217	}
218	if(memcmp(le.magic, LOGENTMAGIC, 4)) {
219	flog(LOG_CRIT, "invalid magic in log");
220	errno = EIO;
221	return(-1);
222	}
223	if(addrcmp(&le.name, at)) {
224	flog(LOG_CRIT, "did not receive correct block from log");
225	errno = EIO;
226	return(-1);
227	}
228	if(readall(fst->logfd, tmpbuf, le.len, ie.off + sizeof(le))) {
229	flog(LOG_CRIT, "could not read log data: %s", strerror(errno));
230	errno = EIO;
231	return(-1);
232	}
233	hash(tmpbuf, le.len, &v);
234	if(addrcmp(&v, &le.name)) {
235	flog(LOG_CRIT, "log data did not verify against hash");
236	errno = EIO;
237	return(-1);
238	}
239	if(buf != NULL)
240	memcpy(buf, tmpbuf, min(len, le.len));
241	return(le.len);
242	}
243
244	static struct storeops fstops = {
245	.release = releaseg,
246	.put = put,
247	.get = get,
248	};
249
250	struct store newfstore(char dir)
251	{
252	struct store *st;
253	struct fstore *fst;
254	char tbuf[1024];
255	struct loghdr lh;
256	struct idxhdr ih;
257	struct stat64 sb;
258
259	fst = calloc(1, sizeof(*fst));
260	fst->logfd = -1;
261	fst->idxfd = -1;
262
263	snprintf(tbuf, sizeof(tbuf), "%s/log", dir);
264	if((fst->logfd = open(tbuf, O_RDWR \| O_LARGEFILE)) < 0) {
265	flog(LOG_ERR, "could not open log %s: %s", tbuf, strerror(errno));
266	release(fst);
267	return(NULL);
268	}
269	if(fstat64(fst->logfd, &sb)) {
270	flog(LOG_ERR, "could not stat log: %s", strerror(errno));
271	release(fst);
272	return(NULL);
273	}
274	fst->logsize = sb.st_size;
275	if(readall(fst->logfd, &lh, sizeof(lh), 0)) {
276	flog(LOG_ERR, "could not read log header: %s", strerror(errno));
277	release(fst);
278	return(NULL);
279	}
280	if(memcmp(lh.magic, LOGMAGIC, sizeof(LOGMAGIC))) {
281	flog(LOG_ERR, "invalid log magic");
282	release(fst);
283	return(NULL);
284	}
285
286	snprintf(tbuf, sizeof(tbuf), "%s/index", dir);
287	if((fst->idxfd = open(tbuf, O_RDWR \| O_LARGEFILE)) < 0) {
288	flog(LOG_ERR, "could not open index %s: %s", tbuf, strerror(errno));
289	release(fst);
290	return(NULL);
291	}
292	if(fstat64(fst->idxfd, &sb)) {
293	flog(LOG_ERR, "could not stat index: %s", strerror(errno));
294	release(fst);
295	return(NULL);
296	}
297	if(readall(fst->idxfd, &ih, sizeof(ih), 0)) {
298	flog(LOG_ERR, "could not read index header: %s", strerror(errno));
299	release(fst);
300	return(NULL);
301	}
302	if(memcmp(ih.magic, IDXMAGIC, sizeof(IDXMAGIC))) {
303	flog(LOG_ERR, "invalid index magic");
304	release(fst);
305	return(NULL);
306	}
307	if(sb.st_size != (sizeof(struct idxhdr) + ih.size * sizeof(struct idxent))) {
308	flog(LOG_ERR, "invalid index size");
309	release(fst);
310	return(NULL);
311	}
312	fst->idxsize = ih.size;
313
314	st = newstore(&fstops);
315	st->pdata = fst;
316	return(st);
317	}
318
319	int mkfstore(char *dir)
320	{
321	char tbuf[1024];
322	int fd;
323	struct loghdr lh;
324	struct idxhdr ih;
325
326	if(access(dir, F_OK)) {
327	if(mkdir(dir, 0700)) {
328	flog(LOG_ERR, "could not create %s: %s", dir, strerror(errno));
329	return(-1);
330	}
331	}
332
333	snprintf(tbuf, sizeof(tbuf), "%s/log", dir);
334	if((fd = open(tbuf, O_WRONLY \| O_CREAT \| O_EXCL, 0600)) < 0) {
335	flog(LOG_ERR, "could not create log %s: %s", tbuf, strerror(errno));
336	return(-1);
337	}
338	memcpy(lh.magic, LOGMAGIC, sizeof(LOGMAGIC));
339	if(writeall(fd, &lh, sizeof(lh), 0)) {
340	flog(LOG_ERR, "could not write log header: %s", strerror(errno));
341	close(fd);
342	return(-1);
343	}
344	close(fd);
345
346	snprintf(tbuf, sizeof(tbuf), "%s/index", dir);
347	if((fd = open(tbuf, O_WRONLY \| O_CREAT \| O_EXCL, 0600)) < 0) {
348	flog(LOG_ERR, "could not create index %s: %s", tbuf, strerror(errno));
349	return(-1);
350	}
351	memcpy(ih.magic, IDXMAGIC, sizeof(IDXMAGIC));
352	ih.size = 0;
353	if(writeall(fd, &ih, sizeof(ih), 0)) {
354	flog(LOG_ERR, "could not write index header: %s", strerror(errno));
355	close(fd);
356	return(-1);
357	}
358	close(fd);
359	return(0);
360	}