ref: 194a9f6821c6870056be370ea88e62c0e76c8ca3
parent: 14883c018e531ff84ae4996268f3091e2e4cad45
author: Noam Preil <noam@pixelhero.dev>
date: Sun Sep 7 02:33:41 EDT 2025
enhance compatibility
--- a/disk.c
+++ b/disk.c
@@ -79,7 +79,7 @@
u16int n, off;
s_sect = &index.sects[isectforbucket(bucket)];
bucket -= s_sect->start;
- cachelookup((char**)&buf, s_sect->cacheindex, bucket);
+ if(!cachelookup((char**)&buf, s_sect->cacheindex, bucket)){fprint(2, "reading from disk...\n");
if(pread(s_sect->fd, (char*)buf, s_sect->blocksize, s_sect->blockbase + (bucket << s_sect->blocklog)) != s_sect->blocksize){cacheunlock(key, bucket);
@@ -86,7 +86,7 @@
werrstr("Failed to read bucket");return 0;
}
-
+ }
n = U16GET(buf);
fprint(2, "appending to bucket %d, key %08x, n %d\n", bucket, key, n);
if(s_sect->bucketmagic){@@ -125,14 +125,13 @@
u32int bucket = U32GET(score) / index.div;
VtISect *s_sect = &index.sects[isectforbucket(bucket)];
bucket -= s_sect->start;
- cachelookup((char**)&buf, s_sect->cacheindex, bucket);
- fprint(2, "reading from disk for readlookup...\n");
+ if(!cachelookup((char**)&buf, s_sect->cacheindex, bucket)){ if(pread(s_sect->fd, (char*)buf, s_sect->blocksize, s_sect->blockbase + (bucket << s_sect->blocklog)) != s_sect->blocksize){cacheunlock(s_sect->cacheindex, bucket);
werrstr("Failed to read bucket");return 0;
}
-
+ }
if(s_sect->bucketmagic && U32GET(buf + 2) != s_sect->bucketmagic){if(U32GET(buf+2) != 0)
sysfatal("index is corrupt: invalid bucket magic: sect %ux, buck %ux", s_sect->bucketmagic, U32GET(buf + 2));@@ -162,12 +161,6 @@
}
static u64int
-arenadirsize(VtArena *arena, u32int clumps)
-{- return ((clumps * ClumpInfoSize / arena->blocksize) + 1) * arena->blocksize;
-}
-
-static u64int
partlen(int fd, char *path)
{Dir *dir = dirfstat(fd);
@@ -191,11 +184,11 @@
char *buf;
if(arena->blocksize != 8192)
sysfatal("invalid blocksize %d\n", arena->blocksize);- cachelookup(&buf, arena->index, blockindex);
+ if(!cachelookup(&buf, arena->index, blockindex)){ if(pread(arena->fd, buf, arena->blocksize, arena->base+(blockindex*arena->blocksize)) != arena->blocksize){return nil;
}
-
+ }
return buf;
}
@@ -202,13 +195,10 @@
u16int
vtreadarena(VtArena *arena, u64int addr, uchar *dbuf, u16int reqsize)
{- u64int end = arena->size - arenadirsize(arena, arena->arenastats.clumps);
u16int off, n, m, size;
u32int blockindex;
char *buf;
size = reqsize;
- if(addr + reqsize > end)
- size = end - addr;
off = addr & (arena->blocksize-1);
addr -= off;
n = 0;
@@ -319,11 +309,14 @@
}
static void
-clumpwrite(char *wptr, u16int len)
+clumpwrite(char *wptr, u16int len, uchar *score)
{U32PUT(wptr, index.arena->clumpmagic);
+ // Fake a type so venti doesn't complain if someone tries using oldventi to read.
+ wptr[4] = 1;
U16PUT(wptr+5, len);
U16PUT(wptr+7, len);
+ memcpy(wptr+9, score, 20);
wptr[29] = 1;
}
@@ -336,7 +329,7 @@
}
int
-vtwritearenaclump(char *buf, u16int len, u64int *offset)
+vtwritearenaclump(char *buf, u16int len, u64int *offset, uchar *score)
{/* We've got a 38-byte clump header to write, and the contents
* of the clump. We'll be splitting it across blocks. First, find
@@ -354,7 +347,7 @@
*offset = index.arena->base + (index.arena->block)*index.arena->blocksize - index.arena->blocksize + index.arena->offset;
getblock();
wptr = index.arena->buf + index.arena->offset;
- clumpwrite(wptr, len);
+ clumpwrite(wptr, len, score);
blockupdate(38);
wptr += 38;
while(len > n){@@ -375,8 +368,7 @@
index.arena->arenastats.uncsize += len;
index.arena->arenastats.clumps += 1;
blockflush();
- // TODO ctime/wtime? Shouldn't go in here, though, should be at a higher level.
- return 1;
+ return vtarenawb(index.arena);
}
int
@@ -388,11 +380,10 @@
// - Write data to arena
// - Write metadata to arena
// - Add entry to index
- if(!vtwritearenaclump(buf, len, &offset)){+ if(!vtwritearenaclump(buf, len, &offset, score)){wunlock(&index);
return 0;
}
-
if(!vtwriteindexclump(score, offset, len)){ werrstr("failed to write clump to index: %r");return 0;
@@ -438,8 +429,6 @@
p += 8+NameSize;
arena->indexstats.clumps = U32GET(p);
arena->indexstats.cclumps = U32GET(p+4);
- arena->ctime = U32GET(p+8);
- arena->wtime = U32GET(p+12);
arena->clumpmagic = U32GET(p+16);
arena->indexstats.used = U64GET(p+20);
arena->indexstats.uncsize = U64GET(p+28);
@@ -475,6 +464,8 @@
sysfatal("arena name mismatch: %s vs %s", arena->name, buf + 8);if(index.arena == nil)
index.arena = arena;
+ if(arena->indexstats.clumps != arena->arenastats.clumps && !vtarenareaddirectory(arena))
+ sysfatal("unable to load arena directory");}
static void
--- a/mkfile
+++ b/mkfile
@@ -2,7 +2,7 @@
TARG=neoventi checkarena
BIN=/$objtype/bin
-OFILES=unwhack.$O server.$O util.$O disk.$O cache.$O
+OFILES=unwhack.$O server.$O util.$O disk.$O cache.$O arena.$O
CLEANFILES=paper.ps paper.pdf
HFILES=neoventi.h
--- a/neoventi.h
+++ b/neoventi.h
@@ -52,30 +52,32 @@
} VtConn;
typedef struct {+ u32int block;
+ u16int offset;
+ u16int blockremain;
+ char *buf;
+} WriteHead;
+
+typedef struct {+ // Total clump count, and compressed clump count, respectively
+ u32int clumps, cclumps;
+ u64int used, uncsize;
+ int sealed;
+} ArenaStats;
+
+typedef struct {RWLock;
+ WriteHead;
char name[NameSize];
u32int clumpmagic;
u32int blocksize;
- u32int version;
- u32int ctime, wtime;
+ int fd;
// used for caching. only 64K arenas may reside in the cache at a time.
u16int index;
/* disk info */
- u64int size;
- u64int base;
- /* Write head: current block, offset within block, available space in block. */
- u32int block;
- u16int offset;
- u16int blockremain;
- char *buf;
- int fd;
- struct {- // Total clump count, and compressed clump count, respectively
- u32int clumps, cclumps;
- u64int used, uncsize;
- int sealed;
- } indexstats, arenastats;
+ u64int size, base, directory;
+ ArenaStats indexstats, arenastats;
} VtArena;
typedef struct {@@ -152,3 +154,7 @@
extern char *arenapath;
extern char *isectpath;
+/* Directory interface */
+int vtarenareaddirectory(VtArena *);
+int vtarenawritedirectory(VtArena *, char score[20], u16int uncsize);
+int vtarenawb(VtArena *);
--- a/notebook
+++ b/notebook
@@ -3205,3 +3205,31 @@
Just added that assertion to the top and it's immediatley failing. That invariant is fine for arenas, not indices, and I didn't document it a year ago when I was working on this, oops. Fortunately, cache keys are largely arbitrary, they just need to be unique. Can just do bucket+1 and should be fine.
But, that's not great. That's a clunky API. This shouldn't be necessary. Much cleaner: change the 'key missing' value to 0xFFFFFFFF (-1, but it's unsigned, so.), init it that way on startup, and make 0 a valid key entry.
+
+and, now everything's working again without the hacks, perfect :)
+
+TODO major change to the disk layer: write back of dirty cache blocks should be handled by cacheunlock?
+
+...wait a minute.
+
+ arena->directory = (( arena->arenastats.clumps * ClumpInfoSize / arena->blocksize) + 1) * arena->blocksize;
+
+This has to be wrong; we're trying to compute where the directory begins. This is... what?? The directory grows _backwards_. It doesn't have a fixed size. Oops. Just drop that for now.
+
+TODO: enhance bounds checking on write.
+
+Okay, so should be writing clump count at least. Can now detect that we're missing some on startup?
+
+Interesting experiment: what happens if I run venti on it afterwards? ...it rejects the clump because default type is corrupt. For compatibily, _have_ to set type to something. Sigh.
+
+With that fixed, the score is missing. Add that in, and...
+
+2025/0907 02:33:00 venti: conf...venti/venti: mem 18,951,372 bcmem 37,902,744 icmem 56,854,118...init...icache 56,854,118 bytes = 565,311 entries; 7 scache
+sync...arena areans0.0: fix=1 flush=1 0->42 0->1 0->0 0->4
+arena areans0.0: header is out-of-date
+2025/0907 02:33:01 areans0.0: indexing 1 clumps...
+
+% venti/read f1d2d2f924e986ac86fdf7b36c94bcdf32beec15
+foo
+
+Woo!
\ No newline at end of file
--
⑨