ref: 99c05d231cb6f47987777d8f6e0b434a0262871d
parent: d52d3d898746cb77b6607d42497fb3d0fd599cd0
author: Ori Bernstein <ori@eigenstate.org>
date: Sat Sep 5 23:50:01 EDT 2020
git/repack, writepack(): fix pack file generation Fix several bugs in packfile generation that were causing invalid packs to be output, and indexes to be misgenerated.
--- a/delta.c
+++ b/delta.c
@@ -70,7 +70,7 @@
{int probe;
- for(probe = rh % dt->sz; dt->b[probe].buf != nil; probe++)
+ for(probe = rh % dt->sz; dt->b[probe].buf != nil; probe = (probe + 1) % dt->sz)
if(dt->b[probe].rhash == rh)
return &dt->b[probe];
return nil;
--- a/objset.c
+++ b/objset.c
@@ -1,6 +1,5 @@
#include <u.h>
#include <libc.h>
-#include <pool.h>
#include "git.h"
@@ -27,8 +26,10 @@
probe = GETBE32(o->hash.h) % s->sz;
while(s->obj[probe]){- if(hasheq(&s->obj[probe]->hash, &o->hash))
+ if(hasheq(&s->obj[probe]->hash, &o->hash)){+ s->obj[probe] = o;
return;
+ }
probe = (probe + 1) % s->sz;
}
assert(s->obj[probe] == nil);
--- a/pack.c
+++ b/pack.c
@@ -348,7 +348,10 @@
if(d != ed && (c & 0x40)) l |= (*d++ << 16) & 0xff0000;
if(l == 0) l = 0x10000;
- assert(o + l <= base->size);
+ if(o + l > base->size){+ werrstr("garbled delta: out of bounds copy");+ return -1;
+ }
memmove(r, b + o, l);
r += l;
/* inline data */
@@ -829,16 +832,23 @@
char path[Pathmax];
char hbuf[41];
Biobuf *f;
- Object *obj;
+ Object *obj, *new;
int l, i, n;
vlong o;
Dir *d;
if((obj = osfind(&objcache, h)) != nil){- if(obj->flag & Cloaded)
- return obj;
- if(obj->flag & Cidx){- assert(idx != nil);
+ if(flag & Cidx){+ /*
+ * If we're indexing, we need to be careful
+ * to only return objects within this pack,
+ * so we don't load objects from outside the
+ * current pack.
+ */
+ if(!(obj->flag & Cidx))
+ return nil;
+ if(obj->flag & Cloaded)
+ return obj;
o = Boffset(idx);
if(Bseek(idx, obj->off, 0) == -1)
return nil;
@@ -849,11 +859,18 @@
cache(obj);
return obj;
}
- }else{- obj = emalloc(sizeof(Object));
- obj->id = objcache.nobj + 1;
- obj->hash = h;
+ if(obj->flag & Cloaded)
+ return obj;
}
+ if(flag & Cidx)
+ return nil;
+ new = nil;
+ if(obj == nil){+ new = emalloc(sizeof(Object));
+ new->id = objcache.nobj + 1;
+ new->hash = h;
+ obj = new;
+ }
d = nil;
snprint(hbuf, sizeof(hbuf), "%H", h);
@@ -902,8 +919,9 @@
cache(obj);
return obj;
error:
+ Bterm(f);
free(d);
- free(obj);
+ free(new);
return nil;
}
@@ -942,6 +960,8 @@
o->hash = h;
o->type = type;
osadd(&objcache, o);
+ o->id = objcache.nobj;
+ o->flag |= Cexist;
return o;
}
@@ -985,7 +1005,8 @@
indexpack(char *pack, char *idx, Hash ph)
{char hdr[4*3], buf[8];
- int nobj, nvalid, nbig, n, i, pcnt, x;
+ int nobj, npct, nvalid, nbig;
+ int i, n, r, x, pcnt;
Object *o, **objects;
DigestState *st;
char *valid;
@@ -1004,20 +1025,21 @@
return -1;
}
+ npct = 0;
nvalid = 0;
nobj = GETBE32(hdr + 8);
objects = calloc(nobj, sizeof(Object*));
valid = calloc(nobj, sizeof(char));
+ fprint(2, "indexing %d objects: 0%%", nobj);
while(nvalid != nobj){- fprint(2, "indexing %d objects: 0%%", nobj-nvalid);
n = 0;
pcnt = 0;
for(i = 0; i < nobj; i++){+ x = (npct*100) / nobj;
if(valid[i]){n++;
continue;
}
- x = (i*100) / nobj;
if(x > pcnt){pcnt = x;
if(pcnt%10 == 0)
@@ -1029,20 +1051,25 @@
objects[i] = o;
}
o = objects[i];
+ /*
+ * We can seek around when packing delta chains.
+ * Be extra careful while we don't know where all
+ * the objects start.
+ */
Bseek(f, o->off, 0);
- if (readpacked(f, o, Cidx) == -1){- objects[i] = nil;
- free(o);
+ r = readpacked(f, o, Cidx);
+ Bseek(f, o->off + o->len, 0);
+ if (r == -1)
continue;
- }
+ Bseek(f, o->off + o->len, 0);
sha1((uchar*)o->all, o->size + strlen(o->all) + 1, o->hash.h, nil);
valid[i] = 1;
cache(o);
+ npct++;
n++;
if(objectcrc(f, o) == -1)
return -1;
}
- fprint(2, "\b\b\b\b100%%\n");
if(n == nvalid){ sysfatal("fix point reached too early: %d/%d: %r", nvalid, nobj);goto error;
@@ -1049,6 +1076,7 @@
}
nvalid = n;
}
+ fprint(2, "\b\b\b\b100%%\n");
Bterm(f);
st = nil;
@@ -1406,9 +1434,26 @@
}
static int
+packhdr(char *hdr, int ty, int len)
+{+ int i;
+
+ hdr[0] = ty << 4;
+ hdr[0] |= len & 0xf;
+ len >>= 4;
+ for(i = 1; len != 0; i++){+ assert(i < sizeof(hdr));
+ hdr[i-1] |= 0x80;
+ hdr[i] = len & 0x7f;
+ len >>= 7;
+ }
+ return i;
+}
+
+static int
genpack(int fd, Objmeta *meta, int nmeta, Hash *h)
{- int i, j, n, x, res, pcnt, len, ret;
+ int i, nh, nd, x, res, pcnt, ret;
DigestState *st;
Biobuf *bfd;
Objmeta *m;
@@ -1443,34 +1488,16 @@
if((o = readobject(m->hash)) == nil)
return -1;
if(m->delta == nil){- len = o->size;
- buf[0] = o->type << 4;
- buf[0] |= len & 0xf;
- len >>= 4;
- for(j = 1; len != 0; j++){- assert(j < sizeof(buf));
- buf[j-1] |= 0x80;
- buf[j] = len & 0x7f;
- len >>= 7;
- }
- hwrite(bfd, buf, j, &st);
+ nh = packhdr(buf, o->type, o->size);
+ hwrite(bfd, buf, nh, &st);
if(hcompress(bfd, o->data, o->size, &st) == -1)
goto error;
}else{- n = encodedelta(m, o, m->base, &p);
- len = n;
- buf[0] = GRdelta << 4;
- buf[0] |= len & 0xf;
- len >>= 4;
- for(j = 1; len != 0; j++){- assert(j < sizeof(buf));
- buf[j-1] |= 0x80;
- buf[j] = len & 0x7f;
- len >>= 7;
- }
- hwrite(bfd, buf, j, &st);
+ nd = encodedelta(m, o, m->base, &p);
+ nh = packhdr(buf, GRdelta, nd);
+ hwrite(bfd, buf, nh, &st);
hwrite(bfd, m->base->hash.h, sizeof(m->base->hash.h), &st);
- res = hcompress(bfd, p, n, &st);
+ res = hcompress(bfd, p, nd, &st);
free(p);
if(res == -1)
goto error;
--- a/repack.c
+++ b/repack.c
@@ -71,6 +71,7 @@
sysfatal("open %s: %r", TMPPATH("pack.tmp"));if(writepack(fd, obj, nobj, &h) == -1)
sysfatal("writepack: %r");+ free(obj);
if(indexpack(TMPPATH("pack.tmp"), TMPPATH("idx.tmp"), h) == -1) sysfatal("indexpack: %r");close(fd);