shithub: s3

Info  •  Files  •  Log  •  Branches

add faux-factotum for keeping the secret

--- a/README

+++ b/README

@@ -1,10 +1,10 @@

 Basics:

-	Requires $AWS_ACCESS_KEY_ID, $AWS_SECRET_ACCESS_KEY, and $AWS_ENDPOINT_URL_S3 defined

-	Usage: s3 cat s3://bucket/file

-	Usage: s3 cp source s3://bucket/destination

-	Usage: s3 cp s3://bucket/source <destination>

-	Usage: s3 rm s3://bucket/path

-	Usage: s3 ls s3://bucket/prefix

+	Requires $AWS_ACCESS_KEY_ID and $AWS_ENDPOINT_URL_S3 defined

+	Usage: s3/cmd cat s3://bucket/file

+	Usage: s3/cmd cp source s3://bucket/destination

+	Usage: s3/cmd cp s3://bucket/source <destination>

+	Usage: s3/cmd rm s3://bucket/path

+	Usage: s3/cmd ls s3://bucket/prefix

 Specifics/Bugs:

 	Uses webfs(4)

--- /dev/null

+++ b/cmd.c

@@ -1,0 +1,391 @@

+#include <u.h>

+#include <libc.h>

+#include <bio.h>

+#include <mp.h>

+#include <libsec.h>

+#include <auth.h>

+#include "xml.h"

+

+typedef struct {

+	char *endpoint;

+	char *host;

+	char *access;

+	char *bucket;

+	char *region;

+} S3;

+

+typedef struct {

+	char method[16];

+	char time[128];

+	uchar payhash[SHA2_256dlen];

+	char authhdr[512];

+	char mime[32];

+} Hreq;

+

+static void

+datetime(char *date, int ndate, char *time, int ntime)

+{

+	Tm t;

+

+	tmnow(&t, nil);

+	snprint(date, ndate, "%τ", tmfmt(&t, "YYYYMMDD"));

+	snprint(time, ntime, "%sT%τZ", date, tmfmt(&t, "hhmmss"));

+}

+

+static void

+getkey(char *date, char *region, char *service, uchar out[SHA2_256dlen])

+{

+	int fd;

+	AuthRpc *rpc;

+	char buf[256];

+	int n;

+	char keyspec[] = "proto=aws4";

+

+	fd = open("/mnt/factotum/rpc", ORDWR);

+	if(fd < 0)

+		sysfatal("factotum rpc open: %r");

+	rpc = auth_allocrpc(fd);

+	if(auth_rpc(rpc, "start", keyspec, strlen(keyspec)) != ARok)

+		sysfatal("auth_rpc: %r");

+	n = snprint(buf, sizeof buf, "%s %s %s", date, region, service);

+	if(auth_rpc(rpc, "write", buf, n) != ARok)

+		sysfatal("auth_rpc: %r");

+	if(auth_rpc(rpc, "read", nil, 0) != ARok)

+		sysfatal("auth_rpc: %r");

+	if(rpc->narg != SHA2_256dlen)

+		sysfatal("invalid auth_rpc output");

+	memcpy(out, rpc->arg, SHA2_256dlen);

+	auth_freerpc(rpc);

+	close(fd);

+}

+

+static void

+mkhreq(Hreq *hreq, S3 *s3, char *method, char *path)

+{

+	char date[64];

+	uchar key[SHA2_256dlen], sig[SHA2_256dlen];

+	char buf[512], req[512];

+	char *sgndhdr;

+

+	datetime(date, sizeof date, hreq->time, sizeof hreq->time);

+	if(strcmp(method, "PUT") == 0){

+		snprint(buf, sizeof buf, "content-type:%s\nhost:%s\nx-amz-content-sha256:%.*lH\nx-amz-date:%s\n",

+			hreq->mime, s3->host, SHA2_256dlen, hreq->payhash, hreq->time);

+		sgndhdr = "content-type;host;x-amz-content-sha256;x-amz-date";

+	} else if(strcmp(method, "GET") == 0 || strcmp(method, "DELETE")==0){

+		hreq->mime[0] = 0;

+		sha2_256(nil, 0, hreq->payhash, nil);

+		snprint(buf, sizeof buf, "host:%s\nx-amz-date:%s\n", s3->host, hreq->time);

+		sgndhdr = "host;x-amz-date";

+	} else

+		sysfatal("invalid method");

+

+	snprint(req, sizeof req, "%s\n/%s/%s\n%s\n%s\n%s\n%.*lH",

+		method, s3->bucket, path, "", buf, sgndhdr, SHA2_256dlen, hreq->payhash);

+	sha2_256((uchar*)req, strlen(req), key, nil);

+	snprint(buf, sizeof buf, "%s\n%s\n%s/%s/%s/aws4_request\n%.*lH",

+		"AWS4-HMAC-SHA256", hreq->time, date, s3->region, "s3", SHA2_256dlen, key);

+	getkey(date, s3->region, "s3", key);

+	hmac_sha2_256((uchar*)buf, strlen(buf), key, SHA2_256dlen, sig, nil);

+

+	snprint(hreq->authhdr, sizeof hreq->authhdr, "%s Credential=%s/%s/%s/%s/aws4_request, SignedHeaders=%s, Signature=%.*lH",

+		"AWS4-HMAC-SHA256", s3->access, date, s3->region, "s3", sgndhdr, SHA2_256dlen, sig);

+	snprint(hreq->method, sizeof hreq->method, "%s", method);

+}

+

+static void

+prep(S3 *s3, int cfd, char *path, Hreq *hreq)

+{

+	if(fprint(cfd, "url %s/%s/%s", s3->endpoint, s3->bucket, path) < 0)

+		sysfatal("url: %r");

+	if(fprint(cfd, "request %s", hreq->method) < 0)

+		sysfatal("request: %r");

+	if(fprint(cfd, "headers Authorization:%s", hreq->authhdr) < 0)

+		sysfatal("headers2: %r");

+	if(fprint(cfd, "headers x-amz-date:%s\nx-amz-content-sha256:%.*lH", hreq->time, SHA2_256dlen, hreq->payhash) < 0)

+		sysfatal("headers: %r");

+	if(hreq->mime[0] != 0 && fprint(cfd, "contenttype %s", hreq->mime) < 0)

+		sysfatal("contenttype: %r");

+}

+

+static void

+download(S3 *s3, int cfd, char *conn, char *path, Biobuf *local)

+{

+	int bfd;

+	long n;

+	char buf[64];

+	char data[8192];

+	Hreq hreq;

+

+	mkhreq(&hreq, s3, "GET", path);

+	prep(s3, cfd, path, &hreq);

+	snprint(buf, sizeof buf, "/mnt/web/%s/body", conn);

+	bfd = open(buf, OREAD);

+	if(bfd < 0)

+		sysfatal("download body: %r");

+	for(;;){

+		n = read(bfd, data, sizeof data);

+		if(n < 0)

+			sysfatal("download body: %r");

+		if(n == 0)

+			return;

+		Bwrite(local, data, n);

+	}

+}

+

+static void

+mimetype(char *path, char *out, int nout)

+{

+	int p[2];

+	char buf[256];

+	int n;

+

+	pipe(p);

+	switch(fork()){

+	case 0:

+		close(0);

+		dup(p[0], 1);

+		close(2);

+		close(p[0]);

+		close(p[1]);

+		execl("/bin/file", "file", "-m", path, nil);

+		sysfatal("execl: %r");

+	default:

+		close(p[0]);

+		n = read(p[1], buf, sizeof buf - 1);

+		if(n <= 0)

+			sysfatal("no mimetype found");

+		buf[n - 1] = 0;

+		snprint(out, nout, "%s", buf);

+	}

+}

+

+static void

+upload(S3 *s3, int cfd, char *conn, char *localpath, char *remotepath)

+{

+	DigestState *ds;

+	uchar data[8192];

+	long n;

+	int fd, bfd;

+	char buf[256];

+	Hreq hreq;

+

+	mimetype(localpath, hreq.mime, sizeof hreq.mime);

+	fd = open(localpath, OREAD);

+	if(fd < 0)

+		sysfatal("upload open: %r");

+	for(ds = nil;;){

+		n = read(fd, data, sizeof data);

+		if(n < 0)

+			sysfatal("file read: %r");

+		if(n == 0)

+			break;

+		ds = sha2_256(data, n, nil, ds);

+	}

+	sha2_256(nil, 0, hreq.payhash, ds);

+	seek(fd, 0, 0);

+

+	mkhreq(&hreq, s3, "PUT", remotepath);

+	prep(s3, cfd, remotepath, &hreq);

+	snprint(buf, sizeof buf, "/mnt/web/%s/postbody", conn);

+	bfd = open(buf, OWRITE);

+	if(bfd < 0)

+		sysfatal("upload postbody open: %r");

+	for(;;){

+		n = read(fd, buf, sizeof buf);

+		if(n < 0)

+			sysfatal("file read: %r");

+		if(n == 0)

+			break;

+		if(write(bfd, buf, n) < 0)

+			sysfatal("upload write: %r");

+	}

+	close(bfd);

+

+	snprint(buf, sizeof buf, "/mnt/web/%s/body", conn);

+	if((bfd = open(buf, OREAD)) < 0)

+		sysfatal("open3: %r");

+	close(bfd);

+}

+

+static int

+parseuri(S3 *s3, char *path, int npath, char *arg)

+{

+	char *p;

+

+	if(strstr(arg, "s3://") != arg)

+		return -1;

+	arg+=5;

+	p = strchr(arg, '/');

+	if(p == nil || p == arg)

+		return -1;

+	snprint(path, npath, "%s", p+1);

+	s3->bucket = strdup(arg);

+	s3->bucket[p-arg] = 0;

+	return 0;

+}

+

+_Noreturn static void

+usage(void)

+{

+	fprint(2, "Requires $AWS_ACCESS_KEY_ID and $AWS_ENDPOINT_URL_S3 defined\n");

+	fprint(2, "Usage: %s cat s3://bucket/file\n", argv0);

+	fprint(2, "Usage: %s cp source s3://bucket/destination\n", argv0);

+	fprint(2, "Usage: %s cp s3://bucket/source <destination>\n", argv0);

+	fprint(2, "Usage: %s rm s3://bucket/path\n", argv0);

+	fprint(2, "Usage: %s ls s3://bucket/prefix\n", argv0);

+	exits("usage");

+}

+

+static void

+cp(S3 *s3, int cfd, char *conn, int argc, char **argv)

+{

+	char path[512];

+	Biobuf *b;

+	int fd;

+

+	if(argc == 0 || argc > 2)

+		usage();

+	if(parseuri(s3, path, sizeof path, argv[0]) == 0){

+		if(argc > 1 && parseuri(s3, path, sizeof path, argv[1]) == 0)

+			sysfatal("s3:// → s3:// is not implemented");

+		if(argc == 1)

+			fd = 1;

+		else {

+			fd = create(argv[1], OWRITE, 0644);

+			if(fd < 0)

+				sysfatal("create: %r");

+		}

+		b = Bfdopen(fd, OWRITE);

+		if(b == nil)

+			sysfatal("Bfdopen: %r");

+		download(s3, cfd, conn, path, b);

+		return;

+	}

+	if(argc == 1 || parseuri(s3, path, sizeof path, argv[1]) < 0)

+		usage();

+	upload(s3, cfd, conn, argv[0], path);

+}

+

+static void

+delete(S3 *s3, int cfd, char *conn, int argc, char **argv)

+{

+	int fd;

+	char buf[256];

+	Hreq hreq;

+	char path[512];

+

+	if(argc == 0)

+		usage();

+	if(parseuri(s3, path, sizeof path, argv[0]) < 0)

+		usage();

+	mkhreq(&hreq, s3, "DELETE", path);

+	prep(s3, cfd, path, &hreq);

+	snprint(buf, sizeof buf, "/mnt/web/%s/body", conn);

+	fd = open(buf, OREAD);

+	if(fd < 0)

+		sysfatal("delete failed: %r");

+	close(fd);

+}

+

+static void

+list(S3 *s3, int cfd, char *conn, int argc, char **argv)

+{

+	int p[2];

+	Biobuf *b[2];

+	Xelem *x;

+	char path[512];

+

+	if(argc == 0)

+		usage();

+	if(parseuri(s3, path, sizeof path, argv[0]) < 0)

+		usage();

+	if(pipe(p) < 0)

+		sysfatal("pipe: %r");

+	switch(fork()){

+	case -1:

+		sysfatal("fork: %r");

+	case 0:

+		close(p[1]);

+		b[0] = Bfdopen(p[0], OWRITE);

+		if(b[0] == nil)

+			sysfatal("Bfdopen: %r");

+		download(s3, cfd, conn, path, b[0]);

+		exits(nil);

+	default:

+		close(p[0]);

+		break;

+	}

+	b[1] = Bfdopen(p[1], OREAD);

+	if(b[1] == nil)

+		sysfatal("Bfdopen: %r");

+	x = xmlread(b[1], 0);

+	if(x == nil)

+		sysfatal("file was not valid XML, maybe not a prefix?");

+	if((x = xmlget(x, "Contents", nil)) == nil)

+		sysfatal("xml did not have Contents field");

+

+	for(; x != nil && xmlget(x, "Key", nil) != nil; x = x->next)

+		print("%s\n", xmlget(x, "Key", nil)->v);

+}

+

+struct {

+	char *cmd;

+	void (*fn)(S3*,int,char*,int,char**);

+} cmdtab[] = {

+	{ "cp", cp },

+	{ "cat", cp },

+	{ "rm", delete },

+	{ "ls", list },

+};

+

+void

+main(int argc , char **argv)

+{

+	S3 s3;

+	int fd;

+	int i;

+	long n;

+	char buf[64];

+

+	tmfmtinstall();

+	fmtinstall('H', encodefmt);

+	ARGBEGIN{

+	default:

+		usage();

+		break;

+	}ARGEND

+	if(argc == 0)

+		usage();

+

+	s3.access = getenv("AWS_ACCESS_KEY_ID");

+	s3.endpoint = getenv("AWS_ENDPOINT_URL_S3");

+	s3.region = getenv("AWS_DEFAULT_REGION");

+	if(s3.access == nil || s3.endpoint == nil)

+		usage();

+	if(s3.region == nil)

+		s3.region = "auto";

+

+	s3.host = strstr(s3.endpoint, "://");

+	if(s3.host == nil)

+		sysfatal("invalid endpoint url");

+	s3.host += 3;

+

+	fd = open("/mnt/web/clone", ORDWR);

+	if(fd < 0)

+		sysfatal("open: %r");

+	n = read(fd, buf, sizeof buf - 1);

+	if(n <= 0)

+		sysfatal("read: %r");

+	buf[n-1] = 0;

+

+	for(i = 0; i < nelem(cmdtab); i++){

+		if(strcmp(argv[0], cmdtab[i].cmd) != 0)

+			continue;

+		argv++;

+		argc--;

+		cmdtab[i].fn(&s3, fd, buf, argc, argv);

+		exits(nil);

+	}

+	sysfatal("unsupported cmd: %s", argv[0]);

+}

--- /dev/null

+++ b/factotum.c

@@ -1,0 +1,373 @@

+#include <u.h>

+#include <libc.h>

+#include <auth.h>

+#include <thread.h>

+#include <fcall.h>

+#include <9p.h>

+#include <mp.h>

+#include <libsec.h>

+

+static char *user;

+static Attr *creds;

+

+enum {

+	Sneedparam,

+	Shaveparam,

+};

+

+typedef struct State State;

+struct State {

+	uchar buf[SHA2_256dlen];

+	uint phase;

+};

+

+static uint

+aws4write(State *s, char *v, uint l)

+{

+	char input[512];

+	char *args[3];

+	int n;

+	char buf[256];

+

+	switch(s->phase){

+	case Shaveparam:

+		werrstr("%s", "read the results");

+		return ARphase;

+	case Sneedparam:

+		/* date region service */

+		snprint(input, sizeof input, "%.*s", (int)l, v);

+		n = tokenize(input, args, nelem(args));

+		if(n != nelem(args)){

+			werrstr("%s", "invalid rpc format");

+			return ARerror;

+		}

+		/* All lights are green */

+		snprint(buf, sizeof buf, "AWS4%s", _strfindattr(creds, "!secret"));

+		hmac_sha2_256((uchar*)args[0], strlen(args[0]), (uchar*)buf, strlen(buf), s->buf, nil);

+		hmac_sha2_256((uchar*)args[1], strlen(args[1]), s->buf, SHA2_256dlen, (uchar*)buf, nil);

+		hmac_sha2_256((uchar*)args[2], strlen(args[2]), (uchar*)buf, SHA2_256dlen, s->buf, nil);

+		hmac_sha2_256((uchar*)"aws4_request", 12, s->buf, SHA2_256dlen, s->buf, nil);

+		memset(input, 0, sizeof input);

+		memset(buf, 0, sizeof buf);

+		s->phase = Shaveparam;

+		return ARok;

+	}

+	return ARrpcfailure;

+}

+

+static uint

+aws4read(State *s, char *v, uint *l)

+{

+	switch(s->phase){

+	case Sneedparam:

+		werrstr("%s", "write the params");

+		return ARphase;

+	case Shaveparam:

+		assert(*l > SHA2_256dlen);

+		*l = SHA2_256dlen;

+		memmove(v, s->buf, SHA2_256dlen);

+		memset(s->buf, 0, SHA2_256dlen);

+		s->phase = Sneedparam;

+		return ARok;

+	}

+	return ARrpcfailure;

+}

+

+enum {

+	Qroot,

+	Qrpc,

+	Qctl,

+	Nqid,

+

+	Qfile = Qrpc,

+};

+

+static int

+attrfmt(Fmt *fmt)

+{

+	Attr *a;

+	int first = 1;

+

+	for(a=va_arg(fmt->args, Attr*); a != nil; a=a->next){

+		if(a->name == nil)

+			continue;

+		switch(a->type){

+		default:

+			continue;

+		case AttrQuery:

+			fmtprint(fmt, first+" %q?", a->name);

+			break;

+		case AttrNameval:

+		case AttrDefault:

+			if(a->name[0] == '!')

+				fmtprint(fmt, first+" %q", a->name);

+			else

+				fmtprint(fmt, first+" %q=%q", a->name, a->val);

+			break;

+		}

+		first = 0;

+	}

+	return 0;

+}

+

+struct {

+	char *name;

+	int mode;

+	int type;

+} qtab[Nqid] = {

+	"/",

+		DMDIR|0500,

+		QTDIR,

+	"rpc",

+		0600,

+		0,

+	"ctl",

+		0600,

+		0,

+};

+

+static int

+dirgen(int n, Dir *dir, void*)

+{

+	n += Qfile; /* offset to make dirread9p happy */

+

+	if(n >= Nqid)

+		return -1;

+	dir->name = estrdup9p(qtab[n].name);

+	dir->uid = estrdup9p(user);

+	dir->gid = estrdup9p(user);

+	dir->muid = estrdup9p("");

+	dir->qid = (Qid){n, 0, qtab[n].type};

+	dir->mode = qtab[n].mode;

+	return 0;

+}

+

+static void

+fsstat(Req *r)

+{

+	int path;

+

+	path = r->fid->qid.path;

+	assert(r->fid->qid.path < Nqid);

+	dirgen(path-Qfile, &r->d, nil);

+	respond(r, nil);

+}

+

+char Enonexist[] = "file does not exist";

+char Ewalk[] = "walk in non directory";

+

+static char*

+fswalk1(Fid *fid, char *name, Qid *qid)

+{

+	int i;

+	ulong path;

+

+	path = fid->qid.path;

+	switch(path){

+	case Qroot:

+		if(strcmp(name, "..") == 0)

+			name = "/";

+		for(i = path; i<Nqid; i++){

+			if(strcmp(name, qtab[i].name) != 0)

+				continue;

+			*qid = (Qid){i, 0, qtab[i].type};

+			fid->qid = *qid;

+			return nil;

+		}

+		return Enonexist;

+	default:

+		return Ewalk;

+	}

+}

+

+enum {

+	FSwrite,

+	FSread,	

+};

+

+typedef struct Xfid Xfid;

+struct Xfid {

+	State proto;

+	uint expect;

+	uint nmsg;

+	char msg[512];

+};

+

+static void

+fsopen(Req *r)

+{

+	if(r->fid->qid.path == Qrpc)

+		r->fid->aux = mallocz(sizeof(Xfid), 1);

+	respond(r, nil);

+}

+

+static void

+fsclose(Fid *f)

+{

+	if(f->qid.path == Qrpc)

+		free(f->aux);

+}

+

+static void

+fsread(Req *r)

+{

+	ulong path;

+	char buf[512];

+	Xfid *x;

+

+	path = r->fid->qid.path;

+	switch(path){

+	case Qroot:

+		dirread9p(r, dirgen, nil);

+		respond(r, nil);

+		break;

+	case Qctl:

+		snprint(buf, sizeof buf, "%A\n", creds);

+		readstr(r, buf);

+		respond(r, nil);

+		break;

+	case Qrpc:

+		x = r->fid->aux;

+		if(x->expect != FSread){

+			respond(r, "error read without a rpc verb write first");

+			break;

+		}

+		r->ifcall.offset = 0;

+		readbuf(r, x->msg, x->nmsg);

+		respond(r, nil);

+		x->expect = FSwrite;

+		break;

+	default:

+		respond(r, "not implemented");

+		break;

+	}

+}

+

+static void

+fswrite(Req *r)

+{

+	ulong path;

+	char buf[512];

+	char *proto;

+	Xfid *x;

+	uint res;

+	char outbuf[512];

+	uint noutbuf;

+

+	path = r->fid->qid.path;

+	r->ofcall.count = snprint(buf, sizeof buf, "%.*s", r->ifcall.count, r->ifcall.data);

+	switch(path){

+	case Qctl:

+		if(strncmp(buf, "key ", 4) == 0){

+			_freeattr(creds);

+			creds = _parseattr(buf+4);

+			if((proto = _strfindattr(creds, "proto")) == nil || strcmp(proto, "aws4") != 0)

+				respond(r, "proto!=aws4");

+			else

+				respond(r, nil);

+		} else if(strncmp(buf, "delkey", 6) == 0){

+			_freeattr(creds);

+			creds = nil;

+			respond(r, nil);

+		} else

+			respond(r, "unknown ctl msg");

+		memset(buf, 0, sizeof buf);

+		break;

+	case Qrpc:

+		x = r->fid->aux;

+		if(x->expect != FSwrite){

+			respond(r, "error write while there's data for you to read");

+			return;

+		}

+		if(strncmp(buf, "start", 5) == 0)

+			res = ARok;

+		else if(strncmp(buf, "write ", 6) == 0)

+			res = aws4write(&x->proto, buf+6, r->ifcall.count-6);

+		else if(strncmp(buf, "read", 4) == 0){

+			noutbuf = sizeof outbuf;

+			res = aws4read(&x->proto, outbuf, &noutbuf);

+			if(res == ARok){

+				x->nmsg = 2+1+SHA2_256dlen;

+				memcpy(x->msg, "ok ", 3);

+				memcpy(x->msg+3, outbuf, SHA2_256dlen);

+				respond(r, nil);

+				x->expect = FSread;

+				return;

+			}

+		} else {

+			respond(r, "invalid rpc verb");

+			return;

+		}

+		switch(res){

+		case ARok:

+			x->nmsg = snprint(x->msg, sizeof x->msg, "ok");

+			break;

+		case ARphase:

+			x->nmsg = snprint(x->msg, sizeof x->msg, "phase %r");

+			break;

+		default:

+			x->nmsg = snprint(x->msg, sizeof x->msg, "error %r");

+			break;

+		}

+		x->expect = FSread;

+		respond(r, nil);

+		break;

+	default:

+		respond(r, "not implemented");

+	}

+}

+

+static void

+fsattach(Req *r)

+{

+	r->fid->qid = (Qid){Qroot, 0, QTDIR};

+	r->ofcall.qid = r->fid->qid;

+	respond(r, nil);

+}

+

+Srv fs = {

+.read=fsread,

+.write=fswrite,

+.attach=fsattach,

+.stat=fsstat,

+.walk1=fswalk1,

+.open=fsopen,

+.destroyfid=fsclose,

+};

+

+_Noreturn static void

+usage(void)

+{

+	fprint(2, "Usage: %s [-D] [-s srv] [-m mntpt]\n", argv0);

+	exits("usage");

+}

+

+void

+main(int argc, char **argv)

+{

+	char *srv, *mntpt;

+

+	srv = nil;

+	mntpt = "/mnt/factotum";

+	ARGBEGIN{

+	case 'D':

+		chatty9p++;

+		break;

+	case 's':

+		srv = EARGF(usage());

+		break;

+	case 'm':

+		mntpt = EARGF(usage());

+		break;

+	default:

+		usage();

+	}ARGEND

+

+	user = getenv("user");

+	if(user == nil)

+		sysfatal("no $user");

+	quotefmtinstall();

+	fmtinstall('A', attrfmt);

+	postmountsrv(&fs, srv, mntpt, MBEFORE);

+	exits(nil);

+}

--- a/mkfile

+++ b/mkfile

@@ -1,14 +1,15 @@

 </$objtype/mkfile

-BIN=$home/bin/$objtype

+BIN=$home/bin/$objtype/s3

-TARG=s3

+TARG=\

+	cmd\

+	factotum\

 OFILES=\

 	xml.$O\

-	s3.$O\

 HFILES=\

 	xml.h\

-</sys/src/cmd/mkone

+</sys/src/cmd/mkmany

--- a/s3.c

+++ /dev/null

@@ -1,379 +1,0 @@

-#include <u.h>

-#include <libc.h>

-#include <bio.h>

-#include <mp.h>

-#include <libsec.h>

-#include "xml.h"

-

-typedef struct {

-	char *endpoint;

-	char *host;

-	char *access;

-	char *secret;

-	char *bucket;

-	char *region;

-} S3;

-

-typedef struct {

-	char method[16];

-	char time[128];

-	uchar payhash[SHA2_256dlen];

-	char authhdr[512];

-	char mime[32];

-} Hreq;

-

-static void

-datetime(char *date, int ndate, char *time, int ntime)

-{

-	Tm t;

-

-	tmnow(&t, nil);

-	snprint(date, ndate, "%τ", tmfmt(&t, "YYYYMMDD"));

-	snprint(time, ntime, "%sT%τZ", date, tmfmt(&t, "hhmmss"));

-}

-

-#define hmac(data, dlen, key, klen, digest) hmac_sha2_256(data, dlen, key, klen, digest, nil)

-

-static void

-mkkey(char *key, char *date, char *region, char *service, uchar out[SHA2_256dlen])

-{

-	char buf[256];

-

-	snprint(buf, sizeof buf, "AWS4%s", key);

-	hmac((uchar*)date, strlen(date), (uchar*)buf, strlen(buf), out);

-	hmac((uchar*)region, strlen(region), out, SHA2_256dlen, (uchar*)buf);

-	hmac((uchar*)service, strlen(service), (uchar*)buf, SHA2_256dlen, out);

-	hmac((uchar*)"aws4_request", 12, out, SHA2_256dlen, out);

-}

-

-static void

-mkhreq(Hreq *hreq, S3 *s3, char *method, char *path)

-{

-	char date[64];

-	uchar key[SHA2_256dlen], sig[SHA2_256dlen];

-	char buf[512], req[512];

-	char *sgndhdr;

-

-	datetime(date, sizeof date, hreq->time, sizeof hreq->time);

-	if(strcmp(method, "PUT") == 0){

-		snprint(buf, sizeof buf, "content-type:%s\nhost:%s\nx-amz-content-sha256:%.*lH\nx-amz-date:%s\n",

-			hreq->mime, s3->host, SHA2_256dlen, hreq->payhash, hreq->time);

-		sgndhdr = "content-type;host;x-amz-content-sha256;x-amz-date";

-	} else if(strcmp(method, "GET") == 0 || strcmp(method, "DELETE")==0){

-		hreq->mime[0] = 0;

-		sha2_256(nil, 0, hreq->payhash, nil);

-		snprint(buf, sizeof buf, "host:%s\nx-amz-date:%s\n", s3->host, hreq->time);

-		sgndhdr = "host;x-amz-date";

-	} else

-		sysfatal("invalid method");

-

-	snprint(req, sizeof req, "%s\n/%s/%s\n%s\n%s\n%s\n%.*lH",

-		method, s3->bucket, path, "", buf, sgndhdr, SHA2_256dlen, hreq->payhash);

-	sha2_256((uchar*)req, strlen(req), key, nil);

-	snprint(buf, sizeof buf, "%s\n%s\n%s/%s/%s/aws4_request\n%.*lH",

-		"AWS4-HMAC-SHA256", hreq->time, date, s3->region, "s3", SHA2_256dlen, key);

-	mkkey(s3->secret, date, s3->region, "s3", key);

-	hmac((uchar*)buf, strlen(buf), key, SHA2_256dlen, sig);

-

-	snprint(hreq->authhdr, sizeof hreq->authhdr, "%s Credential=%s/%s/%s/%s/aws4_request, SignedHeaders=%s, Signature=%.*lH",

-		"AWS4-HMAC-SHA256", s3->access, date, s3->region, "s3", sgndhdr, SHA2_256dlen, sig);

-	snprint(hreq->method, sizeof hreq->method, "%s", method);

-}

-

-static void

-prep(S3 *s3, int cfd, char *path, Hreq *hreq)

-{

-	if(fprint(cfd, "url %s/%s/%s", s3->endpoint, s3->bucket, path) < 0)

-		sysfatal("url: %r");

-	if(fprint(cfd, "request %s", hreq->method) < 0)

-		sysfatal("request: %r");

-	if(fprint(cfd, "headers Authorization:%s", hreq->authhdr) < 0)

-		sysfatal("headers2: %r");

-	if(fprint(cfd, "headers x-amz-date:%s\nx-amz-content-sha256:%.*lH", hreq->time, SHA2_256dlen, hreq->payhash) < 0)

-		sysfatal("headers: %r");

-	if(hreq->mime[0] != 0 && fprint(cfd, "contenttype %s", hreq->mime) < 0)

-		sysfatal("contenttype: %r");

-}

-

-static void

-download(S3 *s3, int cfd, char *conn, char *path, Biobuf *local)

-{

-	int bfd;

-	long n;

-	char buf[64];

-	char data[8192];

-	Hreq hreq;

-

-	mkhreq(&hreq, s3, "GET", path);

-	prep(s3, cfd, path, &hreq);

-	snprint(buf, sizeof buf, "/mnt/web/%s/body", conn);

-	bfd = open(buf, OREAD);

-	if(bfd < 0)

-		sysfatal("download body: %r");

-	for(;;){

-		n = read(bfd, data, sizeof data);

-		if(n < 0)

-			sysfatal("download body: %r");

-		if(n == 0)

-			return;

-		Bwrite(local, data, n);

-	}

-}

-

-static void

-mimetype(char *path, char *out, int nout)

-{

-	int p[2];

-	char buf[256];

-	int n;

-

-	pipe(p);

-	switch(fork()){

-	case 0:

-		close(0);

-		dup(p[0], 1);

-		close(2);

-		close(p[0]);

-		close(p[1]);

-		execl("/bin/file", "file", "-m", path, nil);

-		sysfatal("execl: %r");

-	default:

-		close(p[0]);

-		n = read(p[1], buf, sizeof buf - 1);

-		if(n <= 0)

-			sysfatal("no mimetype found");

-		buf[n - 1] = 0;

-		snprint(out, nout, "%s", buf);

-	}

-}

-

-static void

-upload(S3 *s3, int cfd, char *conn, char *localpath, char *remotepath)

-{

-	DigestState *ds;

-	uchar data[8192];

-	long n;

-	int fd, bfd;

-	char buf[256];

-	Hreq hreq;

-

-	mimetype(localpath, hreq.mime, sizeof hreq.mime);

-	fd = open(localpath, OREAD);

-	if(fd < 0)

-		sysfatal("upload open: %r");

-	for(ds = nil;;){

-		n = read(fd, data, sizeof data);

-		if(n < 0)

-			sysfatal("file read: %r");

-		if(n == 0)

-			break;

-		ds = sha2_256(data, n, nil, ds);

-	}

-	sha2_256(nil, 0, hreq.payhash, ds);

-	seek(fd, 0, 0);

-

-	mkhreq(&hreq, s3, "PUT", remotepath);

-	prep(s3, cfd, remotepath, &hreq);

-	snprint(buf, sizeof buf, "/mnt/web/%s/postbody", conn);

-	bfd = open(buf, OWRITE);

-	if(bfd < 0)

-		sysfatal("upload postbody open: %r");

-	for(;;){

-		n = read(fd, buf, sizeof buf);

-		if(n < 0)

-			sysfatal("file read: %r");

-		if(n == 0)

-			break;

-		if(write(bfd, buf, n) < 0)

-			sysfatal("upload write: %r");

-	}

-	close(bfd);

-

-	snprint(buf, sizeof buf, "/mnt/web/%s/body", conn);

-	if((bfd = open(buf, OREAD)) < 0)

-		sysfatal("open3: %r");

-	close(bfd);

-}

-

-static int

-parseuri(S3 *s3, char *path, int npath, char *arg)

-{

-	char *p;

-

-	if(strstr(arg, "s3://") != arg)

-		return -1;

-	arg+=5;

-	p = strchr(arg, '/');

-	if(p == nil || p == arg)

-		return -1;

-	snprint(path, npath, "%s", p+1);

-	s3->bucket = strdup(arg);

-	s3->bucket[p-arg] = 0;

-	return 0;

-}

-

-_Noreturn static void

-usage(void)

-{

-	fprint(2, "Requires $AWS_ACCESS_KEY_ID, $AWS_SECRET_ACCESS_KEY, and $AWS_ENDPOINT_URL_S3 defined\n");

-	fprint(2, "Usage: %s cat s3://bucket/file\n", argv0);

-	fprint(2, "Usage: %s cp source s3://bucket/destination\n", argv0);

-	fprint(2, "Usage: %s cp s3://bucket/source <destination>\n", argv0);

-	fprint(2, "Usage: %s rm s3://bucket/path\n", argv0);

-	fprint(2, "Usage: %s ls s3://bucket/prefix\n", argv0);

-	exits("usage");

-}

-

-static void

-cp(S3 *s3, int cfd, char *conn, int argc, char **argv)

-{

-	char path[512];

-	Biobuf *b;

-	int fd;

-

-	if(argc == 0 || argc > 2)

-		usage();

-	if(parseuri(s3, path, sizeof path, argv[0]) == 0){

-		if(argc > 1 && parseuri(s3, path, sizeof path, argv[1]) == 0)

-			sysfatal("s3:// → s3:// is not implemented");

-		if(argc == 1)

-			fd = 1;

-		else {

-			fd = create(argv[1], OWRITE, 0644);

-			if(fd < 0)

-				sysfatal("create: %r");

-		}

-		b = Bfdopen(fd, OWRITE);

-		if(b == nil)

-			sysfatal("Bfdopen: %r");

-		download(s3, cfd, conn, path, b);

-		return;

-	}

-	if(argc == 1 || parseuri(s3, path, sizeof path, argv[1]) < 0)

-		usage();

-	upload(s3, cfd, conn, argv[0], path);

-}

-

-static void

-delete(S3 *s3, int cfd, char *conn, int argc, char **argv)

-{

-	int fd;

-	char buf[256];

-	Hreq hreq;

-	char path[512];

-

-	if(argc == 0)

-		usage();

-	if(parseuri(s3, path, sizeof path, argv[0]) < 0)

-		usage();

-	mkhreq(&hreq, s3, "DELETE", path);

-	prep(s3, cfd, path, &hreq);

-	snprint(buf, sizeof buf, "/mnt/web/%s/body", conn);

-	fd = open(buf, OREAD);

-	if(fd < 0)

-		sysfatal("delete failed: %r");

-	close(fd);

-}

-

-static void

-list(S3 *s3, int cfd, char *conn, int argc, char **argv)

-{

-	int p[2];

-	Biobuf *b[2];

-	Xelem *x;

-	char path[512];

-

-	if(argc == 0)

-		usage();

-	if(parseuri(s3, path, sizeof path, argv[0]) < 0)

-		usage();

-	if(pipe(p) < 0)

-		sysfatal("pipe: %r");

-	switch(fork()){

-	case -1:

-		sysfatal("fork: %r");

-	case 0:

-		close(p[1]);

-		b[0] = Bfdopen(p[0], OWRITE);

-		if(b[0] == nil)

-			sysfatal("Bfdopen: %r");

-		download(s3, cfd, conn, path, b[0]);

-		exits(nil);

-	default:

-		close(p[0]);

-		break;

-	}

-	b[1] = Bfdopen(p[1], OREAD);

-	if(b[1] == nil)

-		sysfatal("Bfdopen: %r");

-	x = xmlread(b[1], 0);

-	if(x == nil)

-		sysfatal("file was not valid XML, maybe not a prefix?");

-	if((x = xmlget(x, "Contents", nil)) == nil)

-		sysfatal("xml did not have Contents field");

-

-	for(; x != nil && xmlget(x, "Key", nil) != nil; x = x->next)

-		print("%s\n", xmlget(x, "Key", nil)->v);

-}

-

-struct {

-	char *cmd;

-	void (*fn)(S3*,int,char*,int,char**);

-} cmdtab[] = {

-	{ "cp", cp },

-	{ "cat", cp },

-	{ "rm", delete },

-	{ "ls", list },

-};

-

-void

-main(int argc , char **argv)

-{

-	S3 s3;

-	int fd;

-	int i;

-	long n;

-	char buf[64];

-

-	tmfmtinstall();

-	fmtinstall('H', encodefmt);

-	ARGBEGIN{

-	default:

-		usage();

-		break;

-	}ARGEND

-	if(argc == 0)

-		usage();

-

-	s3.access = getenv("AWS_ACCESS_KEY_ID");

-	s3.secret = getenv("AWS_SECRET_ACCESS_KEY");

-	s3.endpoint = getenv("AWS_ENDPOINT_URL_S3");

-	s3.region = getenv("AWS_DEFAULT_REGION");

-	if(s3.access == nil || s3.secret == nil || s3.endpoint == nil)

-		usage();

-	if(s3.region == nil)

-		s3.region = "auto";

-

-	s3.host = strstr(s3.endpoint, "://");

-	if(s3.host == nil)

-		sysfatal("invalid endpoint url");

-	s3.host += 3;

-

-	fd = open("/mnt/web/clone", ORDWR);

-	if(fd < 0)

-		sysfatal("open: %r");

-	n = read(fd, buf, sizeof buf - 1);

-	if(n <= 0)

-		sysfatal("read: %r");

-	buf[n-1] = 0;

-

-	for(i = 0; i < nelem(cmdtab); i++){

-		if(strcmp(argv[0], cmdtab[i].cmd) != 0)

-			continue;

-		argv++;

-		argc--;

-		cmdtab[i].fn(&s3, fd, buf, argc, argv);

-		exits(nil);

-	}

-	sysfatal("unsupported cmd: %s", argv[0]);

-}

--- /dev/null

+++ b/test/factotum.c

@@ -1,0 +1,79 @@

+#include <u.h>

+#include <libc.h>

+#include <auth.h>

+#include <mp.h>

+#include <libsec.h>

+

+#define hmac(data, dlen, key, klen, digest) hmac_sha2_256(data, dlen, key, klen, digest, nil)

+

+static void

+mkkey(char *key, char *date, char *region, char *service, uchar out[SHA2_256dlen])

+{

+	char buf[256];

+

+	snprint(buf, sizeof buf, "AWS4%s", key);

+	hmac((uchar*)date, strlen(date), (uchar*)buf, strlen(buf), out);

+	hmac((uchar*)region, strlen(region), out, SHA2_256dlen, (uchar*)buf);

+	hmac((uchar*)service, strlen(service), (uchar*)buf, SHA2_256dlen, out);

+	hmac((uchar*)"aws4_request", 12, out, SHA2_256dlen, out);

+}

+

+void

+main(int,char**)

+{

+	char *o;

+	char *s;

+	int pid;

+	int fd;

+	AuthRpc *rpc;

+	int ret;

+	uchar buf[SHA2_256dlen];

+

+	rfork(RFNAMEG);

+	o = getenv("O");

+	if(o == nil)

+		sysfatal("no $O");

+	switch(pid = fork()){

+	case -1:

+		sysfatal("fork");

+	case 0:

+		execl(smprint("../%s.factotum", o), "factotum", nil);

+		sysfatal("exec");

+	default:

+		while(waitpid() != pid)

+			;

+		break;

+	}

+

+	fmtinstall('H', encodefmt);

+	fd = open("/mnt/factotum/ctl", OWRITE);

+	if(fd < 0)

+		sysfatal("open: %r");

+	fprint(fd, "key proto=aws4 !secret=blah");

+	close(fd);

+

+	fd = open("/mnt/factotum/rpc", ORDWR);

+	if(fd < 0)

+		sysfatal("open: %r");

+

+	rpc = auth_allocrpc(fd);

+	ret = auth_rpc(rpc, "start", "proto=aws4", strlen("proto=aws4"));

+	if(ret != ARok)

+		sysfatal("start: %r");

+

+	s = smprint("somedate auto s3");

+	ret = auth_rpc(rpc, "write", s, strlen(s));

+	if(ret != ARok)

+		sysfatal("write: %r");

+

+	ret = auth_rpc(rpc, "read", nil, 0);

+	if(ret != ARok)

+		sysfatal("read: %r");

+	mkkey("blah", "somedate", "auto", "s3", buf);

+

+	if(memcmp(rpc->arg, buf, SHA2_256dlen) != 0)

+		sysfatal("mismatch %.*lH %.*lH", SHA2_256dlen, rpc->arg, SHA2_256dlen, buf);

+

+	auth_freerpc(rpc);

+	exits(nil);

+}

--- /dev/null

+++ b/test/mkfile

@@ -1,0 +1,6 @@

+</$objtype/mkfile

+

+TEST=\

+	factotum

+

+</sys/src/cmd/mktest

-- 

⑨