shithub: dddb

Download patch

ref: aee91c4ed0277e343a62cd124ee7175f54ffefb3
author: Linux User <l@malkuth.lan>
date: Sat Jul 22 12:17:55 EDT 2023 

wip - main, config, ctlfs


--- /dev/null
+++ b/.gitignore
@@ -1,0 +1,2 @@
+*.dis
+*.sbl
--- /dev/null
+++ b/LICENSE
@@ -1,0 +1,27 @@
+Copyright © 2023, kitzman @ disroot.org
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice,
+      this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright notice,
+      this list of conditions and the following disclaimer in the documentation
+      and/or other materials provided with the distribution.
+    * Neither the name of {{ project }} nor the names of its contributors
+      may be used to endorse or promote products derived from this software
+      without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
--- /dev/null
+++ b/README
@@ -1,0 +1,32 @@
+dddb (Distributed DisVM Database) is a WIP RDBMS, aimed
+to run on DisVM, hosted or native.
+
+TODO Components and functionalities
+
+* dddbctl fs -	authenticated 9p fs to control, request, and query
+				operations related to the database instance; it
+				is the main driver for grid database
+* dddbclient fs -	the tcp port 'dddb' is used such that clients
+					can have a fs in which queries can be opened;
+					it will support ANSI SQL
+* db registry -	each instance will, on startup, connect to
+				other instances defined in an ndb configuration
+				file, register itself in that instance, and
+				will be able to use it for: duplicating data,
+				sharding data, and mount further instances;
+				(i.e if a cluster is behind a firewall, and
+				only one instance exposed, the hidden ones
+				can be mounted outside the fw)
+* data structures (1) -	primitives, pages, sections, indices,
+						tables, rows
+* data structures (2) - B-trees, Bloom filters
+* storage operations -	a mounted directory tree will be used
+						for RDRW operations; memfs will be used
+						for caching the data in RAM (performance?)
+* venti/vac support	-	venti + vac + memfs should suffice for
+						storing and persisting data (ramfossil-like);
+						this means scripts will clean and archive it
+* SQL parser -			obviously
+* statement optimizer -	obviously
+* wm/dddbmon -			a nice admin monitor would be nice
+* dddbjdbc/dddbodbc -	JDBC and ODBC drivers are mandatory
--- /dev/null
+++ b/appl/cmd/config.b
@@ -1,0 +1,118 @@
+include "bufio.m";
+include "attrdb.m";
+	attrdb: Attrdb;
+	Attr, Db, Dbptr, Dbentry: import attrdb;
+
+# Default configuration values
+DCFGPATH:	con "/lib/ndb/dddbcfg";
+
+DADDR:		con "tcp!*!dddbctl";
+DFSWRKS:	con 10;
+
+# Attrdb constants 
+KNAME:		con "nodename";
+KSYSNAME:	con "nodesysn";
+KADDR:		con "addr";
+KSTORAGE:	con "storage";
+KFSWRKS:	con "readworkers";
+
+Config.open(nodename: string, path: string): Config
+{
+	sys = load Sys Sys->PATH;
+
+	n: int;
+	sysname: string;
+	buf := array[Sys->NAMEMAX] of byte;
+
+	attrdb = load Attrdb Attrdb->PATH;
+	if(attrdb == nil)
+		error("Attrb not found");
+	attrdb->init();
+
+	if(len path == 0)
+		path = DCFGPATH;
+
+	if(debug)
+		sys->fprint(stderr, "config: opening %s\n", path);
+	db := Db.open(path);
+
+	if(db == nil)
+		error(sys->sprint("ndb file %s could not be opened", path));
+
+	sysname_fd := sys->open("#c/sysname", Sys->OREAD);
+	if(sysname_fd == nil)
+		error(sys->sprint("#c/sysname could not be read"));
+	n = sys->read(sysname_fd, buf, len buf);
+	if(n <= 0)
+		error(sys->sprint("could not read sysname"));
+	sysname = string buf[0:n];
+
+	if(len nodename == 0) {
+		nodename = sysname;
+	}
+
+	thiscfg: ref NodeConfig;
+	nodecfgs: list of NodeConfig;
+	entry: ref Dbentry;
+	dbptr: ref Dbptr;
+
+	(entry, dbptr) = db.find(dbptr, KNAME);
+	while(entry != nil) {
+		nodecfg := NodeConfig.new(entry);
+
+		if(debug)
+			sys->fprint(stderr, "config: found node %s\n", nodecfg.name);
+		if(nodecfg.name == nodename) {
+			thiscfg = ref nodecfg;
+		}
+		else
+			nodecfgs = nodecfg :: nodecfgs;
+
+		(entry, dbptr) = db.find(dbptr, KNAME);
+	}
+
+	if(thiscfg == nil)
+		error("could not find config for nodename");
+
+	if(thiscfg.addr == "")
+		error("node lacks an address");
+
+	if(thiscfg.storage == "")
+		error("node lacks a storage path");
+
+	if(thiscfg.sysn != sysname)
+		error("node and system sysname do not match");
+
+	return Config(
+		thiscfg.name, thiscfg.sysn, thiscfg.addr,	# own configuration
+		thiscfg.storage, thiscfg.fswrks,
+		nodecfgs);									# configured nodes
+}
+
+NodeConfig.new(entry: ref Dbentry): NodeConfig
+{
+	name := entry.findfirst(KNAME);
+	sysname := entry.findfirst(KSYSNAME);
+	addr := entry.findfirst(KADDR);
+	storage := entry.findfirst(KSTORAGE);
+	fswrks_s := entry.findfirst(KFSWRKS);
+
+	fswrks := DFSWRKS;
+
+	if(len name == 0)
+		error("entry has no name");
+	if(len sysname == 0)
+		error("entry has no sysname");
+	if(len addr == 0)
+		addr = DADDR;
+	if(len fswrks_s != 0) {
+		(fswrks_i, rm) := strm->toint(fswrks_s, 10);
+		if(rm != "")
+			error(sys->sprint("malformed fs workers count: %s", fswrks_s));
+		fswrks = fswrks_i;
+	}
+
+	return NodeConfig(
+		name, sysname, addr, storage,	# basic information
+		fswrks);						# tunable options
+}
--- /dev/null
+++ b/appl/cmd/ctlfs.b
@@ -1,0 +1,222 @@
+include "dial.m";
+	dial: Dial;
+
+include "security.m";
+	auth: Auth;
+
+include "styx.m";
+	styx: Styx;
+	Tmsg, Rmsg: import Styx;
+
+include "styxservers.m";
+	styxservers: Styxservers;
+	Styxserver, Fid, Navigator,
+	Navop, Enotfound, Enotdir: import styxservers;
+
+# FS file index
+Qroot, Qctl, Qstats, Qmax: con iota;
+tab := array[] of {
+	(Qroot, ".", Sys->DMDIR|8r555),
+	(Qctl, "ctl", 8r222),
+	(Qstats, "stats", 8r111),
+};
+
+# create ctlfs and the appropriate listeners
+init_ctlfs(cfg: Config, keyfile: string, algs: list of string)
+{
+	dial = load Dial Dial->PATH;
+	auth = load Auth Auth->PATH;
+	styx = load Styx Styx->PATH;
+
+	if(dial == nil)
+		error("ctlfs: dial module not found");
+	if(auth == nil)
+		error("ctlfs: auth module not found");
+	if(styx == nil)
+		error("ctlfs: styx module not found");
+
+	auth->init();
+
+	styx->init();
+	styxservers->init(styx);
+	styxservers->traceset(chatty);
+
+	# authinfo init
+	if(debug)
+		sys->fprint(stderr, "ctlfs: reading authinfo");
+	authinfo: ref Keyring->Authinfo;
+	if (doauth) {
+		if (keyfile == nil)
+			keyfile = "/usr/" + user() + "/keyring/default";
+		authinfo = keyring->readauthinfo(keyfile);
+		if (authinfo == nil)
+			error(sys->sprint("ctlfs: cannot read %s: %r", keyfile));
+	}
+
+	# announcing
+	if(debug)
+		sys->fprint(stderr, "ctlfs: announcing dddbctl");
+	addr := dial->netmkaddr(cfg.addr, "tcp", "dddbctl");
+	c := dial->announce(addr);
+	if(c == nil)
+		error(sys->sprint("ctlfs: cannot listen on %s\n", addr));
+
+	# bootstrapping
+	if(debug)
+		sys->fprint
+	sys->unmount(nil, "/mnt/keys");
+
+	navch := chan of ref Navop;
+	spawn ctlfs_navigator(navch);
+
+	nav := Navigator.new(navch);
+	(tc, srv) := Styxserver.new(fildes(0), nav, big Qroot);
+
+	# listener entrypoint
+	listener(c, authinfo, algs);
+}
+
+# dddbctl listener loop
+ctlfs_listener(c: ref Dial->Connection, authinfo: ref Keyring->Authinfo, algs: list of string)
+{
+	for (;;) {
+		nc := dial->listen(c);
+		if (nc == nil)
+			error(sys->sprint("listen failed: %r"));
+		if (debug)
+			sys->fprint(stderr, "ctlfs: got connection from %s\n",
+						readfile(nc.dir + "/remote"));
+		dfd := dial->accept(nc);
+		if (dfd != nil) {
+			if(nc.cfd != nil)
+				sys->fprint(nc.cfd, "keepalive");
+			hostname: string;
+			if(passhostnames){
+				hostname = readfile(nc.dir + "/remote");
+				if(hostname != nil)
+					hostname = hostname[0:len hostname - 1];
+			}
+
+			spawn ctlfs_authenticator(dfd, authinfo, algs, hostname);
+		}
+	}
+}
+
+# authenticate a connection and set the user id.
+ctlfs_authenticator(dfd: ref Sys->FD, authinfo: ref Keyring->Authinfo,
+		algs: list of string, hostname: string)
+{
+	# authenticate and change user id appropriately
+	(fd, err) := auth->server(algs, authinfo, dfd, 1);
+	if (fd == nil) {
+		if (debug)
+			sys->fprint(stderr(), "ctlfs: authentication failed: %s\n", err);
+		return;
+	}
+	if (debug)
+		sys->fprint(stderr(), "ctlfs: client authenticated as %s\n", err);
+
+	spawn exportproc(sync, mfd, err, hostname, fd);
+}
+
+ctlfs_loop()
+{
+	# Primary server loop
+	loop:
+	while((tmsg := <-tc) != nil) {
+		# Switch on operations being performed on a given Fid
+		pick msg := tmsg {
+		Open =>
+			srv.default(msg);
+		Read =>
+			fid := srv.getfid(msg.fid);
+
+			if(fid.qtype & Sys->QTDIR) {
+				# This is a directory read
+				srv.default(msg);
+				continue loop;
+			}
+
+			case int fid.path {
+			Qlog =>
+				# A read on our log file, tell them what they've already said ?
+				s := "";
+
+				for(l := log; l != nil; l = tl l)
+					s = hd l + s;
+
+				srv.reply(styxservers->readstr(msg, s));
+
+			* =>
+				srv.default(msg);
+			}
+
+		Write =>
+			fid := srv.getfid(msg.fid);
+
+			case int fid.path {
+			Qctl =>
+				# Don't care about offset
+				cmd := string msg.data;
+
+				reply: ref Rmsg = ref Rmsg.Write(msg.tag, len msg.data);
+
+				case cmd {
+				* =>
+					# Ignore empty writes
+					if(cmd != nil)
+						log = cmd :: log;
+					else
+						reply = ref Rmsg.Error(msg.tag, "empty write!");
+				}
+				srv.reply(reply);
+				
+			* =>
+				srv.default(msg);
+			}
+
+		* =>
+			srv.default(msg);
+		}
+	}
+
+	exit;
+}
+
+# Navigator function for moving around under /
+ctlfs_navigator(c: chan of ref Navop) {
+	loop: 
+	for(;;) {
+		navop := <-c;
+		pick op := navop {
+		Stat =>
+			op.reply <-= (dir(int op.path), nil);
+			
+		Walk =>
+			if(op.name == "..") {
+				op.reply <-= (dir(Qroot), nil);
+				continue loop;
+			}
+
+			case int op.path&16rff {
+
+			Qroot =>
+				for(i := 1; i < Qmax; i++)
+					if(tab[i].t1 == op.name) {
+						op.reply <-= (dir(i), nil);
+						continue loop;
+					}
+
+				op.reply <-= (nil, Enotfound);
+			* =>
+				op.reply <-= (nil, Enotdir);
+			}
+			
+		Readdir =>
+			for(i := 0; i < op.count && i + op.offset < (len tab) - 1; i++)
+				op.reply <-= (dir(Qroot+1+i+op.offset), nil);
+
+			op.reply <-= (nil, nil);
+		}
+	}
+}
--- /dev/null
+++ b/appl/cmd/dddb.b
@@ -1,0 +1,94 @@
+implement Dddb;
+
+include "sys.m";
+	sys: Sys;
+include "arg.m";
+include "draw.m";
+include "string.m";
+	strm: String;
+
+include "config.b";
+include "ctlfs.b";
+
+stderr: ref Sys->FD;
+debug: int;
+
+error(s: string)
+{
+	sys->fprint(stderr, "dddb: %s\n", s);
+	raise "dddb:error";
+}
+
+Dddb: module {
+	init: fn(nil: ref Draw->Context, args: list of string);
+	run_fs: fn(cfg: Config);
+
+	Config: adt {
+		name:		string;
+		sysn:		string;
+		addr:		string;
+		storage:	string;
+		fswrks:		int;
+		nodes:		list of NodeConfig;
+
+		open:		fn(nodename: string, path: string): Config;
+	};
+
+	NodeConfig: adt {
+		name:		string;
+		sysn:		string;
+		addr:		string;
+		storage:	string;
+		fswrks:		int;
+
+		new:		fn(entry: ref Dbentry): NodeConfig;
+	};
+
+};
+
+init(nil: ref Draw->Context, args: list of string)
+{
+	sys = load Sys Sys->PATH;
+	arg := load Arg Arg->PATH;
+	strm = load String String->PATH;
+
+	stderr = sys->fildes(2);
+	cfgpath: string = "";
+
+	arg->init(args);
+	arg->setusage(arg->progname()+ " [-d] [-c config] nodename");
+	while((c := arg->opt()) != 0)
+		case c {
+		'd' => debug++;
+		'c' =>
+			cfgpath = arg->earg();
+		* =>
+			sys->fprint(sys->fildes(2), "bad option: -%c\n", c);
+			arg->usage();
+		}
+
+	args = arg->argv();
+
+	nodename := hd args;
+
+	if(nodename == nil) {
+		sys->fprint(stderr, "dddb: no nodename supplied\n");
+		arg->usage();
+	}
+
+	if(debug)
+		sys->fprint(stderr, "dddb: opening config file\n");
+	cfg := Config.open(nodename, cfgpath);
+
+	if(debug) {
+		sys->fprint(stderr, "dddb: database parms:\n");
+		sys->fprint(stderr, "cfg.name: %s\n", cfg.name);
+		sys->fprint(stderr, "cfg.sysn: %s\n", cfg.sysn);
+		sys->fprint(stderr, "cfg.storage: %s\n", cfg.storage);
+		sys->fprint(stderr, "cfg.fswrks: %d\n", cfg.fswrks);
+	}
+
+	run_fs(cfg);
+}
+
+
--- /dev/null
+++ b/appl/cmd/mkfile
@@ -1,0 +1,18 @@
+<../../mkconfig
+
+TARG=\
+        dddb.dis\
+
+MODULES=\
+
+SYSMODULES=\
+	arg.m\
+	dial.m\
+	draw.m\
+	sys.m\
+	styx.m\
+	styxservers.m\
+
+DISBIN=$home/dis
+
+<$ROOT/mkfiles/mkdis
--- /dev/null
+++ b/appl/lib/mkfile
@@ -1,0 +1,11 @@
+<../../mkconfig
+
+TARG=\
+
+MODULES=\
+
+SYSMODULES=\
+
+DISBIN=$ROOT/dis/lib
+
+<$ROOT/mkfiles/mkdis
--- /dev/null
+++ b/appl/mkfile
@@ -1,0 +1,6 @@
+<../mkconfig
+
+DIRS=\
+	cmd\
+
+<$ROOT/mkfiles/mksubdirs
--- /dev/null
+++ b/mkconfig
@@ -1,0 +1,6 @@
+<$ROOT/mkconfig
+
+# Comment the below lines for hosted builds
+SYSHOST=Inferno
+SYSTARG=Inferno
+ROOT=/
--- /dev/null
+++ b/mkfile
@@ -1,0 +1,6 @@
+<mkconfig
+
+DIRS=\
+	appl\
+
+<$ROOT/mkfiles/mksubdirs