diff --git a/lib/radisk3.js b/lib/radisk3.js index 10028c38..8c5cb1ce 100644 --- a/lib/radisk3.js +++ b/lib/radisk3.js @@ -1,159 +1,166 @@ -;(function(){ // RAD -console.log("Warning: Experimental rewrite of RAD to use Book. It is not API compatible with RAD yet and is very alpha."); -var sT = setTimeout, Book = sT.Book, RAD = sT.RAD || (sT.RAD = function(){ - var r = async function rad(word, is, reply){ - if(!b){ start(word, is, reply); return r } - if(is === undefined || 'function' == typeof is){ // THIS IS A READ: - var page = b.page(word); - if(page.from){ - return is(null, page); +; (function () { // RAD + console.log("Warning: Experimental rewrite of RAD to use Book. It is not API compatible with RAD yet and is very alpha."); + var sT = setTimeout, Book = sT.Book, RAD = sT.RAD || (sT.RAD = function (opt) { + opt = opt || {}; + opt.file = String(opt.file || 'radata'); + var log = opt.log || nope; + + var has = (sT.RAD.has || (sT.RAD.has = {}))[opt.file]; + if (has) { return has } + var r = function rad(word, is, reply) { + if (!b) { start(word, is, reply); return r } + if (is === undefined || 'function' == typeof is) { // THIS IS A READ: + var page = b.page(word); + if (page.from) { + return is(null, page); + } + read(word, is, page); // get from disk + return } - read(word, is, page); // get from disk - return - } - //console.log("OFF");return; - // ON WRITE: - // batch until read from disk is done (and if a write was going, do that first) - b(word, is); - write(word, reply); - return r; - }, b; - - -async function read(word, reply, page){ - var p = page;//b.page(word); - get(p, function(err, disk){ - if(err){ console.log("ERR!"); return } - p.from = disk || p.from; - reply && reply(err, p); - }) -} - -async function write(word, reply){ - var p = b.page(word), tmp; - if(tmp = p.saving){ reply && tmp.push(reply); return } p.saving = [reply]; - var S = +new Date; console.log("writing", p.substring(), 'since last', S - p.saved, RAD.c, 'records', env.count++, 'mid-swap.'); - get(p, function(err, disk){ - if(err){ console.log("ERR!"); return } - console.log("MERGE:", p.substring(), disk); - p.from = disk || p.from; // TODO: NEED TO MERGE! AND HANDLE ERR! - //p.list = p.text = p.from = 0; - //p.first = p.first.word || p.first; - tmp = p.saving; p.saving = []; - put(p, ''+p, function(err, ok){ - env.count--; p.saved = +new Date; //console.log("wrote", p.substring(), (p.saved = +new Date) - S); - if(!p.saving.length){ p.saving = 0; return } p.saving = 0; + //console.log("OFF");return; + // ON WRITE: + // batch until read from disk is done (and if a write was going, do that first) + b(word, is); write(word, reply); - }); - }) -} -function put(file, data, cb){ - file.first && (file = Book.slot(file.first)[0]); - put[file = fname(file)] = {data: data}; - RAD.put(file, data, function(err, ok){ - delete put[file]; - cb && cb(err, ok); - }); -}; -function get(file, cb){ var tmp; - file.first && (file = Book.slot(file.first)[0]); - if(tmp = put[file = fname(file)]){ cb(u, tmp.data); return } - if(tmp = get[file]){ tmp.push(cb); return } get[file] = [cb]; - RAD.get(file, function(err, data){ - tmp = get[file]; delete get[file]; - var i = -1, f; while(f = tmp[++i]){ f(err, data) } // CPU SCHEDULE? - }); -}; + return r; + }, /** @param b the book */ b; -function start(word, is, reply){ - if(b){ r(word, is, reply); return } - get(' ', function(err, d){ - if(err){ reply && reply(err); return } - if(b){ r(word, is, reply); return } - //wrap(b = r.book = Book(d)); - (b = r.book = Book()).list = Book.slot(d); - watch(b).list[0] = "'!'"; - r(word, is, reply); - }) -} -function watch(b){ // SPLIT LOGIC! - var split = b.split; - b.list.toString = function(){ - console.time(); - var i = -1, t = '', p; while(p = this[++i]){ - t += "|"+p.substring(); + + async function read(word, reply, page) { + var p = page;//b.page(word); + reply = reply.call ? reply : () => { }; + log(`read() ${word.slice(0, 40)}`); + get(p, function (err, disk) { + if (err) { log("ERR! in read() get() cb", err); reply(err); return } + p.from = disk || p.from; + reply(null, p, b); + }) } - t += "|"; - console.timeEnd(); - return t; + + async function write(word, reply) { + log('write() word', word); + var p = b.page(word), tmp; + if (tmp = p.saving) { reply && tmp.push(reply); return } p.saving = [reply]; + var S = +new Date; log(" writing", p.substring(), 'since last', S - p.saved, RAD.c, 'records', env.count++, 'mid-swap.'); + get(p, function (err, disk) { + if (err) { log("ERR! in write() get() cb ", err); return } + log(' get() - p.saving ', (p.saving || []).length); + if (p.from && disk) { + log(" get() merge: p.from ", p.toString().slice(0, 40), " disk.length", disk?.length || 0); + } + p.from = disk || p.from; // TODO: NEED TO MERGE! AND HANDLE ERR! + // p.list = p.text = p.from = 0; + // p.first = p.first.word || p.first; + tmp = p.saving; p.saving = []; + put(p, '' + p, function (err, ok) { + env.count--; p.saved = +new Date; log(" ...wrote %d bytes in %dms", ('' + p).length, (p.saved = +new Date) - S); + if (!p.saving.length) { p.saving = 0; reply?.call && reply(err, ok); return; } p.saving = 0; // what? + // log({ tmp }); + write(word, reply); + }); + }) + } + function put(file, data, cb) { + file.first && (file = Book.slot(file.first)[0]); + put[file = fname(file)] = { data: data }; + RAD.put(file, data, function (err, ok) { + delete put[file]; + cb && cb(err, ok); + }); + }; + function get(file, cb) { + var tmp; + file.first && (file = Book.slot(file.first)[0]); + if (tmp = put[file = fname(file)]) { cb(u, tmp.data); return } + if (tmp = get[file]) { tmp.push(cb); return } get[file] = [cb]; + RAD.get(file, function (err, data) { + tmp = get[file]; delete get[file]; + var i = -1, f; while (f = tmp[++i]) { f(err, data) } // CPU SCHEDULE? + }); + }; + + function start(word, is, reply) { + if (b) { r(word, is, reply); return } + get(' ', function (err, d) { + if (err) { log('ERR! in start() get()', err); reply && reply(err); return } + if (b) { r(word, is, reply); return } + //wrap(b = r.book = Book(d)); + (b = r.book = Book()).list = Book.slot(d); + watch(b).list[0] = "'!'"; + r(word, is, reply); + }) + } + function watch(b) { // SPLIT LOGIC! + var split = b.split; + b.list.toString = function () { + console.time(); + var i = -1, t = '', p; while (p = this[++i]) { + t += "|" + p.substring(); + } + t += "|"; + console.timeEnd(); + return t; + } + b.split = function (next, page) { + log("SPLIT!!!!", b.list.length); + put(' ', '' + b.list, function (err, ok) { + if (err) { console.log("ERR!"); return } + // ?? + }); + } + return b; + } + + function ename(t) { return encodeURIComponent(t).replace(/\*/g, '%2A').slice(0, 250) } + function fname(p) { return opt.file + '/' + ename(p.substring()) } + + return r; + }), MAX = 1000/* 300000000 */; + + try { module.exports = RAD } catch (e) { } + + // junk below that needs to be cleaned up and corrected for the actual correct RAD API. + var env = {}, nope = function () { }, nah = function () { return nope }, u; + env.require = (typeof require !== '' + u && require) || nope; + env.process = (typeof process != '' + u && process) || { memoryUsage: nah }; + env.os = env.require('os') || { totalmem: nope, freemem: nope }; + env.v8 = env.require('v8') || { getHeapStatistics: nah }; + env.fs = env.require('fs') || { writeFile: nope, readFile: nope }; + + + env.max = env.v8.getHeapStatistics().total_available_size / (2 ** 12); + + env.count = env.last = 0; + return; + + //if(err && 'ENOENT' === (err.code||'').toUpperCase()){ err = null } + + setInterval(function () { + var stats = { memory: {} }; + + stats.memory.total = env.os.totalmem() / 1024 / 1024; // in MB + stats.memory.free = env.os.freemem() / 1024 / 1024; // in MB + stats.memory.hused = env.v8.getHeapStatistics().used_heap_size / 1024 / 1024; // in MB + stats.memory.used = env.process.memoryUsage().rss / 1024 / 1024; // in MB + console.log(stats.memory); + }, 9); + +}()); + + +; (function () { // temporary fs storage plugin, needs to be refactored to use the actual RAD plugin interface. + var fs; + try { fs = require('fs') } catch (e) { }; + if (!fs) { return } + + var sT = setTimeout, RAD = sT.RAD; + RAD.put = function (file, data, cb) { + fs.writeFile(file, data, cb); } - b.split = function(next, page){ - console.log("SPLIT!!!!", b.list.length); - put(' ', ''+b.list, function(err, ok){ - if(err){ console.log("ERR!"); return } - // ?? + RAD.get = function (file, cb) { + fs.readFile(file, function (err, data) { + if (err && 'ENOENT' === (err.code || '').toUpperCase()) { return cb() } + cb(err, data.toString()); }); } - return b; -} - - - return r; -}), MAX = 1000/* 300000000 */; - -function ename(t){ return encodeURIComponent(t).replace(/\*/g, '%2A').slice(0,250) } -function fname(p){ return opt.file+'/'+ename(p.substring()) } - -var opt = {}; -opt.file = String(opt.file || 'radata'); - -try{module.exports=RAD}catch(e){} - -// junk below that needs to be cleaned up and corrected for the actual correct RAD API. -var env = {}, nope = function(){}, nah = function(){ return nope }, u; -env.require = (typeof require !== ''+u && require) || nope; -env.process = (typeof process != ''+u && process) || {memoryUsage: nah}; -env.os = env.require('os') || {totalmem: nope, freemem: nope}; -env.v8 = env.require('v8') || {getHeapStatistics: nah}; -env.fs = env.require('fs') || {writeFile: nope, readFile: nope}; - - -env.max = env.v8.getHeapStatistics().total_available_size / (2**12); - -env.count = env.last = 0; -return; - -//if(err && 'ENOENT' === (err.code||'').toUpperCase()){ err = null } - -setInterval(function(){ - var stats = {memory: {}}; - - stats.memory.total = env.os.totalmem() / 1024 / 1024; // in MB - stats.memory.free = env.os.freemem() / 1024 / 1024; // in MB - stats.memory.hused = env.v8.getHeapStatistics().used_heap_size / 1024 / 1024; // in MB - stats.memory.used = env.process.memoryUsage().rss / 1024 / 1024; // in MB - console.log(stats.memory); -}, 9); - -}()); - - - - - -;(function(){ // temporary fs storage plugin, needs to be refactored to use the actual RAD plugin interface. -var fs; -try{fs = require('fs')}catch(e){}; -if(!fs){ return } - -var sT = setTimeout, RAD = sT.RAD; -RAD.put = function(file, data, cb){ - fs.writeFile(file, data, cb); -} -RAD.get = function(file, cb){ - fs.readFile(file, function(err, data){ - if(err && 'ENOENT' === (err.code||'').toUpperCase()){ return cb() } - cb(err, data.toString()); - }); -} }()); diff --git a/test/rad/rad.book.spec.js b/test/rad/rad.book.spec.js new file mode 100644 index 00000000..47bbf663 --- /dev/null +++ b/test/rad/rad.book.spec.js @@ -0,0 +1,355 @@ +/** + * radisk3/book sanity tests + * - long paths + * - long data + * - special characters + * - escape sequences + * - __proto__ pollution + */ +const expect = require('../expect'); +const path = require('path'); +const fs = require('fs'); + +const ENABLE_GUN_LOGGING = false; + +const PATH_TO_TEST_FOLDER = path.resolve(__dirname, 'booktestdata'); +const PATH_TO_OLD_DB = path.resolve(PATH_TO_TEST_FOLDER, 'oldradata'); +const PATH_TO_TEST_DB = path.resolve(PATH_TO_TEST_FOLDER, 'radatatest'); +const PATH_TO_TEST_DB2 = path.resolve(PATH_TO_TEST_FOLDER, 'radatatest2'); + +if (!fs.existsSync(PATH_TO_TEST_FOLDER)) { + fs.mkdirSync(PATH_TO_TEST_FOLDER); +} +fs.rmdirSync(PATH_TO_TEST_DB, { recursive: true }); + +const BOOK_SPECIAL_CHARS = ['\'|', '|\'', '|', '\'', '\n']; +const BOOK_PAGE_SIZE = 3000; + +const RAD_PATH_SEPARATORS = ['/', '.']; +const GUN_PRIMITIVES = [ + null, + 'string', + 728858, + BigInt(1000000000000000000000000000000000000000000000000000000000n), + true, + false, + -Infinity, + Infinity, + NaN, + -0 +] + + +var root; +var Gun; +var Radix; +var Radisk; +var RFS; + +(function () { + var env; + if (typeof global !== 'undefined') { env = global } + if (typeof window !== 'undefined') { env = window } + root = env.window ? env.window : global; + try { env.window && root.localStorage && root.localStorage.clear() } catch (e) { } + //try{ indexedDB.deleteDatabase('radatatest') }catch(e){} + if (root.Gun) { + root.Gun = root.Gun; + // root.Gun.TESTING = true; + } else { + try { require('fs').unlinkSync('data.json') } catch (e) { } + try { require('../../lib/fsrm')(PATH_TO_TEST_DB) } catch (e) { } + root.Gun = require('../../gun'); + root.Gun.TESTING = true; + } + + try { var expect = global.expect = require("../expect") } catch (e) { } + + if (!root.Gun.SEA) { + require('../../sea.js'); + } +}(this)); + +Gun = root.Gun; +require("../../lib/book"); +Radix = Gun?.window?.Radix || require("../../lib/radix"); +Radisk = Gun?.window?.Radisk || require("../../lib/radisk3"); +RFS = require('../../lib/rfs'); +require('../../lib/store'); +require('../../lib/rindexed'); + +const RE_UNPRINTABLE = /[^\x20-\x7E]/; +const RE_APOSTROPHES = /'/g; + +const SKIP_CHARS = false; +const CHAR_MAX = 330; +const UNPRINTABLE_MAX = 128; //65536; + +const DATA_LENGTHS = [ + // 100, + // 1000, // 1kb + // 3000, // 3kb + // 10000, + 100000, + 1000000, // 1 mb + 10000000, // 10mb + 50000000, // 50mb + 500000000, // 500mb +]; +const LONG_DATAS = DATA_LENGTHS.map(l => ''.padEnd(l, '012345')); + +const ALL_CHARS = []; +for (let i = 0; i < CHAR_MAX; i++) { + ALL_CHARS.push(String.fromCharCode(i)); +} +ALL_CHARS.push('\u001b'); +const PRINTABLE_CHARS = ALL_CHARS.filter(c => !RE_UNPRINTABLE.test(c)); +const UNPRINTABLE_CHARS = ALL_CHARS.filter(c => RE_UNPRINTABLE.test(c)).slice(0, UNPRINTABLE_MAX); +const ESCAPE_CHARS = [`\0`, `\u001b`, `\\`, `\\\\`, `\\\\\\`, `\\\\\\\\`]; +const NORMAL_CHARS = ['A', 'B', 'C']; + +const ESCAPE_FACTORIES = { + ESCAPE_AS_NODE: (before, ec, c, after) => { + ec = ec.length ? [ec] : []; + return [...before, ...ec, c, ...after]; + }, + ESCAPE_IN_PATH: (before, ec, c, after) => [...before, `${ec}${c}`, ...after], + ESCAPE_AT_END: (before, ec, c, after) => { + ec = ec.length ? [ec] : []; + return [...before, `${c}`, ...after, ...ec]; + } +}; + + +function buildSoul(separator, ...args) { + return args.join(separator); +} + +const SOUL_GAUNTLET = RAD_PATH_SEPARATORS + .reduce((pc, ps) => [...pc, + ...[...BOOK_SPECIAL_CHARS, ...ESCAPE_CHARS].reduce((pb, cb) => + [...pb, + NORMAL_CHARS.map((nc) => `${nc}${cb}`), + NORMAL_CHARS.map((nc) => `${cb}${nc}`), + NORMAL_CHARS.map((nc) => `${cb}${cb}${nc}`), + ], [])], []); +// console.log(SOUL_GAUNTLET); +// const SOUL_GAUNTLET = [RAD_PATH_SEPARATORS, [PRINTABLE_CHARS], ESCAPE_FACTORIES, [BOOK_SPECIAL_CHARS, ESCAPE_CHARS]]; +// [\x000/9.2%20b, ...] + + +const opt = { + file: PATH_TO_TEST_DB, + localStorage: false, + log: (msg, ...args) => { + if (ENABLE_GUN_LOGGING) { + + console.log(` ${msg}`, ...args); + } + }, + chunk: 250 +}; +opt.store = RFS(opt); + +let seq = 0; + +describe('radisk3 & book', () => { + let r; + + beforeEach(() => { + // unpersist(opt); + // expect(testDirExists()).to.be.false(); + r = buildRad(opt); + }); + + it('can read older file versions', done => { + done('not implemented'); + }); + it('creates a ! file', (done) => { + expect(testDirExists()).to.be.ok(); + done('not implemented'); + }); + it('loads a ! file if one exists', done => { + // done(); + const r2 = buildRad(opt); + // hmm + done('not implemented'); + }); + + describe('path', () => { + it('supports arbitrarily long souls', done => { + done('not implemented'); + }); + it('uses path delimiters interchangeably', done => { + done('not implemented'); + }); + it('supports souls containing special characters', done => { + done('not implemented'); + }); + }); + + describe('data', () => { + it('can write & read data spanning multiple pages', done => { + done('not implemented'); + }); + it('can write & read all primitives', done => { + done('not implemented'); + }); + it('can write & read objects', done => { + done('not implemented'); + }) + }) +}); + +const soulpermutations = []; +describe('RAD book', () => { + let r; + + describe('node content length & paging', () => { + beforeEach(() => { + r = buildRad(opt); + }) + LONG_DATAS.forEach((d) => { + it(`handles data length of ${d.length} bytes`, (done) => { + seq++; + const dk = `root.seq${seq}`; + console.log(`\n#[${dk}]\n`); + r(dk, d, (err, ok) => { + if (err) { + console.log('ERR! waiting for ${dk} to write', err); + done(false); + } + }); + console.time('test'); + r(dk, (err, res, o) => { + const value = getValueFromPage(dk, res); + if (!res) { + console.log('no result', { err, res, o }); + done('nothing returned'); + return; + } + if (d !== value) { + done('result did not match'); + // TODO is this a failure, or because I'm not reading the result properly? + // TODO when a split() happens, should that affect the resulting page? + return; + } + done(); + }); + console.timeEnd('test'); + }); + }); + }); + + const pathSegments = ['a', 'b']; + if (!SKIP_CHARS) { + RAD_PATH_SEPARATORS.forEach((d, di) => { + describe(`\`${d}\`-delimited paths`, () => { + beforeEach(() => { + r = buildRad(opt); + }); + + it('should separate two keys', (done) => { + seq++; + let soul = `a${seq}${d}a`; + let data = `data for '${soul}' ${seq}`; + r(soul, data); + r(soul, (err, res, o) => { + const rd = getValueFromPage(soul, res); + expect(rd).to.be.ok(); + expect(rd).to.eql(data); + done(); + }); + }); + }); + + describe('other delimited paths', () => { + const charCats = { PRINTABLE_CHARS }; + + Object.entries(charCats).forEach((ckv) => { + const PATH_CHARS = ckv[1]; + Object.keys(ESCAPE_FACTORIES).forEach(escapeVariantKey => { + ESCAPE_CHARS.forEach(ec => { + describe(`${ckv[0]} '${escape(ec)}' ${escapeVariantKey}`, () => { + let rootKey; + + beforeEach(() => { + r = buildRad(opt); + }); + + PATH_CHARS.forEach((c, i) => { + for (let si = 0; si <= pathSegments.length; si++) { + const before = pathSegments.slice(0, si); + const after = pathSegments.slice(si); + const variant = [...ESCAPE_FACTORIES[escapeVariantKey](before, ec, c, after)]; + variant[0] = `${variant[0]}${seq}`; + const rootKey = `${path[0]}${seq}`; + const soul = variant.join(d); + soulpermutations.push(soul); + let data = `data for ${soul} ${seq}`; + + const numericSoul = soul.split('').map(x => `${x.charCodeAt(0)}`.padStart(6, ' ')).join(''); + + it(`${seq} ${numericSoul} # (${soul}) => ${data}`, (done) => { + seq++; + r(soul, data); + + r(soul, (err, res, book) => { + if (err) { + done(err); + return; + } + + const value = book(soul); // getValueFromPage(soul, res); // TODO mark fix this + + if (value !== data) { + debugger; + } + expect(value).to.eql(data); + done(); + }); + }); + } + }); + }); + }); + }); + }); + }); + }); + } + console.log(soulpermutations); +}); + +function getValueFromPage(soul, res) { + // return res; + // console.log(`getting ${soul} from`, res); + const bookList = res?.list || res?.book.list; + if (!bookList) { + return null; + } + const book = findInList(bookList, soul); + if (!book || !book.is) { + return null; + } + return book.is; +} + +function findInList(list, word) { + return (list || []).find(b => b.word === word); +} + + +function testDirExists() { + return fs.readdirSync(PATH_TO_TEST_FOLDER).includes('radatatest'); +} + +function unpersist(opt) { + opt = opt || { file: PATH_TO_TEST_DB }; + opt.file = opt.file || PATH_TO_TEST_DB; + fs.rmdirSync(opt.file, { recursive: true }); +} + +function buildRad(opt) { + return Radisk(opt); +} \ No newline at end of file