From 5cfa492d7dfead3b6425472e85cce50bc6976601 Mon Sep 17 00:00:00 2001 From: masterex1000 Date: Wed, 26 Sep 2018 17:32:22 -0600 Subject: [PATCH 1/5] Add les.js (Memory Evict) --- lib/les.js | 121 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 lib/les.js diff --git a/lib/les.js b/lib/les.js new file mode 100644 index 00000000..e6f3cb16 --- /dev/null +++ b/lib/les.js @@ -0,0 +1,121 @@ +; +(function() { + + // _ _____ ____ _ + // | | | ____/ ___| (_)___ + // | | | _| \___ \ | / __| + // | |___| |___ ___) | | \__ \ + // |_____|_____|____(_)/ |___/ + // ---------------------------- + // LES.js (Last rEcently uSed) + // ---------------------------- + // A Small, lightweight, queue-based + // Garbage Collector for Gun + // Originally By: Collin Conrad (@masterex1000) + + //NOTE: set to false is running from file in YOUR code + var USELOCALGUN = true; + + //NOTE: adds some debug messages + var DEBUG = false; + + + var Gun = (typeof window !== "undefined") ? window.Gun : (USELOCALGUN ? require('../gun') : require("gun")); + var ev = {}; + var empty = {}; + + Gun.on('opt', function(root) { + this.to.next(root); + if (root.once) + return; + if (typeof process == 'undefined') + return + var mem = process.memoryUsage; + + if (!mem) //exit because we are in the browser + return; + + //Figure out the most amount of memory we can use. TODO: make configurable? + ev.max = parseFloat(root.opt.memory || process.env.WEB_MEMORY || 512) * 0.8; + + var nodes = {}; //checks if the node already exists + var nodesArray = []; //used to easily sort everything and store info about the nodes + var memoryUpdate = 0; // last time we printed the current memory stats + + var check = function() { + ev.used = mem().rss / 1024 / 1024; //Contains the amt. of used ram in MB + setTimeout(function() { // So we can handle requests etc. before we start collecting + GC(ev.used / ev.max); // Calculate the memory ratio, and execute the garbage collector + }, 1); + } + + setInterval(check, 1000); // set the garbage collector to run every second, TODO: make configurable + + //Executed every time a node gets modifyed + root.on("put", function(e) { + var ctime = Date.now(); + var souls = Object.keys(e.put || empty); + for (var i = 0; i < souls.length; i++) { + enqueueNode(souls[i], ctime); + } + }); + + //Adds a soul the garbage collectors "freeing" queue + function enqueueNode(soul, ctime) { + if (nodes[soul] == true) { //The node already exists in the queue + var index = nodesArray.findIndex(function(e) { + return e[0] === soul; + }); + if (index == -1) { + console.err("Something happened and the node '" + soul + "' won't get garbage collection unless the value is updated agian"); + return; + } else { + nodesArray.splice(index, 1); // remove the existing ref. + nodesArray.push([soul, ctime]); // push the new instance + } + } else { + nodesArray.push([soul, ctime]); + nodes[soul] = true; + } + } + + //The main garbage collecting routine + function GC(memRatio) { + var curTime = Date.now(); // get the current time + + if (curTime - memoryUpdate >= 5000) { + console.log("|GC| %s | Current Memory Ratio: %d | Current Ram Usage %sMB | Nodes in Memory %s", new Date().toLocaleString(), round(memRatio, 2), round(ev.used, 2), Object.keys(root.graph || empty).length); + memoryUpdate = curTime; + } + + var freed = 0; + + while (nodesArray.length > 0) { + var soul = nodesArray[0][0]; + var nts = nodesArray[0][1]; + if (DEBUG) + console.log("Soul: " + soul + " | Remove Importance: " + calcRemoveImportance(nts, curTime, memRatio) + + " | Memory Ratio: " + memRatio + " | Time Existed: " + (curTime - nts) / 1000); + if (calcRemoveImportance(nodesArray[0][1], curTime, memRatio) >= 100) { + root.gun.get(nodesArray[0][0]).off(); //Remove the node + delete nodes[nodesArray[0][0]]; // remove the lookup value + nodesArray.splice(0, 1); + freed++; + } else + break; + } + if (freed > 0) + console.log("|GC| Removed %s nodes in %s seconds-----------------------------------------------------------------", freed, (Date.now() - curTime) * 0.001); + } + + //Generates a number that, after it hits a threshold, the node gets removed + function calcRemoveImportance(timestamp, ctime, memoryUsageRatio) { + var time = (ctime - timestamp) * 0.001; + return time * 10 * (memoryUsageRatio * memoryUsageRatio) + } + + function round(value, decimals) { //a basic rounding function + return Number(Math.round(value + 'e' + decimals) + 'e-' + decimals); + } + }); +}()); \ No newline at end of file From 38b65ff9ead2410f5fe8f22b3dce298698614db4 Mon Sep 17 00:00:00 2001 From: masterex1000 Date: Mon, 7 Jan 2019 08:09:01 -0700 Subject: [PATCH 2/5] add a bunch options for les.js --- lib/les.js | 78 ++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 67 insertions(+), 11 deletions(-) diff --git a/lib/les.js b/lib/les.js index e6f3cb16..2ac5a4f5 100644 --- a/lib/les.js +++ b/lib/les.js @@ -13,18 +13,56 @@ // Garbage Collector for Gun // Originally By: Collin Conrad (@masterex1000) - //NOTE: set to false is running from file in YOUR code + /** + * + * Usage: require the file in your application + * + * Gun Params: these are passed to the new gun constructor + * + * - gc_enable : enables the gc, good if you are running multiple instances of gun, etc... def. true + * - gc_delay : sets the amount of time between attempted garbage collections in milliseconds + * - gc_info_enable : Enables or Disables the info printout + * - gc_info : sets the ~ amount of time between info messages + * this is checked everytime the gc is ran + * - gc_info_mini : this will use a smaller, less user friendly info printout + * - gc_importance_func : This will be the function used for finding the importance of a potental collect + * takes the form of func(timestamp, ctime, memoryUsageRatio) {return val} + * Collects when returned value is 100 + */ + + //NOTE: set to false is running from file in YOUR code DEFUALT: false var USELOCALGUN = true; + - //NOTE: adds some debug messages - var DEBUG = false; + //NOTE: adds some debug messages DEFUALT: false + var DEBUG = true; + if(!(typeof window !== "undefined") && USELOCALGUN) + console.log("NOTE: You currently have LES.js set to use the 'local' file version of gun, This might crash if set wrong!"); var Gun = (typeof window !== "undefined") ? window.Gun : (USELOCALGUN ? require('../gun') : require("gun")); var ev = {}; var empty = {}; Gun.on('opt', function(root) { + + //Setup various options + + const gc_enable = root.opt.gc_enable ? root.opt.gc_enable : true; + const gc_delay = root.opt.gc_delay ? root.opt.gc_delay : 1000; + + const gc_info_enable = root.opt.gc_info_enable ? root.opt.gc_info_enable : true; + const gc_info = root.opt.gc_info ? root.opt.gc_info : 5000; + const gc_info_mini = root.opt.gc_info_mini ? root.opt.gc_info_mini : false; + + //This is kindof long but it works + const calcRemoveImportance = root.opt.gc_importance_func ? root.opt.gc_importance_func : function (timestamp, ctime, memoryUsageRatio) { + var time = (ctime - timestamp) * 0.001; + return time * 10 * (memoryUsageRatio * memoryUsageRatio); + } + + if(DEBUG) console.log(root.opt); + this.to.next(root); if (root.once) return; @@ -32,6 +70,9 @@ return var mem = process.memoryUsage; + if(!gc_enable) // exit because the gc is disabled + return; + if (!mem) //exit because we are in the browser return; @@ -49,7 +90,7 @@ }, 1); } - setInterval(check, 1000); // set the garbage collector to run every second, TODO: make configurable + setInterval(check, gc_delay); // set the garbage collector to run every second, TODO: make configurable //Executed every time a node gets modifyed root.on("put", function(e) { @@ -60,6 +101,18 @@ } }); + //Removes a node from the queue + function dequeueNode(soul) { + if (nodes[soul] == true) { //The node already exists in the queue + var index = nodesArray.findIndex(function(e) { + return e[0] === soul; + }); + if (index != -1) { + nodesArray.splice(index, 1); // remove the existing ref. + } + } + } + //Adds a soul the garbage collectors "freeing" queue function enqueueNode(soul, ctime) { if (nodes[soul] == true) { //The node already exists in the queue @@ -70,7 +123,7 @@ console.err("Something happened and the node '" + soul + "' won't get garbage collection unless the value is updated agian"); return; } else { - nodesArray.splice(index, 1); // remove the existing ref. + nodesArray.splice(index, 1); // remove the existing ref. faster than dequeue nodesArray.push([soul, ctime]); // push the new instance } } else { @@ -83,8 +136,11 @@ function GC(memRatio) { var curTime = Date.now(); // get the current time - if (curTime - memoryUpdate >= 5000) { - console.log("|GC| %s | Current Memory Ratio: %d | Current Ram Usage %sMB | Nodes in Memory %s", new Date().toLocaleString(), round(memRatio, 2), round(ev.used, 2), Object.keys(root.graph || empty).length); + if (gc_info_enable && curTime - memoryUpdate >= gc_info) { + if(!gc_info_mini) + console.log("|GC| %s | Current Memory Ratio: %d | Current Ram Usage %sMB | Nodes in Memory %s", new Date().toLocaleString(), round(memRatio, 2), round(ev.used, 2), Object.keys(root.graph || empty).length); + else + console.log("|GC| %s, Mem Ratio %d, Ram %sMB, Nodes in mem %s", new Date().toLocaleString(), round(memRatio, 2), round(ev.used, 2), Object.keys(root.graph || empty).length); memoryUpdate = curTime; } @@ -109,10 +165,10 @@ } //Generates a number that, after it hits a threshold, the node gets removed - function calcRemoveImportance(timestamp, ctime, memoryUsageRatio) { - var time = (ctime - timestamp) * 0.001; - return time * 10 * (memoryUsageRatio * memoryUsageRatio) - } + //function calcRemoveImportance(timestamp, ctime, memoryUsageRatio) { + // var time = (ctime - timestamp) * 0.001; + // return time * 10 * (memoryUsageRatio * memoryUsageRatio) + //} function round(value, decimals) { //a basic rounding function return Number(Math.round(value + 'e' + decimals) + 'e-' + decimals); From 810692e2c12d59472042a5f23d8b704faa71dcf2 Mon Sep 17 00:00:00 2001 From: masterex1000 Date: Mon, 7 Jan 2019 08:12:35 -0700 Subject: [PATCH 3/5] add options to les.js --- lib/les.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/les.js b/lib/les.js index 2ac5a4f5..7a41d479 100644 --- a/lib/les.js +++ b/lib/les.js @@ -31,11 +31,11 @@ */ //NOTE: set to false is running from file in YOUR code DEFUALT: false - var USELOCALGUN = true; + var USELOCALGUN = false; //NOTE: adds some debug messages DEFUALT: false - var DEBUG = true; + var DEBUG = false; if(!(typeof window !== "undefined") && USELOCALGUN) console.log("NOTE: You currently have LES.js set to use the 'local' file version of gun, This might crash if set wrong!"); From cb9b4b8013fc70f0d03cbc77b01e2b8d874728c1 Mon Sep 17 00:00:00 2001 From: masterex1000 Date: Thu, 10 Jan 2019 08:20:52 -0700 Subject: [PATCH 4/5] More Changes To Les! --- lib/les.js | 123 ++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 84 insertions(+), 39 deletions(-) diff --git a/lib/les.js b/lib/les.js index 7a41d479..ab3b6133 100644 --- a/lib/les.js +++ b/lib/les.js @@ -30,8 +30,8 @@ * Collects when returned value is 100 */ - //NOTE: set to false is running from file in YOUR code DEFUALT: false - var USELOCALGUN = false; + //NOTE: set to false to use require for getting gun DEFUALT: false + var USELOCALGUN = true; //NOTE: adds some debug messages DEFUALT: false @@ -41,11 +41,31 @@ console.log("NOTE: You currently have LES.js set to use the 'local' file version of gun, This might crash if set wrong!"); var Gun = (typeof window !== "undefined") ? window.Gun : (USELOCALGUN ? require('../gun') : require("gun")); - var ev = {}; - var empty = {}; - - Gun.on('opt', function(root) { + + //Removes a node from the garbage collection until next write + Gun.chain.gcDequeue = function() { + //console.log(this._.root.dequeueNode); + if(this._.root.dequeueNode) { // check that we actually have the dequeue command on this node + let ctx = this; + this.get(function (soul) { + ctx._.root.dequeueNode(soul); + }, true); + } + } + + //Puts node at the front for garbage collection, NOTE: only collects when it is hit it's time + Gun.chain.gcCollect = function() { + if(this._.root.collectNode) { // check that we actually have the dequeue command on this node + let ctx = this; + + this.get(function (soul) { + ctx._.root.collectNode(soul); + }, true); + } + } + + Gun.on('opt', function(root) { //Setup various options const gc_enable = root.opt.gc_enable ? root.opt.gc_enable : true; @@ -55,7 +75,7 @@ const gc_info = root.opt.gc_info ? root.opt.gc_info : 5000; const gc_info_mini = root.opt.gc_info_mini ? root.opt.gc_info_mini : false; - //This is kindof long but it works + //This is long, but it works well const calcRemoveImportance = root.opt.gc_importance_func ? root.opt.gc_importance_func : function (timestamp, ctime, memoryUsageRatio) { var time = (ctime - timestamp) * 0.001; return time * 10 * (memoryUsageRatio * memoryUsageRatio); @@ -64,6 +84,7 @@ if(DEBUG) console.log(root.opt); this.to.next(root); + if (root.once) return; if (typeof process == 'undefined') @@ -76,6 +97,9 @@ if (!mem) //exit because we are in the browser return; + var ev = {}; //stores the environment + var empty = {}; //An empty list used to prevent crashes + //Figure out the most amount of memory we can use. TODO: make configurable? ev.max = parseFloat(root.opt.memory || process.env.WEB_MEMORY || 512) * 0.8; @@ -83,36 +107,33 @@ var nodesArray = []; //used to easily sort everything and store info about the nodes var memoryUpdate = 0; // last time we printed the current memory stats + root.dequeueNode = (soul) => { //forward the call to our gc + dequeueNode(soul); + } + + root.collectNode = (soul) => { //forward the call to our gc + collectNode(soul); + } + var check = function() { ev.used = mem().rss / 1024 / 1024; //Contains the amt. of used ram in MB setTimeout(function() { // So we can handle requests etc. before we start collecting GC(ev.used / ev.max); // Calculate the memory ratio, and execute the garbage collector + //GC(0.99); }, 1); } - setInterval(check, gc_delay); // set the garbage collector to run every second, TODO: make configurable + setInterval(check, gc_delay); // set the garbage collector to run every second - //Executed every time a node gets modifyed + //Executed every time a node gets modified root.on("put", function(e) { var ctime = Date.now(); - var souls = Object.keys(e.put || empty); - for (var i = 0; i < souls.length; i++) { + var souls = Object.keys(e.put || empty); // get all of the nodes in the update + for (var i = 0; i < souls.length; i++) { // iterate over them and add them enqueueNode(souls[i], ctime); } }); - //Removes a node from the queue - function dequeueNode(soul) { - if (nodes[soul] == true) { //The node already exists in the queue - var index = nodesArray.findIndex(function(e) { - return e[0] === soul; - }); - if (index != -1) { - nodesArray.splice(index, 1); // remove the existing ref. - } - } - } - //Adds a soul the garbage collectors "freeing" queue function enqueueNode(soul, ctime) { if (nodes[soul] == true) { //The node already exists in the queue @@ -120,7 +141,7 @@ return e[0] === soul; }); if (index == -1) { - console.err("Something happened and the node '" + soul + "' won't get garbage collection unless the value is updated agian"); + console.error("Something happened and the node '" + soul + "' won't get garbage collection unless the value is updated again"); return; } else { nodesArray.splice(index, 1); // remove the existing ref. faster than dequeue @@ -132,21 +153,50 @@ } } + //Removes a node from the queue + function dequeueNode(soul) { + if (nodes[soul] == true) { //The node already exists in the queue + var index = nodesArray.findIndex(function(e) { + return e[0] === soul; + }); + if (index != -1) { + //nodesArray.splice(index, 1); // remove the existing ref. + nodesArray.shift(); + nodes[soul] = false; // store that we no longer have that node in the queue + } + } + } + + //Moves a node to the start of the queue + function collectNode(soul) { + if (nodes[soul] == true) { //The node already exists in the queue + var index = nodesArray.findIndex(function(e) { + return e[0] === soul; + }); + if (index != -1) { + //nodesArray.splice(index, 1); // remove the existing ref. + nodesArray.shift(); // WAY faster than splice + } + nodesArray.unshift([soul, nodesArray[0][1]]); // create a new node with the next nodes time stamp + nodes[soul] = true; // store that we no longer have that node in the queue + } + } + //The main garbage collecting routine function GC(memRatio) { var curTime = Date.now(); // get the current time - if (gc_info_enable && curTime - memoryUpdate >= gc_info) { + if (gc_info_enable && curTime - memoryUpdate >= gc_info) { // check if we need to print info if(!gc_info_mini) console.log("|GC| %s | Current Memory Ratio: %d | Current Ram Usage %sMB | Nodes in Memory %s", new Date().toLocaleString(), round(memRatio, 2), round(ev.used, 2), Object.keys(root.graph || empty).length); else - console.log("|GC| %s, Mem Ratio %d, Ram %sMB, Nodes in mem %s", new Date().toLocaleString(), round(memRatio, 2), round(ev.used, 2), Object.keys(root.graph || empty).length); - memoryUpdate = curTime; + console.log("|GC| %s, Mem Ratio %d, Ram %sMB, Nodes in mem %s, Tracked Nodes %s", new Date().toLocaleString(), round(memRatio, 2), round(ev.used, 2), Object.keys(root.graph || empty).length, nodesArray.length); + memoryUpdate = curTime; // reset the last update time } - var freed = 0; + var freed = 0; // Just a nice performance counter - while (nodesArray.length > 0) { + while (nodesArray.length > 0) { // iterate over all of our nodes var soul = nodesArray[0][0]; var nts = nodesArray[0][1]; if (DEBUG) @@ -155,21 +205,16 @@ if (calcRemoveImportance(nodesArray[0][1], curTime, memRatio) >= 100) { root.gun.get(nodesArray[0][0]).off(); //Remove the node delete nodes[nodesArray[0][0]]; // remove the lookup value - nodesArray.splice(0, 1); - freed++; + //nodesArray.splice(0, 1); + nodesArray.shift(); + freed++; // add one to our perf counter } else - break; + break; // Break out of the loop because we don't have any more nodes to free } if (freed > 0) console.log("|GC| Removed %s nodes in %s seconds-----------------------------------------------------------------", freed, (Date.now() - curTime) * 0.001); } - - //Generates a number that, after it hits a threshold, the node gets removed - //function calcRemoveImportance(timestamp, ctime, memoryUsageRatio) { - // var time = (ctime - timestamp) * 0.001; - // return time * 10 * (memoryUsageRatio * memoryUsageRatio) - //} - + function round(value, decimals) { //a basic rounding function return Number(Math.round(value + 'e' + decimals) + 'e-' + decimals); } From 2462e4f33de80b3417ea3fbab45ee9fcf73c28e0 Mon Sep 17 00:00:00 2001 From: masterex1000 Date: Thu, 10 Jan 2019 08:38:08 -0700 Subject: [PATCH 5/5] Fix USELOCALGUN setting from last commit --- lib/les.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/les.js b/lib/les.js index ab3b6133..75a44ece 100644 --- a/lib/les.js +++ b/lib/les.js @@ -31,7 +31,7 @@ */ //NOTE: set to false to use require for getting gun DEFUALT: false - var USELOCALGUN = true; + var USELOCALGUN = false; //NOTE: adds some debug messages DEFUALT: false