mirror of
https://github.com/amark/gun.git
synced 2025-03-30 15:08:33 +00:00
117 lines
3.5 KiB
JavaScript
117 lines
3.5 KiB
JavaScript
;(function(){
|
|
|
|
if(!process.env.AWS_S3_BUCKET){ return }
|
|
|
|
var Gun = require('../gun');
|
|
var S3 = require('./aws');
|
|
|
|
Gun.on('opt', function(at){
|
|
var opt = at.opt.s3 || (at.opt.s3 = {});
|
|
var s3 = opt.store || S3(opt);
|
|
opt.store = s3;
|
|
this.to.next(at);
|
|
if(!s3 || !s3.config){ return }
|
|
if(at.once){ return }
|
|
var root = at.gun.back(-1);
|
|
opt.prefix = opt.prefix || '';
|
|
opt.batch = opt.batch || 10;
|
|
opt.throttle = opt.throttle || process.env.AWS_S3_THROTTLE || 15;
|
|
opt.disconnect = opt.disconnect || 5;
|
|
Gun.on('get', function(at){
|
|
if(!at.get){ return }
|
|
var id = at['#'], soul = at.get['#'], field = at.get['.'];
|
|
var key = opt.prefix+soul;
|
|
//console.log("g3t", soul);
|
|
s3.GET(key, function(err, data, text, meta){
|
|
meta = meta || {};
|
|
if(err && err.statusCode == 404){
|
|
err = null; // we want a difference between 'unfound' (data is null) and 'error' (auth is wrong).
|
|
}
|
|
if(!data){
|
|
data = root._.graph[soul] || async[soul];
|
|
}
|
|
if(data && !Gun.node.soul(data)){
|
|
err = {err: Gun.log('No soul on S3 node data!')};
|
|
}
|
|
if(err){
|
|
root.on('in', {'@': id, err: err});
|
|
return
|
|
}
|
|
var node = data;
|
|
graph[soul] = true;
|
|
if(data && field){
|
|
node = Gun.state.ify({}, field, Gun.state.is(node, field), node[field], soul);
|
|
}
|
|
//console.log("got", soul, node);
|
|
root.on('in', {'@': id, put: Gun.graph.node(node)});
|
|
});
|
|
});
|
|
Gun.on('put', function(at){
|
|
var id = at['#'], check = {}, next = s3.next, err, u;
|
|
Gun.graph.is(at.put, function(node, soul){
|
|
check[soul] = 1;
|
|
/*if(!graph[soul]){
|
|
// need to read before writing
|
|
return;
|
|
}*/
|
|
async[soul] = node;
|
|
batch[soul] = (batch[soul] || 0) + 1;
|
|
s3.on(next + ':' + soul, function(arg){
|
|
var reply = arg[1];
|
|
err = arg[0];
|
|
check[soul] = 0;
|
|
this.off();
|
|
Gun.obj.del(async, soul);
|
|
if(Gun.obj.map(check, function(v){
|
|
if(v){ return true }
|
|
})){ return }
|
|
root.on('in', {'@': id, err: err, ok: err? u : reply});
|
|
});
|
|
});
|
|
s3.batching += 1;
|
|
if(opt.batch < s3.batching){
|
|
return now();
|
|
}
|
|
if(!opt.throttle){
|
|
return now();
|
|
}
|
|
s3.wait = s3.wait || setTimeout(now, opt.throttle * 1000); // in seconds
|
|
});
|
|
function now(){
|
|
clearTimeout(s3.wait);
|
|
var keep = batch;
|
|
batch = {};
|
|
s3.batching = 0;
|
|
var now = s3.next;
|
|
s3.next = Gun.time.is();
|
|
s3.wait = null;
|
|
Gun.obj.map(keep, function put(exists, soul, count){
|
|
//console.log("s3ving...", soul);
|
|
var node = root._.graph[soul] || async[soul]; // the batch does not actually have the nodes, but what happens when we do cold data? Could this be gone?
|
|
s3.PUT(opt.prefix+soul, node, function(err, reply){
|
|
if(count < 5 && (err || !reply)){
|
|
put(exists, soul, (count || 0) + 1); // naive implementation of retry TODO: BUG: need backoff and anti-infinite-loop!
|
|
return;
|
|
}
|
|
//console.log("S3VED", soul);
|
|
s3.on(now + ':' + soul, [err, reply]);
|
|
});
|
|
});
|
|
}
|
|
var graph = {}, batch = {}, ids = {}, async = {};
|
|
s3.next = s3.next || Gun.time.is();
|
|
s3.on = s3.on || Gun.on;
|
|
});
|
|
|
|
;(function(){return;
|
|
global.Gun = require('../gun');
|
|
|
|
process.env.AWS_S3_BUCKET = 'test-s3';
|
|
process.env.AWS_ACCESS_KEY_ID = 'asdf';
|
|
process.env.AWS_SECRET_ACCESS_KEY = 'fdsa';
|
|
process.env.fakes3 = 'http://localhost:4567';
|
|
process.env.AWS_S3_THROTTLE = 0;
|
|
|
|
require('../test/abc');
|
|
}());
|
|
}()); |