test('test github repos that use `standard`', function (t) {
t.plan(URLS.length)
mkdirp.sync(TMP)
// test an empty repo
mkdirp.sync(path.join(TMP, 'empty'))
parallelLimit(Object.keys(MODULES).map(function (name) {
var url = MODULES[name]
var folder = path.join(TMP, name)
return function (cb) {
fsAccess(path.join(TMP, name), fs.R_OK | fs.W_OK, function (err) {
var args = err
? [ 'clone', '--depth', 1, url, path.join(TMP, name) ]
: [ 'pull' ]
var opts = err
? {}
: { cwd: folder }
spawn(GIT, args, opts, function (err) {
if (err) return cb(err)
spawn(STANDARD, [], { cwd: folder }, function (err) {
t.error(err, name)
cb(null)
})
})
})
}
}), PARALLEL_LIMIT, function (err) {
if (err) throw err
})
})
var results = await new Promise((resolve, reject) => {
parallelLimit(filez2.map(entry => {
return function(cb) {
exports.renderDocument(
entry.result.config,
entry.result.basedir,
entry.result.fpath,
entry.result.renderTo,
entry.result.renderToPlus,
entry.result.renderBaseMetadata
)
.then((result) => {
// log(`render renderDocument ${result}`);
cb(undefined, { result });
})
.catch(err => {
// console.error(`render renderDocument ${err} ${err.stack}`);
cb(undefined, { error: err });
});
};
}),
config.concurrency, // Concurrency count
function(err, results) {
// gets here on final results
if (err) reject(err);
else resolve(results);
});
});
Torrent.prototype._verifyPieces = function () {
var self = this
parallelLimit(self.pieces.map(function (_, index) {
return function (cb) {
if (self.destroyed) return cb(new Error('torrent is destroyed'))
self.store.get(index, function (err, buf) {
if (err) return process.nextTick(cb, null) // ignore error
sha1(buf, function (hash) {
if (hash === self._hashes[index]) {
if (!self.pieces[index]) return
self._debug('piece verified %s', index)
self._markVerified(index)
} else {
self._debug('piece invalid %s', index)
}
cb(null)
})
})
}
}), FILESYSTEM_CONCURRENCY, function (err) {
if (err) return self._destroy(err)
self._debug('done verifying')
self._onStore()
})
}
return function status () {
var c = get('config')
if (get('args').length) {
throw new Error('unknown arg')
}
var ret = {
thoughts: 0,
ticks: 0,
logs: 0
}
var tasks = []
Object.keys(ret).forEach(function (k) {
tasks.push(function (done) {
get('db').collection(k).count({app_name: get('app_name')}, function (err, result) {
if (err) return done(err)
ret[k] = result
done()
})
})
})
parallel(tasks, c.parallel_limit, function (err) {
if (err) throw err
get('logger').info('status', ret, {feed: 'status'})
get('app').close(function () {
process.exit()
})
})
}
test('test github repos that use `standard`', function (t) {
t.plan(testPackages.length)
mkdirp.sync(TMP)
parallelLimit(testPackages.map(function (pkg) {
var name = pkg.name
var url = pkg.repo + '.git'
var folder = path.join(TMP, name)
return function (cb) {
access(path.join(TMP, name), fs.R_OK | fs.W_OK, function (err) {
if (argv.offline) {
if (err) {
t.pass('SKIPPING (offline): ' + name + ' (' + pkg.repo + ')')
return cb(null)
}
runStandard(cb)
} else {
downloadPackage(function (err) {
if (err) return cb(err)
runStandard(cb)
})
}
function downloadPackage (cb) {
if (err) gitClone(cb)
else gitPull(cb)
}
function gitClone (cb) {
var args = [ 'clone', '--depth', 1, url, path.join(TMP, name) ]
spawn(GIT, args, { stdio: 'ignore' }, function (err) {
if (err) err.message += ' (git clone) (' + name + ')'
cb(err)
})
}
function gitPull (cb) {
var args = [ 'pull' ]
spawn(GIT, args, { cwd: folder, stdio: 'ignore' }, function (err) {
if (err) err.message += ' (git pull) (' + name + ')'
cb(err)
})
}
function runStandard (cb) {
var args = [ '--verbose' ]
if (pkg.args) args.push.apply(args, pkg.args)
spawn(STANDARD, args, { cwd: folder }, function (err) {
var str = name + ' (' + pkg.repo + ')'
if (err) { t.fail(str) } else { t.pass(str) }
cb(null)
})
}
})
}
}), PARALLEL_LIMIT, function (err) {
if (err) throw err
})
})
parallelLimit(tasks, 10, function (err, results) {
if (err) {
return cb(err);
}
opts.dev = false;
// remove purgeExcess result
results.pop();
var incorrects = results.filter(function (dep) {
return !dep.correct;
});
var corrects = results.filter(function (dep) {
return dep.correct;
});
/* for each incorrect
- install it
- remove excess
- force install all children
*/
var inCorrectTasks = incorrects.map(function (incorrect) {
var name = incorrect.name;
var folder = path.join(nodeModules,
name, 'node_modules');
return series.bind(null, [
installModule.bind(
null, nodeModules, incorrect, opts),
forceInstall.bind(null, folder, incorrect, opts)
]);
});
var correctTasks = corrects.map(function (correct) {
var name = correct.name;
var folder = path.join(nodeModules, name,
'node_modules');
return forceInstall.bind(
null, folder, correct, opts);
});
/* for each correct
- force install all children
*/
var tasks = [].concat(inCorrectTasks, correctTasks);
parallelLimit(tasks, 10, cb);
});
Torrent.prototype.getFileModtimes = function (cb) {
var self = this
var ret = []
parallelLimit(self.files.map(function (file, index) {
return function (cb) {
fs.stat(path.join(self.path, file.path), function (err, stat) {
if (err && err.code !== 'ENOENT') return cb(err)
ret[index] = stat && stat.mtime.getTime()
cb(null)
})
}
}), FILESYSTEM_CONCURRENCY, function (err) {
self._debug('done getting file modtimes')
cb(err, ret)
})
}
glob(path.join(initDocsPath, '/**/*'), (err, res) => {
if (err) {
throw err
}
const index = __dirname.lastIndexOf('/')
parallelLimit(res.map((element) => (callback) => {
const addPath = element.substring(index + 1, element.length)
if (!fs.statSync(element).isDirectory()) {
const rs = new Readable()
rs.push(fs.readFileSync(element))
rs.push(null)
const filePair = {path: addPath, content: rs}
i.write(filePair)
}
callback()
}), 10, (err) => {
if (err) {
throw err
}
i.end()
})
})
// Reads telemetry log files in parallel, then produces a summary array, one
// entry for each day: [{date, actives, retention, ...}, ...]
function loadTelemetrySummary (logFiles, cb) {
console.log('Summarizing ' + logFiles.length + ' telemetry log files')
const tasks = logFiles.map(function (filename) {
return function (cb) {
// Read each telemetry log file, one per day...
const filePath = path.join(TELEMETRY_PATH, filename)
console.log('Reading ' + filename)
fs.readFile(filePath, 'utf8', function (err, json) {
if (err) return cb(err)
// Each log file contains one JSON record per line
console.log('Parsing ' + filename)
const lines = json.trim().split('\n')
const records = lines
.map(function (line, i) {
try {
return JSON.parse(line)
} catch (err) {
console.error('Skipping invalid line %s:%d', filename, i + 1)
console.error(err)
return null
}
})
.filter(Boolean)
console.log('Read ' + records.length + ' rows from ' + filename)
cb(null, summarizeDailyTelemetryLog(filename, records))
})
}
})
runParallelLimit(tasks, PARALLEL_LIMIT, function (err, days) {
if (err) return cb(err)
cb(null, combineDailyTelemetrySummaries(days))
})
}
Machine.command(args, (err, stdout) => {
if (err) return done(err)
const machines = stdout.split(NEWLINE).filter(Boolean).map(line => {
const values = line.split(LIST_COLUMNS_SEP)
const machine = {}
LIST_COLUMNS.forEach((name, i) => {
const key = camelCase(name)
const val = values[i]
machine[key] = val === '' ? null : decodeURIComponent(val)
})
// ResponseTime is in nanoseconds
machine.responseTime = parseInt(machine.responseTime) / 1e6
machine.state = machine.state.toLowerCase()
machine.activeHost = machine.activeHost === 'true'
machine.activeSwarm = machine.activeSwarm === 'true'
if (machine.dockerVersion === 'Unknown') {
machine.dockerVersion = null
}
return machine
})
if (!opts.inspect) return done(null, machines)
// Add additional metadata from `docker-machine inspect <name>`
parallel(machines.map(machine => next => {
Machine.inspect(machine.name, (err, data) => {
if (err) next(err)
else next(null, xtend(machine, data))
})
}), 4, done)
})
test('test github repos that use `standard`', function (t) {
t.plan(testPackages.length)
mkdirp.sync(TMP)
// test an empty repo
mkdirp.sync(path.join(TMP, 'empty'))
parallelLimit(testPackages.map(function (pkg) {
var name = pkg.name
var url = pkg.repo + '.git'
var folder = path.join(TMP, name)
return function (cb) {
fsAccess(path.join(TMP, name), fs.R_OK | fs.W_OK, function (err) {
var gitArgs = err
? [ 'clone', '--depth', 1, url, path.join(TMP, name) ]
: [ 'pull' ]
var gitOpts = err
? {}
: { cwd: folder }
spawn(GIT, gitArgs, gitOpts, function (err) {
if (err) {
err.message += ' (' + name + ')'
return cb(err)
}
spawn(STANDARD, [ '--verbose' ], { cwd: folder }, function (err) {
t.error(err, name)
cb(null)
})
})
})
}
}), PARALLEL_LIMIT, function (err) {
if (err) throw err
})
})
glob(path.join(initDocsPath, '/**/*'), function (err, res) {
if (err) {
throw err;
}
var index = __dirname.lastIndexOf('/');
parallelLimit(res.map(function (element) {
return function (callback) {
var addPath = element.substring(index + 1, element.length);
if (!fs.statSync(element).isDirectory()) {
var rs = new Readable();
rs.push(fs.readFileSync(element));
rs.push(null);
var filePair = { path: addPath, content: rs };
i.write(filePair);
}
callback();
};
}), 10, function (err) {
if (err) {
throw err;
}
i.end();
});
});
utils.getIPFS((err, ipfs) => {
if (err) {
throw err
}
const i = ipfs.files.add()
var filePair
i.on('data', (file) => {
console.log('added', bs58.encode(file.multihash).toString(), file.path)
})
i.once('end', () => {
return
})
if (res.length !== 0) {
const index = inPath.lastIndexOf('/')
parallelLimit(res.map((element) => (callback) => {
if (!fs.statSync(element).isDirectory()) {
i.write({
path: element.substring(index + 1, element.length),
stream: fs.createReadStream(element)
})
}
callback()
}), 10, (err) => {
if (err) {
throw err
}
i.end()
})
} else {
rs = fs.createReadStream(inPath)
inPath = inPath.substring(inPath.lastIndexOf('/') + 1, inPath.length)
filePair = {path: inPath, stream: rs}
i.write(filePair)
i.end()
}
})
parallel(tasks, PARALLEL_LIMIT, (err, results) => {
console.log("\nBacktesting complete, saving results...");
results = results.filter(function (r) {
return !!r
})
results.sort((a,b) => (a.roi < b.roi) ? 1 : ((b.roi < a.roi) ? -1 : 0));
let fileName = `backtesting_${Math.round(+new Date()/1000)}.csv`;
let filedsGeneral = ['roi', 'vsBuyHold', 'errorRate', 'wlRatio', 'frequency', 'endBalance', 'buyHold', 'wins', 'losses', 'period', 'min_periods', 'days'];
let filedNamesGeneral = ['ROI (%)', 'VS Buy Hold (%)', 'Error Rate (%)', 'Win/Loss Ratio', '# Trades/Day', 'Ending Balance ($)', 'Buy Hold ($)', '# Wins', '# Losses', 'Period', 'Min Periods', '# Days'];
let fields = {
cci_srsi: filedsGeneral.concat(['cciPeriods', 'rsiPeriods', 'srsiPeriods', 'srsiK', 'srsiD', 'oversoldRsi', 'overboughtRsi', 'oversoldCci', 'overboughtCci', 'Constant', 'params']),
srsi_macd: filedsGeneral.concat(['rsiPeriods', 'srsiPeriods', 'srsiK', 'srsiD', 'oversoldRsi', 'overboughtRsi', 'emaShortPeriod', 'emaLongPeriod', 'signalPeriod', 'upTrendThreshold', 'downTrendThreshold', 'params']),
macd: filedsGeneral.concat([ 'emaShortPeriod', 'emaLongPeriod', 'signalPeriod', 'upTrendThreshold', 'downTrendThreshold', 'overboughtRsiPeriods', 'overboughtRsi', 'params']),
rsi: filedsGeneral.concat(['rsiPeriods', 'oversoldRsi', 'overboughtRsi', 'rsiRecover', 'rsiDrop', 'rsiDivsor', 'params']),
sar: filedsGeneral.concat(['sarAf', 'sarMaxAf', 'params']),
speed: filedsGeneral.concat(['baselinePeriods', 'triggerFactor', 'params']),
trend_ema: filedsGeneral.concat(['trendEma', 'neutralRate', 'oversoldRsiPeriods', 'oversoldRsi', 'params'])
};
let fieldNames = {
cci_srsi: filedNamesGeneral.concat(['CCI Periods', 'RSI Periods', 'SRSI Periods', 'SRSI K', 'SRSI D', 'Oversold RSI', 'Overbought RSI', 'Oversold CCI', 'Overbought CCI', 'Constant', 'Full Parameters']),
srsi_macd: filedNamesGeneral.concat(['RSI Periods', 'SRSI Periods', 'SRSI K', 'SRSI D', 'Oversold RSI', 'Overbought RSI', 'EMA Short Period', 'EMA Long Period', 'Signal Period', 'Up Trend Threshold', 'Down Trend Threshold', 'Full Parameters']),
macd: filedNamesGeneral.concat(['EMA Short Period', 'EMA Long Period', 'Signal Period', 'Up Trend Threshold', 'Down Trend Threshold', 'Overbought Rsi Periods', 'Overbought Rsi', 'Full Parameters']),
rsi: filedNamesGeneral.concat(['RSI Periods', 'Oversold RSI', 'Overbought RSI', 'RSI Recover', 'RSI Drop', 'RSI Divisor', 'Full Parameters']),
sar: filedNamesGeneral.concat(['SAR AF', 'SAR MAX AF', 'Full Parameters']),
speed: filedNamesGeneral.concat(['Baseline Periods', 'Trigger Factor', 'Full Parameters']),
trend_ema: filedNamesGeneral.concat(['Trend EMA', 'Neutral Rate', 'Oversold RSI Periods', 'Oversold RSI', 'Full Parameters'])
};
let csv = json2csv({
data: results,
fields: fields[strategyName],
fieldNames: fieldNames[strategyName]
});
fs.writeFile(fileName, csv, err => {
if (err) throw err;
console.log(`\nResults successfully saved to ${fileName}!\n`);
});
});
function forceInstall(nodeModules, shrinkwrap, opts, cb) {
if (typeof opts === 'function') {
cb = opts;
opts = {};
}
// if no dependencies object then terminate recursion
if (shrinkwrap.name && !shrinkwrap.dependencies) {
return purgeExcess(nodeModules, shrinkwrap, opts, cb);
}
var deps = shrinkwrap.dependencies;
// console.log('shrinkwrap', shrinkwrap);
var tasks = Object.keys(deps).map(function (key) {
var dep = deps[key];
if (!dep.name) {
dep.name = key;
}
var filePath = path.join(nodeModules, key);
return isCorrect.bind(null, filePath, dep, opts);
});
tasks.push(purgeExcess.bind(
null, nodeModules, shrinkwrap, opts));
parallelLimit(tasks, 10, function (err, results) {
if (err) {
return cb(err);
}
opts.dev = false;
// remove purgeExcess result
results.pop();
var incorrects = results.filter(function (dep) {
return !dep.correct;
});
var corrects = results.filter(function (dep) {
return dep.correct;
});
/* for each incorrect
- install it
- remove excess
- force install all children
*/
var inCorrectTasks = incorrects.map(function (incorrect) {
var name = incorrect.name;
var folder = path.join(nodeModules,
name, 'node_modules');
return series.bind(null, [
installModule.bind(
null, nodeModules, incorrect, opts),
forceInstall.bind(null, folder, incorrect, opts)
]);
});
var correctTasks = corrects.map(function (correct) {
var name = correct.name;
var folder = path.join(nodeModules, name,
'node_modules');
return forceInstall.bind(
null, folder, correct, opts);
});
/* for each correct
- force install all children
*/
var tasks = [].concat(inCorrectTasks, correctTasks);
parallelLimit(tasks, 10, cb);
});
}
let simulateGeneration = () => {
console.log(`\n\n=== Simulating generation ${++generationCount} ===\n`)
let days = argv.days
if (!days) {
if (argv.start) {
var start = moment(argv.start, 'YYYYMMDDhhmm')
days = Math.max(1, moment().diff(start, 'days'))
}
else {
var end = moment(argv.end, 'YYYYMMDDhhmm')
days = moment().diff(end, 'days') + 1
}
}
runUpdate(days, argv.selector)
iterationCount = 1
let tasks = selectedStrategies.map(v => pools[v]['pool'].population().map(phenotype => {
return cb => {
runCommand(v, phenotype, cb)
}
})).reduce((a, b) => a.concat(b))
parallel(tasks, PARALLEL_LIMIT, (err, results) => {
console.log('\nGeneration complete, saving results...')
results = results.filter(function(r) {
return !!r
})
results.sort((a, b) => (a.fitness < b.fitness) ? 1 : ((b.fitness < a.fitness) ? -1 : 0))
let fieldsGeneral = ['selector.normalized', 'fitness', 'vsBuyHold', 'wlRatio', 'frequency', 'strategy', 'order_type', 'endBalance', 'buyHold', 'wins', 'losses', 'period_length', 'min_periods', 'days', 'params']
let fieldNamesGeneral = ['Selector', 'Fitness', 'VS Buy Hold (%)', 'Win/Loss Ratio', '# Trades/Day', 'Strategy', 'Order Type', 'Ending Balance ($)', 'Buy Hold ($)', '# Wins', '# Losses', 'Period', 'Min Periods', '# Days', 'Full Parameters']
let dataCSV = json2csv({
data: results,
fields: fieldsGeneral,
fieldNames: fieldNamesGeneral
})
let fileDate = Math.round(+new Date() / 1000)
let csvFileName = `simulations/backtesting_${fileDate}.csv`
let poolData = {}
selectedStrategies.forEach(function(v) {
poolData[v] = pools[v]['pool'].population()
})
let jsonFileName = `simulations/generation_data_${fileDate}_gen_${generationCount}.json`
let dataJSON = JSON.stringify(poolData, null, 2)
var filesSaved = 0
saveGenerationData(csvFileName, jsonFileName, dataCSV, dataJSON, (id)=>{
filesSaved++
if(filesSaved == 2){
console.log('\n\nGeneration\'s Best Results')
selectedStrategies.forEach((v)=> {
let best = pools[v]['pool'].best()
if(best.sim){
console.log(`\t(${v}) Sim Fitness ${best.sim.fitness}, VS Buy and Hold: ${best.sim.vsBuyHold} End Balance: ${best.sim.endBalance}, Wins/Losses ${best.sim.wins}/${best.sim.losses}.`)
} else {
console.log(`\t(${v}) Result Fitness ${results[0].fitness}, VS Buy and Hold: ${results[0].vsBuyHold}, End Balance: ${results[0].endBalance}, Wins/Losses ${results[0].wins}/${results[0].losses}.`)
}
// prepare command snippet from top result for this strat
let prefix = './zenbot.sh sim '
let bestCommand = generateCommandParams(results[0])
bestCommand = prefix + bestCommand
bestCommand = bestCommand + ' --asset_capital=' + argv.asset_capital + ' --currency_capital=' + argv.currency_capital
console.log(bestCommand + '\n')
let nextGen = pools[v]['pool'].evolve()
})
simulateGeneration()
}
})
})
}
Torrent.prototype._onMetadata = function (metadata) {
var self = this
if (self.metadata || self.destroyed) return
debug('got metadata')
var parsedTorrent
if (metadata && metadata.infoHash) {
// `metadata` is a parsed torrent (from parse-torrent module)
parsedTorrent = metadata
} else {
try {
parsedTorrent = parseTorrent(metadata)
} catch (err) {
return self._onError(err)
}
}
self._processParsedTorrent(parsedTorrent)
self.metadata = self.torrentFile
// pass full torrent metadata to discovery module
self.discovery.setTorrent(self)
// add web seed urls (BEP19)
if (self.urlList) self.urlList.forEach(self.addWebSeed.bind(self))
self.rarityMap = new RarityMap(self.swarm, self.pieces.length)
self.store = new ImmediateChunkStore(
new self._store(self.pieceLength, {
files: self.files.map(function (file) {
return {
path: path.join(self.path, file.path),
length: file.length,
offset: file.offset
}
}),
length: self.length
})
)
self.files = self.files.map(function (file) {
return new File(self, file)
})
self._hashes = self.pieces
self.pieces = self.pieces.map(function (hash, i) {
var pieceLength = (i === self.pieces.length - 1)
? self.lastPieceLength
: self.pieceLength
return new Piece(pieceLength)
})
self._reservations = self.pieces.map(function () {
return []
})
self.bitfield = new BitField(self.pieces.length)
self.swarm.wires.forEach(function (wire) {
// If we didn't have the metadata at the time ut_metadata was initialized for this
// wire, we still want to make it available to the peer in case they request it.
if (wire.ut_metadata) wire.ut_metadata.setMetadata(self.metadata)
self._onWireWithMetadata(wire)
})
debug('verifying existing torrent data')
parallelLimit(self.pieces.map(function (piece, index) {
return function (cb) {
self.store.get(index, function (err, buf) {
if (err) return cb(null) // ignore error
sha1(buf, function (hash) {
if (hash === self._hashes[index]) {
if (!self.pieces[index]) return
debug('piece verified %s', index)
self.pieces[index] = null
self._reservations[index] = null
self.bitfield.set(index, true)
} else {
debug('piece invalid %s', index)
}
cb(null)
})
})
}
}), cpus().length, function (err) {
if (err) return self._onError(err)
debug('done verifying')
self._onStore()
})
self.emit('metadata')
}