function addDevSamplePkgs(opts, cb) {
var papi = opts.sdcadm.papi;
var samplePkgsPath = path.resolve(__dirname,
'../../etc/sample-packages.json');
var samplePkgs = JSON.parse(fs.readFileSync(samplePkgsPath, 'utf-8'));
vasync.forEachPipeline({
inputs: samplePkgs,
func: function importPkg(pkgData, nextPkg) {
papi.list({name: pkgData.name}, {}, function (err, pkgs) {
if (err) {
nextPkg(new errors.SDCClientError(err, 'papi'));
return;
} else if (pkgs.length !== 0) {
opts.progress('Already have package %s (%s).',
pkgData.name, pkgs[0].uuid);
nextPkg();
return;
}
papi.add(pkgData, function (addErr, pkg) {
if (addErr) {
nextPkg(new errors.SDCClientError(addErr, 'papi'));
return;
}
opts.progress('Added package %s (%s)', pkg.name, pkg.uuid);
nextPkg();
});
});
}
}, cb);
}
exec(cmd, function (err, stdout, stderr) {
t.ifError(err);
vasync.forEachPipeline({
func: function checkOne(check, cb) {
var mpath = path.join(TSTDIR, check.path);
self.client.info(mpath, function (err2, info) {
t.ifError(err2, err2);
if (!err2) {
t.equal(info.type, check.type, format(
'%s is expected type (%s): %s',
mpath, check.type, info.type));
if (check.size) {
t.equal(info.size, check.size, format(
'%s is expected size (%s): %s',
mpath, check.size, info.size));
}
if (check.md5) {
t.equal(info.md5, check.md5, format(
'%s is expected md5 (%s): %s',
mpath, check.md5, info.md5));
}
}
cb();
});
},
inputs: c.checks
}, function (err3, results) {
self.client.rmr(TSTDIR, function (rmErr) {
t.ifError(rmErr, rmErr);
t.done();
});
});
});
/*
* Clean out package field cruft from some of the services that we
* created in sapi post-setup (e.g. nat, hostvolume, and docker).
*/
function cleanPkgCruftFromSomeSvcs(ctx, next) {
var svcNames = ['docker', 'hostvolume', 'nat'];
var paramUpdatesFromSvcName = {};
svcNames.forEach(function (svcName) {
var svc = ctx.svcFromName[svcName];
if (!svc) {
return;
}
var paramUpdates = {};
if (svc.params.customer_metadata &&
Object.keys(svc.params.customer_metadata).length === 0) {
paramUpdates.customer_metadata = null;
}
['cpu_shares',
'cpu_cap',
'zfs_io_priority',
'max_lwps',
'max_physical_memory',
'max_locked_memory',
'max_swap',
'quota',
'package_version',
'package_name'].forEach(function (field) {
if (svc.params.hasOwnProperty(field)) {
paramUpdates[field] = null;
}
});
// 'nat' zone service shouldn't have a "networks" (TOOLS-1101)
if (svcName === 'nat') {
if (svc.params.hasOwnProperty('networks')) {
paramUpdates['networks'] = null;
}
}
if (Object.keys(paramUpdates).length !== 0) {
paramUpdatesFromSvcName[svcName] = paramUpdates;
}
});
vasync.forEachPipeline({
inputs: Object.keys(paramUpdatesFromSvcName),
func: function updateSvc(svcName, nextSvc) {
var paramUpdates = paramUpdatesFromSvcName[svcName];
log.debug({paramUpdates: paramUpdates},
'docker service param cruft');
progress('Remove package cruft from "%s" svc params: %s',
svcName, Object.keys(paramUpdates).join(', '));
var svcUuid = ctx.svcFromName[svcName].uuid;
self.sdcadm.sapi.updateService(svcUuid, {
action: 'delete',
params: paramUpdates
}, errors.sdcClientErrWrap(nextSvc, 'sapi'));
}
}, next);
},
function doZone(zname, zcb) {
var addRecords = [];
vasync.forEachPipeline({
func: doEntry,
inputs: newZones[zname]
}, function (err) {
self.parsedRecords[zname] = addRecords;
zcb(null);
});
function doEntry(entry, ecb) {
addRecords[entry.name] = [];
vasync.forEachPipeline({
func: doRecord,
inputs: entry.records
}, ecb);
function doRecord(rec) {
var constructor = Object.keys(rec)[0];
var newRec = {
"constructor": constructor,
"args": rec[constructor],
"id": genuuid()
};
addRecords[entry.name].push(newRec);
ecb(null);
}
}
}
}, function (histErr, history) {
if (histErr) {
callback(new errors.DockerError(histErr,
'could not create image layer'));
return;
}
vasync.forEachPipeline({
inputs: history,
func: addImageHead
}, function (err) {
if (err) {
callback(new errors.DockerError(err,
'could not create image layer'));
return;
}
callback();
});
function addImageHead(img, next) {
log.debug('addImageHead img id: %s, head: %s',
img.docker_id, finalImageId);
var addHeadOpts = {
heads: [finalImageId],
id: img.docker_id,
index_name: img.index_name,
req: req
};
req.log.debug(format('Adding %j to %j image heads',
finalImageId, img.docker_id));
req.backend.addImageHeads(addHeadOpts, next);
}
});
SlaveConnection.prototype.state_setup.npm = function (S) {
var self = this;
mod_vasync.forEachPipeline({
func: processPkgsrc,
inputs: this.sc_config.slaves.npm || []
}, function (err) {
if (err) {
self.sc_log.error(err, 'failed to setup zone');
S.gotoState('closing');
return;
}
S.gotoState('setup.clean_old');
});
function processPkgsrc(instr, cb) {
var cmd = 'pfexec';
var args = ['/opt/local/bin/npm', 'install', '-g', instr];
var kid = self.spawn(cmd, args);
var errOut = '';
S.on(kid.stderr, 'data', function (data) {
errOut = errOut + data.toString('utf-8');
});
S.on(kid, 'close', function (exitStatus) {
if (exitStatus === 0) {
cb();
return;
}
self.sc_log.error('failed to run npm cmd in zone',
{args: args, stderr: errOut});
cb(new Error('npm command failed'));
});
}
};
return ufds.listRoles(id, function (err, groups) {
if (err) {
return next(err);
}
return vasync.forEachPipeline({
func: function (group, cb) {
translateGroup(req, group, function (err2, grp) {
if (err) {
return cb(err2);
}
grps.push(grp);
return cb(null);
});
},
inputs: groups
}, function (error, results) {
if (error) {
return next(error);
}
log.debug('GET %s => %j', req.path(), grps);
res.send(grps);
return next();
});
});
function gatherDataInteractive(_, next) {
if (opts.file) {
return next();
} else if (!process.stdin.isTTY) {
return next(new errors.UsageError('cannot interactively ' +
'create a user: stdin is not a TTY'));
} else if (!process.stdout.isTTY) {
return next(new errors.UsageError('cannot interactively ' +
'create a user: stdout is not a TTY'));
}
// TODO: confirm password
// TODO: some validation on login, email, password complexity
// TODO: retries on failure
// TODO: on failure write out to a tmp file with cmd to add it
data = {};
vasync.forEachPipeline({
inputs: CREATE_USER_FIELDS,
func: function getField(field, nextField) {
common.promptField(field, function (err, value) {
if (value) {
data[field.key] = value;
}
nextField(err);
});
}
}, function (err) {
console.log();
next(err);
});
},
}, function (cb) {
var nic_index = 0;
var nics;
nics = Object.keys(payload['Network Interfaces']);
vasync.forEachPipeline({
inputs: nics,
func: function (n, c) {
var nic = payload['Network Interfaces'][n];
addMAC(nic, mock_oui, cn_index, nic_index++, c);
if (nic.hasOwnProperty('NIC Names') &&
nic['NIC Names'].indexOf('admin') !== -1) {
admin_nic = nic;
} else if (nic_index === 1 && !admin_nic) {
// default to first one in case we don't have specified
// one
admin_nic = nic;
}
}
}, function (e) {
cb(e);
});
}, function (cb) {
exports.create_vm_invalid_volumes_params = testIfVolapiPresent(function (t) {
var INVALID_VOLUMES_PARAMS = [
null,
[],
[ {} ],
[ {foo: 'bar'} ],
[ {name: '---,*;'} ],
[ {name: 'foo', mountpoint: 'invalidmountpoint'} ],
[ {name: 'foo', mountpoint: '/bar', mode: 'invalidmode'} ]
];
vasync.forEachPipeline({
func: function provisionVmWithInvalidVol(volumesParam, next) {
var vmPayload = getVmPayloadTemplate();
vmPayload.volumes = volumesParam;
client.post({
path: '/vms'
}, vmPayload, function onVmCreated(createVmErr, req, res, body) {
var expectedResStatusCode = 409;
t.ok(createVmErr, 'VM creation should error');
t.equal(res.statusCode, expectedResStatusCode,
'HTTP status code should be ' + expectedResStatusCode);
next();
});
},
inputs: INVALID_VOLUMES_PARAMS
}, function onAllInvalidVolParamsTestsDone(err) {
t.done();
});
});
load_build_specs: function (base_file, optional_file, cb) {
mod_assert.string(base_file, 'base_file');
mod_assert.string(optional_file, 'optional_file');
mod_assert.func(cb, 'cb');
var bs = new BuildSpec();
mod_vasync.forEachPipeline({
func: function (_, next) {
bs.load_file(_.path, _.optional, next);
},
inputs: [
{ optional: false, path: base_file },
{ optional: true, path: optional_file }
]
}, function (err) {
if (err) {
cb(new VError(err, 'failed to load build ' +
'specs'));
return;
}
cb(null, bs);
});
}
function main() {
executed = 0;
failed = 0;
assertplus.string(config_file);
if (!fs.existsSync(config_file)) {
console.log('tests for %s could not be run, since the' +
'configuration file needed at %s is missing',
target_path, config_file);
process.exit(1);
}
vasync.forEachPipeline({
'func': runTestCases,
'inputs': cases
}, function (err) {
if (err) {
console.error('test failed with error: ', err);
}
});
process.on('exit', function () {
assertplus.equal(executed, cases.length);
console.log('%d case(s) executed, %d case(s) failed',
executed, failed);
if (failed > 0) {
process.exit(1);
}
});
}
function _runMigrationsForModel(modelName, dataMigrations, callback) {
assert.string(modelName, 'modelName');
assert.arrayOfObject(dataMigrations, 'dataMigrations');
assert.func(callback, 'callback');
assert.object(this._log, 'this._log');
var log = this._log;
var self = this;
log.info('Starting data migrations for model %s', modelName);
self._latestCompletedMigrations = {};
vasync.forEachPipeline({
func: function runSingleMigration(migration, next) {
assert.number(migration.DATA_VERSION, 'migration.DATA_VERSION');
assert.ok(migration.DATA_VERSION >= 1,
'migration.DATA_VERSION >= 1');
self._runSingleMigration(modelName, migration, {
log: log
}, function onMigration(migrationErr) {
if (migrationErr) {
if (self._latestErrors === undefined) {
self._latestErrors = {};
}
self._latestErrors[modelName] = migrationErr;
log.error({err: migrationErr},
'Error when running migration to data version: ' +
migration.DATA_VERSION);
} else {
self._latestCompletedMigrations[modelName] =
migration.DATA_VERSION;
if (self._latestErrors && self._latestErrors[modelName]) {
delete self._latestErrors[modelName];
if (Object.keys(self._latestErrors).length === 0) {
self._latestErrors = undefined;
}
}
log.info('Data migration to data version: ' +
migration.DATA_VERSION + ' ran successfully');
}
next(migrationErr);
});
},
inputs: dataMigrations
}, function onAllMigrationsDone(migrationsErr, results) {
var err;
if (migrationsErr) {
err = new VError(migrationsErr, 'Failed to run data migrations');
}
callback(err);
});
};
function executeTestCases(callback) {
vasync.forEachPipeline({
inputs: testcases,
func: executeCopyOutTestcase
},
function (err) {
callback(err);
});
}
Server.prototype._emitErrorEvents = function _emitErrorEvents(
req,
res,
route,
err,
cb
) {
var self = this;
// Error can be of any type: undefined, Error, Number, etc. so we need
// to protect ourselves from trying to resolve names from non Error objects
var errName = err && err.name;
var normalizedErrName = errName && errEvtNameFromError(err);
req.log.trace(
{
err: err,
errName: normalizedErrName
},
'entering emitErrorEvents',
errName
);
var errEvtNames = [];
// if we have listeners for the specific error, fire those first.
// if there's no error name, we should not emit an event
if (normalizedErrName && self.listeners(normalizedErrName).length > 0) {
errEvtNames.push(normalizedErrName);
}
// or if we have a generic error listener. always fire generic error event
// listener afterwards.
if (self.listeners('restifyError').length > 0) {
errEvtNames.push('restifyError');
}
// kick off the async listeners
return vasync.forEachPipeline(
{
inputs: errEvtNames,
func: function emitError(errEvtName, vasyncCb) {
self.emit(errEvtName, req, res, err, function emitErrDone() {
// the error listener may return arbitrary objects, throw
// them away and continue on. don't want vasync to take
// that error and stop, we want to emit every single event.
return vasyncCb();
});
}
},
// eslint-disable-next-line handle-callback-err
function onResult(nullErr, results) {
// vasync will never return error here. callback with the original
// error to pass it on.
return cb(err);
}
);
};
function populateDirs(subcallback) {
log.info('populating directories', dirs);
mod_vasync.forEachPipeline({
'inputs': dirs,
'func': function doMkdir(dirname, subsubcb) {
api.manta.mkdirp(dirname, subsubcb);
}
}, function (err) { subcallback(err); });
},
function rmAgentDirs(_, next) {
vasync.forEachPipeline({
inputs: agents,
func: function rmOneAgent(agent, nextAgent) {
var agentDir = path.join(agentsDir, agent);
rimraf(agentDir, nextAgent);
}
}, next);
},
DNSServer.prototype.garbageCollect = function (serials, peers, cb) {
var self = this;
/* First build the lookup of minimum required serial for each zone. */
var minSerial = Object.create(serials);
Object.keys(peers).forEach(function (peer) {
if (peer === '127.0.0.1' || peer === '::1')
return;
var zones = peers[peer];
Object.keys(zones).forEach(function (zone) {
if (minSerial[zone] !== undefined &&
zones[zone] < minSerial[zone]) {
minSerial[zone] = zones[zone];
}
});
});
vasync.forEachPipeline({
func: doZone,
inputs: Object.keys(serials)
}, function (err, res) {
if (cb)
cb(err);
});
function doZone(zone, zcb) {
var k = 'zone:' + zone + ':all';
self.redis.lrange(k, 0, -1, function (err2, zserials) {
if (err2 || zserials === null || zserials === []) {
zcb(null);
return;
}
zserials = zserials.map(function (v) {
return (parseInt(v, 10));
}).filter(function (v) {
return (v < minSerial[zone]);
});
/*
* Policy: all serials < the minimum required serial
* for this zone will be dropped, except the last 2.
*/
zserials = zserials.slice(0, -2);
if (zserials.length > 0) {
self.log.debug({zone: zone},
'garbage collecting %d serials',
zserials.length);
}
vasync.forEachParallel({
func: doSerial,
inputs: zserials
}, zcb);
function doSerial(serial, scb) {
self.dropSerial(zone, serial, scb);
}
});
}
};
morayFind(BUCKET, '(uuid=*)', function (err, objs) {
assert.ifError(err);
console.log('%s objects found', objs.length);
vasync.forEachPipeline({
inputs: objs,
func: updatePackage
}, cb);
});
var remove = function(app) {
app.emit('pre-remove');
var components = _.toArray(_.cloneDeep(app.components));
vasync.forEachPipeline({
'func': cmp.remove,
'inputs': components
}, function(err, results) {
app.emit('post-remove');
});
};
var kill = function(app) {
app.emit('pre-kill');
var components = _.toArray(_.cloneDeep(app.components));
vasync.forEachPipeline({
'func': cmp.kill,
'inputs': components
}, function(err, container) {
app.emit('post-kill');
});
};
}, function (err) {
vasync.forEachPipeline({
func: doAdds,
inputs: Object.keys(ec.parsedRecords)
}, function (err2) {
if (err2 === null)
console.log("Zones added.");
ec.client.quit();
});
});
function main() {
vasync.forEachPipeline({
'func': runTestCase,
'inputs': testCases
}, function (err) {
assertplus.ok(!err);
assertplus.equal(nrun, testCases.length);
console.error('%d test cases run', nrun);
});
}
UpdateBinderV1.prototype.execute = function ushiExecute(opts, cb) {
assert.object(opts, 'opts');
assert.object(opts.sdcadm, 'opts.sdcadm');
assert.object(opts.plan, 'opts.plan');
assert.object(opts.log, 'opts.log');
assert.func(opts.progress, 'opts.progress');
assert.string(opts.wrkDir, 'opts.wrkDir');
assert.func(cb, 'cb');
var self = this;
var log = opts.log;
// var progress = opts.progress;
// Mirroring UpdateStatelessServicesV1 above, even though here we should
// only have one instance.
vasync.forEachPipeline({
inputs: self.changes,
func: updateBinder
}, cb);
function updateBinder(change, nextSvc) {
var arg = {
change: change,
opts: opts,
userScript: false
};
var inst = change.inst;
vasync.pipeline({funcs: [
function bailIfBinderHasNoDelegate(_, next) {
vmadm.vmGet(inst.zonename, {log: log}, function (err, vm) {
if (err) {
return next(err);
}
var expectedDs = sprintf('zones/%s/data', inst.zonename);
log.debug({expectedDs: expectedDs, vm: vm}, 'binder vm');
if (vm.datasets.indexOf(expectedDs) === -1) {
return next(new errors.UpdateError(format(
'binder vm %s has no "%s" delegate dataset, ' +
'upgrading it would lose image file data',
vm.uuid, expectedDs)));
}
next();
});
},
s.getUserScript,
s.writeOldUserScriptForRollback,
s.updateSvcUserScript,
s.updateVmUserScript,
s.updateSapiSvc,
s.imgadmInstall,
s.reprovision,
s.waitForInstToBeUp
], arg: arg}, nextSvc);
}
};
var pull = function(app) {
app.emit('pre-pull');
var components = _.filter(app.components, function(component) { return !component.image.build });
var images = _.pluck(components, 'image');
vasync.forEachPipeline({
'func': image.pull,
'inputs': images
}, function(err, results) {
app.emit('post-pull');
});
};
var init = function(app) {
// TODO: Validate if <appname>_ containers exist already
app.emit('pre-init');
var components = _.toArray(_.cloneDeep(app.components));
vasync.forEachPipeline({
'func': cmp.create,
'inputs': components
}, function(err, results) {
app.emit('post-init');
});
};
function (_, subcb) {
vasync.forEachPipeline({
'func': oldRaft.clientRequest.bind(oldRaft),
'inputs': [
{ 'command': 'foo' },
{ 'command': 'bar' },
{ 'command': 'baz' },
{ 'command': 'bang' }
]
}, subcb);
},
var build = function(app) {
app.emit('pre-build');
var components = _.filter(app.components, function(component) { return component.image.build === true});
var images = _.pluck(components, 'image');
vasync.forEachPipeline({
'func': image.build,
'inputs': images
}, function(err, results) {
app.emit('post-build');
});
};
function verifyAndLinkFiles(_, cb) {
vasync.forEachPipeline({
'func': verifyAndLinkFile,
'inputs': _.cruft.map(function (c) {
//This is a little wonky but I need the _ to propagate
return ({
'cruft': c,
'opts': _
});
})
}, cb);
}
function copyFilesIn(local, remote, cb) {
vasync.forEachPipeline({
inputs: remoteFilenames,
func: function (filename, next) {
copyFileIn(tt, local,
remote + filename, CONTAINER_NAME_ALPINE, next);
}
}, function (err) {
tt.ifErr(err, 'no error copying test files into container');
cb();
});
}