/**
* Mark a value as seen, only actually updating the Azure table row
* occasionally. This takes the approach of updating the row immediately on
* the first call, then not updating until `updateFrequency` has elapsed.
* Thus the `expires` value on a row may be out-of-date by `updateFrequency`.
*
* Note that the cache is never purged of outdated entries; this assumes that
* the process is restarted on a daily basis, so there is not too much time
* for stale cache entries to accumulate.
*/
async valueSeen(key, updateExpires) {
let now = new Date();
let nextUpdate = this.nextUpdateAt[key];
if (!nextUpdate || nextUpdate < now) {
this.nextUpdateAt[key] = taskcluster.fromNow(this.updateFrequency);
await updateExpires();
}
}
test('success', async function() {
const res = await helper.client().claimNamespace('tcpulse-test-sample', {
expires: taskcluster.fromNow('1 day'),
contact: '*****@*****.**',
});
assert.equal(res.namespace, 'tcpulse-test-sample');
// check that the connection string is in the proper vhost
assert(res.connectionString.endsWith(encodeURIComponent('/test')));
});
test('returns namespace', async () => {
await helper.client().claimNamespace('tcpulse-test-sample', {
expires: taskcluster.fromNow('1 day'),
contact: '*****@*****.**',
});
let res = await helper.client().namespace('tcpulse-test-sample');
assert.equal(res.namespace, 'tcpulse-test-sample');
});
test('charlene revokes role3', async () => {
let newClient = await charlene.updateClient('test-users/charlene/travis-tests', {
description: 'Permacred created by test',
expires: taskcluster.fromNow('3 hours'),
scopes: [
'assume:test-role:role2',
],
});
});
test.skip('charlene replaces role3 with one of its constituent scopes', async () => {
let newClient = await charlene.updateClient('test-users/charlene/travis-tests', {
description: 'Permacred created by test',
expires: taskcluster.fromNow('3 hours'),
scopes: [
'scope3a',
],
});
});
test('root grants role3 again', async () => {
let newClient = await helper.apiClient.updateClient('test-users/charlene/travis-tests', {
description: 'Permacred created by test',
expires: taskcluster.fromNow('3 hours'),
scopes: [
'assume:test-role:role3',
],
});
});
test('charlene grants role2 and removes role1', async () => {
let newClient = await charlene.updateClient('test-users/charlene/travis-tests', {
description: 'Permacred created by test',
expires: taskcluster.fromNow('3 hours'),
scopes: [
'assume:test-role:role2',
],
});
assume(newClient.scopes).to.contain('assume:test-role:role2');
});
test("root grants role3", async () => {
let newClient = await helper.auth.updateClient('test-users/charlene/travis-tests', {
description: "Permacred created by test",
expires: taskcluster.fromNow("3 hours"),
scopes: [
'assume:test-role:role2',
'assume:test-role:role3',
],
});
});
test('char invalid symbols', () => {
return helper.client().claimNamespace('tcpulse-test-%', {
expires: taskcluster.fromNow('1 day'),
contact: '*****@*****.**',
}).then(function() {
assert(false, 'This shouldn\'t have worked');
}, function(err) {
assert(err.statusCode === 400, 'Should have returned 400');
});
});
test('char limit over', () => {
const longname = 'tcpulse-test-samplenamespacesamplenamespacesamplenamespacesamplenamespace';
return helper.client().claimNamespace(longname, {
expires: taskcluster.fromNow('1 day'),
contact: '*****@*****.**',
}).then(function() {
assert(false, 'This shouldn\'t have worked');
}, function(err) {
assert(err.statusCode === 400, 'Should have returned 400');
});
});
WorkerType.prototype.declareWorkerType = function() {
// keep the worker around well over 24 hours, as we will likely re-declare
// at least that often
const expires = taskcluster.fromNow('36 hours');
log.info(`declaring workerType ${this.workerType} to queue`);
return this.queue.declareWorkerType(this.provisionerId, this.workerType, {
stability: 'stable',
expires,
description: `${this.description}\n\nOwner: ${this.owner}`,
});
};
test('idempotency - return same namespace', async () => {
let expires = taskcluster.fromNow('1 day');
let a = await helper.client().claimNamespace('tcpulse-test-sample', {
expires,
contact: '*****@*****.**',
});
let b = await helper.client().claimNamespace('tcpulse-test-sample', {
expires,
contact: '*****@*****.**',
});
assert(_.isEqual(a, b));
});
test('header auth (temp creds)', async () => {
let myClient2 = new helper.TestClient({
rootUrl: helper.rootUrl,
credentials: taskcluster.createTemporaryCredentials({
expiry: taskcluster.fromNow('10 min'),
scopes: ['myapi:*'],
credentials: rootCredentials,
}),
});
let result = await myClient2.resource();
assert(result.message === 'Hello World');
});
setup: async ({cfg, TaskGroup, influx}) => {
var now = taskcluster.fromNow(cfg.app.taskExpirationDelay);
assert(!_.isNaN(now), "Can't have NaN as now");
// Expire task-groups using delay
debug("Expiring task-groups at: %s, from before %s", new Date(), now);
let count = await TaskGroup.expire(now);
debug("Expired %s task-groups", count);
// Stop recording statistics and send any stats that we have
base.stats.stopProcessUsageReporting();
return influx.close();
}
setup: async ({cfg, Artifact, influx}) => {
// Find an artifact expiration delay
let now = taskcluster.fromNow(cfg.app.artifactExpirationDelay);
assert(!_.isNaN(now), "Can't have NaN as now");
debug("Expiring artifacts at: %s, from before %s", new Date(), now);
let count = await Artifact.expire(now);
debug("Expired %s artifacts", count);
// Stop recording statistics and send any stats that we have
base.stats.stopProcessUsageReporting();
return influx.close();
}
setup: async ({cfg, monitor, IndexedTask, Namespace}) => {
let now = taskcluster.fromNow(cfg.app.expirationDelay);
debug('Expiring namespaces');
let namespaces = await Namespace.expireEntries(now);
debug(`Expired ${namespaces} namespaces`);
debug('Expiring indexed tasks');
let tasks = await IndexedTask.expireTasks(now);
debug(`Expired ${tasks} tasks`);
monitor.count('expire.done');
monitor.stopResourceMonitoring();
await monitor.flush();
},
test('update contact', async () => {
let expires = taskcluster.fromNow('1 day');
let a = await helper.client().claimNamespace('tcpulse-test-sample', {
expires,
contact: '*****@*****.**',
});
let b = await helper.client().claimNamespace('tcpulse-test-sample', {
expires,
contact: '*****@*****.**',
});
// contact info isn't returned (ever)
assert(!b.contact);
});
test("auth.currentScopes with temp credentials", async () => {
let auth = new helper.Auth({
baseUrl: helper.baseUrl,
credentials: taskcluster.createTemporaryCredentials({
expiry: taskcluster.fromNow('10 min'),
scopes: ['myapi:x', 'myapi:y'],
credentials: {
clientId: 'root',
accessToken: helper.cfg.app.rootAccessToken,
},
}),
});
assumeScopesetsEqual(await auth.currentScopes(),
{scopes: ['myapi:x', 'myapi:y']});
});
test('entry creation', async () => {
for (let i = 0; i < 10; i++) {
await helper.client().claimNamespace('tcpulse-test-sample', {
expires: taskcluster.fromNow('1 day'),
contact: '*****@*****.**',
});
}
let count = 0;
await helper.Namespace.scan({},
{
limit: 250,
handler: ns => count++,
});
assert.equal(count, 1);
});
test('charlene tries to grant role3 (which she does not have) to her client', async () => {
try {
await charlene.updateClient('test-users/charlene/travis-tests', {
description: 'Permacred created by test',
expires: taskcluster.fromNow('3 hours'),
scopes: [
'assume:test-role:role1',
'assume:test-role:role3',
],
});
throw new Error('did not get expected error');
} catch (err) {
assume(err.statusCode).to.equal(403);
}
});
test('header auth (temp creds - wrong scope)', async () => {
let myClient2 = new helper.TestClient({
rootUrl: helper.rootUrl,
credentials: taskcluster.createTemporaryCredentials({
expiry: taskcluster.fromNow('10 min'),
scopes: ['myapi--'],
credentials: rootCredentials,
}),
});
await myClient2.resource().then(() => {
assert(false, 'expected an error!');
}, err => {
assert(err.statusCode === 403, 'expected 403');
});
});
test("charlene creates permanent credentials for her tests", async () => {
let travisClient = await charlene.createClient('test-users/charlene/travis-tests', {
description: "Permacred created by test",
expires: taskcluster.fromNow("3 hours"), // N.B. longer than temp creds
scopes: [
'assume:test-role:role1',
],
});
travisTests = new helper.Auth({
credentials: {
clientId: 'test-users/charlene/travis-tests',
accessToken: travisClient.accessToken
}
});
});
describe('secrets api', () => {
let client;
var token = slugid.v4();
var secretToAdd = {
workerType: 'workerType',
secrets: {
key1: true,
key2: 123,
key3: 'sample',
key4: {a: 123},
},
scopes: ['ascope'],
token: token,
expiration: taskcluster.fromNow('1 day'),
};
before(async () => {
client = helper.getClient();
});
beforeEach(async () => {
await main('tableCleaner', {process: 'tableCleaner', profile: 'test'});
});
it('should be able to create a secret (idempotent)', async () => {
await client.createSecret(token, secretToAdd);
await client.createSecret(token, secretToAdd);
});
it('should be able to load a secret', async () => {
await client.createSecret(token, secretToAdd);
var loadedSecret = await client.getSecret(token);
assume(loadedSecret.data).to.eql(secretToAdd.secrets);
});
it('should be able to remove a secret (idempotent)', async () => {
await client.removeSecret(token);
await client.removeSecret(token);
try {
await client.getSecret(token);
} catch (err) {
assume(err.statusCode).equals(404);
}
});
});
async killed(task) {
debug('switching live log redirect to backing log...')
try {
await this.container.stop();
} catch (e) {
// Catch any errors when stopping the container, but don't throw an exception
// that would cause the task to fail.
debug('Caught error when stopping live log container. %s %s', e, e.stack);
}
// Can't create artifacts for a task that's been canceled
if (task.isCanceled()) {
// Cleanup all references to the live logging server...
task.runtime.gc.removeContainer(this.container.id);
return;
}
// Note here we don't wait or care for the live logging to complete
// correctly we simply let it pass/fail to finish since we are going to kill
// the connection anyway...
let backingUrl = await this.bulkLog.killed(task);
// Switch references to the new log file on s3 rather then the local worker
// server...
let expiration = taskcluster.fromNow(task.runtime.logging.bulkLogExpires);
expiration = new Date(Math.min(expiration, new Date(task.task.expires)));
await task.queue.createArtifact(
task.status.taskId,
task.runId,
this.logsLocations.live,
{
storageType: 'reference',
expires: expiration,
contentType: 'text/plain',
url: backingUrl
}
);
// Cleanup all references to the live logging server...
task.runtime.gc.removeContainer(this.container.id);
}
test('Expiring Indexed Tasks', async function() {
// Create expiration
var expiry = new Date();
expiry.setDate(expiry.getDate() - 1);
var myns = slugid.v4();
var taskId = slugid.v4();
var taskId2 = slugid.v4();
await helper.index.insertTask(myns + '.my-task', {
taskId: taskId,
rank: 41,
data: {hello: 'world'},
expires: expiry.toJSON(),
});
let result = await helper.index.findTask(myns + '.my-task');
assert(result.taskId === taskId, 'Wrong taskId');
expiry.setDate(expiry.getDate() + 1);
await helper.index.insertTask(myns + '.my-task2', {
taskId: taskId2,
rank: 42,
data: {hello: 'world two'},
expires: expiry.toJSON(),
});
result = await helper.index.findTask(myns + '.my-task2');
assert(result.taskId === taskId2, 'Wrong taskId');
// Set now to one day in the past
var now = taskcluster.fromNow('- 1 day');
debug('Expiring indexed tasks at: %s, from before %s', new Date(), now);
await helper.handlers.IndexedTask.expireTasks(now);
try {
await helper.index.findTask(myns + '.my-task');
} catch (err) {
assert(err.statusCode === 404, 'Should have returned 404');
return;
}
assert(false, 'This shouldn\'t have worked');
});
test("create temporary credentials for charlene's browser login", async () => {
charlene = new helper.Auth({
credentials: taskcluster.createTemporaryCredentials({
start: new Date(),
expiry: taskcluster.fromNow("1 hour"),
credentials: {
clientId: 'test-users',
accessToken: identityProviderToken
},
scopes: [
'auth:create-client:test-users/charlene/*',
'auth:update-client:test-users/charlene/*',
'auth:delete-client:test-users/charlene/*',
'auth:reset-access-token:test-users/charlene/*',
'assume:test-role:role1',
'assume:test-role:role2',
],
}),
});
});
}, async function(req, res) {
let {namespace} = req.params;
// NOTE: at the moment we do not confirm that the user has permission to send
// the given notification. The possibility of abuse is remote (and would
// involve abusing pulse), but we can solve this problem if and when we have
// it.
let expires = req.body.expires ? new Date(req.body.expires) : taskcluster.fromNow('4 hours');
let contact = req.body.contact || '';
let newNamespace = await maintenance.claim({
Namespace: this.Namespace,
cfg: this.cfg,
rabbitManager: this.rabbitManager,
namespace,
contact,
expires,
});
res.reply(newNamespace.json({cfg: this.cfg, includePassword: true}));
});
Client.ensureRootClient = function(accessToken) {
assert(typeof(accessToken) === 'string',
"Expected accessToken to be a string");
let Client = this;
// Create client resolving conflicts by overwriting
return Client.create({
clientId: 'root',
description: "Automatically created `root` client with star scopes " +
"for bootstrapping API access",
accessToken: accessToken,
expires: taskcluster.fromNow('24 hours'),
details: {
created: new Date().toJSON(),
lastModified: new Date().toJSON(),
lastDateUsed: new Date().toJSON(),
lastRotated: new Date().toJSON()
},
scopes: ['*'],
disabled: 0,
}, true);
};
async upsertWorkerType({provisionerId, workerType, stability, description, expires}) {
const wType = await this.WorkerType.load({provisionerId, workerType}, true);
let result;
if (wType) {
result = await wType.modify((entity) => {
entity.stability = stability || entity.stability;
entity.description = description || entity.description;
entity.expires = new Date(expires || entity.expires);
});
} else {
result = await this.WorkerType.create({
provisionerId,
workerType,
expires: new Date(expires || taskcluster.fromNow('5 days')),
lastDateActive: new Date(),
description: description || '',
stability: stability || 'experimental',
});
}
return result;
}
test("add a client for the identity provider", async () => {
let idp = await helper.auth.createClient('test-users', {
description: "Test users identity provider",
expires: taskcluster.fromNow("2 hours"),
scopes: [
'auth:create-client:test-users/*',
'auth:update-client:test-users/*',
'auth:enable-client:test-users/*',
'auth:disable-client:test-users/*',
'auth:delete-client:test-users/*',
'auth:reset-access-token:test-users/*',
'assume:test-role:*',
],
});
identityProviderToken = idp.accessToken;
identityProvider = new helper.Auth({
credentials: {
clientId: 'test-users',
accessToken: identityProviderToken
}
});
});