var getProjectId = function*(request, projectName) {
var names = projectName.split('/');
var response;
yield retry(function*(){
response = yield server.get(request('/projects/search/' + names[1] + '?per_page=100'));
assertStatus(response, 'response.statusCode < 300');
}, {retries: 10});
for(var i = 0; i < response.body.length; i++) {
if(response.body[i].path_with_namespace === projectName) {
return response.body[i].id;
}
}
return null;
};
/**
* send to bigquery with retry
* @property params
* @property {object} params.bq_client - bigquery client
* @property {string} params.dataset_id - target dataset_id
* @property {string} params.table_id - target table_id
* @property {string} params.schema - schema of table
* @property {string} params.data - data to send
* @returns {*}
*/
sendToBigQuery(params) {
var self = this;
return retry(function* () {
var dataset = params.bq_client.dataset(params.dataset_id);
var table = dataset.table(params.table_id);
try {
return yield table.insertAsync(params.data);
} catch (e) {
if (e.code === 404) {
if (e.message.includes("Not found: Table")) {
yield dataset.createTableAsync({
id: params.table_id,
schema: {
fields: params.table_definition.fields
},
expirationTime: moment().add(params.table_definition.ttl, 'days').valueOf()
});
return yield table.insertAsync(params.data);
} else if (e.message.includes("Not found: Dataset")) {
yield params.bq_client.createDatasetAsync(params.dataset_id);
yield dataset.createTableAsync({
id: params.table_id,
schema: {
fields: params.table_definition.fields
},
expirationTime: moment().add(params.table_definition.ttl, 'days').valueOf()
});
return yield table.insertAsync(params.data);
} else {
self.logger.error(new Date(), e);
throw e;
}
} else {
self.logger.error(new Date(), e);
throw e;
}
}
}, {retries: params.retries || 10, interval: params.retry_interval || 500, factor: 2});
}