SyncModuleWorker.prototype.syncUpstream = function* (name) {
if (config.sourceNpmRegistry.indexOf('registry.npmjs.org') >= 0 ||
config.sourceNpmRegistry.indexOf('registry.npmjs.com') >= 0 ||
config.sourceNpmRegistry.indexOf('replicate.npmjs.com') >= 0) {
this.log('----------------- upstream is npm registry: %s, ignore it -------------------',
config.sourceNpmRegistry);
return;
}
var syncname = name;
if (this.type === 'user') {
syncname = this.type + ':' + syncname;
}
var url = config.sourceNpmRegistry + '/' + syncname + '/sync?sync_upstream=true';
if (this.noDep) {
url += '&nodeps=true';
}
var r = yield urllib.request(url, {
method: 'put',
timeout: 20000,
headers: {
'content-length': 0
},
dataType: 'json',
gzip: true,
});
if (r.status !== 201 || !r.data.ok) {
return this.log('sync upstream %s error, status: %s, response: %j',
url, r.status, r.data);
}
var logURL = config.sourceNpmRegistry + '/' + name + '/sync/log/' + r.data.logId;
var offset = 0;
this.log('----------------- Syncing upstream %s -------------------', logURL);
var count = 0;
while (true) {
count++;
var synclogURL = logURL + '?offset=' + offset;
var rs = yield urllib.request(synclogURL, {
timeout: 20000,
dataType: 'json',
gzip: true,
});
if (rs.status !== 200 || !rs.data.ok) {
this.log('sync upstream %s error, status: %s, response: %j',
synclogURL, rs.status, rs.data);
break;
}
var data = rs.data;
var syncDone = false;
if (data.log && data.log.indexOf('[done] Sync') >= 0) {
syncDone = true;
data.log = data.log.replace('[done] Sync', '[Upstream done] Sync');
}
if (data.log) {
this.log(data.log);
}
if (syncDone) {
break;
}
if (count >= 30) {
this.log('sync upstream %s fail, give up', logURL);
break;
}
if (data.log) {
offset += data.log.split('\n').length;
}
yield sleep(2000);
}
this.log('----------------- Synced upstream %s -------------------', logURL);
};
yield _.each([2,3], function*(x) {
yield sleep(100);
vals.push(x);
});
middleware: function* () {
yield sleep(100);
return this.params;
}
function* commonSleepError() {
yield sleep(50);
fooAfterSleep();
}
SyncModuleWorker.prototype._sync = function* (name, pkg) {
var that = this;
var hasModules = false;
var result = yield [
packageService.listModulesByName(name),
packageService.listModuleTags(name),
_listStarUsers(name),
packageService.listPublicModuleMaintainers(name),
packageService.listModuleAbbreviatedsByName(name),
];
var moduleRows = result[0];
var tagRows = result[1];
var existsStarUsers = result[2];
var existsNpmMaintainers = result[3];
var existsModuleAbbreviateds = result[4];
if (common.isLocalModule(moduleRows)) {
// publish on cnpm, dont sync this version package
that.log(' [%s] publish on local cnpm registry, don\'t sync', name);
return [];
}
var missingModuleAbbreviateds = [];
var existsModuleAbbreviatedsMap = {};
for (var item of existsModuleAbbreviateds) {
existsModuleAbbreviatedsMap[item.version] = item;
}
hasModules = moduleRows.length > 0;
// localPackage
var map = {};
var localVersionNames = [];
for (var i = 0; i < moduleRows.length; i++) {
var r = moduleRows[i];
if (!r.package || !r.package.dist) {
// package json parse error
continue;
}
if (!map.latest) {
map.latest = r;
}
map[r.version] = r;
localVersionNames.push(r.version);
}
var latestVersionPackageReadme = {
version: pkg['dist-tags'].latest,
readme: pkg.readme,
};
var tags = {};
for (var i = 0; i < tagRows.length; i++) {
var r = tagRows[i];
if (!r.module_id) {
// no module_id, need to sync tags
continue;
}
tags[r.tag] = r.version;
}
// get package AbbreviatedMetadata
var remoteAbbreviatedMetadatas = {};
if (config.enableAbbreviatedMetadata) {
var packageUrl = '/' + name.replace('/', '%2f');
var result = yield npmSerivce.request(packageUrl, {
dataType: 'text',
registry: config.sourceNpmRegistry,
headers: {
Accept: 'application/vnd.npm.install-v1+json',
},
});
if (result.status === 200) {
var data;
try {
data = JSON.parse(result.data);
} catch (err) {
that.log(' [%s] get abbreviated meta error: %s, headers: %j, %j',
name, err, result.headers, result.data);
}
if (data) {
var versions = data && data.versions || {};
for (var version in versions) {
const item = versions[version];
if (item && typeof item._hasShrinkwrap === 'boolean') {
remoteAbbreviatedMetadatas[version] = { _hasShrinkwrap: item._hasShrinkwrap };
}
}
}
}
}
// any properties changed versions
var changedVersions = {};
var missingVersions = [];
var missingTags = [];
var missingDescriptions = [];
var missingReadmes = [];
var missingStarUsers = [];
var npmUsernames = {};
var missingDeprecateds = [];
// [[user, 'add or remove'], ...]
var diffNpmMaintainers = [];
// [
// { name, version, _hasShrinkwrap }
// ]
var missingAbbreviatedMetadatas = [];
// [
// { name, version, deprecated },
// ]
var missingDeprecatedsOnExistsModuleAbbreviated = [];
// find out new maintainers
var pkgMaintainers = pkg.maintainers || [];
if (Array.isArray(pkgMaintainers)) {
var existsMap = {};
var originalMap = {};
for (var i = 0; i < existsNpmMaintainers.length; i++) {
var user = existsNpmMaintainers[i];
existsMap[user] = true;
}
for (var i = 0; i < pkgMaintainers.length; i++) {
var item = pkgMaintainers[i];
originalMap[item.name] = item;
npmUsernames[item.name.toLowerCase()] = 1;
}
// find add users
for (var i = 0; i < pkgMaintainers.length; i++) {
var item = pkgMaintainers[i];
if (!existsMap[item.name]) {
diffNpmMaintainers.push([item.name, 'add']);
}
}
// find remove users
for (var i = 0; i < existsNpmMaintainers.length; i++) {
var user = existsNpmMaintainers[i];
if (!originalMap[user]) {
diffNpmMaintainers.push([user, 'remove']);
}
}
}
// find out all user names
for (var v in pkg.versions) {
var p = pkg.versions[v];
var maintainers = p.maintainers || [];
if (!Array.isArray(maintainers)) {
// http://r.cnpmjs.org/jasmine-node
// TODO: "maintainers": "Martin H膫陇ger <*****@*****.**>",
maintainers = [maintainers];
}
for (var i = 0; i < maintainers.length; i++) {
var m = maintainers[i];
if (m.name) {
npmUsernames[m.name.toLowerCase()] = 1;
}
}
}
// get the missing star users
var starUsers = pkg.users || {};
for (var k in starUsers) {
if (!existsStarUsers[k]) {
missingStarUsers.push(k);
}
npmUsernames[k.toLowerCase()] = 1;
}
that.log(' [%s] found %d missing star users', name, missingStarUsers.length);
var times = pkg.time || {};
pkg.versions = pkg.versions || {};
var remoteVersionNames = Object.keys(pkg.versions);
var remoteVersionNameMap = {};
// find out missing versions
for (var i = 0; i < remoteVersionNames.length; i++) {
var v = remoteVersionNames[i];
remoteVersionNameMap[v] = v;
var exists = map[v] || {};
var version = pkg.versions[v];
if (!version || !version.dist || !version.dist.tarball) {
continue;
}
// remove readme
if (config.enableAbbreviatedMetadata) {
version.readme = undefined;
} else {
// patch for readme
if (!version.readme) {
version.readme = pkg.readme;
}
}
var publish_time = times[v];
version.publish_time = publish_time ? Date.parse(publish_time) : null;
if (!version.maintainers || !version.maintainers[0]) {
version.maintainers = pkg.maintainers;
}
var abbreviatedMetadata = remoteAbbreviatedMetadatas[version.version];
if (exists.package && exists.package.dist.shasum === version.dist.shasum) {
var existsModuleAbbreviated = existsModuleAbbreviatedsMap[exists.package.version];
if (!existsModuleAbbreviated) {
missingModuleAbbreviateds.push(exists);
} else {
// sync missing deprecated on existsModuleAbbreviated
if (exists.package.deprecated && exists.package.deprecated !== existsModuleAbbreviated.package.deprecated) {
// add deprecated
missingDeprecatedsOnExistsModuleAbbreviated.push({
name,
version: exists.package.version,
deprecated: exists.package.deprecated,
});
} else if (existsModuleAbbreviated.package.deprecated && !exists.package.deprecated) {
// remove deprecated
missingDeprecatedsOnExistsModuleAbbreviated.push({
name,
version: exists.package.version,
deprecated: undefined,
});
}
}
// * shasum make sure equal
if ((version.publish_time === exists.publish_time) ||
(!version.publish_time && exists.publish_time)) {
// debug(' [%s] %s publish_time equal: %s, %s',
// name, version.version, version.publish_time, exists.publish_time);
// * publish_time make sure equal
if (exists.description === null && version.description) {
// * make sure description exists
missingDescriptions.push({
id: exists.id,
description: version.description
});
changedVersions[v] = 1;
}
if (config.enableAbbreviatedMetadata) {
// remove readme
if (exists.package.readme) {
missingReadmes.push({
id: exists.id,
readme: undefined,
});
}
} else {
if (!exists.package.readme && version.readme) {
// * make sure readme exists
missingReadmes.push({
id: exists.id,
readme: version.readme,
});
changedVersions[v] = 1;
}
}
if (version.deprecated && version.deprecated !== exists.package.deprecated) {
// need to sync deprecated field
missingDeprecateds.push({
id: exists.id,
deprecated: version.deprecated,
});
changedVersions[v] = 1;
}
if (exists.package.deprecated && !version.deprecated) {
// remove deprecated info
missingDeprecateds.push({
id: exists.id,
deprecated: undefined,
});
changedVersions[v] = 1;
}
// find missing abbreviatedMetadata
if (abbreviatedMetadata) {
for (var key in abbreviatedMetadata) {
if (!(key in exists.package) || abbreviatedMetadata[key] !== exists.package[key]) {
missingAbbreviatedMetadatas.push(Object.assign({
id: exists.id,
name: exists.package.name,
version: exists.package.version,
}, abbreviatedMetadata));
break;
}
}
}
continue;
}
}
// set abbreviatedMetadata to version package
if (abbreviatedMetadata) {
Object.assign(version, abbreviatedMetadata);
}
missingVersions.push(version);
changedVersions[v] = 1;
}
// find out deleted versions
var deletedVersionNames = [];
for (var i = 0; i < localVersionNames.length; i++) {
var v = localVersionNames[i];
if (!remoteVersionNameMap[v]) {
deletedVersionNames.push(v);
}
}
// delete local abbreviatedMetadata data too
for (var item of existsModuleAbbreviateds) {
if (!remoteVersionNameMap[item.version] && deletedVersionNames.indexOf(item.version) === -1) {
deletedVersionNames.push(item.version);
}
}
// find out missing tags
var sourceTags = pkg['dist-tags'] || {};
for (var t in sourceTags) {
var sourceTagVersion = sourceTags[t];
if (sourceTagVersion && tags[t] !== sourceTagVersion) {
missingTags.push([t, sourceTagVersion]);
}
}
// find out deleted tags
var deletedTags = [];
for (var t in tags) {
if (!sourceTags[t]) {
// not in remote tags, delete it from local registry
deletedTags.push(t);
}
}
if (missingVersions.length === 0) {
that.log(' [%s] all versions are exists', name);
} else {
missingVersions.sort(function (a, b) {
return a.publish_time - b.publish_time;
});
that.log(' [%s] %d versions need to sync', name, missingVersions.length);
}
var syncedVersionNames = [];
var syncIndex = 0;
// sync missing versions
while (missingVersions.length) {
var index = syncIndex++;
var syncModule = missingVersions.shift();
if (!syncModule.dist.tarball) {
continue;
}
// retry 3 times
var tries = 3;
while (true) {
try {
yield that._syncOneVersion(index, syncModule);
syncedVersionNames.push(syncModule.version);
break;
} catch (err) {
var delay = Date.now() - syncModule.publish_time;
that.log(' [%s:%d] tries: %d, delay: %s ms, sync error, version: %s, %s: %s',
syncModule.name, index, tries, delay, syncModule.version, err.name, err.stack);
var maxDelay = 3600000;
if (tries-- > 0 && delay < maxDelay) {
that.log(' [%s:%d] retry after 30s', syncModule.name, index);
yield sleep(30000);
} else {
break;
}
}
}
}
if (deletedVersionNames.length === 0) {
that.log(' [%s] no versions need to deleted', name);
} else {
if (config.syncDeletedVersions) {
that.log(' [%s] %d versions: %j need to deleted, because config.syncDeletedVersions=true',
name, deletedVersionNames.length, deletedVersionNames);
try {
yield packageService.removeModulesByNameAndVersions(name, deletedVersionNames);
} catch (err) {
that.log(' [%s] delete error, %s: %s', name, err.name, err.message);
}
} else {
const downloadCount = yield downloadTotalService.getTotalByName(name);
if (downloadCount >= 10000) {
// find deleted in 24 hours versions
var oneDay = 3600000 * 24;
var now = Date.now();
var deletedIn24HoursVersions = [];
var oldVersions = [];
for (var i = 0; i < deletedVersionNames.length; i++) {
var v = deletedVersionNames[i];
var exists = map[v];
if (exists && now - exists.publish_time < oneDay) {
deletedIn24HoursVersions.push(v);
} else {
oldVersions.push(v);
}
}
if (deletedIn24HoursVersions.length > 0) {
that.log(' [%s] %d versions: %j need to deleted, because they are deleted in 24 hours',
name, deletedIn24HoursVersions.length, deletedIn24HoursVersions);
try {
yield packageService.removeModulesByNameAndVersions(name, deletedIn24HoursVersions);
} catch (err) {
that.log(' [%s] delete error, %s: %s', name, err.name, err.message);
}
}
that.log(' [%s] %d versions: %j no need to delete, because `config.syncDeletedVersions=false`',
name, oldVersions.length, oldVersions);
} else {
that.log(' [%s] %d versions: %j need to deleted, because downloads %s < 10000',
name, deletedVersionNames.length, deletedVersionNames, downloadCount);
try {
yield packageService.removeModulesByNameAndVersions(name, deletedVersionNames);
} catch (err) {
that.log(' [%s] delete error, %s: %s', name, err.name, err.message);
}
}
}
}
// sync missing descriptions
function* syncDes() {
if (missingDescriptions.length === 0) {
return;
}
that.log(' [%s] saving %d descriptions', name, missingDescriptions.length);
var res = yield gather(missingDescriptions.map(function (item) {
return packageService.updateModuleDescription(item.id, item.description);
}));
for (var i = 0; i < res.length; i++) {
var item = missingDescriptions[i];
var r = res[i];
if (r.error) {
that.log(' save error, id: %s, description: %s, error: %s',
item.id, item.description, r.error.message);
} else {
that.log(' saved, id: %s, description length: %d',
item.id, item.description.length);
}
}
}
// sync missing tags
function* syncTag() {
if (deletedTags.length > 0) {
yield packageService.removeModuleTagsByNames(name, deletedTags);
that.log(' [%s] deleted %d tags: %j',
name, deletedTags.length, deletedTags);
}
if (missingTags.length === 0) {
return;
}
that.log(' [%s] adding %d tags', name, missingTags.length);
// sync tags
var res = yield gather(missingTags.map(function (item) {
return packageService.addModuleTag(name, item[0], item[1]);
}));
for (var i = 0; i < res.length; i++) {
var item = missingTags[i];
var r = res[i];
if (r.error) {
that.log(' add tag %s:%s error, error: %s',
item.id, item.description, r.error.message);
} else {
that.log(' added tag %s:%s, module_id: %s',
item[0], item[1], r.value && r.value.module_id);
}
}
}
// sycn missing readme
function* syncReadme() {
if (missingReadmes.length === 0) {
return;
}
that.log(' [%s] saving %d readmes', name, missingReadmes.length);
var res = yield gather(missingReadmes.map(function (item) {
return packageService.updateModuleReadme(item.id, item.readme);
}));
for (var i = 0; i < res.length; i++) {
var item = missingReadmes[i];
var r = res[i];
if (r.error) {
that.log(' save error, id: %s, error: %s', item.id, r.error.message);
} else {
that.log(' saved, id: %s', item.id);
}
}
}
function* syncModuleAbbreviateds() {
if (missingModuleAbbreviateds.length === 0) {
return;
}
that.log(' [%s] saving %d missing moduleAbbreviateds', name, missingModuleAbbreviateds.length);
var res = yield gather(missingModuleAbbreviateds.map(function (item) {
return packageService.saveModuleAbbreviated(item);
}));
for (var i = 0; i < res.length; i++) {
var item = missingModuleAbbreviateds[i];
var r = res[i];
if (r.error) {
that.log(' save moduleAbbreviateds error, module: %s@%s, error: %s', item.name, item.version, r.error.message);
} else {
that.log(' saved moduleAbbreviateds, module: %s@%s', item.name, item.version);
}
}
}
function* syncAbbreviatedMetadatas() {
if (missingAbbreviatedMetadatas.length === 0) {
return;
}
that.log(' [%s] saving %d abbreviated meta datas', name, missingAbbreviatedMetadatas.length);
var res = yield gather(missingAbbreviatedMetadatas.map(function (item) {
return packageService.updateModuleAbbreviatedPackage(item);
}));
for (var i = 0; i < res.length; i++) {
var item = missingAbbreviatedMetadatas[i];
var r = res[i];
if (r.error) {
that.log(' save error, module_abbreviated: %s@%s, error: %s',
item.name, item.version, r.error.stack);
} else {
that.log(' saved, module_abbreviated: %s@%s, %j', item.name, item.version, item);
}
}
var res = yield gather(missingAbbreviatedMetadatas.map(function (item) {
var fields = {};
for (var key in item) {
if (key === 'id' || key === 'name' || key === 'version') {
continue;
}
fields[key] = item[key];
}
return packageService.updateModulePackageFields(item.id, fields);
}));
for (var i = 0; i < res.length; i++) {
var item = missingAbbreviatedMetadatas[i];
var r = res[i];
if (r.error) {
that.log(' save error, module id: %s, error: %s', item.id, r.error.stack);
} else {
that.log(' saved, module id: %s, %j', item.id, item);
}
}
}
function *syncDeprecatedsOnExistsModuleAbbreviated() {
if (missingDeprecatedsOnExistsModuleAbbreviated.length === 0) {
return;
}
that.log(' [%s] saving %d module abbreviated deprecated fields',
name, missingDeprecatedsOnExistsModuleAbbreviated.length);
var res = yield gather(missingDeprecatedsOnExistsModuleAbbreviated.map(function (item) {
return packageService.updateModuleAbbreviatedPackage(item);
}));
for (var i = 0; i < res.length; i++) {
var item = missingDeprecatedsOnExistsModuleAbbreviated[i];
var r = res[i];
if (r.error) {
that.log(' save error, module abbreviated: %s@%s, error: %s', item.name, item.version, r.error.message);
} else {
that.log(' saved, module abbreviated: %s@%s, deprecated: %j', item.name, item.version, item.deprecated);
}
}
}
function *syncDeprecateds() {
if (missingDeprecateds.length === 0) {
return;
}
that.log(' [%s] saving %d Deprecated fields', name, missingDeprecateds.length);
var res = yield gather(missingDeprecateds.map(function (item) {
return packageService.updateModulePackageFields(item.id, {
deprecated: item.deprecated
});
}));
for (var i = 0; i < res.length; i++) {
var item = missingDeprecateds[i];
var r = res[i];
if (r.error) {
that.log(' save error, id: %s, error: %s', item.id, r.error.message);
} else {
that.log(' saved, id: %s, deprecated: %j', item.id, item.deprecated);
}
}
}
function* syncMissingUsers() {
var missingUsers = [];
var names = Object.keys(npmUsernames);
if (names.length === 0) {
return;
}
var rows = yield User.listByNames(names);
var map = {};
rows.forEach(function (r) {
map[r.name] = r;
});
names.forEach(function (username) {
var r = map[username];
if (!r || !r.json) {
if (username[0] !== '"' && username[0] !== "'") {
missingUsers.push(username);
}
}
});
if (missingUsers.length === 0) {
that.log(' [%s] all %d npm users exists', name, names.length);
return;
}
that.log(' [%s] saving %d/%d missing npm users: %j',
name, missingUsers.length, names.length, missingUsers);
var res = yield gather(missingUsers.map(function (username) {
return _saveNpmUser(username);
}));
for (var i = 0; i < res.length; i++) {
var r = res[i];
if (r.error) {
that.log(' save npm user error, %s', r.error.message);
}
}
}
// sync missing star users
function* syncMissingStarUsers() {
if (missingStarUsers.length === 0) {
return;
}
that.log(' [%s] saving %d star users', name, missingStarUsers.length);
var res = yield gather(missingStarUsers.map(function (username) {
return packageService.addStar(name, username);
}));
for (var i = 0; i < res.length; i++) {
var r = res[i];
if (r.error) {
that.log(' add star user error, %s', r.error.stack);
}
}
}
// sync diff npm package maintainers
function* syncNpmPackageMaintainers() {
if (diffNpmMaintainers.length === 0) {
return;
}
that.log(' [%s] syncing %d diff package maintainers: %j',
name, diffNpmMaintainers.length, diffNpmMaintainers);
var res = yield gather(diffNpmMaintainers.map(function (item) {
return _saveMaintainer(name, item[0], item[1]);
}));
for (var i = 0; i < res.length; i++) {
var r = res[i];
if (r.error) {
that.log(' save package maintainer error, %s', r.error.stack);
}
}
}
if (latestVersionPackageReadme.version && latestVersionPackageReadme.readme) {
var existsPackageReadme = yield packageService.getPackageReadme(name, true);
if (!existsPackageReadme ||
existsPackageReadme.version !== latestVersionPackageReadme.version ||
existsPackageReadme.readme !== latestVersionPackageReadme.readme) {
var r = yield packageService.savePackageReadme(name, latestVersionPackageReadme.readme, latestVersionPackageReadme.version);
that.log(' save packageReadme: %s %s %s', r.id, r.name, r.version);
}
}
yield syncDes();
yield syncTag();
yield syncReadme();
yield syncDeprecateds();
yield syncMissingStarUsers();
yield syncMissingUsers();
yield syncNpmPackageMaintainers();
yield syncModuleAbbreviateds();
yield syncAbbreviatedMetadatas();
yield syncDeprecatedsOnExistsModuleAbbreviated();
changedVersions = Object.keys(changedVersions);
// hooks
const envelope = {
event: 'package:sync',
name: name,
type: 'package',
version: null,
hookOwner: null,
payload: {
changedVersions,
},
change: null,
};
hook.trigger(envelope);
return syncedVersionNames;
};