C.SETS.forEach(function(SET, i)
{
var setWithExtras = JSON.parse(args[i]);
setWithExtras.cards.forEach(function(card)
{
Object.forEach(C.FIELD_TYPES, function(fieldName, fieldType)
{
if(!allCardsWithExtras.hasOwnProperty(card.name))
allCardsWithExtras[card.name] = {};
if(C.SET_SPECIFIC_FIELDS.contains(fieldName))
return;
if(!previousSeenSetCodes.hasOwnProperty(card.name))
previousSeenSetCodes[card.name] = {};
if(!previousSeenSetCodes[card.name].hasOwnProperty(fieldName))
previousSeenSetCodes[card.name][fieldName] = [];
var fieldValue = card[fieldName];
if(fieldName==="imageName") // Modify for AllCards.json the imageName field to match the mtgimage.com /card/ prefix syntax
fieldValue = card.name.toLowerCase().strip(":\"?").replaceAll("/", " ").trim("0123456789 .").replaceAll(" token card", "");
if(C.ORACLE_FIELDS.contains(fieldName) && !["B.F.M. (Big Furry Monster)"].contains(card.name) && fieldName!=="foreignNames")
{
if(!card.hasOwnProperty(fieldName))
{
if(allCardsWithExtras[card.name].hasOwnProperty(fieldName))
{
base.warn("Card [%s] mismatch with field [%s] between current set [%s] and previous [%s] with values:\n\tNO VALUE\n\t%s", card.code, fieldName, SET.code, previousSeenSetCodes[card.name][fieldName].join(" "), allCardsWithExtras[card.name][fieldName]);
taintedCards.push({card:card, fieldName:fieldName});
previousSeenSetCodes[card.name][fieldName].forEach(function(prevSetCode)
{
allSetsWithExtras[prevSetCode].cards.forEach(function(prevCard) { if(prevCard.name===card.name) { taintedCards.push({card:prevCard, fieldName:fieldName}); } });
});
[].concat(previousSeenSetCodes[card.name][fieldName], [SET.code]).forEach(function(taintedCode)
{
if(!C.SETS.filter(function(taintedSet) { return taintedSet.code===taintedCode; })[0].isMCISet && !C.SETS_NOT_ON_GATHERER.contains(taintedCode))
taintedSetCodes.push(taintedCode);
});
}
return;
}
if(allCardsWithExtras[card.name].hasOwnProperty(fieldName))
{
var fieldDifference = diffUtil.diff(fieldValue, allCardsWithExtras[card.name][fieldName]);
if(fieldDifference)
{
base.warn("Card [%s] mismatch with field [%s] between current set [%s] and previous [%s] with : %s", card.name, fieldName, SET.code, previousSeenSetCodes[card.name][fieldName].join(" "), fieldDifference);
taintedCards.push({card:card, fieldName:fieldName});
previousSeenSetCodes[card.name][fieldName].forEach(function(prevSetCode)
{
allSetsWithExtras[prevSetCode].cards.forEach(function(prevCard) { if(prevCard.name===card.name) { taintedCards.push({card:prevCard, fieldName:fieldName}); } });
});
[].concat(previousSeenSetCodes[card.name][fieldName], [SET.code]).forEach(function(taintedCode)
{
if(!C.SETS.filter(function(taintedSet) { return taintedSet.code===taintedCode; })[0].isMCISet && !C.SETS_NOT_ON_GATHERER.contains(taintedCode))
taintedSetCodes.push(taintedCode);
});
}
}
}
previousSeenSetCodes[card.name][fieldName].push(setWithExtras.code);
allCardsWithExtras[card.name][fieldName] = fieldValue;
});
});
// Strip out internal only data
delete setWithExtras.isMCISet;
delete setWithExtras.magicRaritiesCode;
delete setWithExtras.essentialMagicCode;
delete setWithExtras.useMagicRaritiesNumber;
allSetsWithExtras[SET.code] = setWithExtras;
allSetsArrayWithExtras.push(setWithExtras);
var set = base.clone(setWithExtras, true);
set.cards.forEach(function(card)
{
// Strip out extras
C.EXTRA_FIELDS.forEach(function(EXTRA_FIELD)
{
delete card[EXTRA_FIELD];
});
});
allSets[SET.code] = set;
allSetsArray.push(set);
fs.writeFile(path.join(__dirname, "json", SET.code + ".json"), JSON.stringify(set), {encoding : "utf8"}, this.parallel());
fs.writeFile(path.join(__dirname, "json", SET.code + ".jsonp"), JSONP_PREFIX + JSON.stringify(set) + ', "' + SET.code + '"' + JSONP_SUFFIX, {encoding : "utf8"}, this.parallel());
if(SET.code==="CON")
{
fs.writeFile(path.join(__dirname, "json", "_" + SET.code + ".json"), JSON.stringify(set), {encoding : "utf8"}, this.parallel());
fs.writeFile(path.join(__dirname, "json", "_" + SET.code + ".jsonp"), JSONP_PREFIX + JSON.stringify(set) + ', "' + SET.code + '"' + JSONP_SUFFIX, {encoding : "utf8"}, this.parallel());
}
fs.writeFile(path.join(__dirname, "json", SET.code + "-x.json"), JSON.stringify(setWithExtras), {encoding : "utf8"}, this.parallel());
fs.writeFile(path.join(__dirname, "json", SET.code + "-x.jsonp"), JSONP_PREFIX + JSON.stringify(setWithExtras) + ', "' + SET.code + '-x"' + JSONP_SUFFIX, {encoding : "utf8"}, this.parallel());
var setSize = printUtil.toSize(JSON.stringify(set).length, 0);
setSize = " ".repeat(6-setSize.length) + setSize;
var setXSize = printUtil.toSize(JSON.stringify(setWithExtras).length, 0);
setXSize = " ".repeat(6-setXSize.length) + setXSize;
var dustSetData = {code : SET.code, name : SET.name, releaseDate : SET.releaseDate, size : setSize, sizeX : setXSize};
if(SET.code==="CON")
dustSetData.isCON = true;
if(SET.code.length===3)
dustSetData.shortCode = true;
dustData.sets.push(dustSetData);
}.bind(this));
function saveSets()
{
var args=arguments;
var allSets = {};
var allSetsArray = [];
var allSetsWithExtras = {};
var allSetsArrayWithExtras = [];
var allCardsWithExtras = {};
var previousSeenSetCodes = {};
var taintedSetCodes = [];
var taintedCards = [];
base.info("Creating JSON files...");
C.SETS.forEach(function(SET, i)
{
var setWithExtras = JSON.parse(args[i]);
setWithExtras.cards.forEach(function(card)
{
Object.forEach(C.FIELD_TYPES, function(fieldName, fieldType)
{
if(!allCardsWithExtras.hasOwnProperty(card.name))
allCardsWithExtras[card.name] = {};
if(C.SET_SPECIFIC_FIELDS.contains(fieldName))
return;
if(!previousSeenSetCodes.hasOwnProperty(card.name))
previousSeenSetCodes[card.name] = {};
if(!previousSeenSetCodes[card.name].hasOwnProperty(fieldName))
previousSeenSetCodes[card.name][fieldName] = [];
var fieldValue = card[fieldName];
if(fieldName==="imageName") // Modify for AllCards.json the imageName field to match the mtgimage.com /card/ prefix syntax
fieldValue = card.name.toLowerCase().strip(":\"?").replaceAll("/", " ").trim("0123456789 .").replaceAll(" token card", "");
if(C.ORACLE_FIELDS.contains(fieldName) && !["B.F.M. (Big Furry Monster)"].contains(card.name) && fieldName!=="foreignNames")
{
if(!card.hasOwnProperty(fieldName))
{
if(allCardsWithExtras[card.name].hasOwnProperty(fieldName))
{
base.warn("Card [%s] mismatch with field [%s] between current set [%s] and previous [%s] with values:\n\tNO VALUE\n\t%s", card.name, fieldName, SET.code, previousSeenSetCodes[card.name][fieldName].join(" "), allCardsWithExtras[card.name][fieldName]);
taintedCards.push({card:card, fieldName:fieldName});
previousSeenSetCodes[card.name][fieldName].forEach(function(prevSetCode)
{
allSetsWithExtras[prevSetCode].cards.forEach(function(prevCard) { if(prevCard.name===card.name) { taintedCards.push({card:prevCard, fieldName:fieldName}); } });
});
[].concat(previousSeenSetCodes[card.name][fieldName], [SET.code]).forEach(function(taintedCode)
{
if(!C.SETS.filter(function(taintedSet) { return taintedSet.code===taintedCode; })[0].isMCISet && !C.SETS_NOT_ON_GATHERER.contains(taintedCode))
taintedSetCodes.push(taintedCode);
});
}
return;
}
if(allCardsWithExtras[card.name].hasOwnProperty(fieldName))
{
var fieldDifference = diffUtil.diff(fieldValue, allCardsWithExtras[card.name][fieldName]);
if(fieldDifference)
{
base.warn("Card [%s] mismatch with field [%s] between current set [%s] and previous [%s] with : %s", card.name, fieldName, SET.code, previousSeenSetCodes[card.name][fieldName].join(" "), fieldDifference);
taintedCards.push({card:card, fieldName:fieldName});
previousSeenSetCodes[card.name][fieldName].forEach(function(prevSetCode)
{
allSetsWithExtras[prevSetCode].cards.forEach(function(prevCard) { if(prevCard.name===card.name) { taintedCards.push({card:prevCard, fieldName:fieldName}); } });
});
[].concat(previousSeenSetCodes[card.name][fieldName], [SET.code]).forEach(function(taintedCode)
{
if(!C.SETS.filter(function(taintedSet) { return taintedSet.code===taintedCode; })[0].isMCISet && !C.SETS_NOT_ON_GATHERER.contains(taintedCode))
taintedSetCodes.push(taintedCode);
});
}
}
}
previousSeenSetCodes[card.name][fieldName].push(setWithExtras.code);
allCardsWithExtras[card.name][fieldName] = fieldValue;
});
});
// Strip out internal only data
delete setWithExtras.isMCISet;
delete setWithExtras.magicRaritiesCode;
delete setWithExtras.essentialMagicCode;
delete setWithExtras.useMagicRaritiesNumber;
allSetsWithExtras[SET.code] = setWithExtras;
allSetsArrayWithExtras.push(setWithExtras);
var set = base.clone(setWithExtras, true);
set.cards.forEach(function(card)
{
// Strip out extras
C.EXTRA_FIELDS.forEach(function(EXTRA_FIELD)
{
delete card[EXTRA_FIELD];
});
});
allSets[SET.code] = set;
allSetsArray.push(set);
fs.writeFile(path.join(__dirname, "json", SET.code + ".json"), JSON.stringify(set), {encoding : "utf8"}, this.parallel());
fs.writeFile(path.join(__dirname, "json", SET.code + ".jsonp"), JSONP_PREFIX + JSON.stringify(set) + ', "' + SET.code + '"' + JSONP_SUFFIX, {encoding : "utf8"}, this.parallel());
if (SET.code === "CON") {
fs.writeFile(path.join(__dirname, "json", "_" + SET.code + ".json"), JSON.stringify(set), {encoding : "utf8"}, this.parallel());
fs.writeFile(path.join(__dirname, "json", "_" + SET.code + ".jsonp"), JSONP_PREFIX + JSON.stringify(set) + ', "' + SET.code + '"' + JSONP_SUFFIX, {encoding : "utf8"}, this.parallel());
}
fs.writeFile(path.join(__dirname, "json", SET.code + "-x.json"), JSON.stringify(setWithExtras), {encoding : "utf8"}, this.parallel());
fs.writeFile(path.join(__dirname, "json", SET.code + "-x.jsonp"), JSONP_PREFIX + JSON.stringify(setWithExtras) + ', "' + SET.code + '-x"' + JSONP_SUFFIX, {encoding : "utf8"}, this.parallel());
var setSize = printUtil.toSize(JSON.stringify(set).length, 0);
var setXSize = printUtil.toSize(JSON.stringify(setWithExtras).length, 0);
var dustSetData = {code : SET.code, lcCode : SET.code.toLowerCase(), name : SET.name, releaseDate : SET.releaseDate, size : setSize, sizeX : setXSize};
if(SET.code==="CON")
dustSetData.isCON = true;
if(SET.code.length===3)
dustSetData.shortCode = true;
if(SET.isMCISet)
dustSetData.isMCISet = true;
dustData.sets.push(dustSetData);
}.bind(this));
taintedSetCodes = taintedSetCodes.unique();
if(taintedSetCodes.length>0) {
base.info("Tainted set codes: %s", taintedSetCodes.join(" "));
//this.data.taintedCards = taintedCards;
// *** CLEAR TAINTED DATA ***
taintedSetCodes = [];
taintedCards = [];
// **************************
}
var allCards = base.clone(allCardsWithExtras, true);
Object.values(allCards).forEach(function(card) {
// Strip out extras
C.EXTRA_FIELDS.forEach(function(EXTRA_FIELD) {
delete card[EXTRA_FIELD];
});
});
dustData.sets = dustData.sets.sort(function(a, b) { return moment(a.releaseDate, "YYYY-MM-DD").unix()-moment(b.releaseDate, "YYYY-MM-DD").unix(); });
dustData.allSize = printUtil.toSize(JSON.stringify(allSets).length, 1);
dustData.allSizeX = printUtil.toSize(JSON.stringify(allSetsWithExtras).length, 1);
dustData.allCardsSize = printUtil.toSize(JSON.stringify(allCards).length, 1);
dustData.allCardsSizeX = printUtil.toSize(JSON.stringify(allCardsWithExtras).length, 1);
var changeLog = JSON.parse(fs.readFileSync(path.join(__dirname, "changelog.json"), {encoding : "utf8"}));
dustData.changeLog = changeLog.map(function(o, i) {
o.whenAtom = moment(o.when, "YYYY-MM-DD").format("YYYY-MM-DDTHH:mm:ss");
o.whenSiteMap = o.when;
o.when = moment(o.when, "YYYY-MM-DD").format("MMM D, YYYY");
o.uniqueID = changeLog.length-i;
o.atomContent = "<p>Changes:<br><ul>" + o.changes.map(function(change) { return "<li>" + change + "</li>"; }).join("") + "</ul></p>";
return o;
});
dustData.changeLogAtom = dustData.changeLog.slice(0, 9);
dustData.lastUpdatedAtom = dustData.changeLog[0].whenAtom;
dustData.lastUpdated = dustData.changeLog[0].when;
dustData.lastUpdatedSiteMap = dustData.changeLog[0].whenSiteMap;
dustData.version = dustData.changeLog[0].version;
dustData.setSpecificFields = C.SET_SPECIFIC_FIELDS.sort().join(", ");
// Save stuff.
tiptoe(
function() {
fs.writeFile(path.join(__dirname, "json", "AllSets.json"), JSON.stringify(allSets), {encoding : "utf8"}, this.parallel());
fs.writeFile(path.join(__dirname, "json", "AllSets.jsonp"), JSONP_PREFIX + JSON.stringify(allSets) + ', "AllSets"' + JSONP_SUFFIX, {encoding : "utf8"}, this.parallel());
},
function() {
fs.writeFile(path.join(__dirname, "json", "AllSetsArray.json"), JSON.stringify(allSetsArray), {encoding : "utf8"}, this.parallel());
fs.writeFile(path.join(__dirname, "json", "AllSetsArray.jsonp"), JSONP_PREFIX + JSON.stringify(allSetsArray) + ', "AllSetsArray"' + JSONP_SUFFIX, {encoding : "utf8"}, this.parallel());
},
function() {
fs.writeFile(path.join(__dirname, "json", "AllSets-x.json"), JSON.stringify(allSetsWithExtras), {encoding : "utf8"}, this.parallel());
fs.writeFile(path.join(__dirname, "json", "AllSets-x.jsonp"), JSONP_PREFIX + JSON.stringify(allSetsWithExtras) + ', "AllSets-x"' + JSONP_SUFFIX, {encoding : "utf8"}, this.parallel());
},
function() {
fs.writeFile(path.join(__dirname, "json", "AllSetsArray-x.json"), JSON.stringify(allSetsArrayWithExtras), {encoding : "utf8"}, this.parallel());
fs.writeFile(path.join(__dirname, "json", "AllSetsArray-x.jsonp"), JSONP_PREFIX + JSON.stringify(allSetsArrayWithExtras) + ', "AllSetsArray-x"' + JSONP_SUFFIX, {encoding : "utf8"}, this.parallel());
},
function() {
fs.writeFile(path.join(__dirname, "json", "AllCards.json"), JSON.stringify(allCards), {encoding : "utf8"}, this.parallel());
fs.writeFile(path.join(__dirname, "json", "AllCards.jsonp"), JSONP_PREFIX + JSON.stringify(allCards) + ', "AllCards"' + JSONP_SUFFIX, {encoding : "utf8"}, this.parallel());
},
function() {
fs.writeFile(path.join(__dirname, "json", "AllCards-x.json"), JSON.stringify(allCardsWithExtras), {encoding : "utf8"}, this.parallel());
fs.writeFile(path.join(__dirname, "json", "AllCards-x.jsonp"), JSONP_PREFIX + JSON.stringify(allCardsWithExtras) + ', "AllCards-x"' + JSONP_SUFFIX, {encoding : "utf8"}, this.parallel());
},
function() {
fs.writeFile(path.join(__dirname, "json", "SetCodes.json"), JSON.stringify(C.SETS.map(function(SET) { return SET.code; })), {encoding : "utf8"}, this.parallel());
fs.writeFile(path.join(__dirname, "json", "SetCodes.jsonp"), JSONP_PREFIX + JSON.stringify(C.SETS.map(function(SET) { return SET.code; })) + ', "SetCodes"' + JSONP_SUFFIX, {encoding : "utf8"}, this.parallel());
},
function() {
fs.writeFile(path.join(__dirname, "json", "SetList.json"), JSON.stringify(C.SETS.map(function(SET) { return {name : SET.name, code : SET.code, releaseDate : SET.releaseDate}; })), {encoding : "utf8"}, this.parallel());
fs.writeFile(path.join(__dirname, "json", "SetList.jsonp"), JSONP_PREFIX + JSON.stringify(C.SETS.map(function(SET) { return {name : SET.name, code : SET.code, releaseDate : SET.releaseDate}; })) + ', "SetList"' + JSONP_SUFFIX, {encoding : "utf8"}, this.parallel());
},
function() {
fs.writeFile(path.join(__dirname, "json", "version-full.json"), JSON.stringify({version:dustData.version}), {encoding : "utf8"}, this.parallel());
fs.writeFile(path.join(__dirname, "json", "version-full.jsonp"), JSONP_PREFIX + JSON.stringify({version:dustData.version}) + ', "version-full"' + JSONP_SUFFIX, {encoding : "utf8"}, this.parallel());
},
function() {
fs.writeFile(path.join(__dirname, "json", "version.json"), JSON.stringify(dustData.version), {encoding : "utf8"}, this.parallel());
fs.writeFile(path.join(__dirname, "json", "version.jsonp"), JSONP_PREFIX + JSON.stringify(dustData.version) + ', "version"' + JSONP_SUFFIX, {encoding : "utf8"}, this.parallel());
},
function() {
fs.writeFile(path.join(__dirname, "json", "changelog.json"), fs.readFileSync(path.join(__dirname, "changelog.json"), {encoding : "utf8"}), {encoding : "utf8"}, this.parallel());
fs.writeFile(path.join(__dirname, "json", "changelog.jsonp"), JSONP_PREFIX + fs.readFileSync(path.join(__dirname, "changelog.json"), {encoding : "utf8"}) + ', "changelog"' + JSONP_SUFFIX, {encoding : "utf8"}, this.parallel());
},
function finish(err) {
this(err);
}
);
},
C.SETS.forEach(function(SET, i)
{
dustData.sets[i].sizeZip = printUtil.toSize(fs.statSync(path.join(__dirname, "json", SET.code + ".json.zip")).size, 1);
dustData.sets[i].sizeXZip = printUtil.toSize(fs.statSync(path.join(__dirname, "json", SET.code + "-x.json.zip")).size, 1);
});