keyboards.forEach(function(keyboard) {
if(program.keyboards.length == 0 || program.keyboards.indexOf(keyboard.shortname+'/'+keyboard.id) >= 0) {
code += `
describe('Test keyboard ${keyboard.id}', () => {
const shortname='${keyboard.shortname}';
const id='${keyboard.id}';
const locator = shortname+'/'+id;
it('should generate a set of results for "all" possible inputs', function() {
return windowLoad
.then(() => testRunner.loadTests(locator))
.then((shouldRun) => { if(shouldRun) { return testRunner.runTests(id); } })
.then(() => testRunner.saveTestResults(locator, (testRunner.keyboards[id] || {}).results));
}).timeout(0);
});
`;
}
});
keyboards.forEach((keyboard) => {
if(!program.keyboards.length || program.keyboards.indexOf(keyboard.shortname+'/'+keyboard.id) >= 0) {
// Validate each of the test files against the first tested compiler+engine version
let localFailCount = 0;
const localFail = knownFailures.hasOwnProperty([keyboard.id]) ?
(msg) => { if(++localFailCount == 1) console.warn(`WARN: Not failing test because ${keyboard.id} is in known-failures.`); console.warn(`WARN: ${msg}`); } :
fail;
try {
const baseResultFilename = path.join(KEYBOARDS_ROOT, keyboard.shortname, keyboard.id, 'tests', `${keyboard.id}-${baseCompilerVersion}-${baseEngineVersion}.results`);
const baseResult = fs.readFileSync(baseResultFilename, 'utf8');
const baseResultJSON = JSON.parse(baseResult);
const testsFilename = path.join(KEYBOARDS_ROOT, keyboard.shortname, keyboard.id, 'tests', `${keyboard.id}.tests`);
const testsJSON = JSON.parse(fs.readFileSync(testsFilename, 'utf8'));
const numTests = Object.keys(testsJSON.inputTests).length;
// First, check the base result for errors
for(let k in baseResultJSON) {
if(typeof baseResultJSON[k] !== 'string') {
let input = `${testsJSON.inputTests[k].context ? `"${testsJSON.inputTests[k].context}" ` : ""}+ ${keyname(testsJSON.inputTests[k].modifier, testsJSON.inputTests[k].key)}`;
localFail(`${keyboard.shortname}/${keyboard.id}[${k}]: error in test: ${input}: ${baseResultJSON[k].error}`, 5);
}
}
// Make sure every test is in the base result set
if(Object.keys(baseResultJSON).length != numTests) {
localFail(`${keyboard.shortname}/${keyboard.id}: base result set has fewer results (${Object.keys(baseResultJSON).length}) than expected (${numTests})`, 6);
}
//console.log(baseResultFilename, baseResult);
testedCompilerVersions.forEach((cv) => {
testedEngineVersions.forEach((ev) => {
const resultFilename = path.join(KEYBOARDS_ROOT, keyboard.shortname, keyboard.id, 'tests', `${keyboard.id}-${cv}-${ev}.results`);
const result = fs.readFileSync(resultFilename, 'utf8');
//console.log(resultFilename, result);
// Naive string test first
if(result !== baseResult) {
// Now, report first mismatch and total number of mismatches after parsing JSON
const resultJSON = JSON.parse(result);
let errors = 0, prefix = `${keyboard.shortname}/${keyboard.id}`;
for(let k in baseResultJSON) {
if(resultJSON[k] !== baseResultJSON[k]) {
if(++errors == 1 || program.logAllFailures) {
let
ix = k.toString(),
whitespace = ' '.repeat(prefix.length + ix.length + 6),
input = `${testsJSON.inputTests[k].context ? `"${testsJSON.inputTests[k].context}" ` : ""}+ ${keyname(testsJSON.inputTests[k].modifier, testsJSON.inputTests[k].key)}`;
console.error(`${prefix}[${ix}]: expected: ${input} > "${baseResultJSON[k]}"`);
console.error(`${whitespace}actual: ${input} > "${resultJSON[k]}"`);
}
}
}
localFail(`${keyboard.shortname}/${keyboard.id} ${errors}/${numTests} test(s) mismatched between (${baseCompilerVersion} / ${baseEngineVersion}) and (${cv} / ${ev})`, 4);
}
});
});
} catch(e) {
localFail(`Failed to load test results for ${keyboard.shortname}/${keyboard.id}: ${typeof e == 'object' ? e.message : e}`, 7);
}
}
});