let runAnalyze = runCompile.then(() => { // If we are first run, then build the .tests files if(firstCompile && !program.skipAnalysis) { firstCompile = false; let analyzeKeyboard = function(keyboard) { const locator = keyboard.shortname+'/'+keyboard.id; const kmx = path.join(config.KEYBOARDS_ROOT, config.KEYBOARDS_GROUP, locator, 'build', keyboard.id+'.kmx'); const testsPath = path.join(config.KEYBOARDS_ROOT, config.KEYBOARDS_GROUP, locator, 'tests'); const tests = path.join(testsPath, keyboard.id+'.tests'); if(!fs.existsSync(kmx)) { // TODO: kmanalyze should really call the compiler to build an intermediate .kmx // for use with web. This should not be that hard to do... console.log('Cannot currently generate tests without a .kmx for '+keyboard.id); return true; } if(!fs.existsSync(testsPath)) { fs.mkdirSync(testsPath); } console.log(`Building test cases for ${locator}`); // TODO: Find kmanalyze outside the repo. This forces Windows-dependence right now return util.runProcess('../../../windows/bin/developer/kmanalyze.exe', [kmx, tests], {}, true); }; let keyboards = program.keyboards.length ? program.keyboards.map((locator) => util.parseLocator(locator)) : util.getKeyboardFolders(KEYBOARDS_ROOT, false); return forEachPromise(keyboards, analyzeKeyboard); } });
keyboards.forEach(function(keyboard) { if(program.keyboards.length == 0 || program.keyboards.indexOf(keyboard.shortname+'/'+keyboard.id) >= 0) { code += ` describe('Test keyboard ${keyboard.id}', () => { const shortname='${keyboard.shortname}'; const id='${keyboard.id}'; const locator = shortname+'/'+id; it('should generate a set of results for "all" possible inputs', function() { return windowLoad .then(() => testRunner.loadTests(locator)) .then((shouldRun) => { if(shouldRun) { return testRunner.runTests(id); } }) .then(() => testRunner.saveTestResults(locator, (testRunner.keyboards[id] || {}).results)); }).timeout(0); }); `; } });
return getEngine.then(() => { console.log('Testing compiler version '+compilerVersion+', engine version '+engineVersion); if(testedEngineVersions.indexOf(engineVersion) < 0) { testedEngineVersions.push(engineVersion); } return util.runProcess( `node`, [].concat( ['test-builder.js', '--compiler-version', compilerVersion, '--engine-version', engineVersion], program.keyboards.length ? ['--keyboards', program.keyboards.join(',')] : [] ) ); });
keyboards.forEach((keyboard) => { if(!program.keyboards.length || program.keyboards.indexOf(keyboard.shortname+'/'+keyboard.id) >= 0) { // Validate each of the test files against the first tested compiler+engine version let localFailCount = 0; const localFail = knownFailures.hasOwnProperty([keyboard.id]) ? (msg) => { if(++localFailCount == 1) console.warn(`WARN: Not failing test because ${keyboard.id} is in known-failures.`); console.warn(`WARN: ${msg}`); } : fail; try { const baseResultFilename = path.join(KEYBOARDS_ROOT, keyboard.shortname, keyboard.id, 'tests', `${keyboard.id}-${baseCompilerVersion}-${baseEngineVersion}.results`); const baseResult = fs.readFileSync(baseResultFilename, 'utf8'); const baseResultJSON = JSON.parse(baseResult); const testsFilename = path.join(KEYBOARDS_ROOT, keyboard.shortname, keyboard.id, 'tests', `${keyboard.id}.tests`); const testsJSON = JSON.parse(fs.readFileSync(testsFilename, 'utf8')); const numTests = Object.keys(testsJSON.inputTests).length; // First, check the base result for errors for(let k in baseResultJSON) { if(typeof baseResultJSON[k] !== 'string') { let input = `${testsJSON.inputTests[k].context ? `"${testsJSON.inputTests[k].context}" ` : ""}+ ${keyname(testsJSON.inputTests[k].modifier, testsJSON.inputTests[k].key)}`; localFail(`${keyboard.shortname}/${keyboard.id}[${k}]: error in test: ${input}: ${baseResultJSON[k].error}`, 5); } } // Make sure every test is in the base result set if(Object.keys(baseResultJSON).length != numTests) { localFail(`${keyboard.shortname}/${keyboard.id}: base result set has fewer results (${Object.keys(baseResultJSON).length}) than expected (${numTests})`, 6); } //console.log(baseResultFilename, baseResult); testedCompilerVersions.forEach((cv) => { testedEngineVersions.forEach((ev) => { const resultFilename = path.join(KEYBOARDS_ROOT, keyboard.shortname, keyboard.id, 'tests', `${keyboard.id}-${cv}-${ev}.results`); const result = fs.readFileSync(resultFilename, 'utf8'); //console.log(resultFilename, result); // Naive string test first if(result !== baseResult) { // Now, report first mismatch and total number of mismatches after parsing JSON const resultJSON = JSON.parse(result); let errors = 0, prefix = `${keyboard.shortname}/${keyboard.id}`; for(let k in baseResultJSON) { if(resultJSON[k] !== baseResultJSON[k]) { if(++errors == 1 || program.logAllFailures) { let ix = k.toString(), whitespace = ' '.repeat(prefix.length + ix.length + 6), input = `${testsJSON.inputTests[k].context ? `"${testsJSON.inputTests[k].context}" ` : ""}+ ${keyname(testsJSON.inputTests[k].modifier, testsJSON.inputTests[k].key)}`; console.error(`${prefix}[${ix}]: expected: ${input} > "${baseResultJSON[k]}"`); console.error(`${whitespace}actual: ${input} > "${resultJSON[k]}"`); } } } localFail(`${keyboard.shortname}/${keyboard.id} ${errors}/${numTests} test(s) mismatched between (${baseCompilerVersion} / ${baseEngineVersion}) and (${cv} / ${ev})`, 4); } }); }); } catch(e) { localFail(`Failed to load test results for ${keyboard.shortname}/${keyboard.id}: ${typeof e == 'object' ? e.message : e}`, 7); } } });