function createTokenizer () {
var t = new Tokenizer();
// we only use built-in rules
t.addRule(Tokenizer.number);
t.addRule(Tokenizer.word);
t.addRule(Tokenizer.whitespace);
t.ignore('whitespace');
t.on('token', function(token, type) {
//console.log('got token %s of type ', JSON.stringify(token), type);
});
return t;
}
module.exports = function JsonTokenizer(options) {
var t = new Tokenizer();
t.addRule(/^,$/, 'comma');
t.addRule(/^:$/, 'end-label');
t.addRule(/^\{$/, 'begin-object');
t.addRule(/^\}$/, 'end-object');
t.addRule(/^\[$/, 'begin-array');
t.addRule(/^\]$/, 'end-array');
t.addRule(/^"(\\["\\/bfnrtu"]|[^"\\"])*"$/, 'string');
t.addRule(/^"([^"]|\\")*$/, 'maybe-string');
t.addRule(/^null$/, 'null');
t.addRule(/^(true|false)$/, 'boolean');
t.addRule(/^-?\d+(\.\d+)?([eE]-?\d+)?$/, 'number');
t.addRule(/^-?\d+\.$/, 'maybe-decimal-number');
t.addRule(/^-$/, 'maybe-negative-number');
t.addRule(/^-?\d+(\.\d+)?([eE])?$/, 'maybe-exponential-number');
t.addRule(/^-?\d+(\.\d+)?([eE]-)?$/, 'maybe-exponential-number-negative');
t.addRule(/^\w+$/, 'symbol');
t.addRule(Tokenizer.whitespace);
if (options == null || !options.whitespace) {
t.ignore('whitespace');
}
// if we had comments tokens, we would ignore them as well
// but the JSON spec doesn't allow comments!
return t;
}
module.exports = function (cb) {
var t = new Tokenizer(cb)
t.addRule(/^"(?:(?:\\\n|\\"|[^"\n]))*?"?$/, "string1")
t.addRule(/^'(?:(?:\\\n|\\'|[^'\n]))*?'?$/, "string2")
t.addRule(/^\/\*[\s\S]*?\*\/$/, "comment1")
t.addRule(/^\/\/.*?\n$/, "comment2")
t.addRule(Tokenizer.whitespace)
t.addRule(/^\b(?:var|let|for|if|else|in|class|function|return|with|case|break|switch|export|new|while|do|throw|catch)\b$/, "keyword")
t.addRule(/^\/(?:(?:\\\/|[^\/]))*?\/?$/, "regexp")
t.addRule(/^[a-zA-Z_\$][a-zA-Z_\$0-9]*$/, "name")
t.addRule(/^\d+(?:\.\d+)?(?:e[+-]?\d+)?$/, "number")
t.addRule(/^[\(\)]$/, "parens")
t.addRule(/^[{}]$/, "curly")
t.addRule(/^[\[\]]$/, "square")
t.addRule(/^[;.:\?\^%<>=!&|+\-,]$/, "punct")
return t
}
var parse = function (input) {
var tokenizer = new Tokenizer();
tokenizer.addRule(/^'([^']|\\')+':$/, 'key');
tokenizer.addRule(/^'([^']|\\')*'$/, 'string');
tokenizer.addRule(/^\s+$/, 'whitespace');
tokenizer.addRule(/^(\d*\.)?\d+$/, 'number');
tokenizer.addRule(/^(true|false)$/, 'boolean');
tokenizer.addRule(/^null$/, 'null');
tokenizer.addRule(/^(\[|,|\])$/, 'symbol');
tokenizer.addRule(/^;$/, 'terminator');
tokenizer.addRule(/^'([^']|\\')*$/, 'maybe-string');
tokenizer.addRule(/^\d*\.$/, 'maybe-number');
tokenizer.addRule(/^(t(r(ue?)?)?|f(a(l(se?)?)?)?)$/, 'maybe-boolean');
tokenizer.addRule(/^n(u(ll?)?)?$/, 'maybe-null');
var result = [];
var key = 0;
var cursors = [ result ]
tokenizer.on('token', function (token, type) {
var cursor = cursors[cursors.length - 1];
switch (type) {
case 'symbol':
switch (token) {
case '[':
var obj = {};
cursor[key] = obj;
cursors.push(obj);
break;
case ']':
cursors.pop();
break;
}
break;
case 'key':
key = token.substr(1).replace(/'\s*:$/, '');
break;
case 'string':
cursor[key] = token.substr(1, token.length - 2);
break;
case 'number':
cursor[key] = Number(token);
break;
case 'boolean':
cursor[key] = (token === 'true');
break;
case 'null':
cursor[key] = null;
break;
}
});
tokenizer.write(input + ';');
return result[0];
};