/
JsonTokenizer.js
34 lines (27 loc) · 1.1 KB
/
JsonTokenizer.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
var Tokenizer = require('tokenizer');
module.exports = function JsonTokenizer(options) {
var t = new Tokenizer();
t.addRule(/^,$/, 'comma');
t.addRule(/^:$/, 'end-label');
t.addRule(/^\{$/, 'begin-object');
t.addRule(/^\}$/, 'end-object');
t.addRule(/^\[$/, 'begin-array');
t.addRule(/^\]$/, 'end-array');
t.addRule(/^"(\\["\\/bfnrtu"]|[^"\\"])*"$/, 'string');
t.addRule(/^"([^"]|\\")*$/, 'maybe-string');
t.addRule(/^null$/, 'null');
t.addRule(/^(true|false)$/, 'boolean');
t.addRule(/^-?\d+(\.\d+)?([eE]-?\d+)?$/, 'number');
t.addRule(/^-?\d+\.$/, 'maybe-decimal-number');
t.addRule(/^-$/, 'maybe-negative-number');
t.addRule(/^-?\d+(\.\d+)?([eE])?$/, 'maybe-exponential-number');
t.addRule(/^-?\d+(\.\d+)?([eE]-)?$/, 'maybe-exponential-number-negative');
t.addRule(/^\w+$/, 'symbol');
t.addRule(Tokenizer.whitespace);
if (options == null || !options.whitespace) {
t.ignore('whitespace');
}
// if we had comments tokens, we would ignore them as well
// but the JSON spec doesn't allow comments!
return t;
}