diff options
Diffstat (limited to 'Source/WebCore/inspector/front-end/SourceJavaScriptTokenizer.re2js')
| -rw-r--r-- | Source/WebCore/inspector/front-end/SourceJavaScriptTokenizer.re2js | 178 |
1 files changed, 178 insertions, 0 deletions
diff --git a/Source/WebCore/inspector/front-end/SourceJavaScriptTokenizer.re2js b/Source/WebCore/inspector/front-end/SourceJavaScriptTokenizer.re2js new file mode 100644 index 0000000..ae71efe --- /dev/null +++ b/Source/WebCore/inspector/front-end/SourceJavaScriptTokenizer.re2js @@ -0,0 +1,178 @@ +/* + * Copyright (C) 2009 Google Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +// Generate js file as follows: +// +// re2c -isc WebCore/inspector/front-end/SourceJavaScriptTokenizer.re2js \ +// | sed 's|^yy\([^:]*\)*\:|case \1:|' \ +// | sed 's|[*]cursor[+][+]|this._charAt(cursor++)|' \ +// | sed 's|[[*][+][+]cursor|this._charAt(++cursor)|' \ +// | sed 's|[*]cursor|this._charAt(cursor)|' \ +// | sed 's|yych = \*\([^;]*\)|yych = this._charAt\1|' \ +// | sed 's|goto case \([^;]*\)|{ gotoCase = \1; continue; }|' \ +// | sed 's|unsigned\ int|var|' \ +// | sed 's|var\ yych|case 1: var yych|' + +WebInspector.SourceJavaScriptTokenizer = function() +{ + WebInspector.SourceTokenizer.call(this); + + this._keywords = [ + "null", "true", "false", "break", "case", "catch", "const", "default", "finally", "for", + "instanceof", "new", "var", "continue", "function", "return", "void", "delete", "if", + "this", "do", "while", "else", "in", "switch", "throw", "try", "typeof", "debugger", + "class", "enum", "export", "extends", "import", "super", "get", "set", "with" + ].keySet(); + + this._lexConditions = { + DIV: 0, + NODIV: 1, + COMMENT: 2, + DSTRING: 3, + SSTRING: 4, + REGEX: 5 + }; + + this.case_DIV = 1000; + this.case_NODIV = 1001; + this.case_COMMENT = 1002; + this.case_DSTRING = 1003; + this.case_SSTRING = 1004; + this.case_REGEX = 1005; + + this.initialCondition = { lexCondition: this._lexConditions.NODIV } + this.condition = this.initialCondition; +} + +WebInspector.SourceJavaScriptTokenizer.prototype = { + nextToken: function(cursor) + { + var cursorOnEnter = cursor; + var gotoCase = 1; + while (1) { + switch (gotoCase) + // Following comment is replaced with generated state machine. + /*!re2c + re2c:define:YYCTYPE = "var"; + re2c:define:YYCURSOR = cursor; + re2c:define:YYGETCONDITION = "this.getLexCondition"; + re2c:define:YYSETCONDITION = "this.setLexCondition"; + re2c:condprefix = "case this.case_"; + re2c:condenumprefix = "this._lexConditions."; + re2c:yyfill:enable = 0; + re2c:labelprefix = "case "; + re2c:indent:top = 2; + re2c:indent:string = " "; + + LineComment = "//" [^\r\n]*; + CommentContent = ([^*\r\n] | ("*"+[^/*]))*; + Comment = "/*" CommentContent "*"+ "/"; + CommentStart = "/*" CommentContent [\r\n]; + CommentEnd = CommentContent "*"+ "/"; + + DecimalDigit = [0-9]; + NonZeroDigit = [1-9]; + OctalDigit = [0-7]; + HexDigit = [0-9a-fA-F]; + SignedInteger = ("+"|"-")? DecimalDigit+; + ExponentPart = ("e" | "E") SignedInteger; + DecimalIntegerLiteral = "0" | NonZeroDigit DecimalDigit*; + DecimalLiteral = DecimalIntegerLiteral "." DecimalDigit* ExponentPart? | "." DecimalDigit+ ExponentPart? | DecimalIntegerLiteral ExponentPart?; + HexIntegerLiteral = "0" ("x"|"X") HexDigit+; + OctalIntegerLiteral = "0" OctalDigit+; + NumericLiteral = DecimalLiteral | HexIntegerLiteral | OctalIntegerLiteral; + + Punctuation = [\!\%\&\(\*\+\,\-\.\:\;\<\=\>\?\[\]\^\{\|\}\~] | "!=" | "!==" | "%=" | "&&" | "&=" | "*=" | "++" | "+=" | "--" | "-=" | "<<" | "<<=" | "<=" | "==" | "===" | ">=" | ">>" | ">>=" | ">>>" | ">>>=" | "^=" | "|=" | "||"; + Division = "/" | "/="; + RightParen = ")"; + + Letter = [a-zA-Z\x80-\xFF]; + UnicodeEscapeSequence = "\\u" HexDigit HexDigit HexDigit HexDigit; + + IdentifierStart = Letter | "_" | "$" | UnicodeEscapeSequence; + IdentifierPart = IdentifierStart | DecimalDigit; + Identifier = IdentifierStart IdentifierPart *; + + DoubleStringContent = ([^\r\n\"\\] | UnicodeEscapeSequence | "\\" ['"\\bfnrtv])*; + SingleStringContent = ([^\r\n\'\\] | UnicodeEscapeSequence | "\\" ['"\\bfnrtv])*; + StringLiteral = "\"" DoubleStringContent "\"" | "'" SingleStringContent "'"; + DoubleStringStart = "\"" DoubleStringContent "\\" [\r\n]; + DoubleStringEnd = DoubleStringContent "\""; + SingleStringStart = "'" SingleStringContent "\\" [\r\n]; + SingleStringEnd = SingleStringContent "'"; + + BackslashSequence = "\\" [^\r\n]; + RegexSet = "[" ([^\r\n*\\/] | BackslashSequence)* "]"; + RegexFirstChar = [^\r\n*\\/\[\]] | BackslashSequence | RegexSet; + RegexChar = [^\r\n\\/\[\]] | BackslashSequence | RegexSet; + RegexContent = RegexChar*; + Regex = "/" RegexFirstChar RegexContent "/" [igm]*; + RegexStart = "/" RegexFirstChar RegexContent "\\"; + RegexEnd = RegexContent "/" [igm]*; + + <DIV,NODIV> LineComment { this.tokenType = "javascript-comment"; return cursor; } + <DIV,NODIV> Comment { this.tokenType = "javascript-comment"; return cursor; } + <DIV,NODIV> CommentStart => COMMENT { this.tokenType = "javascript-comment"; return cursor; } + <COMMENT> CommentContent => COMMENT { this.tokenType = "javascript-comment"; return cursor; } + <COMMENT> CommentEnd => NODIV { this.tokenType = "javascript-comment"; return cursor; } + + <DIV,NODIV> StringLiteral { this.tokenType = "javascript-string"; return cursor; } + <DIV,NODIV> DoubleStringStart => DSTRING { this.tokenType = "javascript-string"; return cursor; } + <DSTRING> DoubleStringContent => DSTRING { this.tokenType = "javascript-string"; return cursor; } + <DSTRING> DoubleStringEnd => NODIV { this.tokenType = "javascript-string"; return cursor; } + <DIV,NODIV> SingleStringStart => SSTRING { this.tokenType = "javascript-string"; return cursor; } + <SSTRING> SingleStringContent => SSTRING { this.tokenType = "javascript-string"; return cursor; } + <SSTRING> SingleStringEnd => NODIV { this.tokenType = "javascript-string"; return cursor; } + + <NODIV> Regex { this.tokenType = "javascript-regexp"; return cursor; } + <NODIV> RegexStart => REGEX { this.tokenType = "javascript-regexp"; return cursor; } + <REGEX> RegexContent => REGEX { this.tokenType = "javascript-regexp"; return cursor; } + <REGEX> RegexEnd => NODIV { this.tokenType = "javascript-regexp"; return cursor; } + + <DIV,NODIV> NumericLiteral => DIV { this.tokenType = "javascript-number"; return cursor; } + <DIV,NODIV> Identifier => DIV + { + var token = this._line.substring(cursorOnEnter, cursor); + if (token in this._keywords) + this.tokenType = "javascript-keyword"; + else + this.tokenType = "javascript-ident"; + return cursor; + } + <DIV,NODIV> RightParen => DIV { this.tokenType = null; return cursor; } + <DIV,NODIV> Punctuation => NODIV { this.tokenType = null; return cursor; } + <DIV> Division => NODIV { this.tokenType = null; return cursor; } + <*> [^] { this.tokenType = null; return cursor; } + */ + } + } +} + +WebInspector.SourceJavaScriptTokenizer.prototype.__proto__ = WebInspector.SourceTokenizer.prototype; |