29 people like it.

Lexer

Hand-written efficient JavaScript lexer from the IronJS project https://github.com/fholm/IronJS

  1: 
  2: 
  3: 
  4: 
  5: 
  6: 
  7: 
  8: 
  9: 
 10: 
 11: 
 12: 
 13: 
 14: 
 15: 
 16: 
 17: 
 18: 
 19: 
 20: 
 21: 
 22: 
 23: 
 24: 
 25: 
 26: 
 27: 
 28: 
 29: 
 30: 
 31: 
 32: 
 33: 
 34: 
 35: 
 36: 
 37: 
 38: 
 39: 
 40: 
 41: 
 42: 
 43: 
 44: 
 45: 
 46: 
 47: 
 48: 
 49: 
 50: 
 51: 
 52: 
 53: 
 54: 
 55: 
 56: 
 57: 
 58: 
 59: 
 60: 
 61: 
 62: 
 63: 
 64: 
 65: 
 66: 
 67: 
 68: 
 69: 
 70: 
 71: 
 72: 
 73: 
 74: 
 75: 
 76: 
 77: 
 78: 
 79: 
 80: 
 81: 
 82: 
 83: 
 84: 
 85: 
 86: 
 87: 
 88: 
 89: 
 90: 
 91: 
 92: 
 93: 
 94: 
 95: 
 96: 
 97: 
 98: 
 99: 
100: 
101: 
102: 
103: 
104: 
105: 
106: 
107: 
108: 
109: 
110: 
111: 
112: 
113: 
114: 
115: 
116: 
117: 
118: 
119: 
120: 
121: 
122: 
123: 
124: 
125: 
126: 
127: 
128: 
129: 
130: 
131: 
132: 
133: 
134: 
135: 
136: 
137: 
138: 
139: 
140: 
141: 
142: 
143: 
144: 
145: 
146: 
147: 
148: 
149: 
150: 
151: 
152: 
153: 
154: 
155: 
156: 
157: 
158: 
159: 
160: 
161: 
162: 
163: 
164: 
165: 
166: 
167: 
168: 
169: 
170: 
171: 
172: 
173: 
174: 
175: 
176: 
177: 
178: 
179: 
180: 
181: 
182: 
183: 
184: 
185: 
186: 
187: 
188: 
189: 
190: 
191: 
192: 
193: 
194: 
195: 
196: 
197: 
198: 
199: 
200: 
201: 
202: 
203: 
204: 
205: 
206: 
207: 
208: 
209: 
210: 
211: 
212: 
213: 
214: 
215: 
216: 
217: 
218: 
219: 
220: 
221: 
222: 
223: 
224: 
225: 
226: 
227: 
228: 
229: 
230: 
231: 
232: 
233: 
234: 
235: 
236: 
237: 
238: 
239: 
240: 
241: 
242: 
243: 
244: 
245: 
246: 
247: 
248: 
249: 
250: 
251: 
252: 
253: 
254: 
255: 
256: 
257: 
258: 
259: 
260: 
261: 
262: 
263: 
264: 
265: 
266: 
267: 
268: 
269: 
270: 
271: 
272: 
273: 
274: 
275: 
276: 
277: 
278: 
279: 
280: 
281: 
282: 
283: 
284: 
285: 
286: 
287: 
288: 
289: 
290: 
291: 
292: 
293: 
294: 
295: 
296: 
297: 
298: 
299: 
300: 
301: 
302: 
303: 
304: 
305: 
306: 
307: 
308: 
309: 
310: 
311: 
312: 
313: 
314: 
315: 
316: 
317: 
318: 
319: 
320: 
321: 
322: 
323: 
324: 
325: 
326: 
327: 
328: 
329: 
330: 
331: 
332: 
333: 
334: 
335: 
336: 
337: 
338: 
339: 
340: 
341: 
342: 
343: 
344: 
345: 
346: 
347: 
348: 
349: 
350: 
351: 
352: 
353: 
354: 
355: 
356: 
357: 
358: 
359: 
360: 
361: 
362: 
363: 
364: 
365: 
366: 
367: 
368: 
369: 
370: 
371: 
372: 
373: 
374: 
375: 
376: 
377: 
378: 
379: 
380: 
381: 
382: 
383: 
384: 
385: 
386: 
387: 
388: 
389: 
390: 
391: 
392: 
393: 
394: 
395: 
396: 
397: 
398: 
399: 
400: 
401: 
402: 
403: 
404: 
405: 
406: 
407: 
408: 
409: 
410: 
411: 
412: 
413: 
414: 
415: 
416: 
417: 
418: 
419: 
420: 
421: 
422: 
423: 
424: 
425: 
426: 
427: 
428: 
429: 
430: 
431: 
432: 
433: 
434: 
435: 
436: 
437: 
438: 
439: 
440: 
441: 
442: 
443: 
444: 
445: 
446: 
447: 
448: 
449: 
450: 
451: 
452: 
453: 
454: 
455: 
456: 
457: 
458: 
459: 
460: 
461: 
462: 
463: 
464: 
465: 
466: 
467: 
468: 
469: 
470: 
471: 
472: 
473: 
474: 
namespace IronJS.Compiler
module Lexer =
  
  open System

  module private Char =
    
    type private Cat = 
      Globalization.UnicodeCategory

    let inline isCRLF cr lf = cr = '\r' && lf = '\n'
    let inline isAlpha c = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
    let inline isDecimal c = c >= '0' && c <= '9'
    let inline isOctal c = c >= '0' && c <= '7'
    let inline isHex c = isDecimal c || (c >= 'A' && c <= 'F') || (c >= 'a' && c <= 'f')
    let inline isQuote c = c = '"' || c = '\''
    let inline isLineTerminator c = 
      match c with
      | '\n'|'\r'|'\u2028'|'\u2029' -> true
      | _ -> false

    let inline isWhiteSpace (c:Char) =
      match int c with
      | 0x09   | 0x0B   | 0x0C   
      | 0x20   | 0xA0   | 0x1680 
      | 0x180E | 0x202F | 0x205F 
      | 0x3000 | 0xFEFF -> true
      | c -> c >= 8192 && c <= 8202;

    let isUnicodeIdentifierStart c =
      match c |> Char.GetUnicodeCategory with
      | Cat.UppercaseLetter
      | Cat.LowercaseLetter
      | Cat.TitlecaseLetter
      | Cat.ModifierLetter
      | Cat.OtherLetter 
      | Cat.LetterNumber -> true
      | _ -> false

    let inline isIdentifierStart c =
      if c |> isAlpha || c = '_' || c = '$' 
        then true
        else c |> isUnicodeIdentifierStart

    let isUnicodeIdentifier c =
      match c |> Char.GetUnicodeCategory with
      | Cat.UppercaseLetter 
      | Cat.LowercaseLetter
      | Cat.TitlecaseLetter
      | Cat.ModifierLetter
      | Cat.OtherLetter
      | Cat.LetterNumber 
      | Cat.NonSpacingMark
      | Cat.SpacingCombiningMark
      | Cat.DecimalDigitNumber
      | Cat.ConnectorPunctuation -> true
      | _ -> int c = 0x200C || int c = 0x200D

    let inline isIdentifier c = 
      if c |> isAlpha || c = '_' || c = '$' || c |> isDecimal 
        then true
        else c |> isUnicodeIdentifier

    let inline isPunctuation_Simple c =
      match c with
      | '{' | '}' | '(' | ')' | '[' | ']' 
      | ';' | ',' | '?' | ':' | '~' -> true
      | _ -> false

    let inline isPunctuation c =
      match c with
      | '<' | '>' | '=' | '+' | '-' | '!'
      | '%' | '&' | '|' | '^' | '*' -> true
      | _ -> false

  type Symbol
    //Keywords
    = Break = 0
    | Case = 1
    | Catch = 2
    | Continue = 3
    | Default = 4
    | Delete = 5
    | Do = 6
    | Else = 7
    | Finally = 9
    | Function = 10
    | If = 11
    | In = 12
    | InstanceOf = 13
    | New = 14
    | Return = 16
    | Switch = 17
    | This = 18
    | Throw = 19
    | Try = 20
    | TypeOf = 22
    | Var = 23
    | Void = 24
    | While = 25
    | With = 26
    | For = 84

    //Punctuation
    | LeftBrace = 27
    | RightBrace = 28
    | LeftParenthesis = 29
    | RightParenthesis = 30
    | LeftBracket = 31
    | RightBracket = 32
    | Semicolon = 33
    | Comma = 34
    | Equal = 35
    | NotEqual = 36
    | StrictEqual = 37
    | StrictNotEqual = 38
    | LessThan = 39
    | GreaterThan = 40
    | LessThanOrEqual = 41
    | GreaterThanOrEqual = 42
    | Plus = 43
    | Minus = 44
    | Multiply = 45
    | Divide = 46
    | Modulo = 47
    | Increment = 48
    | Decrement = 49
    | LeftShift = 50
    | RightShift = 51
    | URightShift = 52
    | BitwiseAnd = 53
    | BitwiseOr = 54
    | BitwiseXor = 55
    | BitwiseNot = 56
    | LogicalNot = 57
    | LogicalAnd = 58
    | LogicalOr = 59
    | Condition = 60
    | Colon = 61
    | Assign = 62
    | AssignAdd = 63
    | AssignSubtract = 64
    | AssignMultiply = 65
    | AssignDivide = 66
    | AssignModulo = 67
    | AssignLeftShift = 68
    | AssignSignedRightShift = 69
    | AssignUnsignedRightShift = 70
    | AssignBitwiseAnd = 71
    | AssignBitwiseOr = 72
    | AssignBitwiseXor = 73
    | Dot = 74

    //Literals
    | True = 75
    | False = 76
    | Null = 77
    | String = 78
    | Number = 79
    | RegExp = 80
    | LineTerminator = 81
    | Identifier = 82
    | Comment = 83

    //Special
    | StartOfInput = 100
    | EndOfInput = 101

  type Token = Symbol * string * int * int

  module private Input = 
    
    [<NoComparison>]
    type T = 
      val mutable Source : string
      val mutable Index : int
      val mutable Line : int
      val mutable Column : int
      val mutable Buffer : Text.StringBuilder
      val mutable Previous : Symbol

      new (source) = {
        Source = source
        Index = 0
        Line = 1
        Column = 0
        Buffer = Text.StringBuilder(1024)
        Previous = Symbol.StartOfInput
      }

    let create (input:string) = T(input)
    let inline newline (t:T) = t.Line <- t.Line + 1
    let inline current (t:T) = t.Source.[t.Index]
    let inline previous (t:T) = t.Source.[t.Index-1]
    let inline peek (t:T) = t.Source.[t.Index+1]
    let inline canPeek (t:T) = t.Index+1 < t.Source.Length
    let inline continue' (t:T) = t.Index < t.Source.Length
    let inline position (t:T) = t.Line, t.Column
    let inline rewind (t:T) = t.Index <- t.Index - 1
    let inline advance (t:T) = 
      t.Index <- t.Index + 1
      t.Column <- t.Column + 1

    let inline skip n (t:T) = 
      t.Index <- t.Index + n
      t.Column <- t.Column + n

    let inline buffer (t:T) (c:Char) = t.Buffer.Append(c) |> ignore
    let inline clearBuffer (t:T) = t.Buffer.Clear() |> ignore
    let inline bufferValue (t:T) = t.Buffer.ToString()
    let inline nextLine (t:T) =
      if t |> current = '\r' && t |> canPeek && t |> peek = '\n' 
        then t |> advance
        
      t.Line <- t.Line + 1
      t.Column <- 0

    let inline setupBuffer (t:T) =
      t |> advance
      t |> clearBuffer
      t.Line, t.Column

    let inline output symbol value line col (t:T) =
      t.Previous <- symbol
      symbol, value, line, col

    let endOfInput (t:T) =
      t |> output Symbol.EndOfInput null -1 -1
      
  open Char
  open Input
  open System.Collections.Generic

  [<Literal>]
  let private empty:string = null

  let private keywordMap = 
    [
      ("break", Symbol.Break)
      ("case", Symbol.Case)
      ("catch", Symbol.Catch)
      ("continue", Symbol.Continue)
      ("default", Symbol.Default)
      ("delete", Symbol.Delete)
      ("do", Symbol.Do)
      ("else", Symbol.Else)
      ("finally", Symbol.Finally)
      ("function", Symbol.Function)
      ("if", Symbol.If)
      ("in", Symbol.In)
      ("instanceof", Symbol.InstanceOf)
      ("new", Symbol.New)
      ("return", Symbol.Return)
      ("switch", Symbol.Switch)
      ("this", Symbol.This)
      ("throw", Symbol.Throw)
      ("try", Symbol.Try)
      ("typeof", Symbol.TypeOf)
      ("var", Symbol.Var)
      ("void", Symbol.Void)
      ("while", Symbol.While)
      ("with", Symbol.With)
      ("for", Symbol.For)
    ] |> Map.ofList

  let private punctuationMap =
    new Dictionary<string, Symbol>(
      [
        ("==", Symbol.Equal) 
        ("!=", Symbol.NotEqual) 
        ("===", Symbol.StrictEqual) 
        ("!==", Symbol.StrictNotEqual)
        ("<", Symbol.LessThan) 
        (">", Symbol.GreaterThan) 
        ("<=", Symbol.LessThanOrEqual)
        (">=", Symbol.GreaterThanOrEqual)
        ("+", Symbol.Plus)
        ("-", Symbol.Minus) 
        ("*", Symbol.Multiply) 
        ("%", Symbol.Modulo) 
        ("++", Symbol.Increment) 
        ("--", Symbol.Decrement)
        ("<<", Symbol.LeftShift) 
        (">>", Symbol.RightShift) 
        (">>>", Symbol.URightShift) 
        ("&", Symbol.BitwiseAnd) 
        ("|", Symbol.BitwiseOr)
        ("^", Symbol.BitwiseXor) 
        ("&&", Symbol.LogicalAnd) 
        ("||", Symbol.LogicalOr) 
        ("=", Symbol.Assign) 
        ("+=", Symbol.AssignAdd)
        ("-=", Symbol.AssignSubtract) 
        ("*=", Symbol.AssignMultiply) 
        ("%=", Symbol.AssignModulo) 
        ("<<=", Symbol.AssignLeftShift) 
        (">>=", Symbol.AssignSignedRightShift)
        (">>>=", Symbol.AssignUnsignedRightShift)
        ("&=", Symbol.AssignBitwiseAnd)
        ("|=", Symbol.AssignBitwiseOr)
        ("^=", Symbol.AssignBitwiseXor)
      ] |> Map.ofList
    )

  let private simplePunctuation (s:Input.T) c =
    s |> advance

    let symbol =
      match c with
      | '{' -> Symbol.LeftBrace
      | '}' -> Symbol.RightBrace
      | '(' -> Symbol.LeftParenthesis
      | ')' -> Symbol.RightParenthesis
      | '[' -> Symbol.LeftBracket
      | ']' -> Symbol.RightBracket
      | ';' -> Symbol.Semicolon
      | ',' -> Symbol.Comma
      | '?' -> Symbol.Condition
      | ':' -> Symbol.Colon
      | '~' -> Symbol.BitwiseNot
      | _ -> failwithf "Invalid simple punctuation %c" c

    symbol, empty, s.Line, s.Column

  let private identifier (s:Input.T) (first:char) =
    let line = s.Line
    let column = s.Column

    s |> advance
    s |> clearBuffer
    first |> buffer s

    while s |> continue' && s |> current |> isIdentifier do
      s |> current |> buffer s
      s |> advance

    let identifier = s |> bufferValue
    match keywordMap.TryFind identifier with
    | None -> s |> output Symbol.Identifier identifier line column
    | Some keyword -> s |> output keyword empty line column

  let private singleLineComment (s:Input.T) =
    let line = s.Line
    let column = s.Column

    s |> clearBuffer
    s |> advance

    while s |> continue' && s |> current |> isLineTerminator |> not do
      s |> current |> buffer s
      s |> advance

    s |> output Symbol.Comment (s |> bufferValue) line column

  let private multiLineComment (s:Input.T) =
    let line = s.Line
    let column = s.Column

    s |> clearBuffer
    s |> advance

    while current s <> '*' && peek s <> '/' do
      if s |> current |> isLineTerminator then
        s |> nextLine

      s |> current |> buffer s
      s |> advance
      
    s |> skip 2
    s |> output Symbol.Comment (s |> bufferValue) line column

  let private punctuation (s:Input.T) (first:char) =
    
    let inline makeToken (s:Input.T) (buffer:string) =
      let column = s.Column - buffer.Length
      s |> output punctuationMap.[buffer] empty s.Line column

    let rec punctuation (s:Input.T) (buffer:string) =
      s |> advance

      if s |> continue' then
        let newBuffer = buffer + (s |> current |> string)

        if punctuationMap.ContainsKey(newBuffer) 
          then newBuffer |> punctuation s
          else buffer |> makeToken s

      else
        buffer |> makeToken s

    first |> string |> punctuation s

  let private literalString (s:Input.T) (stop:char) =
    let line = s.Line
    let column = s.Column

    s |> clearBuffer

    let rec literalString (s:Input.T) =
      s |> advance

      if s |> continue' then
        match s |> current with
        | c when c |> isLineTerminator -> 
          failwith "Unexpected newline in literal string"

        | '\\' -> 
          s |> advance

          match s |> current with
          | 'n' -> '\n' |> buffer s
          | 'r' -> '\r' |> buffer s
          | 'b' -> '\b' |> buffer s
          | 'f' -> '\f' |> buffer s
          | 't' -> '\t' |> buffer s
          | 'v' -> '\v' |> buffer s
          | c -> c |> buffer s

          s |> literalString

        | c when c = stop ->
          s |> advance
          s |> output Symbol.String (s |> bufferValue) line column

        | c -> 
          c |> buffer s
          s |> literalString

      else
        failwith "Unexpected end of source in string"

    s |> literalString

  let create (source:string) =
    let buffer = Text.StringBuilder(512)
    let s = source |> Input.create

    let rec lexer () = 
      if s |> continue' then
        match s |> current with
        | c when c |> isWhiteSpace -> 
          s |> advance; lexer()

        | c when c |> isPunctuation_Simple ->
          c |> simplePunctuation s

        | c when c |> isPunctuation ->
          c |> punctuation s

        | c when c |> isIdentifierStart ->
          c |> identifier s

        | c when c |> isQuote ->
          c |> literalString s

        // Deal with comments, division and regexp literals
        | '/' ->
          s |> advance
          match s |> current with
          | '/' -> s |> singleLineComment
          | '*' -> s |> multiLineComment

        | c when c |> isLineTerminator ->
          s |> nextLine
          s |> advance
          s |> output Symbol.LineTerminator empty s.Line s.Column

        | c -> 
          failwithf "Incorrect input %c" c

      else
        s |> endOfInput

    lexer
namespace System
type Char =
  struct
    member CompareTo : value:obj -> int + 1 overload
    member Equals : obj:obj -> bool + 1 overload
    member GetHashCode : unit -> int
    member GetTypeCode : unit -> TypeCode
    member ToString : unit -> string + 1 overload
    static val MaxValue : char
    static val MinValue : char
    static member ConvertFromUtf32 : utf32:int -> string
    static member ConvertToUtf32 : highSurrogate:char * lowSurrogate:char -> int + 1 overload
    static member GetNumericValue : c:char -> float + 1 overload
    ...
  end

Full name: System.Char
type private Cat = Globalization.UnicodeCategory

Full name: IronJS.Compiler.Lexer.Char.Cat
namespace System.Globalization
type UnicodeCategory =
  | UppercaseLetter = 0
  | LowercaseLetter = 1
  | TitlecaseLetter = 2
  | ModifierLetter = 3
  | OtherLetter = 4
  | NonSpacingMark = 5
  | SpacingCombiningMark = 6
  | EnclosingMark = 7
  | DecimalDigitNumber = 8
  | LetterNumber = 9
  ...

Full name: System.Globalization.UnicodeCategory
val private isCRLF : cr:char -> lf:char -> bool

Full name: IronJS.Compiler.Lexer.Char.isCRLF
val cr : char
val lf : char
val private isAlpha : c:char -> bool

Full name: IronJS.Compiler.Lexer.Char.isAlpha
val c : char
val private isDecimal : c:char -> bool

Full name: IronJS.Compiler.Lexer.Char.isDecimal
val private isOctal : c:char -> bool

Full name: IronJS.Compiler.Lexer.Char.isOctal
val private isHex : c:char -> bool

Full name: IronJS.Compiler.Lexer.Char.isHex
val private isQuote : c:char -> bool

Full name: IronJS.Compiler.Lexer.Char.isQuote
val private isLineTerminator : c:char -> bool

Full name: IronJS.Compiler.Lexer.Char.isLineTerminator
val private isWhiteSpace : c:Char -> bool

Full name: IronJS.Compiler.Lexer.Char.isWhiteSpace
val c : Char
Multiple items
val int : value:'T -> int (requires member op_Explicit)

Full name: Microsoft.FSharp.Core.Operators.int

--------------------
type int = int32

Full name: Microsoft.FSharp.Core.int

--------------------
type int<'Measure> = int

Full name: Microsoft.FSharp.Core.int<_>
val c : int
val private isUnicodeIdentifierStart : c:char -> bool

Full name: IronJS.Compiler.Lexer.Char.isUnicodeIdentifierStart
Char.GetUnicodeCategory(c: char) : Globalization.UnicodeCategory
Char.GetUnicodeCategory(s: string, index: int) : Globalization.UnicodeCategory
field Globalization.UnicodeCategory.UppercaseLetter = 0
field Globalization.UnicodeCategory.LowercaseLetter = 1
field Globalization.UnicodeCategory.TitlecaseLetter = 2
field Globalization.UnicodeCategory.ModifierLetter = 3
field Globalization.UnicodeCategory.OtherLetter = 4
field Globalization.UnicodeCategory.LetterNumber = 9
val private isIdentifierStart : c:char -> bool

Full name: IronJS.Compiler.Lexer.Char.isIdentifierStart
val private isUnicodeIdentifier : c:char -> bool

Full name: IronJS.Compiler.Lexer.Char.isUnicodeIdentifier
field Globalization.UnicodeCategory.NonSpacingMark = 5
field Globalization.UnicodeCategory.SpacingCombiningMark = 6
field Globalization.UnicodeCategory.DecimalDigitNumber = 8
field Globalization.UnicodeCategory.ConnectorPunctuation = 18
val private isIdentifier : c:char -> bool

Full name: IronJS.Compiler.Lexer.Char.isIdentifier
val private isPunctuation_Simple : c:char -> bool

Full name: IronJS.Compiler.Lexer.Char.isPunctuation_Simple
val private isPunctuation : c:char -> bool

Full name: IronJS.Compiler.Lexer.Char.isPunctuation
type Symbol =
  | Break = 0
  | Case = 1
  | Catch = 2
  | Continue = 3
  | Default = 4
  | Delete = 5
  | Do = 6
  | Else = 7
  | Finally = 9
  | Function = 10
  | If = 11
  | In = 12
  | InstanceOf = 13
  | New = 14
  | Return = 16
  | Switch = 17
  | This = 18
  | Throw = 19
  | Try = 20
  | TypeOf = 22
  | Var = 23
  | Void = 24
  | While = 25
  | With = 26
  | For = 84
  | LeftBrace = 27
  | RightBrace = 28
  | LeftParenthesis = 29
  | RightParenthesis = 30
  | LeftBracket = 31
  | RightBracket = 32
  | Semicolon = 33
  | Comma = 34
  | Equal = 35
  | NotEqual = 36
  | StrictEqual = 37
  | StrictNotEqual = 38
  | LessThan = 39
  | GreaterThan = 40
  | LessThanOrEqual = 41
  | GreaterThanOrEqual = 42
  | Plus = 43
  | Minus = 44
  | Multiply = 45
  | Divide = 46
  | Modulo = 47
  | Increment = 48
  | Decrement = 49
  | LeftShift = 50
  | RightShift = 51
  | URightShift = 52
  | BitwiseAnd = 53
  | BitwiseOr = 54
  | BitwiseXor = 55
  | BitwiseNot = 56
  | LogicalNot = 57
  | LogicalAnd = 58
  | LogicalOr = 59
  | Condition = 60
  | Colon = 61
  | Assign = 62
  | AssignAdd = 63
  | AssignSubtract = 64
  | AssignMultiply = 65
  | AssignDivide = 66
  | AssignModulo = 67
  | AssignLeftShift = 68
  | AssignSignedRightShift = 69
  | AssignUnsignedRightShift = 70
  | AssignBitwiseAnd = 71
  | AssignBitwiseOr = 72
  | AssignBitwiseXor = 73
  | Dot = 74
  | True = 75
  | False = 76
  | Null = 77
  | String = 78
  | Number = 79
  | RegExp = 80
  | LineTerminator = 81
  | Identifier = 82
  | Comment = 83
  | StartOfInput = 100
  | EndOfInput = 101

Full name: IronJS.Compiler.Lexer.Symbol
Symbol.Break: Symbol = 0
Symbol.Case: Symbol = 1
Symbol.Catch: Symbol = 2
Symbol.Continue: Symbol = 3
Symbol.Default: Symbol = 4
Symbol.Delete: Symbol = 5
Symbol.Do: Symbol = 6
Symbol.Else: Symbol = 7
Symbol.Finally: Symbol = 9
Symbol.Function: Symbol = 10
Symbol.If: Symbol = 11
Symbol.In: Symbol = 12
Symbol.InstanceOf: Symbol = 13
Symbol.New: Symbol = 14
Symbol.Return: Symbol = 16
Symbol.Switch: Symbol = 17
Symbol.This: Symbol = 18
Symbol.Throw: Symbol = 19
Symbol.Try: Symbol = 20
Symbol.TypeOf: Symbol = 22
Symbol.Var: Symbol = 23
Multiple items
Symbol.Void: Symbol = 24

--------------------
type Void =

Full name: System.Void
Symbol.While: Symbol = 25
Symbol.With: Symbol = 26
Symbol.For: Symbol = 84
Symbol.LeftBrace: Symbol = 27
Symbol.RightBrace: Symbol = 28
Symbol.LeftParenthesis: Symbol = 29
Symbol.RightParenthesis: Symbol = 30
Symbol.LeftBracket: Symbol = 31
Symbol.RightBracket: Symbol = 32
Symbol.Semicolon: Symbol = 33
Symbol.Comma: Symbol = 34
Symbol.Equal: Symbol = 35
Symbol.NotEqual: Symbol = 36
Symbol.StrictEqual: Symbol = 37
Symbol.StrictNotEqual: Symbol = 38
Symbol.LessThan: Symbol = 39
Symbol.GreaterThan: Symbol = 40
Symbol.LessThanOrEqual: Symbol = 41
Symbol.GreaterThanOrEqual: Symbol = 42
Symbol.Plus: Symbol = 43
Symbol.Minus: Symbol = 44
Symbol.Multiply: Symbol = 45
Symbol.Divide: Symbol = 46
Symbol.Modulo: Symbol = 47
Symbol.Increment: Symbol = 48
Symbol.Decrement: Symbol = 49
Symbol.LeftShift: Symbol = 50
Symbol.RightShift: Symbol = 51
Symbol.URightShift: Symbol = 52
Symbol.BitwiseAnd: Symbol = 53
Symbol.BitwiseOr: Symbol = 54
Symbol.BitwiseXor: Symbol = 55
Symbol.BitwiseNot: Symbol = 56
Symbol.LogicalNot: Symbol = 57
Symbol.LogicalAnd: Symbol = 58
Symbol.LogicalOr: Symbol = 59
Symbol.Condition: Symbol = 60
Symbol.Colon: Symbol = 61
Symbol.Assign: Symbol = 62
Symbol.AssignAdd: Symbol = 63
Symbol.AssignSubtract: Symbol = 64
Symbol.AssignMultiply: Symbol = 65
Symbol.AssignDivide: Symbol = 66
Symbol.AssignModulo: Symbol = 67
Symbol.AssignLeftShift: Symbol = 68
Symbol.AssignSignedRightShift: Symbol = 69
Symbol.AssignUnsignedRightShift: Symbol = 70
Symbol.AssignBitwiseAnd: Symbol = 71
Symbol.AssignBitwiseOr: Symbol = 72
Symbol.AssignBitwiseXor: Symbol = 73
Symbol.Dot: Symbol = 74
Symbol.True: Symbol = 75
Symbol.False: Symbol = 76
Symbol.Null: Symbol = 77
Multiple items
Symbol.String: Symbol = 78

--------------------
type String =
  new : value:char -> string + 7 overloads
  member Chars : int -> char
  member Clone : unit -> obj
  member CompareTo : value:obj -> int + 1 overload
  member Contains : value:string -> bool
  member CopyTo : sourceIndex:int * destination:char[] * destinationIndex:int * count:int -> unit
  member EndsWith : value:string -> bool + 2 overloads
  member Equals : obj:obj -> bool + 2 overloads
  member GetEnumerator : unit -> CharEnumerator
  member GetHashCode : unit -> int
  ...

Full name: System.String

--------------------
String(value: nativeptr<char>) : unit
String(value: nativeptr<sbyte>) : unit
String(value: char []) : unit
String(c: char, count: int) : unit
String(value: nativeptr<char>, startIndex: int, length: int) : unit
String(value: nativeptr<sbyte>, startIndex: int, length: int) : unit
String(value: char [], startIndex: int, length: int) : unit
String(value: nativeptr<sbyte>, startIndex: int, length: int, enc: Text.Encoding) : unit
Symbol.Number: Symbol = 79
Symbol.RegExp: Symbol = 80
Symbol.LineTerminator: Symbol = 81
Symbol.Identifier: Symbol = 82
Symbol.Comment: Symbol = 83
Symbol.StartOfInput: Symbol = 100
Symbol.EndOfInput: Symbol = 101
type Token = Symbol * string * int * int

Full name: IronJS.Compiler.Lexer.Token
Multiple items
val string : value:'T -> string

Full name: Microsoft.FSharp.Core.Operators.string

--------------------
type string = String

Full name: Microsoft.FSharp.Core.string
Multiple items
type NoComparisonAttribute =
  inherit Attribute
  new : unit -> NoComparisonAttribute

Full name: Microsoft.FSharp.Core.NoComparisonAttribute

--------------------
new : unit -> NoComparisonAttribute
type private T =
  new : source:string -> T
  val mutable Source: string
  val mutable Index: int
  val mutable Line: int
  val mutable Column: int
  val mutable Buffer: StringBuilder
  val mutable Previous: Symbol

Full name: IronJS.Compiler.Lexer.Input.T
T.Source: string
T.Index: int
T.Line: int
T.Column: int
Multiple items
T.Buffer: Text.StringBuilder

--------------------
type Buffer =
  static member BlockCopy : src:Array * srcOffset:int * dst:Array * dstOffset:int * count:int -> unit
  static member ByteLength : array:Array -> int
  static member GetByte : array:Array * index:int -> byte
  static member SetByte : array:Array * index:int * value:byte -> unit

Full name: System.Buffer
namespace System.Text
Multiple items
type StringBuilder =
  new : unit -> StringBuilder + 5 overloads
  member Append : value:string -> StringBuilder + 18 overloads
  member AppendFormat : format:string * arg0:obj -> StringBuilder + 4 overloads
  member AppendLine : unit -> StringBuilder + 1 overload
  member Capacity : int with get, set
  member Chars : int -> char with get, set
  member Clear : unit -> StringBuilder
  member CopyTo : sourceIndex:int * destination:char[] * destinationIndex:int * count:int -> unit
  member EnsureCapacity : capacity:int -> int
  member Equals : sb:StringBuilder -> bool
  ...

Full name: System.Text.StringBuilder

--------------------
Text.StringBuilder() : unit
Text.StringBuilder(capacity: int) : unit
Text.StringBuilder(value: string) : unit
Text.StringBuilder(value: string, capacity: int) : unit
Text.StringBuilder(capacity: int, maxCapacity: int) : unit
Text.StringBuilder(value: string, startIndex: int, length: int, capacity: int) : unit
T.Previous: Symbol
val source : string
type Buffer =
  static member BlockCopy : src:Array * srcOffset:int * dst:Array * dstOffset:int * count:int -> unit
  static member ByteLength : array:Array -> int
  static member GetByte : array:Array * index:int -> byte
  static member SetByte : array:Array * index:int * value:byte -> unit

Full name: System.Buffer
val private create : input:string -> T

Full name: IronJS.Compiler.Lexer.Input.create
val input : string
private new : source:string -> T
val private newline : t:T -> unit

Full name: IronJS.Compiler.Lexer.Input.newline
val t : T
val private current : t:T -> char

Full name: IronJS.Compiler.Lexer.Input.current
val private previous : t:T -> char

Full name: IronJS.Compiler.Lexer.Input.previous
val private peek : t:T -> char

Full name: IronJS.Compiler.Lexer.Input.peek
val private canPeek : t:T -> bool

Full name: IronJS.Compiler.Lexer.Input.canPeek
property String.Length: int
val private continue' : t:T -> bool

Full name: IronJS.Compiler.Lexer.Input.continue'
val private position : t:T -> int * int

Full name: IronJS.Compiler.Lexer.Input.position
val private rewind : t:T -> unit

Full name: IronJS.Compiler.Lexer.Input.rewind
val private advance : t:T -> unit

Full name: IronJS.Compiler.Lexer.Input.advance
val private skip : n:int -> t:T -> unit

Full name: IronJS.Compiler.Lexer.Input.skip
val n : int
val private buffer : t:T -> c:Char -> unit

Full name: IronJS.Compiler.Lexer.Input.buffer
T.Buffer: Text.StringBuilder
Text.StringBuilder.Append(value: char []) : Text.StringBuilder
   (+0 other overloads)
Text.StringBuilder.Append(value: obj) : Text.StringBuilder
   (+0 other overloads)
Text.StringBuilder.Append(value: uint64) : Text.StringBuilder
   (+0 other overloads)
Text.StringBuilder.Append(value: uint32) : Text.StringBuilder
   (+0 other overloads)
Text.StringBuilder.Append(value: uint16) : Text.StringBuilder
   (+0 other overloads)
Text.StringBuilder.Append(value: decimal) : Text.StringBuilder
   (+0 other overloads)
Text.StringBuilder.Append(value: float) : Text.StringBuilder
   (+0 other overloads)
Text.StringBuilder.Append(value: float32) : Text.StringBuilder
   (+0 other overloads)
Text.StringBuilder.Append(value: int64) : Text.StringBuilder
   (+0 other overloads)
Text.StringBuilder.Append(value: int) : Text.StringBuilder
   (+0 other overloads)
val ignore : value:'T -> unit

Full name: Microsoft.FSharp.Core.Operators.ignore
val private clearBuffer : t:T -> unit

Full name: IronJS.Compiler.Lexer.Input.clearBuffer
Text.StringBuilder.Clear() : Text.StringBuilder
val private bufferValue : t:T -> string

Full name: IronJS.Compiler.Lexer.Input.bufferValue
Text.StringBuilder.ToString() : string
Text.StringBuilder.ToString(startIndex: int, length: int) : string
val private nextLine : t:T -> unit

Full name: IronJS.Compiler.Lexer.Input.nextLine
val private setupBuffer : t:T -> int * int

Full name: IronJS.Compiler.Lexer.Input.setupBuffer
val private output : symbol:Symbol -> value:'a -> line:'b -> col:'c -> t:T -> Symbol * 'a * 'b * 'c

Full name: IronJS.Compiler.Lexer.Input.output
val symbol : Symbol
val value : 'a
val line : 'b
val col : 'c
val private endOfInput : t:T -> Symbol * 'a * int * int (requires 'a : null)

Full name: IronJS.Compiler.Lexer.Input.endOfInput
module Input

from IronJS.Compiler.Lexer
namespace System.Collections
namespace System.Collections.Generic
Multiple items
type LiteralAttribute =
  inherit Attribute
  new : unit -> LiteralAttribute

Full name: Microsoft.FSharp.Core.LiteralAttribute

--------------------
new : unit -> LiteralAttribute
val private empty : string

Full name: IronJS.Compiler.Lexer.empty
val private keywordMap : Map<string,Symbol>

Full name: IronJS.Compiler.Lexer.keywordMap
Symbol.Void: Symbol = 24
Multiple items
module Map

from Microsoft.FSharp.Collections

--------------------
type Map<'Key,'Value (requires comparison)> =
  interface IEnumerable
  interface IComparable
  interface IEnumerable<KeyValuePair<'Key,'Value>>
  interface ICollection<KeyValuePair<'Key,'Value>>
  interface IDictionary<'Key,'Value>
  new : elements:seq<'Key * 'Value> -> Map<'Key,'Value>
  member Add : key:'Key * value:'Value -> Map<'Key,'Value>
  member ContainsKey : key:'Key -> bool
  override Equals : obj -> bool
  member Remove : key:'Key -> Map<'Key,'Value>
  ...

Full name: Microsoft.FSharp.Collections.Map<_,_>

--------------------
new : elements:seq<'Key * 'Value> -> Map<'Key,'Value>
val ofList : elements:('Key * 'T) list -> Map<'Key,'T> (requires comparison)

Full name: Microsoft.FSharp.Collections.Map.ofList
val private punctuationMap : Dictionary<string,Symbol>

Full name: IronJS.Compiler.Lexer.punctuationMap
Multiple items
type Dictionary<'TKey,'TValue> =
  new : unit -> Dictionary<'TKey, 'TValue> + 5 overloads
  member Add : key:'TKey * value:'TValue -> unit
  member Clear : unit -> unit
  member Comparer : IEqualityComparer<'TKey>
  member ContainsKey : key:'TKey -> bool
  member ContainsValue : value:'TValue -> bool
  member Count : int
  member GetEnumerator : unit -> Enumerator<'TKey, 'TValue>
  member GetObjectData : info:SerializationInfo * context:StreamingContext -> unit
  member Item : 'TKey -> 'TValue with get, set
  ...
  nested type Enumerator
  nested type KeyCollection
  nested type ValueCollection

Full name: System.Collections.Generic.Dictionary<_,_>

--------------------
Dictionary() : unit
Dictionary(capacity: int) : unit
Dictionary(comparer: IEqualityComparer<'TKey>) : unit
Dictionary(dictionary: IDictionary<'TKey,'TValue>) : unit
Dictionary(capacity: int, comparer: IEqualityComparer<'TKey>) : unit
Dictionary(dictionary: IDictionary<'TKey,'TValue>, comparer: IEqualityComparer<'TKey>) : unit
val private simplePunctuation : s:T -> c:char -> Symbol * string * int * int

Full name: IronJS.Compiler.Lexer.simplePunctuation
val s : T
val failwithf : format:Printf.StringFormat<'T,'Result> -> 'T

Full name: Microsoft.FSharp.Core.ExtraTopLevelOperators.failwithf
val private identifier : s:T -> first:char -> Symbol * string * int * int

Full name: IronJS.Compiler.Lexer.identifier
val first : char
Multiple items
val char : value:'T -> char (requires member op_Explicit)

Full name: Microsoft.FSharp.Core.Operators.char

--------------------
type char = Char

Full name: Microsoft.FSharp.Core.char
val line : int
val column : int
val identifier : string
member Map.TryFind : key:'Key -> 'Value option
union case Option.None: Option<'T>
union case Option.Some: Value: 'T -> Option<'T>
val keyword : Symbol
val private singleLineComment : s:T -> Symbol * string * int * int

Full name: IronJS.Compiler.Lexer.singleLineComment
val not : value:bool -> bool

Full name: Microsoft.FSharp.Core.Operators.not
val private multiLineComment : s:T -> Symbol * string * int * int

Full name: IronJS.Compiler.Lexer.multiLineComment
val private punctuation : s:T -> first:char -> Symbol * string * int * int

Full name: IronJS.Compiler.Lexer.punctuation
val makeToken : (T -> string -> Symbol * string * int * int)
val buffer : string
val punctuation : (T -> string -> Symbol * string * int * int)
val newBuffer : string
Dictionary.ContainsKey(key: string) : bool
val private literalString : s:T -> stop:char -> Symbol * string * int * int

Full name: IronJS.Compiler.Lexer.literalString
val stop : char
val literalString : (T -> Symbol * string * int * int)
val failwith : message:string -> 'T

Full name: Microsoft.FSharp.Core.Operators.failwith
Symbol.String: Symbol = 78
val create : source:string -> (unit -> Symbol * string * int * int)

Full name: IronJS.Compiler.Lexer.create
val buffer : Text.StringBuilder
val lexer : (unit -> Symbol * string * int * int)
Next Version Raw view Test code New version

More information

Link:http://fssnip.net/3w
Posted:13 years ago
Author:fholm
Tags: lexer , javascript