Remove block comments from a list of tokens

Cleaning up source code for a Forth like language before the compiling stage

 1: 
 2: 
 3: 
 4: 
 5: 
 6: 
 7: 
 8: 
 9: 
10: 
11: 
12: 
13: 
14: 
15: 
16: 
17: 
18: 
19: 
20: 
21: 
22: 
23: 
24:

// Remove block comment(s) from a list of tokens
let removeComments listIn start stop =
    let rec loop listIn listOut depth =   
        match listIn with
        | [] -> match depth with
                | 0 -> listOut
                | depth when depth > 0 -> failwith (sprintf "Too many '%s' tokens\n" stop)
                | _ -> failwith (sprintf "Too many '%s' tokens\n" start)
        | head::tail when depth > 0    -> loop [] listOut depth
        | head::tail when head = start -> loop tail listOut (depth - 1)
        | head::tail when head = stop  -> loop tail listOut (depth + 1)
        | head::tail when depth = 0    -> loop tail (head::listOut) depth
        | head::tail                   -> loop tail listOut depth
    loop listIn [] 0 |> List.rev

 // Tokenize string by splitting at whitespace and removing empty cells
let tokenize (t:string) = t.Split () |> Array.toList |> List.filter (fun x -> x <> "")

let source = ": CLSB ( n -- n ) dup 1 - and ; ( Clear least significant bit )"
printfn "Source: %A" source

let tokens = tokenize source
let noCommentTokens = removeComments tokens "(" ")"
printfn "Clean source %A" (List.fold (fun a t -> a + t + " ") " " noCommentTokens)

val removeComments : listIn:string list -> start:string -> stop:string -> string list

Full name: Script.removeComments

val listIn : string list

val start : string

val stop : string

val loop : (string list -> string list -> int -> string list)

val listOut : string list

val depth : int

val failwith : message:string -> 'T

Full name: Microsoft.FSharp.Core.Operators.failwith

val sprintf : format:Printf.StringFormat<'T> -> 'T

Full name: Microsoft.FSharp.Core.ExtraTopLevelOperators.sprintf

val head : string

val tail : string list

Multiple items
module List

from Microsoft.FSharp.Collections

--------------------
type List<'T> =
  | ( [] )
  | ( :: ) of Head: 'T * Tail: 'T list
  interface IEnumerable
  interface IEnumerable<'T>
  member Head : 'T
  member IsEmpty : bool
  member Item : index:int -> 'T with get
  member Length : int
  member Tail : 'T list
  static member Cons : head:'T * tail:'T list -> 'T list
  static member Empty : 'T list

Full name: Microsoft.FSharp.Collections.List<_>

val rev : list:'T list -> 'T list

Full name: Microsoft.FSharp.Collections.List.rev

val tokenize : t:string -> string list

Full name: Script.tokenize

val t : string

Multiple items
val string : value:'T -> string

Full name: Microsoft.FSharp.Core.Operators.string

--------------------
type string = System.String

Full name: Microsoft.FSharp.Core.string

System.String.Split([<System.ParamArray>] separator: char []) : string []
System.String.Split(separator: string [], options: System.StringSplitOptions) : string []
System.String.Split(separator: char [], options: System.StringSplitOptions) : string []
System.String.Split(separator: char [], count: int) : string []
System.String.Split(separator: string [], count: int, options: System.StringSplitOptions) : string []
System.String.Split(separator: char [], count: int, options: System.StringSplitOptions) : string []

module Array

from Microsoft.FSharp.Collections

val toList : array:'T [] -> 'T list

Full name: Microsoft.FSharp.Collections.Array.toList

val filter : predicate:('T -> bool) -> list:'T list -> 'T list

Full name: Microsoft.FSharp.Collections.List.filter

val x : string

val source : string

Full name: Script.source

val printfn : format:Printf.TextWriterFormat<'T> -> 'T

Full name: Microsoft.FSharp.Core.ExtraTopLevelOperators.printfn

val tokens : string list

Full name: Script.tokens

val noCommentTokens : string list

Full name: Script.noCommentTokens

val fold : folder:('State -> 'T -> 'State) -> state:'State -> list:'T list -> 'State

Full name: Microsoft.FSharp.Collections.List.fold

val a : string

Raw view Test code New version

More information

Link:	http://fssnip.net/ih
Posted:	10 years ago
Author:	Bjørn Bæverfjord
Tags:	forth , block comments