3 people like it.

Log file parser for log4Net files

Log file parser for log4Net files

  1: 
  2: 
  3: 
  4: 
  5: 
  6: 
  7: 
  8: 
  9: 
 10: 
 11: 
 12: 
 13: 
 14: 
 15: 
 16: 
 17: 
 18: 
 19: 
 20: 
 21: 
 22: 
 23: 
 24: 
 25: 
 26: 
 27: 
 28: 
 29: 
 30: 
 31: 
 32: 
 33: 
 34: 
 35: 
 36: 
 37: 
 38: 
 39: 
 40: 
 41: 
 42: 
 43: 
 44: 
 45: 
 46: 
 47: 
 48: 
 49: 
 50: 
 51: 
 52: 
 53: 
 54: 
 55: 
 56: 
 57: 
 58: 
 59: 
 60: 
 61: 
 62: 
 63: 
 64: 
 65: 
 66: 
 67: 
 68: 
 69: 
 70: 
 71: 
 72: 
 73: 
 74: 
 75: 
 76: 
 77: 
 78: 
 79: 
 80: 
 81: 
 82: 
 83: 
 84: 
 85: 
 86: 
 87: 
 88: 
 89: 
 90: 
 91: 
 92: 
 93: 
 94: 
 95: 
 96: 
 97: 
 98: 
 99: 
100: 
101: 
102: 
103: 
104: 
105: 
106: 
107: 
108: 
109: 
110: 
111: 
112: 
113: 
114: 
115: 
116: 
117: 
118: 
119: 
120: 
121: 
122: 
123: 
124: 
125: 
126: 
127: 
128: 
129: 
130: 
131: 
132: 
133: 

(* Sample log:

2013-10-26 17:11:49$$INFO$$some text$$more text$$etc$$
2013-10-26 17:12:40$$INFO$$some text$$more text$$etc$$
2013-10-26 17:20:39$$INFO$$some text$$more text$$etc$$
Some-unimportant-info...
2013-10-26 17:20:50$$INFO$$some text$$more text$$etc$$
2013-10-27 14:21:24$$ERROR$$some text$$more text$$Operation failed: $$
System.InvalidOperationException: My error
   at MyCompany.MyClass.Method(String id)
   at System.ServiceModel.Dispatcher.ImmutableDispatchRuntime.ProcessMessage2(MessageRpc& rpc)
   at System.ServiceModel.Dispatcher.MessageRpc.Process(Boolean isOperationContextSet)
2013-10-27 14:21:47$$INFO$$some text$$more text$$etc$$
2013-10-27 14:22:20$$ERROR$$some text$$more text$$Operation failed: $$
System.InvalidOperationException: My error
   at MyCompany.MyClass.Method(String id)
   at System.ServiceModel.Dispatcher.ImmutableDispatchRuntime.ProcessMessage2(MessageRpc& rpc)
   at System.ServiceModel.Dispatcher.MessageRpc.Process(Boolean isOperationContextSet)
2013-10-27 14:22:45$$ERROR$$some text$$more text$$Timeout expired: $$
System.Data.SqlClient.SqlException: Timeout expired.  The timeout period...
   --- End of inner exception stack trace ---
   at System.Data.SqlClient.SqlConnection.OnError(SqlException exception, Boolean breakConnection, Action wrapCloseInAction)
   at MyCompany.MyClass2.OtherMethod(String param)

*)

#if INTERACTIVE
#else
module LogReader
#endif
//Log file parser for log4Net files.

open System
open System.IO

type CollectData =
| NotIntresting
| SeekStackTraceLine of string
| SeekCompanyStackTraceLine of string * string

///Log4Net item separator
[<Literal>] 
let separator = "$$"

///Beginig for a normal line. e.g. date like 2013-10-27
[<Literal>] 
let normalLineBegin = "201"

///Company namespace in the stacktrace
[<Literal>] 
let companyStackTraceSign = "MyCompany"

///Detect error line in the log: contains this string
let errorsign = separator + "ERROR" + separator

///This will parse the lines to resultset of errors
let collectErrorInfo lines =
    let rec readLines (myLines:string list) (mode:CollectData) resultdata =

        // Add current findings to collection, just in case
        let collectedResult = 
            match mode with
            | NotIntresting -> resultdata 
            | SeekStackTraceLine(a) -> 
                let info = "No-StackTrace", "", a
                (info::resultdata)
            | SeekCompanyStackTraceLine(a, b) ->
                let info = "Non-"+companyStackTraceSign+"-StackTrace", a, b
                (info::resultdata)
                
        match myLines, mode with
        //New error-line: try to seek stacktrace, store current findings
        | h::t, _ when h.StartsWith(normalLineBegin) && h.Contains(errorsign) -> 
            readLines t (SeekStackTraceLine(h)) collectedResult
        //New info-line: dont seek stacktrace, store current findings
        | h::t, _ when h.StartsWith(normalLineBegin) && not (h.Contains(errorsign)) -> 
            readLines t NotIntresting collectedResult
                
        //New stacktrace-line: try to seek Company-stack, dont store yet
        | h::t, SeekStackTraceLine(l) when not (h.StartsWith(normalLineBegin)) ->
            readLines t (SeekCompanyStackTraceLine(h, l)) resultdata

        //New Company-stack-line: everything ok, lets store the result and continue
        | h::t, SeekCompanyStackTraceLine(a, b) when not (h.StartsWith(normalLineBegin)) && h.Contains(companyStackTraceSign) ->
            readLines t NotIntresting collectedResult

        //All the other cases: continue to next line
        | h::t, _  -> readLines t mode resultdata

        //End of file: return all with the last one
        | [], _ -> collectedResult

    readLines (lines |> Seq.toList) NotIntresting []

///Function to break-down lines by separator.
///e.g. for csv-import: by default Excel doesn't support multiple character separators
let breakLineDetails mySeq = 
    let breakLineDetail (a,b,c:string) = 
        a,b,c.Split([|separator|],StringSplitOptions.None)
    mySeq |> Seq.map breakLineDetail

let filesByPath path = Directory.EnumerateFiles(path,"*.log*",SearchOption.AllDirectories)

open System.Linq

/// This will read the file, parse it and give results.
let processFiles files =
    let result = 
        files
        |> Seq.map (fun (fileOrPath:string) -> 
            match fileOrPath with
            | path when path.EndsWith(@"\") || path.EndsWith(@"/") -> 
                    filesByPath path |> Seq.map File.ReadLines |> Seq.concat
            | file -> File.ReadLines file)
        |> Seq.concat
        |> collectErrorInfo
        |> breakLineDetails
    result.GroupBy(fun (a,b,c) -> a + " " + b).OrderByDescending(fun k -> k.Count())

///For console program, we can use this
// For good UI we would input fileName and output results
// (results is a grouped list of errors by type)
[<EntryPoint>]
let main argv = 
    let filtered = argv |> Array.filter (fun i -> not (i = ""))
    match filtered with
    | [||] -> printfn "Please input filename as argument."
    | fileNames -> 
        //let fileNames = [|@"C:\...\MyProgram.log"|]
        let results = processFiles fileNames
        for i in results
            do Console.WriteLine("Count: " + i.Count().ToString() + ", Item: " + i.Key)
    0 // return an integer exit code
namespace System
namespace System.IO
type CollectData =
  | NotIntresting
  | SeekStackTraceLine of string
  | SeekCompanyStackTraceLine of string * string

Full name: Script.CollectData
union case CollectData.NotIntresting: CollectData
union case CollectData.SeekStackTraceLine: string -> CollectData
Multiple items
val string : value:'T -> string

Full name: Microsoft.FSharp.Core.Operators.string

--------------------
type string = String

Full name: Microsoft.FSharp.Core.string
union case CollectData.SeekCompanyStackTraceLine: string * string -> CollectData
Multiple items
type LiteralAttribute =
  inherit Attribute
  new : unit -> LiteralAttribute

Full name: Microsoft.FSharp.Core.LiteralAttribute

--------------------
new : unit -> LiteralAttribute
val separator : string

Full name: Script.separator


Log4Net item separator
val normalLineBegin : string

Full name: Script.normalLineBegin


Beginig for a normal line. e.g. date like 2013-10-27
val companyStackTraceSign : string

Full name: Script.companyStackTraceSign


Company namespace in the stacktrace
val errorsign : string

Full name: Script.errorsign


Detect error line in the log: contains this string
val collectErrorInfo : lines:seq<string> -> (string * string * string) list

Full name: Script.collectErrorInfo


This will parse the lines to resultset of errors
val lines : seq<string>
val readLines : (string list -> CollectData -> (string * string * string) list -> (string * string * string) list)
val myLines : string list
type 'T list = List<'T>

Full name: Microsoft.FSharp.Collections.list<_>
val mode : CollectData
val resultdata : (string * string * string) list
val collectedResult : (string * string * string) list
val a : string
val info : string * string * string
val b : string
val h : string
val t : string list
String.StartsWith(value: string) : bool
String.StartsWith(value: string, comparisonType: StringComparison) : bool
String.StartsWith(value: string, ignoreCase: bool, culture: Globalization.CultureInfo) : bool
String.Contains(value: string) : bool
val not : value:bool -> bool

Full name: Microsoft.FSharp.Core.Operators.not
val l : string
module Seq

from Microsoft.FSharp.Collections
val toList : source:seq<'T> -> 'T list

Full name: Microsoft.FSharp.Collections.Seq.toList
val breakLineDetails : mySeq:seq<'a * 'b * string> -> seq<'a * 'b * string []>

Full name: Script.breakLineDetails


Function to break-down lines by separator.
e.g. for csv-import: by default Excel doesn't support multiple character separators
val mySeq : seq<'a * 'b * string>
val breakLineDetail : ('c * 'd * string -> 'c * 'd * string [])
val a : 'c
val b : 'd
val c : string
String.Split([<ParamArray>] separator: char []) : string []
String.Split(separator: string [], options: StringSplitOptions) : string []
String.Split(separator: char [], options: StringSplitOptions) : string []
String.Split(separator: char [], count: int) : string []
String.Split(separator: string [], count: int, options: StringSplitOptions) : string []
String.Split(separator: char [], count: int, options: StringSplitOptions) : string []
type StringSplitOptions =
  | None = 0
  | RemoveEmptyEntries = 1

Full name: System.StringSplitOptions
field StringSplitOptions.None = 0
val map : mapping:('T -> 'U) -> source:seq<'T> -> seq<'U>

Full name: Microsoft.FSharp.Collections.Seq.map
val filesByPath : path:string -> Collections.Generic.IEnumerable<string>

Full name: Script.filesByPath
val path : string
type Directory =
  static member CreateDirectory : path:string -> DirectoryInfo + 1 overload
  static member Delete : path:string -> unit + 1 overload
  static member EnumerateDirectories : path:string -> IEnumerable<string> + 2 overloads
  static member EnumerateFileSystemEntries : path:string -> IEnumerable<string> + 2 overloads
  static member EnumerateFiles : path:string -> IEnumerable<string> + 2 overloads
  static member Exists : path:string -> bool
  static member GetAccessControl : path:string -> DirectorySecurity + 1 overload
  static member GetCreationTime : path:string -> DateTime
  static member GetCreationTimeUtc : path:string -> DateTime
  static member GetCurrentDirectory : unit -> string
  ...

Full name: System.IO.Directory
Directory.EnumerateFiles(path: string) : Collections.Generic.IEnumerable<string>
Directory.EnumerateFiles(path: string, searchPattern: string) : Collections.Generic.IEnumerable<string>
Directory.EnumerateFiles(path: string, searchPattern: string, searchOption: SearchOption) : Collections.Generic.IEnumerable<string>
type SearchOption =
  | TopDirectoryOnly = 0
  | AllDirectories = 1

Full name: System.IO.SearchOption
field SearchOption.AllDirectories = 1
namespace System.Linq
val processFiles : files:seq<string> -> IOrderedEnumerable<IGrouping<string,(string * string * string [])>>

Full name: Script.processFiles


 This will read the file, parse it and give results.
val files : seq<string>
val result : seq<string * string * string []>
val fileOrPath : string
String.EndsWith(value: string) : bool
String.EndsWith(value: string, comparisonType: StringComparison) : bool
String.EndsWith(value: string, ignoreCase: bool, culture: Globalization.CultureInfo) : bool
type File =
  static member AppendAllLines : path:string * contents:IEnumerable<string> -> unit + 1 overload
  static member AppendAllText : path:string * contents:string -> unit + 1 overload
  static member AppendText : path:string -> StreamWriter
  static member Copy : sourceFileName:string * destFileName:string -> unit + 1 overload
  static member Create : path:string -> FileStream + 3 overloads
  static member CreateText : path:string -> StreamWriter
  static member Decrypt : path:string -> unit
  static member Delete : path:string -> unit
  static member Encrypt : path:string -> unit
  static member Exists : path:string -> bool
  ...

Full name: System.IO.File
File.ReadLines(path: string) : Collections.Generic.IEnumerable<string>
File.ReadLines(path: string, encoding: Text.Encoding) : Collections.Generic.IEnumerable<string>
val concat : sources:seq<#seq<'T>> -> seq<'T>

Full name: Microsoft.FSharp.Collections.Seq.concat
val file : string
(extension) Collections.Generic.IEnumerable.GroupBy<'TSource,'TKey>(keySelector: Func<'TSource,'TKey>) : Collections.Generic.IEnumerable<IGrouping<'TKey,'TSource>>
(extension) Collections.Generic.IEnumerable.GroupBy<'TSource,'TKey>(keySelector: Func<'TSource,'TKey>, comparer: Collections.Generic.IEqualityComparer<'TKey>) : Collections.Generic.IEnumerable<IGrouping<'TKey,'TSource>>
(extension) Collections.Generic.IEnumerable.GroupBy<'TSource,'TKey,'TElement>(keySelector: Func<'TSource,'TKey>, elementSelector: Func<'TSource,'TElement>) : Collections.Generic.IEnumerable<IGrouping<'TKey,'TElement>>
(extension) Collections.Generic.IEnumerable.GroupBy<'TSource,'TKey,'TResult>(keySelector: Func<'TSource,'TKey>, resultSelector: Func<'TKey,Collections.Generic.IEnumerable<'TSource>,'TResult>) : Collections.Generic.IEnumerable<'TResult>
(extension) Collections.Generic.IEnumerable.GroupBy<'TSource,'TKey,'TElement>(keySelector: Func<'TSource,'TKey>, elementSelector: Func<'TSource,'TElement>, comparer: Collections.Generic.IEqualityComparer<'TKey>) : Collections.Generic.IEnumerable<IGrouping<'TKey,'TElement>>
(extension) Collections.Generic.IEnumerable.GroupBy<'TSource,'TKey,'TElement,'TResult>(keySelector: Func<'TSource,'TKey>, elementSelector: Func<'TSource,'TElement>, resultSelector: Func<'TKey,Collections.Generic.IEnumerable<'TElement>,'TResult>) : Collections.Generic.IEnumerable<'TResult>
(extension) Collections.Generic.IEnumerable.GroupBy<'TSource,'TKey,'TResult>(keySelector: Func<'TSource,'TKey>, resultSelector: Func<'TKey,Collections.Generic.IEnumerable<'TSource>,'TResult>, comparer: Collections.Generic.IEqualityComparer<'TKey>) : Collections.Generic.IEnumerable<'TResult>
(extension) Collections.Generic.IEnumerable.GroupBy<'TSource,'TKey,'TElement,'TResult>(keySelector: Func<'TSource,'TKey>, elementSelector: Func<'TSource,'TElement>, resultSelector: Func<'TKey,Collections.Generic.IEnumerable<'TElement>,'TResult>, comparer: Collections.Generic.IEqualityComparer<'TKey>) : Collections.Generic.IEnumerable<'TResult>
val c : string []
val k : IGrouping<string,(string * string * string [])>
(extension) Collections.Generic.IEnumerable.Count<'TSource>() : int
(extension) Collections.Generic.IEnumerable.Count<'TSource>(predicate: Func<'TSource,bool>) : int
Multiple items
type EntryPointAttribute =
  inherit Attribute
  new : unit -> EntryPointAttribute

Full name: Microsoft.FSharp.Core.EntryPointAttribute

--------------------
new : unit -> EntryPointAttribute
val main : argv:string [] -> int

Full name: Script.main


For console program, we can use this
val argv : string []
val filtered : string []
type Array =
  member Clone : unit -> obj
  member CopyTo : array:Array * index:int -> unit + 1 overload
  member GetEnumerator : unit -> IEnumerator
  member GetLength : dimension:int -> int
  member GetLongLength : dimension:int -> int64
  member GetLowerBound : dimension:int -> int
  member GetUpperBound : dimension:int -> int
  member GetValue : [<ParamArray>] indices:int[] -> obj + 7 overloads
  member Initialize : unit -> unit
  member IsFixedSize : bool
  ...

Full name: System.Array
val filter : predicate:('T -> bool) -> array:'T [] -> 'T []

Full name: Microsoft.FSharp.Collections.Array.filter
val i : string
val printfn : format:Printf.TextWriterFormat<'T> -> 'T

Full name: Microsoft.FSharp.Core.ExtraTopLevelOperators.printfn
val fileNames : string []
val results : IOrderedEnumerable<IGrouping<string,(string * string * string [])>>
val i : IGrouping<string,(string * string * string [])>
type Console =
  static member BackgroundColor : ConsoleColor with get, set
  static member Beep : unit -> unit + 1 overload
  static member BufferHeight : int with get, set
  static member BufferWidth : int with get, set
  static member CapsLock : bool
  static member Clear : unit -> unit
  static member CursorLeft : int with get, set
  static member CursorSize : int with get, set
  static member CursorTop : int with get, set
  static member CursorVisible : bool with get, set
  ...

Full name: System.Console
Console.WriteLine() : unit
   (+0 other overloads)
Console.WriteLine(value: string) : unit
   (+0 other overloads)
Console.WriteLine(value: obj) : unit
   (+0 other overloads)
Console.WriteLine(value: uint64) : unit
   (+0 other overloads)
Console.WriteLine(value: int64) : unit
   (+0 other overloads)
Console.WriteLine(value: uint32) : unit
   (+0 other overloads)
Console.WriteLine(value: int) : unit
   (+0 other overloads)
Console.WriteLine(value: float32) : unit
   (+0 other overloads)
Console.WriteLine(value: float) : unit
   (+0 other overloads)
Console.WriteLine(value: decimal) : unit
   (+0 other overloads)
property IGrouping.Key: string

More information

Link:http://fssnip.net/kw
Posted:11 years ago
Author:Tuomas Hietanen
Tags: log , parsing

This web site is created using F# and Suave web server. It is hosted on Azure and the source code is on GitHub. Contributions are welcome!

The first version of fssnip.net has been created by @tomaspetricek back in 2010. This web site is a new, open-source and contribution-friendly version.