AdaBoost in F#

I have provided a class that implements adaptive boosting (AdaBoost) which is a very elegant technique to create a strong classifier from a set of weak classifiers. The strong classifier is the sign of the weighted average of the best N weak classifiers where N is a user specified parameter. The training process consists of N selection rounds. During each round, the training samples are given a weight distribution. The classifier selected is the one that has the least error which is equal to the weighted average of incorrectly classified samples. Training samples that have been incorrectly classified are given a larger weight for the next round. This increases the likelihood of drafting a weak classifier that can help with as yet misclassified samples.

Copy Source
Copy Link
Tools:
 1: namespace AdaBooster
 2: 
 3:     type LabelType = 
 4:         | NEG = -1
 5:         | POS = 1
 6: 
 7:     type AdaBooster(trainingSamples : (obj * LabelType) [], weakClassifiers : (obj -> LabelType) [], weakClassifierMaxCount : int) =
 8:         
 9:         let trainingSamplesCount = trainingSamples.Length
10:         let weakClassifierCount = weakClassifiers.Length
11: 
12:         let weightMatrix = Array2D.create trainingSamplesCount weakClassifierMaxCount 1.0
13: 
14:         let scoutingMatrix = Array2D.create trainingSamplesCount weakClassifierCount 0
15: 
16:         let mutable strongClassifier: (int * float) list = []
17: 
18:         let initializeScoutingMatrix = 
19:             for j in 0 .. weakClassifiers.Length - 1 do
20:                 for i in 0 .. trainingSamplesCount - 1 do
21:                     let (sample, label) = trainingSamples.[i]
22:                     scoutingMatrix.[i, j] <- 
23:                         if ( weakClassifiers.[j] sample ) <> label then 1 else 0
24: 
25:         let calculateError weakClassifierIndex weights = 
26:             weights     
27:             |> List.mapi (fun i wt -> float scoutingMatrix.[i, weakClassifierIndex] * wt)
28:             |> List.sum
29: 
30:         let selectBestWeakClassifier weights weakClassifierIndices = 
31:             weakClassifierIndices
32:             |> List.map (fun weakClassifierIndex -> (weakClassifierIndex, calculateError weakClassifierIndex weights))
33:             |> List.minBy (fun (x, y) -> y)
34: 
35:         let rec train m weights weakClassifierIndices =
36: 
37:             if m >= weakClassifierMaxCount then []
38:             else
39: 
40:                 let nextM = m + 1 
41:                 let (bestWeakClassifierIndex, classifierWeight) = selectBestWeakClassifier weights weakClassifierIndices
42:                 let error = classifierWeight / List.sum weights
43:                 let weakClassifierCoefficient = 0.5 * log ((1.0 - error) / error)
44:                 let wm1 = sqrt(( 1.0 - error) / error)
45: 
46:                 let newWeights =
47:                     weights 
48:                     |> List.mapi ( fun i wm -> if scoutingMatrix.[ i, bestWeakClassifierIndex]  = 1 then wm * wm1 else wm / wm1)
49: 
50:                 for i in 0 .. (trainingSamplesCount - 1) do 
51:                     weightMatrix.[i, m] <- weights.[i]
52: 
53:                 let weakClassifiersToScanNext = 
54:                     weakClassifierIndices 
55:                     |> List.filter (fun index -> index <> bestWeakClassifierIndex)
56: 
57:                 (bestWeakClassifierIndex, weakClassifierCoefficient) :: train nextM newWeights weakClassifiersToScanNext
58: 
59:         member this.Predict sample  = 
60:             strongClassifier 
61:             |> List.sumBy (fun (weakClassifierIndex,  weakClassifierCoefficient) ->  weakClassifierCoefficient * float ( weakClassifiers.[weakClassifierIndex] sample) )
62:             |> (fun x -> if x > 0.0 then (x, LabelType.POS) else (x,LabelType.NEG))
63: 
64:         member this.Train  = 
65:             let initialWeights = List.init trainingSamplesCount (fun x -> 1.0)
66:             
67:             strongClassifier <- train 0 initialWeights [0 .. (weakClassifierCount - 1)]
68:             strongClassifier
69: 
70:     
71:         member this.ScoutingMatrix = 
72:             scoutingMatrix
73: 
74:         member this.WeightMatrix = 
75:             weightMatrix
namespace AdaBooster
LabelType.NEG: LabelType = -1
LabelType.POS: LabelType = 1
type AdaBooster =
  class
    new : trainingSamples:(obj * LabelType) [] * weakClassifiers:(obj -> LabelType) [] * weakClassifierMaxCount:int -> AdaBooster
    member Predict : sample:obj -> float * LabelType
    member ScoutingMatrix : int [,]
    member Train : (int * float) list
    member WeightMatrix : float [,]
  end

Full name: AdaBooster.AdaBooster
val trainingSamples : (obj * LabelType) []

  type: (obj * LabelType) []
  implements: System.ICloneable
  implements: System.Collections.IList
  implements: System.Collections.ICollection
  implements: System.Collections.IStructuralComparable
  implements: System.Collections.IStructuralEquatable
  implements: System.Collections.Generic.IList<obj * LabelType>
  implements: System.Collections.Generic.ICollection<obj * LabelType>
  implements: seq<obj * LabelType>
  implements: System.Collections.IEnumerable
  inherits: System.Array
type obj = System.Object

Full name: Microsoft.FSharp.Core.obj
type LabelType =
  | NEG = -1
  | POS = 1

Full name: AdaBooster.LabelType

  type: LabelType
  inherits: System.Enum
  inherits: System.ValueType
val weakClassifiers : (obj -> LabelType) []

  type: (obj -> LabelType) []
  implements: System.ICloneable
  implements: System.Collections.IList
  implements: System.Collections.ICollection
  implements: System.Collections.IStructuralComparable
  implements: System.Collections.IStructuralEquatable
  implements: System.Collections.Generic.IList<(obj -> LabelType)>
  implements: System.Collections.Generic.ICollection<(obj -> LabelType)>
  implements: seq<(obj -> LabelType)>
  implements: System.Collections.IEnumerable
  inherits: System.Array
val weakClassifierMaxCount : int

  type: int
  implements: System.IComparable
  implements: System.IFormattable
  implements: System.IConvertible
  implements: System.IComparable<int>
  implements: System.IEquatable<int>
  inherits: System.ValueType
Multiple items
val int : 'T -> int (requires member op_Explicit)

Full name: Microsoft.FSharp.Core.Operators.int

--------------------
type int<'Measure> = int

Full name: Microsoft.FSharp.Core.int<_>

  type: int<'Measure>
  implements: System.IComparable
  implements: System.IConvertible
  implements: System.IFormattable
  implements: System.IComparable<int<'Measure>>
  implements: System.IEquatable<int<'Measure>>
  inherits: System.ValueType


--------------------
type int = int32

Full name: Microsoft.FSharp.Core.int

  type: int
  implements: System.IComparable
  implements: System.IFormattable
  implements: System.IConvertible
  implements: System.IComparable<int>
  implements: System.IEquatable<int>
  inherits: System.ValueType
val trainingSamplesCount : int

  type: int
  implements: System.IComparable
  implements: System.IFormattable
  implements: System.IConvertible
  implements: System.IComparable<int>
  implements: System.IEquatable<int>
  inherits: System.ValueType
property System.Array.Length: int
val weakClassifierCount : int

  type: int
  implements: System.IComparable
  implements: System.IFormattable
  implements: System.IConvertible
  implements: System.IComparable<int>
  implements: System.IEquatable<int>
  inherits: System.ValueType
val weightMatrix : float [,]

  type: float [,]
  implements: System.ICloneable
  implements: System.Collections.IList
  implements: System.Collections.ICollection
  implements: System.Collections.IEnumerable
  implements: System.Collections.IStructuralComparable
  implements: System.Collections.IStructuralEquatable
  inherits: System.Array
module Array2D

from Microsoft.FSharp.Collections
val create : int -> int -> 'T -> 'T [,]

Full name: Microsoft.FSharp.Collections.Array2D.create
val scoutingMatrix : int [,]

  type: int [,]
  implements: System.ICloneable
  implements: System.Collections.IList
  implements: System.Collections.ICollection
  implements: System.Collections.IEnumerable
  implements: System.Collections.IStructuralComparable
  implements: System.Collections.IStructuralEquatable
  inherits: System.Array
val mutable strongClassifier : (int * float) list

  type: (int * float) list
  implements: System.Collections.IStructuralEquatable
  implements: System.IComparable<List<int * float>>
  implements: System.IComparable
  implements: System.Collections.IStructuralComparable
  implements: System.Collections.Generic.IEnumerable<int * float>
  implements: System.Collections.IEnumerable
Multiple items
val float : 'T -> float (requires member op_Explicit)

Full name: Microsoft.FSharp.Core.Operators.float

--------------------
type float<'Measure> = float

Full name: Microsoft.FSharp.Core.float<_>

  type: float<'Measure>
  implements: System.IComparable
  implements: System.IConvertible
  implements: System.IFormattable
  implements: System.IComparable<float<'Measure>>
  implements: System.IEquatable<float<'Measure>>
  inherits: System.ValueType


--------------------
type float = System.Double

Full name: Microsoft.FSharp.Core.float

  type: float
  implements: System.IComparable
  implements: System.IFormattable
  implements: System.IConvertible
  implements: System.IComparable<float>
  implements: System.IEquatable<float>
  inherits: System.ValueType
type 'T list = List<'T>

Full name: Microsoft.FSharp.Collections.list<_>

  type: 'T list
  implements: System.Collections.IStructuralEquatable
  implements: System.IComparable<List<'T>>
  implements: System.IComparable
  implements: System.Collections.IStructuralComparable
  implements: System.Collections.Generic.IEnumerable<'T>
  implements: System.Collections.IEnumerable
val initializeScoutingMatrix : unit

  type: unit
  implements: System.IComparable
val j : int32

  type: int32
  implements: System.IComparable
  implements: System.IFormattable
  implements: System.IConvertible
  implements: System.IComparable<int>
  implements: System.IEquatable<int>
  inherits: System.ValueType
val i : int32

  type: int32
  implements: System.IComparable
  implements: System.IFormattable
  implements: System.IConvertible
  implements: System.IComparable<int>
  implements: System.IEquatable<int>
  inherits: System.ValueType
val sample : obj
val label : LabelType

  type: LabelType
  inherits: System.Enum
  inherits: System.ValueType
val calculateError : (int -> float list -> float)
val weakClassifierIndex : int

  type: int
  implements: System.IComparable
  implements: System.IFormattable
  implements: System.IConvertible
  implements: System.IComparable<int>
  implements: System.IEquatable<int>
  inherits: System.ValueType
val weights : float list

  type: float list
  implements: System.Collections.IStructuralEquatable
  implements: System.IComparable<List<float>>
  implements: System.IComparable
  implements: System.Collections.IStructuralComparable
  implements: System.Collections.Generic.IEnumerable<float>
  implements: System.Collections.IEnumerable
Multiple items
module List

from Microsoft.FSharp.Collections

--------------------
type List<'T> =
  | ( [] )
  | ( :: ) of 'T * 'T list
  with
    interface System.Collections.IEnumerable
    interface System.Collections.Generic.IEnumerable<'T>
    member Head : 'T
    member IsEmpty : bool
    member Item : index:int -> 'T with get
    member Length : int
    member Tail : 'T list
    static member Cons : head:'T * tail:'T list -> 'T list
    static member Empty : 'T list
  end

Full name: Microsoft.FSharp.Collections.List<_>

  type: List<'T>
  implements: System.Collections.IStructuralEquatable
  implements: System.IComparable<List<'T>>
  implements: System.IComparable
  implements: System.Collections.IStructuralComparable
  implements: System.Collections.Generic.IEnumerable<'T>
  implements: System.Collections.IEnumerable
val mapi : (int -> 'T -> 'U) -> 'T list -> 'U list

Full name: Microsoft.FSharp.Collections.List.mapi
val i : int

  type: int
  implements: System.IComparable
  implements: System.IFormattable
  implements: System.IConvertible
  implements: System.IComparable<int>
  implements: System.IEquatable<int>
  inherits: System.ValueType
val wt : float

  type: float
  implements: System.IComparable
  implements: System.IFormattable
  implements: System.IConvertible
  implements: System.IComparable<float>
  implements: System.IEquatable<float>
  inherits: System.ValueType
val sum : 'T list -> 'T (requires member ( + ) and member get_Zero)

Full name: Microsoft.FSharp.Collections.List.sum
val selectBestWeakClassifier : (float list -> int list -> int * float)
val weakClassifierIndices : int list

  type: int list
  implements: System.Collections.IStructuralEquatable
  implements: System.IComparable<List<int>>
  implements: System.IComparable
  implements: System.Collections.IStructuralComparable
  implements: System.Collections.Generic.IEnumerable<int>
  implements: System.Collections.IEnumerable
val map : ('T -> 'U) -> 'T list -> 'U list

Full name: Microsoft.FSharp.Collections.List.map
val minBy : ('T -> 'U) -> 'T list -> 'T (requires comparison)

Full name: Microsoft.FSharp.Collections.List.minBy
val x : int

  type: int
  implements: System.IComparable
  implements: System.IFormattable
  implements: System.IConvertible
  implements: System.IComparable<int>
  implements: System.IEquatable<int>
  inherits: System.ValueType
val y : float

  type: float
  implements: System.IComparable
  implements: System.IFormattable
  implements: System.IConvertible
  implements: System.IComparable<float>
  implements: System.IEquatable<float>
  inherits: System.ValueType
val train : (int -> float list -> int list -> (int * float) list)
val m : int

  type: int
  implements: System.IComparable
  implements: System.IFormattable
  implements: System.IConvertible
  implements: System.IComparable<int>
  implements: System.IEquatable<int>
  inherits: System.ValueType
val nextM : int

  type: int
  implements: System.IComparable
  implements: System.IFormattable
  implements: System.IConvertible
  implements: System.IComparable<int>
  implements: System.IEquatable<int>
  inherits: System.ValueType
val bestWeakClassifierIndex : int

  type: int
  implements: System.IComparable
  implements: System.IFormattable
  implements: System.IConvertible
  implements: System.IComparable<int>
  implements: System.IEquatable<int>
  inherits: System.ValueType
val classifierWeight : float

  type: float
  implements: System.IComparable
  implements: System.IFormattable
  implements: System.IConvertible
  implements: System.IComparable<float>
  implements: System.IEquatable<float>
  inherits: System.ValueType
val error : float

  type: float
  implements: System.IComparable
  implements: System.IFormattable
  implements: System.IConvertible
  implements: System.IComparable<float>
  implements: System.IEquatable<float>
  inherits: System.ValueType
val weakClassifierCoefficient : float

  type: float
  implements: System.IComparable
  implements: System.IFormattable
  implements: System.IConvertible
  implements: System.IComparable<float>
  implements: System.IEquatable<float>
  inherits: System.ValueType
val log : 'T -> 'T (requires member Log)

Full name: Microsoft.FSharp.Core.Operators.log
val wm1 : float

  type: float
  implements: System.IComparable
  implements: System.IFormattable
  implements: System.IConvertible
  implements: System.IComparable<float>
  implements: System.IEquatable<float>
  inherits: System.ValueType
val sqrt : 'T -> 'U (requires member Sqrt)

Full name: Microsoft.FSharp.Core.Operators.sqrt
val newWeights : float list

  type: float list
  implements: System.Collections.IStructuralEquatable
  implements: System.IComparable<List<float>>
  implements: System.IComparable
  implements: System.Collections.IStructuralComparable
  implements: System.Collections.Generic.IEnumerable<float>
  implements: System.Collections.IEnumerable
val wm : float

  type: float
  implements: System.IComparable
  implements: System.IFormattable
  implements: System.IConvertible
  implements: System.IComparable<float>
  implements: System.IEquatable<float>
  inherits: System.ValueType
val weakClassifiersToScanNext : int list

  type: int list
  implements: System.Collections.IStructuralEquatable
  implements: System.IComparable<List<int>>
  implements: System.IComparable
  implements: System.Collections.IStructuralComparable
  implements: System.Collections.Generic.IEnumerable<int>
  implements: System.Collections.IEnumerable
val filter : ('T -> bool) -> 'T list -> 'T list

Full name: Microsoft.FSharp.Collections.List.filter
val index : int

  type: int
  implements: System.IComparable
  implements: System.IFormattable
  implements: System.IConvertible
  implements: System.IComparable<int>
  implements: System.IEquatable<int>
  inherits: System.ValueType
val this : AdaBooster
member AdaBooster.Predict : sample:obj -> float * LabelType

Full name: AdaBooster.AdaBooster.Predict
val sumBy : ('T -> 'U) -> 'T list -> 'U (requires member ( + ) and member get_Zero)

Full name: Microsoft.FSharp.Collections.List.sumBy
val x : float

  type: float
  implements: System.IComparable
  implements: System.IFormattable
  implements: System.IConvertible
  implements: System.IComparable<float>
  implements: System.IEquatable<float>
  inherits: System.ValueType
member AdaBooster.Train : (int * float) list

Full name: AdaBooster.AdaBooster.Train
val initialWeights : float list

  type: float list
  implements: System.Collections.IStructuralEquatable
  implements: System.IComparable<List<float>>
  implements: System.IComparable
  implements: System.Collections.IStructuralComparable
  implements: System.Collections.Generic.IEnumerable<float>
  implements: System.Collections.IEnumerable
val init : int -> (int -> 'T) -> 'T list

Full name: Microsoft.FSharp.Collections.List.init
member AdaBooster.ScoutingMatrix : int [,]

Full name: AdaBooster.AdaBooster.ScoutingMatrix
member AdaBooster.WeightMatrix : float [,]

Full name: AdaBooster.AdaBooster.WeightMatrix

More information

Link: http://fssnip.net/hD
Posted: 1 years ago
Author: Suzanna (website)
Tags: AdaBoost