9 people like it.
Like the snippet!
Async File Crawl
Async File Crawl
1:
2:
3:
4:
5:
6:
7:
8:
9:
10:
11:
12:
13:
14:
15:
16:
17:
18:
19:
20:
21:
22:
23:
24:
25:
26:
27:
28:
29:
30:
31:
32:
33:
34:
35:
36:
37:
|
#light
#time
open System
open System.IO
open System.Threading
open System.Threading.Tasks
type CList<'a> = System.Collections.Concurrent.ConcurrentBag<'a>
let readFile (file:string) (result:CList<'a>) (f:byte[] -> 'a) =
async {
let stream = File.OpenRead(file)
let! content = stream.AsyncRead(int stream.Length)
result.Add(f content)
stream.Close()
} |> Async.StartAsTask
let rec readDir (dir:string) (result:CList<'a>) (f:byte[] -> 'a) =
seq {
for file in Directory.GetFiles(dir) do
yield readFile file result f
for dir in Directory.GetDirectories(dir) do
yield! readDir dir result f
}
let rec collectFiles<'a> (dir:string) (f:byte[] -> 'a) =
let result = new CList<'a>()
let tasks = readDir dir result f |> Seq.cast<Task> |> Array.ofSeq
Task.WaitAll(tasks)
result
let result =
collectFiles<int> @"C:\Users\Fredrik\Projects\IronJS" (fun b -> b.Length)
|
namespace System
namespace System.IO
namespace System.Threading
namespace System.Threading.Tasks
type CList<'a> = Collections.Concurrent.ConcurrentBag<'a>
Full name: Script.CList<_>
namespace System.Collections
namespace System.Collections.Concurrent
Multiple items
type ConcurrentBag<'T> =
new : unit -> ConcurrentBag<'T> + 1 overload
member Add : item:'T -> unit
member CopyTo : array:'T[] * index:int -> unit
member Count : int
member GetEnumerator : unit -> IEnumerator<'T>
member IsEmpty : bool
member ToArray : unit -> 'T[]
member TryPeek : result:'T -> bool
member TryTake : result:'T -> bool
Full name: System.Collections.Concurrent.ConcurrentBag<_>
--------------------
Collections.Concurrent.ConcurrentBag() : unit
Collections.Concurrent.ConcurrentBag(collection: Collections.Generic.IEnumerable<'T>) : unit
val readFile : file:string -> result:CList<'a> -> f:(byte [] -> 'a) -> Task<unit>
Full name: Script.readFile
val file : string
Multiple items
val string : value:'T -> string
Full name: Microsoft.FSharp.Core.Operators.string
--------------------
type string = String
Full name: Microsoft.FSharp.Core.string
val result : CList<'a>
val f : (byte [] -> 'a)
Multiple items
val byte : value:'T -> byte (requires member op_Explicit)
Full name: Microsoft.FSharp.Core.Operators.byte
--------------------
type byte = Byte
Full name: Microsoft.FSharp.Core.byte
val async : AsyncBuilder
Full name: Microsoft.FSharp.Core.ExtraTopLevelOperators.async
val stream : FileStream
type File =
static member AppendAllLines : path:string * contents:IEnumerable<string> -> unit + 1 overload
static member AppendAllText : path:string * contents:string -> unit + 1 overload
static member AppendText : path:string -> StreamWriter
static member Copy : sourceFileName:string * destFileName:string -> unit + 1 overload
static member Create : path:string -> FileStream + 3 overloads
static member CreateText : path:string -> StreamWriter
static member Decrypt : path:string -> unit
static member Delete : path:string -> unit
static member Encrypt : path:string -> unit
static member Exists : path:string -> bool
...
Full name: System.IO.File
File.OpenRead(path: string) : FileStream
val content : byte []
member Stream.AsyncRead : count:int -> Async<byte []>
member Stream.AsyncRead : buffer:byte [] * ?offset:int * ?count:int -> Async<int>
Multiple items
val int : value:'T -> int (requires member op_Explicit)
Full name: Microsoft.FSharp.Core.Operators.int
--------------------
type int = int32
Full name: Microsoft.FSharp.Core.int
--------------------
type int<'Measure> = int
Full name: Microsoft.FSharp.Core.int<_>
property FileStream.Length: int64
Collections.Concurrent.ConcurrentBag.Add(item: 'a) : unit
Stream.Close() : unit
Multiple items
type Async
static member AsBeginEnd : computation:('Arg -> Async<'T>) -> ('Arg * AsyncCallback * obj -> IAsyncResult) * (IAsyncResult -> 'T) * (IAsyncResult -> unit)
static member AwaitEvent : event:IEvent<'Del,'T> * ?cancelAction:(unit -> unit) -> Async<'T> (requires delegate and 'Del :> Delegate)
static member AwaitIAsyncResult : iar:IAsyncResult * ?millisecondsTimeout:int -> Async<bool>
static member AwaitTask : task:Task<'T> -> Async<'T>
static member AwaitWaitHandle : waitHandle:WaitHandle * ?millisecondsTimeout:int -> Async<bool>
static member CancelDefaultToken : unit -> unit
static member Catch : computation:Async<'T> -> Async<Choice<'T,exn>>
static member FromBeginEnd : beginAction:(AsyncCallback * obj -> IAsyncResult) * endAction:(IAsyncResult -> 'T) * ?cancelAction:(unit -> unit) -> Async<'T>
static member FromBeginEnd : arg:'Arg1 * beginAction:('Arg1 * AsyncCallback * obj -> IAsyncResult) * endAction:(IAsyncResult -> 'T) * ?cancelAction:(unit -> unit) -> Async<'T>
static member FromBeginEnd : arg1:'Arg1 * arg2:'Arg2 * beginAction:('Arg1 * 'Arg2 * AsyncCallback * obj -> IAsyncResult) * endAction:(IAsyncResult -> 'T) * ?cancelAction:(unit -> unit) -> Async<'T>
static member FromBeginEnd : arg1:'Arg1 * arg2:'Arg2 * arg3:'Arg3 * beginAction:('Arg1 * 'Arg2 * 'Arg3 * AsyncCallback * obj -> IAsyncResult) * endAction:(IAsyncResult -> 'T) * ?cancelAction:(unit -> unit) -> Async<'T>
static member FromContinuations : callback:(('T -> unit) * (exn -> unit) * (OperationCanceledException -> unit) -> unit) -> Async<'T>
static member Ignore : computation:Async<'T> -> Async<unit>
static member OnCancel : interruption:(unit -> unit) -> Async<IDisposable>
static member Parallel : computations:seq<Async<'T>> -> Async<'T []>
static member RunSynchronously : computation:Async<'T> * ?timeout:int * ?cancellationToken:CancellationToken -> 'T
static member Sleep : millisecondsDueTime:int -> Async<unit>
static member Start : computation:Async<unit> * ?cancellationToken:CancellationToken -> unit
static member StartAsTask : computation:Async<'T> * ?taskCreationOptions:TaskCreationOptions * ?cancellationToken:CancellationToken -> Task<'T>
static member StartChild : computation:Async<'T> * ?millisecondsTimeout:int -> Async<Async<'T>>
static member StartChildAsTask : computation:Async<'T> * ?taskCreationOptions:TaskCreationOptions -> Async<Task<'T>>
static member StartImmediate : computation:Async<unit> * ?cancellationToken:CancellationToken -> unit
static member StartWithContinuations : computation:Async<'T> * continuation:('T -> unit) * exceptionContinuation:(exn -> unit) * cancellationContinuation:(OperationCanceledException -> unit) * ?cancellationToken:CancellationToken -> unit
static member SwitchToContext : syncContext:SynchronizationContext -> Async<unit>
static member SwitchToNewThread : unit -> Async<unit>
static member SwitchToThreadPool : unit -> Async<unit>
static member TryCancelled : computation:Async<'T> * compensation:(OperationCanceledException -> unit) -> Async<'T>
static member CancellationToken : Async<CancellationToken>
static member DefaultCancellationToken : CancellationToken
Full name: Microsoft.FSharp.Control.Async
--------------------
type Async<'T>
Full name: Microsoft.FSharp.Control.Async<_>
static member Async.StartAsTask : computation:Async<'T> * ?taskCreationOptions:TaskCreationOptions * ?cancellationToken:CancellationToken -> Task<'T>
val readDir : dir:string -> result:CList<'a> -> f:(byte [] -> 'a) -> seq<Task<unit>>
Full name: Script.readDir
val dir : string
Multiple items
val seq : sequence:seq<'T> -> seq<'T>
Full name: Microsoft.FSharp.Core.Operators.seq
--------------------
type seq<'T> = Collections.Generic.IEnumerable<'T>
Full name: Microsoft.FSharp.Collections.seq<_>
type Directory =
static member CreateDirectory : path:string -> DirectoryInfo + 1 overload
static member Delete : path:string -> unit + 1 overload
static member EnumerateDirectories : path:string -> IEnumerable<string> + 2 overloads
static member EnumerateFileSystemEntries : path:string -> IEnumerable<string> + 2 overloads
static member EnumerateFiles : path:string -> IEnumerable<string> + 2 overloads
static member Exists : path:string -> bool
static member GetAccessControl : path:string -> DirectorySecurity + 1 overload
static member GetCreationTime : path:string -> DateTime
static member GetCreationTimeUtc : path:string -> DateTime
static member GetCurrentDirectory : unit -> string
...
Full name: System.IO.Directory
Directory.GetFiles(path: string) : string []
Directory.GetFiles(path: string, searchPattern: string) : string []
Directory.GetFiles(path: string, searchPattern: string, searchOption: SearchOption) : string []
Directory.GetDirectories(path: string) : string []
Directory.GetDirectories(path: string, searchPattern: string) : string []
Directory.GetDirectories(path: string, searchPattern: string, searchOption: SearchOption) : string []
val collectFiles : dir:string -> f:(byte [] -> 'a) -> CList<'a>
Full name: Script.collectFiles
val tasks : Task []
module Seq
from Microsoft.FSharp.Collections
val cast : source:Collections.IEnumerable -> seq<'T>
Full name: Microsoft.FSharp.Collections.Seq.cast
Multiple items
type Task =
new : action:Action -> Task + 7 overloads
member AsyncState : obj
member ContinueWith : continuationAction:Action<Task> -> Task + 9 overloads
member CreationOptions : TaskCreationOptions
member Dispose : unit -> unit
member Exception : AggregateException
member Id : int
member IsCanceled : bool
member IsCompleted : bool
member IsFaulted : bool
...
Full name: System.Threading.Tasks.Task
--------------------
type Task<'TResult> =
inherit Task
new : function:Func<'TResult> -> Task<'TResult> + 7 overloads
member ContinueWith : continuationAction:Action<Task<'TResult>> -> Task + 9 overloads
member Result : 'TResult with get, set
static member Factory : TaskFactory<'TResult>
Full name: System.Threading.Tasks.Task<_>
--------------------
Task(action: Action) : unit
Task(action: Action, cancellationToken: CancellationToken) : unit
Task(action: Action, creationOptions: TaskCreationOptions) : unit
Task(action: Action<obj>, state: obj) : unit
Task(action: Action, cancellationToken: CancellationToken, creationOptions: TaskCreationOptions) : unit
Task(action: Action<obj>, state: obj, cancellationToken: CancellationToken) : unit
Task(action: Action<obj>, state: obj, creationOptions: TaskCreationOptions) : unit
Task(action: Action<obj>, state: obj, cancellationToken: CancellationToken, creationOptions: TaskCreationOptions) : unit
--------------------
Task(function: Func<'TResult>) : unit
Task(function: Func<'TResult>, cancellationToken: CancellationToken) : unit
Task(function: Func<'TResult>, creationOptions: TaskCreationOptions) : unit
Task(function: Func<obj,'TResult>, state: obj) : unit
Task(function: Func<'TResult>, cancellationToken: CancellationToken, creationOptions: TaskCreationOptions) : unit
Task(function: Func<obj,'TResult>, state: obj, cancellationToken: CancellationToken) : unit
Task(function: Func<obj,'TResult>, state: obj, creationOptions: TaskCreationOptions) : unit
Task(function: Func<obj,'TResult>, state: obj, cancellationToken: CancellationToken, creationOptions: TaskCreationOptions) : unit
type Array =
member Clone : unit -> obj
member CopyTo : array:Array * index:int -> unit + 1 overload
member GetEnumerator : unit -> IEnumerator
member GetLength : dimension:int -> int
member GetLongLength : dimension:int -> int64
member GetLowerBound : dimension:int -> int
member GetUpperBound : dimension:int -> int
member GetValue : [<ParamArray>] indices:int[] -> obj + 7 overloads
member Initialize : unit -> unit
member IsFixedSize : bool
...
Full name: System.Array
val ofSeq : source:seq<'T> -> 'T []
Full name: Microsoft.FSharp.Collections.Array.ofSeq
Task.WaitAll([<ParamArray>] tasks: Task []) : unit
Task.WaitAll(tasks: Task [], cancellationToken: CancellationToken) : unit
Task.WaitAll(tasks: Task [], millisecondsTimeout: int) : bool
Task.WaitAll(tasks: Task [], timeout: TimeSpan) : bool
Task.WaitAll(tasks: Task [], millisecondsTimeout: int, cancellationToken: CancellationToken) : bool
val result : CList<int>
Full name: Script.result
val b : byte []
property Array.Length: int
More information