Parallel Sitemap Test Runner

 1: 
 2: 
 3: 
 4: 
 5: 
 6: 
 7: 
 8: 
 9: 
10: 
11: 
12: 
13: 
14: 
15: 
16: 
17: 
18: 
19: 
20: 
21: 
22: 
23: 
24: 
25: 
26: 
27: 
28: 
29: 
30: 
31: 
32: 
33: 
34: 
35: 
36: 
37: 
38: 
39: 
40: 
41: 
42: 
43: 
44: 
45: 
46: 
47: 
48: 
49: 
50: 
51: 
52: 
53: 
54: 
55: 
56: 
57: 
58: 
59: 
60: 
61: 
62: 
63: 
64: 
65: 
66: 
67: 
68: 
69: 
70: 
71: 
72: 
73: 
74: 
75: 

#r "FSharp.PowerPack"
#r "System.Xml"
#r "System.Xml.Linq"

open System
open System.Diagnostics
open System.IO
open System.Net
open System.Xml.Linq
open System.Xml.XPath
open Microsoft.FSharp.Control.WebExtensions

let loadUrl (url:string) = async {
    let request = WebRequest.Create(url)
    use! response = request.AsyncGetResponse()
    use stream = response.GetResponseStream()
    use reader = new StreamReader(stream)
    return reader.ReadToEnd() }

let (!!) xn = XName.Get(xn)

let getTestUrls (sitemapUrl:string) = async {
    let! sitemap = loadUrl sitemapUrl
    let doc = XDocument.Parse(sitemap, LoadOptions.None)
    let locs = doc.Root.Descendants(!!"{http://www.sitemaps.org/schemas/sitemap/0.9}loc")
    let urls = seq { for loc in locs do yield loc.Value }
    return urls }

let getAllTestUrls sitemapUrls =
    seq { for sitemapUrl in sitemapUrls do yield! getTestUrls sitemapUrl |> Async.RunSynchronously }

let testUrl url = async {
    let sw = Stopwatch.StartNew()
    try
        let! page = loadUrl url
        sw.Stop()
        if page.Contains("404") || page.Contains("Site Feedback") then
            return url + " failed to load correctly in " + sw.ElapsedMilliseconds.ToString() + " ms"
        else
            return url + " loaded successfully in " + sw.ElapsedMilliseconds.ToString() + " ms"
    with
    | :? WebException as exn -> sw.Stop()
                                return url + " raised an exception after " + sw.ElapsedMilliseconds.ToString() + ": " + exn.Message }

let run host =
    printfn "Running tests (this may take awhile) ..."
    let stopwatch = Stopwatch()
    stopwatch.Start()
    
    let results =
        [| host + "/sitemap-0.xml"
           host + "/sitemap-1.xml"
           host + "/sitemap-2.xml" |]
        |> getAllTestUrls
        |> Seq.map testUrl
        |> Async.Parallel
        |> Async.RunSynchronously
    
    stopwatch.Stop()
    printfn "Checked %i urls in %d seconds" (results |> Seq.length) (stopwatch.Elapsed.Seconds)
        
    // Print all request results
    results |> Seq.iter (printfn "%s")

//    // Print the bad requests
//    let badUrl (result:string) = result.EndsWith("failed to load correctly") ||
//                                 result.Contains("raised an exception")
//    let badRequests = results |> Seq.filter badUrl
//    
//    badRequests |> Seq.iter (printfn "%s")
//    printfn "Found %i bad requests" (badRequests |> Seq.length)
    
    printfn "Finished!"

run "http://your.site.com"

namespace System

namespace System.Diagnostics

namespace System.IO

namespace System.Net

namespace System.Xml

namespace System.Xml.Linq

namespace System.Xml.XPath

namespace Microsoft

namespace Microsoft.FSharp

namespace Microsoft.FSharp.Control

module WebExtensions

from Microsoft.FSharp.Control

val loadUrl : url:string -> Async<string>

Full name: Script.loadUrl

val url : string

Multiple items
val string : value:'T -> string

Full name: Microsoft.FSharp.Core.Operators.string

--------------------
type string = String

Full name: Microsoft.FSharp.Core.string

val async : AsyncBuilder

Full name: Microsoft.FSharp.Core.ExtraTopLevelOperators.async

val request : WebRequest

type WebRequest =
  inherit MarshalByRefObject
  member Abort : unit -> unit
  member AuthenticationLevel : AuthenticationLevel with get, set
  member BeginGetRequestStream : callback:AsyncCallback * state:obj -> IAsyncResult
  member BeginGetResponse : callback:AsyncCallback * state:obj -> IAsyncResult
  member CachePolicy : RequestCachePolicy with get, set
  member ConnectionGroupName : string with get, set
  member ContentLength : int64 with get, set
  member ContentType : string with get, set
  member Credentials : ICredentials with get, set
  member EndGetRequestStream : asyncResult:IAsyncResult -> Stream
  ...

Full name: System.Net.WebRequest

WebRequest.Create(requestUri: Uri) : WebRequest
WebRequest.Create(requestUriString: string) : WebRequest

val response : WebResponse

member WebRequest.AsyncGetResponse : unit -> Async<WebResponse>

val stream : Stream

WebResponse.GetResponseStream() : Stream

val reader : StreamReader

Multiple items
type StreamReader =
  inherit TextReader
  new : stream:Stream -> StreamReader + 9 overloads
  member BaseStream : Stream
  member Close : unit -> unit
  member CurrentEncoding : Encoding
  member DiscardBufferedData : unit -> unit
  member EndOfStream : bool
  member Peek : unit -> int
  member Read : unit -> int + 1 overload
  member ReadLine : unit -> string
  member ReadToEnd : unit -> string
  ...

Full name: System.IO.StreamReader

--------------------
StreamReader(stream: Stream) : unit
StreamReader(path: string) : unit
StreamReader(stream: Stream, detectEncodingFromByteOrderMarks: bool) : unit
StreamReader(stream: Stream, encoding: Text.Encoding) : unit
StreamReader(path: string, detectEncodingFromByteOrderMarks: bool) : unit
StreamReader(path: string, encoding: Text.Encoding) : unit
StreamReader(stream: Stream, encoding: Text.Encoding, detectEncodingFromByteOrderMarks: bool) : unit
StreamReader(path: string, encoding: Text.Encoding, detectEncodingFromByteOrderMarks: bool) : unit
StreamReader(stream: Stream, encoding: Text.Encoding, detectEncodingFromByteOrderMarks: bool, bufferSize: int) : unit
StreamReader(path: string, encoding: Text.Encoding, detectEncodingFromByteOrderMarks: bool, bufferSize: int) : unit

StreamReader.ReadToEnd() : string

val xn : string

type XName =
  member Equals : obj:obj -> bool
  member GetHashCode : unit -> int
  member LocalName : string
  member Namespace : XNamespace
  member NamespaceName : string
  member ToString : unit -> string
  static member Get : expandedName:string -> XName + 1 overload

Full name: System.Xml.Linq.XName

XName.Get(expandedName: string) : XName
XName.Get(localName: string, namespaceName: string) : XName

val getTestUrls : sitemapUrl:string -> Async<seq<string>>

Full name: Script.getTestUrls

val sitemapUrl : string

val sitemap : string

val doc : XDocument

Multiple items
type XDocument =
  inherit XContainer
  new : unit -> XDocument + 3 overloads
  member Declaration : XDeclaration with get, set
  member DocumentType : XDocumentType
  member NodeType : XmlNodeType
  member Root : XElement
  member Save : fileName:string -> unit + 6 overloads
  member WriteTo : writer:XmlWriter -> unit
  static member Load : uri:string -> XDocument + 7 overloads
  static member Parse : text:string -> XDocument + 1 overload

Full name: System.Xml.Linq.XDocument

--------------------
XDocument() : unit
XDocument([<ParamArray>] content: obj []) : unit
XDocument(other: XDocument) : unit
XDocument(declaration: XDeclaration, [<ParamArray>] content: obj []) : unit

XDocument.Parse(text: string) : XDocument
XDocument.Parse(text: string, options: LoadOptions) : XDocument

type LoadOptions =
  | None = 0
  | PreserveWhitespace = 1
  | SetBaseUri = 2
  | SetLineInfo = 4

Full name: System.Xml.Linq.LoadOptions

field LoadOptions.None = 0

val locs : Collections.Generic.IEnumerable<XElement>

property XDocument.Root: XElement

XContainer.Descendants() : Collections.Generic.IEnumerable<XElement>
XContainer.Descendants(name: XName) : Collections.Generic.IEnumerable<XElement>

val urls : seq<string>

Multiple items
val seq : sequence:seq<'T> -> seq<'T>

Full name: Microsoft.FSharp.Core.Operators.seq

--------------------
type seq<'T> = Collections.Generic.IEnumerable<'T>

Full name: Microsoft.FSharp.Collections.seq<_>

val loc : XElement

property XElement.Value: string

val getAllTestUrls : sitemapUrls:seq<string> -> seq<string>

Full name: Script.getAllTestUrls

val sitemapUrls : seq<string>

Multiple items
type Async
static member AsBeginEnd : computation:('Arg -> Async<'T>) -> ('Arg * AsyncCallback * obj -> IAsyncResult) * (IAsyncResult -> 'T) * (IAsyncResult -> unit)
static member AwaitEvent : event:IEvent<'Del,'T> * ?cancelAction:(unit -> unit) -> Async<'T> (requires delegate and 'Del :> Delegate)
static member AwaitIAsyncResult : iar:IAsyncResult * ?millisecondsTimeout:int -> Async<bool>
static member AwaitTask : task:Task<'T> -> Async<'T>
static member AwaitWaitHandle : waitHandle:WaitHandle * ?millisecondsTimeout:int -> Async<bool>
static member CancelDefaultToken : unit -> unit
static member Catch : computation:Async<'T> -> Async<Choice<'T,exn>>
static member FromBeginEnd : beginAction:(AsyncCallback * obj -> IAsyncResult) * endAction:(IAsyncResult -> 'T) * ?cancelAction:(unit -> unit) -> Async<'T>
static member FromBeginEnd : arg:'Arg1 * beginAction:('Arg1 * AsyncCallback * obj -> IAsyncResult) * endAction:(IAsyncResult -> 'T) * ?cancelAction:(unit -> unit) -> Async<'T>
static member FromBeginEnd : arg1:'Arg1 * arg2:'Arg2 * beginAction:('Arg1 * 'Arg2 * AsyncCallback * obj -> IAsyncResult) * endAction:(IAsyncResult -> 'T) * ?cancelAction:(unit -> unit) -> Async<'T>
static member FromBeginEnd : arg1:'Arg1 * arg2:'Arg2 * arg3:'Arg3 * beginAction:('Arg1 * 'Arg2 * 'Arg3 * AsyncCallback * obj -> IAsyncResult) * endAction:(IAsyncResult -> 'T) * ?cancelAction:(unit -> unit) -> Async<'T>
static member FromContinuations : callback:(('T -> unit) * (exn -> unit) * (OperationCanceledException -> unit) -> unit) -> Async<'T>
static member Ignore : computation:Async<'T> -> Async<unit>
static member OnCancel : interruption:(unit -> unit) -> Async<IDisposable>
static member Parallel : computations:seq<Async<'T>> -> Async<'T []>
static member RunSynchronously : computation:Async<'T> * ?timeout:int * ?cancellationToken:CancellationToken -> 'T
static member Sleep : millisecondsDueTime:int -> Async<unit>
static member Start : computation:Async<unit> * ?cancellationToken:CancellationToken -> unit
static member StartAsTask : computation:Async<'T> * ?taskCreationOptions:TaskCreationOptions * ?cancellationToken:CancellationToken -> Task<'T>
static member StartChild : computation:Async<'T> * ?millisecondsTimeout:int -> Async<Async<'T>>
static member StartChildAsTask : computation:Async<'T> * ?taskCreationOptions:TaskCreationOptions -> Async<Task<'T>>
static member StartImmediate : computation:Async<unit> * ?cancellationToken:CancellationToken -> unit
static member StartWithContinuations : computation:Async<'T> * continuation:('T -> unit) * exceptionContinuation:(exn -> unit) * cancellationContinuation:(OperationCanceledException -> unit) * ?cancellationToken:CancellationToken -> unit
static member SwitchToContext : syncContext:SynchronizationContext -> Async<unit>
static member SwitchToNewThread : unit -> Async<unit>
static member SwitchToThreadPool : unit -> Async<unit>
static member TryCancelled : computation:Async<'T> * compensation:(OperationCanceledException -> unit) -> Async<'T>
static member CancellationToken : Async<CancellationToken>
static member DefaultCancellationToken : CancellationToken

Full name: Microsoft.FSharp.Control.Async

--------------------
type Async<'T>

Full name: Microsoft.FSharp.Control.Async<_>

static member Async.RunSynchronously : computation:Async<'T> * ?timeout:int * ?cancellationToken:Threading.CancellationToken -> 'T

val testUrl : url:string -> Async<string>

Full name: Script.testUrl

val sw : Stopwatch

Multiple items
type Stopwatch =
  new : unit -> Stopwatch
  member Elapsed : TimeSpan
  member ElapsedMilliseconds : int64
  member ElapsedTicks : int64
  member IsRunning : bool
  member Reset : unit -> unit
  member Restart : unit -> unit
  member Start : unit -> unit
  member Stop : unit -> unit
  static val Frequency : int64
  ...

Full name: System.Diagnostics.Stopwatch

--------------------
Stopwatch() : unit

Stopwatch.StartNew() : Stopwatch

val page : string

Stopwatch.Stop() : unit

String.Contains(value: string) : bool

property Stopwatch.ElapsedMilliseconds: int64

Int64.ToString() : string
Int64.ToString(format: string) : string
Int64.ToString(provider: IFormatProvider) : string
Int64.ToString(format: string, provider: IFormatProvider) : string

Multiple items
type WebException =
  inherit InvalidOperationException
  new : unit -> WebException + 4 overloads
  member GetObjectData : serializationInfo:SerializationInfo * streamingContext:StreamingContext -> unit
  member Response : WebResponse
  member Status : WebExceptionStatus

Full name: System.Net.WebException

--------------------
WebException() : unit
WebException(message: string) : unit
WebException(message: string, innerException: exn) : unit
WebException(message: string, status: WebExceptionStatus) : unit
WebException(message: string, innerException: exn, status: WebExceptionStatus, response: WebResponse) : unit

Multiple items
val exn : WebException

--------------------
type exn = Exception

Full name: Microsoft.FSharp.Core.exn

property Exception.Message: string

val run : host:string -> unit

Full name: Script.run

val host : string

val printfn : format:Printf.TextWriterFormat<'T> -> 'T

Full name: Microsoft.FSharp.Core.ExtraTopLevelOperators.printfn

val stopwatch : Stopwatch

Stopwatch.Start() : unit

val results : string []

module Seq

from Microsoft.FSharp.Collections

val map : mapping:('T -> 'U) -> source:seq<'T> -> seq<'U>

Full name: Microsoft.FSharp.Collections.Seq.map

static member Async.Parallel : computations:seq<Async<'T>> -> Async<'T []>

val length : source:seq<'T> -> int

Full name: Microsoft.FSharp.Collections.Seq.length

property Stopwatch.Elapsed: TimeSpan

property TimeSpan.Seconds: int

val iter : action:('T -> unit) -> source:seq<'T> -> unit

Full name: Microsoft.FSharp.Collections.Seq.iter