2 people like it.

Multithreaded Proxy Checker in F#

The text file containing proxies named "proxlist.txt", in the form of "ip:port" per line is the input source, the program will attempt to establish a connection through each proxy to a provided URL. Proxies that passed the test will be written out to the file "results.txt" in the form "ip:port duration_of_connection_attempt"

  1: 
  2: 
  3: 
  4: 
  5: 
  6: 
  7: 
  8: 
  9: 
 10: 
 11: 
 12: 
 13: 
 14: 
 15: 
 16: 
 17: 
 18: 
 19: 
 20: 
 21: 
 22: 
 23: 
 24: 
 25: 
 26: 
 27: 
 28: 
 29: 
 30: 
 31: 
 32: 
 33: 
 34: 
 35: 
 36: 
 37: 
 38: 
 39: 
 40: 
 41: 
 42: 
 43: 
 44: 
 45: 
 46: 
 47: 
 48: 
 49: 
 50: 
 51: 
 52: 
 53: 
 54: 
 55: 
 56: 
 57: 
 58: 
 59: 
 60: 
 61: 
 62: 
 63: 
 64: 
 65: 
 66: 
 67: 
 68: 
 69: 
 70: 
 71: 
 72: 
 73: 
 74: 
 75: 
 76: 
 77: 
 78: 
 79: 
 80: 
 81: 
 82: 
 83: 
 84: 
 85: 
 86: 
 87: 
 88: 
 89: 
 90: 
 91: 
 92: 
 93: 
 94: 
 95: 
 96: 
 97: 
 98: 
 99: 
100: 
open System
open System.IO
open System.Net
open System.Text
open System.Threading
open System.Net.Http
open System.Timers

let readFile(filePath:string):string =
    try
    let encoding = System.Text.Encoding.UTF8
    let rr = File.OpenRead(filePath)
    use reader = new IO.StreamReader(rr,encoding)
    let content = reader.ReadToEnd()
    content
    with error ->  "Error"

let writeFile (filePath:string, fileContent:string):unit =
    try        
       let encoding = System.Text.Encoding.UTF8 
       let rr = File.OpenWrite(filePath) 
       use writer = new IO.StreamWriter(rr,encoding)  
       writer.Write(fileContent)        
    with error -> eprintf "An exception occurred with message %s" error.Message

let getProxiesListFromFile(path:string):string[]=
   //--------------something like this------------------
   (*
        1.10.141.220:54620
        77.236.236.38:10000
        185.255.46.121:8000
        85.195.104.71:80
        141.98.112.87:8080
        154.64.211.145:999
        195.187.63.42:8080
        109.173.44.42:8000
        94.180.106.94:32767
        213.157.47.38:8080   
   *)
   let strContent = readFile(path)
   let strs:string[] = strContent.Split([|"\r\n"|], StringSplitOptions.RemoveEmptyEntries)
   let sfiltered = strs |>  Array.map (fun x -> x.Replace("=> success","").Replace("=> failure","").Trim())
   sfiltered

let defaultValue(proxyUrl:string)=
  async{
   return (proxyUrl, false,0.)
  }

let runHttpClient(url: string,urlProxy:string)=
   async {
    let stopWatch = System.Diagnostics.Stopwatch.StartNew()
    try
        let ur = Uri(url)
        let proxy = new WebProxy(urlProxy, true, null, null)
        let handler = new HttpClientHandler()
        handler.Proxy <- proxy
        handler.UseCookies <- false        
        let client = new HttpClient(handler)
        client.Timeout <- TimeSpan.FromSeconds(4.)
        let! responseString = Async.AwaitTask <| client.GetStringAsync(ur)  
        stopWatch.Stop()
        if responseString <> null && responseString <> "" 
             then              
             let ms =Math.Round( stopWatch.Elapsed.TotalSeconds,2)
             return (urlProxy, true,ms)
             else return (urlProxy, false,0.)        
    with  error -> return (urlProxy, false,0.)        
      }

let runAsyncWithTimeout(urlPoint: string,urlProxy:string)=
  async {                                      
   try                                          
    let! child = Async.Catch (Async.StartChild( runHttpClient(urlPoint,urlProxy), 10000 ))
    match child with
        | Choice1Of2 myValue -> return! myValue
        | Choice2Of2 error -> return! defaultValue(urlPoint)
   with error ->  return! defaultValue(urlPoint)
  }

[<EntryPoint>]
let main argv =  
    let stopWatch = System.Diagnostics.Stopwatch.StartNew()
    let sf = getProxiesListFromFile("./proxlist.txt")
    let tasks = [for urlProxy in sf ->  runAsyncWithTimeout ("https://httpbin.org/get",urlProxy) ]
    let parallelTasks = Async.Parallel tasks
    Console.WriteLine("----------------")
    let str = Async.RunSynchronously parallelTasks    
    let sm = str |> Array.filter (fun (x,y,z) -> y = true)  |> Array.sortBy (fun (x,y,z) -> z)
    sm |> Array.iter(fun (x,y,z)-> Console.WriteLine(x+" : "+y.ToString()+" : "+z.ToString()))
    let results = sm |> Array.map(fun (x,y,z)-> x.ToString()+" "+z.ToString())|> String.concat "\r\n"
    writeFile("./results.txt",results)
    stopWatch.Stop()
    let ms =Math.Round( stopWatch.Elapsed.TotalSeconds,2)
    let count:int = Array.length sf
    Console.WriteLine("Total elapsed time in seconds : "+stopWatch.Elapsed.Seconds.ToString())
    Console.WriteLine("Total count of checked proxies "+count.ToString())
    Console.WriteLine("--------END--------")
    Console.ReadLine() |> ignore
    0
namespace System
namespace System.IO
namespace System.Net
namespace System.Text
namespace System.Threading
namespace System.Net.Http
namespace System.Timers
val readFile : filePath:string -> string
val filePath : string
Multiple items
val string : value:'T -> string

--------------------
type string = String
val encoding : Encoding
type Encoding =
  member BodyName : string
  member Clone : unit -> obj
  member CodePage : int
  member DecoderFallback : DecoderFallback with get, set
  member EncoderFallback : EncoderFallback with get, set
  member EncodingName : string
  member Equals : value:obj -> bool
  member GetByteCount : chars:char[] -> int + 5 overloads
  member GetBytes : chars:char[] -> byte[] + 7 overloads
  member GetCharCount : bytes:byte[] -> int + 3 overloads
  ...
property Encoding.UTF8: Encoding with get
val rr : FileStream
type File =
  static member AppendAllLines : path:string * contents:IEnumerable<string> -> unit + 1 overload
  static member AppendAllLinesAsync : path:string * contents:IEnumerable<string> * ?cancellationToken:CancellationToken -> Task + 1 overload
  static member AppendAllText : path:string * contents:string -> unit + 1 overload
  static member AppendAllTextAsync : path:string * contents:string * ?cancellationToken:CancellationToken -> Task + 1 overload
  static member AppendText : path:string -> StreamWriter
  static member Copy : sourceFileName:string * destFileName:string -> unit + 1 overload
  static member Create : path:string -> FileStream + 2 overloads
  static member CreateText : path:string -> StreamWriter
  static member Decrypt : path:string -> unit
  static member Delete : path:string -> unit
  ...
File.OpenRead(path: string) : FileStream
val reader : StreamReader
Multiple items
type StreamReader =
  inherit TextReader
  new : stream:Stream -> StreamReader + 10 overloads
  member BaseStream : Stream
  member Close : unit -> unit
  member CurrentEncoding : Encoding
  member DiscardBufferedData : unit -> unit
  member EndOfStream : bool
  member Peek : unit -> int
  member Read : unit -> int + 2 overloads
  member ReadAsync : buffer:Memory<char> * ?cancellationToken:CancellationToken -> ValueTask<int> + 1 overload
  member ReadBlock : buffer:Span<char> -> int + 1 overload
  ...

--------------------
StreamReader(stream: Stream) : StreamReader
   (+0 other overloads)
StreamReader(path: string) : StreamReader
   (+0 other overloads)
StreamReader(stream: Stream, detectEncodingFromByteOrderMarks: bool) : StreamReader
   (+0 other overloads)
StreamReader(stream: Stream, encoding: Encoding) : StreamReader
   (+0 other overloads)
StreamReader(path: string, detectEncodingFromByteOrderMarks: bool) : StreamReader
   (+0 other overloads)
StreamReader(path: string, encoding: Encoding) : StreamReader
   (+0 other overloads)
StreamReader(stream: Stream, encoding: Encoding, detectEncodingFromByteOrderMarks: bool) : StreamReader
   (+0 other overloads)
StreamReader(path: string, encoding: Encoding, detectEncodingFromByteOrderMarks: bool) : StreamReader
   (+0 other overloads)
StreamReader(stream: Stream, encoding: Encoding, detectEncodingFromByteOrderMarks: bool, bufferSize: int) : StreamReader
   (+0 other overloads)
StreamReader(path: string, encoding: Encoding, detectEncodingFromByteOrderMarks: bool, bufferSize: int) : StreamReader
   (+0 other overloads)
val content : string
val error : exn
val writeFile : filePath:string * fileContent:string -> unit
val fileContent : string
type unit = Unit
File.OpenWrite(path: string) : FileStream
val writer : StreamWriter
Multiple items
type StreamWriter =
  inherit TextWriter
  new : stream:Stream -> StreamWriter + 7 overloads
  member AutoFlush : bool with get, set
  member BaseStream : Stream
  member Close : unit -> unit
  member DisposeAsync : unit -> ValueTask
  member Encoding : Encoding
  member Flush : unit -> unit
  member FlushAsync : unit -> Task
  member Write : value:char -> unit + 8 overloads
  member WriteAsync : value:char -> Task + 3 overloads
  ...

--------------------
StreamWriter(stream: Stream) : StreamWriter
StreamWriter(path: string) : StreamWriter
StreamWriter(stream: Stream, encoding: Encoding) : StreamWriter
StreamWriter(path: string, append: bool) : StreamWriter
StreamWriter(stream: Stream, encoding: Encoding, bufferSize: int) : StreamWriter
StreamWriter(path: string, append: bool, encoding: Encoding) : StreamWriter
StreamWriter(stream: Stream,?encoding: Encoding,?bufferSize: int,?leaveOpen: bool) : StreamWriter
StreamWriter(path: string, append: bool, encoding: Encoding, bufferSize: int) : StreamWriter
val eprintf : format:Printf.TextWriterFormat<'T> -> 'T
val getProxiesListFromFile : path:string -> string []
val path : string
val strContent : string
val strs : string []
type StringSplitOptions =
  | None = 0
  | RemoveEmptyEntries = 1
field StringSplitOptions.RemoveEmptyEntries: StringSplitOptions = 1
val sfiltered : string []
type Array =
  member Clone : unit -> obj
  member CopyTo : array:Array * index:int -> unit + 1 overload
  member GetEnumerator : unit -> IEnumerator
  member GetLength : dimension:int -> int
  member GetLongLength : dimension:int -> int64
  member GetLowerBound : dimension:int -> int
  member GetUpperBound : dimension:int -> int
  member GetValue : [<ParamArray>] indices:int[] -> obj + 7 overloads
  member Initialize : unit -> unit
  member IsFixedSize : bool
  ...
val map : mapping:('T -> 'U) -> array:'T [] -> 'U []
val x : string
val defaultValue : proxyUrl:string -> Async<string * bool * float>
val proxyUrl : string
val async : AsyncBuilder
val runHttpClient : url:string * urlProxy:string -> Async<string * bool * float>
val url : string
val urlProxy : string
val stopWatch : Diagnostics.Stopwatch
namespace System.Diagnostics
Multiple items
type Stopwatch =
  new : unit -> Stopwatch
  member Elapsed : TimeSpan
  member ElapsedMilliseconds : int64
  member ElapsedTicks : int64
  member IsRunning : bool
  member Reset : unit -> unit
  member Restart : unit -> unit
  member Start : unit -> unit
  member Stop : unit -> unit
  static val Frequency : int64
  ...

--------------------
Diagnostics.Stopwatch() : Diagnostics.Stopwatch
Diagnostics.Stopwatch.StartNew() : Diagnostics.Stopwatch
val ur : Uri
Multiple items
type Uri =
  new : uriString:string -> Uri + 5 overloads
  member AbsolutePath : string
  member AbsoluteUri : string
  member Authority : string
  member DnsSafeHost : string
  member Equals : comparand:obj -> bool
  member Fragment : string
  member GetComponents : components:UriComponents * format:UriFormat -> string
  member GetHashCode : unit -> int
  member GetLeftPart : part:UriPartial -> string
  ...

--------------------
Uri(uriString: string) : Uri
Uri(uriString: string, uriKind: UriKind) : Uri
Uri(baseUri: Uri, relativeUri: string) : Uri
Uri(baseUri: Uri, relativeUri: Uri) : Uri
val proxy : WebProxy
Multiple items
type WebProxy =
  new : unit -> WebProxy + 9 overloads
  member Address : Uri with get, set
  member BypassArrayList : ArrayList
  member BypassList : string[] with get, set
  member BypassProxyOnLocal : bool with get, set
  member Credentials : ICredentials with get, set
  member GetProxy : destination:Uri -> Uri
  member IsBypassed : host:Uri -> bool
  member UseDefaultCredentials : bool with get, set
  static member GetDefaultProxy : unit -> WebProxy

--------------------
WebProxy() : WebProxy
WebProxy(Address: Uri) : WebProxy
WebProxy(Address: string) : WebProxy
WebProxy(Address: Uri, BypassOnLocal: bool) : WebProxy
WebProxy(Host: string, Port: int) : WebProxy
WebProxy(Address: string, BypassOnLocal: bool) : WebProxy
WebProxy(Address: Uri, BypassOnLocal: bool, BypassList: string []) : WebProxy
WebProxy(Address: string, BypassOnLocal: bool, BypassList: string []) : WebProxy
WebProxy(Address: Uri, BypassOnLocal: bool, BypassList: string [], Credentials: ICredentials) : WebProxy
WebProxy(Address: string, BypassOnLocal: bool, BypassList: string [], Credentials: ICredentials) : WebProxy
val handler : HttpClientHandler
Multiple items
type HttpClientHandler =
  inherit HttpMessageHandler
  new : unit -> HttpClientHandler
  member AllowAutoRedirect : bool with get, set
  member AutomaticDecompression : DecompressionMethods with get, set
  member CheckCertificateRevocationList : bool with get, set
  member ClientCertificateOptions : ClientCertificateOption with get, set
  member ClientCertificates : X509CertificateCollection
  member CookieContainer : CookieContainer with get, set
  member Credentials : ICredentials with get, set
  member DefaultProxyCredentials : ICredentials with get, set
  member MaxAutomaticRedirections : int with get, set
  ...

--------------------
HttpClientHandler() : HttpClientHandler
val client : HttpClient
Multiple items
type HttpClient =
  inherit HttpMessageInvoker
  new : unit -> HttpClient + 2 overloads
  member BaseAddress : Uri with get, set
  member CancelPendingRequests : unit -> unit
  member DefaultRequestHeaders : HttpRequestHeaders
  member DefaultRequestVersion : Version with get, set
  member DeleteAsync : requestUri:string -> Task<HttpResponseMessage> + 3 overloads
  member GetAsync : requestUri:string -> Task<HttpResponseMessage> + 7 overloads
  member GetByteArrayAsync : requestUri:string -> Task<byte[]> + 1 overload
  member GetStreamAsync : requestUri:string -> Task<Stream> + 1 overload
  member GetStringAsync : requestUri:string -> Task<string> + 1 overload
  ...

--------------------
HttpClient() : HttpClient
HttpClient(handler: HttpMessageHandler) : HttpClient
HttpClient(handler: HttpMessageHandler, disposeHandler: bool) : HttpClient
Multiple items
type TimeSpan =
  struct
    new : ticks:int64 -> TimeSpan + 3 overloads
    member Add : ts:TimeSpan -> TimeSpan
    member CompareTo : value:obj -> int + 1 overload
    member Days : int
    member Divide : divisor:float -> TimeSpan + 1 overload
    member Duration : unit -> TimeSpan
    member Equals : value:obj -> bool + 1 overload
    member GetHashCode : unit -> int
    member Hours : int
    member Milliseconds : int
    ...
  end

--------------------
TimeSpan ()
TimeSpan(ticks: int64) : TimeSpan
TimeSpan(hours: int, minutes: int, seconds: int) : TimeSpan
TimeSpan(days: int, hours: int, minutes: int, seconds: int) : TimeSpan
TimeSpan(days: int, hours: int, minutes: int, seconds: int, milliseconds: int) : TimeSpan
TimeSpan.FromSeconds(value: float) : TimeSpan
val responseString : string
Multiple items
type Async =
  static member AsBeginEnd : computation:('Arg -> Async<'T>) -> ('Arg * AsyncCallback * obj -> IAsyncResult) * (IAsyncResult -> 'T) * (IAsyncResult -> unit)
  static member AwaitEvent : event:IEvent<'Del,'T> * ?cancelAction:(unit -> unit) -> Async<'T> (requires delegate and 'Del :> Delegate)
  static member AwaitIAsyncResult : iar:IAsyncResult * ?millisecondsTimeout:int -> Async<bool>
  static member AwaitTask : task:Task -> Async<unit>
  static member AwaitTask : task:Task<'T> -> Async<'T>
  static member AwaitWaitHandle : waitHandle:WaitHandle * ?millisecondsTimeout:int -> Async<bool>
  static member CancelDefaultToken : unit -> unit
  static member Catch : computation:Async<'T> -> Async<Choice<'T,exn>>
  static member Choice : computations:seq<Async<'T option>> -> Async<'T option>
  static member FromBeginEnd : beginAction:(AsyncCallback * obj -> IAsyncResult) * endAction:(IAsyncResult -> 'T) * ?cancelAction:(unit -> unit) -> Async<'T>
  ...

--------------------
type Async<'T> =
static member Async.AwaitTask : task:Tasks.Task -> Async<unit>
static member Async.AwaitTask : task:Tasks.Task<'T> -> Async<'T>
val ms : float
type Math =
  static val E : float
  static val PI : float
  static member Abs : value:float -> float + 6 overloads
  static member Acos : d:float -> float
  static member Acosh : d:float -> float
  static member Asin : d:float -> float
  static member Asinh : d:float -> float
  static member Atan : d:float -> float
  static member Atan2 : y:float * x:float -> float
  static member Atanh : d:float -> float
  ...
Math.Round(a: float) : float
Math.Round(d: decimal) : decimal
Math.Round(value: float, mode: MidpointRounding) : float
Math.Round(value: float, digits: int) : float
Math.Round(d: decimal, mode: MidpointRounding) : decimal
Math.Round(d: decimal, decimals: int) : decimal
Math.Round(value: float, digits: int, mode: MidpointRounding) : float
Math.Round(d: decimal, decimals: int, mode: MidpointRounding) : decimal
val runAsyncWithTimeout : urlPoint:string * urlProxy:string -> Async<string * bool * float>
val urlPoint : string
val child : Choice<Async<string * bool * float>,exn>
static member Async.Catch : computation:Async<'T> -> Async<Choice<'T,exn>>
static member Async.StartChild : computation:Async<'T> * ?millisecondsTimeout:int -> Async<Async<'T>>
union case Choice.Choice1Of2: 'T1 -> Choice<'T1,'T2>
val myValue : Async<string * bool * float>
union case Choice.Choice2Of2: 'T2 -> Choice<'T1,'T2>
Multiple items
type EntryPointAttribute =
  inherit Attribute
  new : unit -> EntryPointAttribute

--------------------
new : unit -> EntryPointAttribute
val main : argv:string [] -> int
val argv : string []
val sf : string []
val tasks : Async<string * bool * float> list
val parallelTasks : Async<(string * bool * float) []>
static member Async.Parallel : computations:seq<Async<'T>> -> Async<'T []>
static member Async.Parallel : computations:seq<Async<'T>> * ?maxDegreeOfParallelism:int -> Async<'T []>
type Console =
  static member BackgroundColor : ConsoleColor with get, set
  static member Beep : unit -> unit + 1 overload
  static member BufferHeight : int with get, set
  static member BufferWidth : int with get, set
  static member CapsLock : bool
  static member Clear : unit -> unit
  static member CursorLeft : int with get, set
  static member CursorSize : int with get, set
  static member CursorTop : int with get, set
  static member CursorVisible : bool with get, set
  ...
Console.WriteLine() : unit
   (+0 other overloads)
Console.WriteLine(value: string) : unit
   (+0 other overloads)
Console.WriteLine(value: obj) : unit
   (+0 other overloads)
Console.WriteLine(value: uint64) : unit
   (+0 other overloads)
Console.WriteLine(value: int64) : unit
   (+0 other overloads)
Console.WriteLine(value: uint32) : unit
   (+0 other overloads)
Console.WriteLine(value: int) : unit
   (+0 other overloads)
Console.WriteLine(value: float32) : unit
   (+0 other overloads)
Console.WriteLine(value: float) : unit
   (+0 other overloads)
Console.WriteLine(value: decimal) : unit
   (+0 other overloads)
val str : (string * bool * float) []
static member Async.RunSynchronously : computation:Async<'T> * ?timeout:int * ?cancellationToken:CancellationToken -> 'T
val sm : (string * bool * float) []
val filter : predicate:('T -> bool) -> array:'T [] -> 'T []
val y : bool
val z : float
val sortBy : projection:('T -> 'Key) -> array:'T [] -> 'T [] (requires comparison)
val iter : action:('T -> unit) -> array:'T [] -> unit
val results : string
Multiple items
type String =
  new : value:char[] -> string + 8 overloads
  member Chars : int -> char
  member Clone : unit -> obj
  member CompareTo : value:obj -> int + 1 overload
  member Contains : value:string -> bool + 3 overloads
  member CopyTo : sourceIndex:int * destination:char[] * destinationIndex:int * count:int -> unit
  member EndsWith : value:string -> bool + 3 overloads
  member EnumerateRunes : unit -> StringRuneEnumerator
  member Equals : obj:obj -> bool + 2 overloads
  member GetEnumerator : unit -> CharEnumerator
  ...

--------------------
String(value: char []) : String
String(value: nativeptr<char>) : String
String(value: nativeptr<sbyte>) : String
String(value: ReadOnlySpan<char>) : String
String(c: char, count: int) : String
String(value: char [], startIndex: int, length: int) : String
String(value: nativeptr<char>, startIndex: int, length: int) : String
String(value: nativeptr<sbyte>, startIndex: int, length: int) : String
String(value: nativeptr<sbyte>, startIndex: int, length: int, enc: Encoding) : String
val concat : sep:string -> strings:seq<string> -> string
val count : int
Multiple items
val int : value:'T -> int (requires member op_Explicit)

--------------------
type int = int32

--------------------
type int<'Measure> = int
val length : array:'T [] -> int
Console.ReadLine() : string
val ignore : value:'T -> unit
Next Version Raw view Test code New version

More information

Link:http://fssnip.net/85C
Posted:3 years ago
Author:Vladimir Demydov
Tags: async , http , multithreading