2 people like it.
Like the snippet!
Multithreaded Proxy Checker in F#
The text file containing proxies named "proxlist.txt", in the form of "ip:port" per line is the input source, the program will attempt to establish a connection through each proxy to a provided URL. Proxies that passed the test will be written out to the file "results.txt" in the form "ip:port duration_of_connection_attempt"
1:
2:
3:
4:
5:
6:
7:
8:
9:
10:
11:
12:
13:
14:
15:
16:
17:
18:
19:
20:
21:
22:
23:
24:
25:
26:
27:
28:
29:
30:
31:
32:
33:
34:
35:
36:
37:
38:
39:
40:
41:
42:
43:
44:
45:
46:
47:
48:
49:
50:
51:
52:
53:
54:
55:
56:
57:
58:
59:
60:
61:
62:
63:
64:
65:
66:
67:
68:
69:
70:
71:
72:
73:
74:
75:
76:
77:
78:
79:
80:
81:
82:
83:
84:
85:
86:
87:
88:
89:
90:
91:
92:
93:
94:
95:
96:
97:
98:
99:
100:
|
open System
open System.IO
open System.Net
open System.Text
open System.Threading
open System.Net.Http
open System.Timers
let readFile(filePath:string):string =
try
let encoding = System.Text.Encoding.UTF8
let rr = File.OpenRead(filePath)
use reader = new IO.StreamReader(rr,encoding)
let content = reader.ReadToEnd()
content
with error -> "Error"
let writeFile (filePath:string, fileContent:string):unit =
try
let encoding = System.Text.Encoding.UTF8
let rr = File.OpenWrite(filePath)
use writer = new IO.StreamWriter(rr,encoding)
writer.Write(fileContent)
with error -> eprintf "An exception occurred with message %s" error.Message
let getProxiesListFromFile(path:string):string[]=
//--------------something like this------------------
(*
1.10.141.220:54620
77.236.236.38:10000
185.255.46.121:8000
85.195.104.71:80
141.98.112.87:8080
154.64.211.145:999
195.187.63.42:8080
109.173.44.42:8000
94.180.106.94:32767
213.157.47.38:8080
*)
let strContent = readFile(path)
let strs:string[] = strContent.Split([|"\r\n"|], StringSplitOptions.RemoveEmptyEntries)
let sfiltered = strs |> Array.map (fun x -> x.Replace("=> success","").Replace("=> failure","").Trim())
sfiltered
let defaultValue(proxyUrl:string)=
async{
return (proxyUrl, false,0.)
}
let runHttpClient(url: string,urlProxy:string)=
async {
let stopWatch = System.Diagnostics.Stopwatch.StartNew()
try
let ur = Uri(url)
let proxy = new WebProxy(urlProxy, true, null, null)
let handler = new HttpClientHandler()
handler.Proxy <- proxy
handler.UseCookies <- false
let client = new HttpClient(handler)
client.Timeout <- TimeSpan.FromSeconds(4.)
let! responseString = Async.AwaitTask <| client.GetStringAsync(ur)
stopWatch.Stop()
if responseString <> null && responseString <> ""
then
let ms =Math.Round( stopWatch.Elapsed.TotalSeconds,2)
return (urlProxy, true,ms)
else return (urlProxy, false,0.)
with error -> return (urlProxy, false,0.)
}
let runAsyncWithTimeout(urlPoint: string,urlProxy:string)=
async {
try
let! child = Async.Catch (Async.StartChild( runHttpClient(urlPoint,urlProxy), 10000 ))
match child with
| Choice1Of2 myValue -> return! myValue
| Choice2Of2 error -> return! defaultValue(urlPoint)
with error -> return! defaultValue(urlPoint)
}
[<EntryPoint>]
let main argv =
let stopWatch = System.Diagnostics.Stopwatch.StartNew()
let sf = getProxiesListFromFile("./proxlist.txt")
let tasks = [for urlProxy in sf -> runAsyncWithTimeout ("https://httpbin.org/get",urlProxy) ]
let parallelTasks = Async.Parallel tasks
Console.WriteLine("----------------")
let str = Async.RunSynchronously parallelTasks
let sm = str |> Array.filter (fun (x,y,z) -> y = true) |> Array.sortBy (fun (x,y,z) -> z)
sm |> Array.iter(fun (x,y,z)-> Console.WriteLine(x+" : "+y.ToString()+" : "+z.ToString()))
let results = sm |> Array.map(fun (x,y,z)-> x.ToString()+" "+z.ToString())|> String.concat "\r\n"
writeFile("./results.txt",results)
stopWatch.Stop()
let ms =Math.Round( stopWatch.Elapsed.TotalSeconds,2)
let count:int = Array.length sf
Console.WriteLine("Total elapsed time in seconds : "+stopWatch.Elapsed.Seconds.ToString())
Console.WriteLine("Total count of checked proxies "+count.ToString())
Console.WriteLine("--------END--------")
Console.ReadLine() |> ignore
0
|
namespace System
namespace System.IO
namespace System.Net
namespace System.Text
namespace System.Threading
namespace System.Net.Http
namespace System.Timers
val readFile : filePath:string -> string
val filePath : string
Multiple items
val string : value:'T -> string
--------------------
type string = String
val encoding : Encoding
type Encoding =
member BodyName : string
member Clone : unit -> obj
member CodePage : int
member DecoderFallback : DecoderFallback with get, set
member EncoderFallback : EncoderFallback with get, set
member EncodingName : string
member Equals : value:obj -> bool
member GetByteCount : chars:char[] -> int + 5 overloads
member GetBytes : chars:char[] -> byte[] + 7 overloads
member GetCharCount : bytes:byte[] -> int + 3 overloads
...
property Encoding.UTF8: Encoding with get
val rr : FileStream
type File =
static member AppendAllLines : path:string * contents:IEnumerable<string> -> unit + 1 overload
static member AppendAllLinesAsync : path:string * contents:IEnumerable<string> * ?cancellationToken:CancellationToken -> Task + 1 overload
static member AppendAllText : path:string * contents:string -> unit + 1 overload
static member AppendAllTextAsync : path:string * contents:string * ?cancellationToken:CancellationToken -> Task + 1 overload
static member AppendText : path:string -> StreamWriter
static member Copy : sourceFileName:string * destFileName:string -> unit + 1 overload
static member Create : path:string -> FileStream + 2 overloads
static member CreateText : path:string -> StreamWriter
static member Decrypt : path:string -> unit
static member Delete : path:string -> unit
...
File.OpenRead(path: string) : FileStream
val reader : StreamReader
Multiple items
type StreamReader =
inherit TextReader
new : stream:Stream -> StreamReader + 10 overloads
member BaseStream : Stream
member Close : unit -> unit
member CurrentEncoding : Encoding
member DiscardBufferedData : unit -> unit
member EndOfStream : bool
member Peek : unit -> int
member Read : unit -> int + 2 overloads
member ReadAsync : buffer:Memory<char> * ?cancellationToken:CancellationToken -> ValueTask<int> + 1 overload
member ReadBlock : buffer:Span<char> -> int + 1 overload
...
--------------------
StreamReader(stream: Stream) : StreamReader
(+0 other overloads)
StreamReader(path: string) : StreamReader
(+0 other overloads)
StreamReader(stream: Stream, detectEncodingFromByteOrderMarks: bool) : StreamReader
(+0 other overloads)
StreamReader(stream: Stream, encoding: Encoding) : StreamReader
(+0 other overloads)
StreamReader(path: string, detectEncodingFromByteOrderMarks: bool) : StreamReader
(+0 other overloads)
StreamReader(path: string, encoding: Encoding) : StreamReader
(+0 other overloads)
StreamReader(stream: Stream, encoding: Encoding, detectEncodingFromByteOrderMarks: bool) : StreamReader
(+0 other overloads)
StreamReader(path: string, encoding: Encoding, detectEncodingFromByteOrderMarks: bool) : StreamReader
(+0 other overloads)
StreamReader(stream: Stream, encoding: Encoding, detectEncodingFromByteOrderMarks: bool, bufferSize: int) : StreamReader
(+0 other overloads)
StreamReader(path: string, encoding: Encoding, detectEncodingFromByteOrderMarks: bool, bufferSize: int) : StreamReader
(+0 other overloads)
val content : string
val error : exn
val writeFile : filePath:string * fileContent:string -> unit
val fileContent : string
type unit = Unit
File.OpenWrite(path: string) : FileStream
val writer : StreamWriter
Multiple items
type StreamWriter =
inherit TextWriter
new : stream:Stream -> StreamWriter + 7 overloads
member AutoFlush : bool with get, set
member BaseStream : Stream
member Close : unit -> unit
member DisposeAsync : unit -> ValueTask
member Encoding : Encoding
member Flush : unit -> unit
member FlushAsync : unit -> Task
member Write : value:char -> unit + 8 overloads
member WriteAsync : value:char -> Task + 3 overloads
...
--------------------
StreamWriter(stream: Stream) : StreamWriter
StreamWriter(path: string) : StreamWriter
StreamWriter(stream: Stream, encoding: Encoding) : StreamWriter
StreamWriter(path: string, append: bool) : StreamWriter
StreamWriter(stream: Stream, encoding: Encoding, bufferSize: int) : StreamWriter
StreamWriter(path: string, append: bool, encoding: Encoding) : StreamWriter
StreamWriter(stream: Stream,?encoding: Encoding,?bufferSize: int,?leaveOpen: bool) : StreamWriter
StreamWriter(path: string, append: bool, encoding: Encoding, bufferSize: int) : StreamWriter
val eprintf : format:Printf.TextWriterFormat<'T> -> 'T
val getProxiesListFromFile : path:string -> string []
val path : string
val strContent : string
val strs : string []
type StringSplitOptions =
| None = 0
| RemoveEmptyEntries = 1
field StringSplitOptions.RemoveEmptyEntries: StringSplitOptions = 1
val sfiltered : string []
type Array =
member Clone : unit -> obj
member CopyTo : array:Array * index:int -> unit + 1 overload
member GetEnumerator : unit -> IEnumerator
member GetLength : dimension:int -> int
member GetLongLength : dimension:int -> int64
member GetLowerBound : dimension:int -> int
member GetUpperBound : dimension:int -> int
member GetValue : [<ParamArray>] indices:int[] -> obj + 7 overloads
member Initialize : unit -> unit
member IsFixedSize : bool
...
val map : mapping:('T -> 'U) -> array:'T [] -> 'U []
val x : string
val defaultValue : proxyUrl:string -> Async<string * bool * float>
val proxyUrl : string
val async : AsyncBuilder
val runHttpClient : url:string * urlProxy:string -> Async<string * bool * float>
val url : string
val urlProxy : string
val stopWatch : Diagnostics.Stopwatch
namespace System.Diagnostics
Multiple items
type Stopwatch =
new : unit -> Stopwatch
member Elapsed : TimeSpan
member ElapsedMilliseconds : int64
member ElapsedTicks : int64
member IsRunning : bool
member Reset : unit -> unit
member Restart : unit -> unit
member Start : unit -> unit
member Stop : unit -> unit
static val Frequency : int64
...
--------------------
Diagnostics.Stopwatch() : Diagnostics.Stopwatch
Diagnostics.Stopwatch.StartNew() : Diagnostics.Stopwatch
val ur : Uri
Multiple items
type Uri =
new : uriString:string -> Uri + 5 overloads
member AbsolutePath : string
member AbsoluteUri : string
member Authority : string
member DnsSafeHost : string
member Equals : comparand:obj -> bool
member Fragment : string
member GetComponents : components:UriComponents * format:UriFormat -> string
member GetHashCode : unit -> int
member GetLeftPart : part:UriPartial -> string
...
--------------------
Uri(uriString: string) : Uri
Uri(uriString: string, uriKind: UriKind) : Uri
Uri(baseUri: Uri, relativeUri: string) : Uri
Uri(baseUri: Uri, relativeUri: Uri) : Uri
val proxy : WebProxy
Multiple items
type WebProxy =
new : unit -> WebProxy + 9 overloads
member Address : Uri with get, set
member BypassArrayList : ArrayList
member BypassList : string[] with get, set
member BypassProxyOnLocal : bool with get, set
member Credentials : ICredentials with get, set
member GetProxy : destination:Uri -> Uri
member IsBypassed : host:Uri -> bool
member UseDefaultCredentials : bool with get, set
static member GetDefaultProxy : unit -> WebProxy
--------------------
WebProxy() : WebProxy
WebProxy(Address: Uri) : WebProxy
WebProxy(Address: string) : WebProxy
WebProxy(Address: Uri, BypassOnLocal: bool) : WebProxy
WebProxy(Host: string, Port: int) : WebProxy
WebProxy(Address: string, BypassOnLocal: bool) : WebProxy
WebProxy(Address: Uri, BypassOnLocal: bool, BypassList: string []) : WebProxy
WebProxy(Address: string, BypassOnLocal: bool, BypassList: string []) : WebProxy
WebProxy(Address: Uri, BypassOnLocal: bool, BypassList: string [], Credentials: ICredentials) : WebProxy
WebProxy(Address: string, BypassOnLocal: bool, BypassList: string [], Credentials: ICredentials) : WebProxy
val handler : HttpClientHandler
Multiple items
type HttpClientHandler =
inherit HttpMessageHandler
new : unit -> HttpClientHandler
member AllowAutoRedirect : bool with get, set
member AutomaticDecompression : DecompressionMethods with get, set
member CheckCertificateRevocationList : bool with get, set
member ClientCertificateOptions : ClientCertificateOption with get, set
member ClientCertificates : X509CertificateCollection
member CookieContainer : CookieContainer with get, set
member Credentials : ICredentials with get, set
member DefaultProxyCredentials : ICredentials with get, set
member MaxAutomaticRedirections : int with get, set
...
--------------------
HttpClientHandler() : HttpClientHandler
val client : HttpClient
Multiple items
type HttpClient =
inherit HttpMessageInvoker
new : unit -> HttpClient + 2 overloads
member BaseAddress : Uri with get, set
member CancelPendingRequests : unit -> unit
member DefaultRequestHeaders : HttpRequestHeaders
member DefaultRequestVersion : Version with get, set
member DeleteAsync : requestUri:string -> Task<HttpResponseMessage> + 3 overloads
member GetAsync : requestUri:string -> Task<HttpResponseMessage> + 7 overloads
member GetByteArrayAsync : requestUri:string -> Task<byte[]> + 1 overload
member GetStreamAsync : requestUri:string -> Task<Stream> + 1 overload
member GetStringAsync : requestUri:string -> Task<string> + 1 overload
...
--------------------
HttpClient() : HttpClient
HttpClient(handler: HttpMessageHandler) : HttpClient
HttpClient(handler: HttpMessageHandler, disposeHandler: bool) : HttpClient
Multiple items
type TimeSpan =
struct
new : ticks:int64 -> TimeSpan + 3 overloads
member Add : ts:TimeSpan -> TimeSpan
member CompareTo : value:obj -> int + 1 overload
member Days : int
member Divide : divisor:float -> TimeSpan + 1 overload
member Duration : unit -> TimeSpan
member Equals : value:obj -> bool + 1 overload
member GetHashCode : unit -> int
member Hours : int
member Milliseconds : int
...
end
--------------------
TimeSpan ()
TimeSpan(ticks: int64) : TimeSpan
TimeSpan(hours: int, minutes: int, seconds: int) : TimeSpan
TimeSpan(days: int, hours: int, minutes: int, seconds: int) : TimeSpan
TimeSpan(days: int, hours: int, minutes: int, seconds: int, milliseconds: int) : TimeSpan
TimeSpan.FromSeconds(value: float) : TimeSpan
val responseString : string
Multiple items
type Async =
static member AsBeginEnd : computation:('Arg -> Async<'T>) -> ('Arg * AsyncCallback * obj -> IAsyncResult) * (IAsyncResult -> 'T) * (IAsyncResult -> unit)
static member AwaitEvent : event:IEvent<'Del,'T> * ?cancelAction:(unit -> unit) -> Async<'T> (requires delegate and 'Del :> Delegate)
static member AwaitIAsyncResult : iar:IAsyncResult * ?millisecondsTimeout:int -> Async<bool>
static member AwaitTask : task:Task -> Async<unit>
static member AwaitTask : task:Task<'T> -> Async<'T>
static member AwaitWaitHandle : waitHandle:WaitHandle * ?millisecondsTimeout:int -> Async<bool>
static member CancelDefaultToken : unit -> unit
static member Catch : computation:Async<'T> -> Async<Choice<'T,exn>>
static member Choice : computations:seq<Async<'T option>> -> Async<'T option>
static member FromBeginEnd : beginAction:(AsyncCallback * obj -> IAsyncResult) * endAction:(IAsyncResult -> 'T) * ?cancelAction:(unit -> unit) -> Async<'T>
...
--------------------
type Async<'T> =
static member Async.AwaitTask : task:Tasks.Task -> Async<unit>
static member Async.AwaitTask : task:Tasks.Task<'T> -> Async<'T>
val ms : float
type Math =
static val E : float
static val PI : float
static member Abs : value:float -> float + 6 overloads
static member Acos : d:float -> float
static member Acosh : d:float -> float
static member Asin : d:float -> float
static member Asinh : d:float -> float
static member Atan : d:float -> float
static member Atan2 : y:float * x:float -> float
static member Atanh : d:float -> float
...
Math.Round(a: float) : float
Math.Round(d: decimal) : decimal
Math.Round(value: float, mode: MidpointRounding) : float
Math.Round(value: float, digits: int) : float
Math.Round(d: decimal, mode: MidpointRounding) : decimal
Math.Round(d: decimal, decimals: int) : decimal
Math.Round(value: float, digits: int, mode: MidpointRounding) : float
Math.Round(d: decimal, decimals: int, mode: MidpointRounding) : decimal
val runAsyncWithTimeout : urlPoint:string * urlProxy:string -> Async<string * bool * float>
val urlPoint : string
val child : Choice<Async<string * bool * float>,exn>
static member Async.Catch : computation:Async<'T> -> Async<Choice<'T,exn>>
static member Async.StartChild : computation:Async<'T> * ?millisecondsTimeout:int -> Async<Async<'T>>
union case Choice.Choice1Of2: 'T1 -> Choice<'T1,'T2>
val myValue : Async<string * bool * float>
union case Choice.Choice2Of2: 'T2 -> Choice<'T1,'T2>
Multiple items
type EntryPointAttribute =
inherit Attribute
new : unit -> EntryPointAttribute
--------------------
new : unit -> EntryPointAttribute
val main : argv:string [] -> int
val argv : string []
val sf : string []
val tasks : Async<string * bool * float> list
val parallelTasks : Async<(string * bool * float) []>
static member Async.Parallel : computations:seq<Async<'T>> -> Async<'T []>
static member Async.Parallel : computations:seq<Async<'T>> * ?maxDegreeOfParallelism:int -> Async<'T []>
type Console =
static member BackgroundColor : ConsoleColor with get, set
static member Beep : unit -> unit + 1 overload
static member BufferHeight : int with get, set
static member BufferWidth : int with get, set
static member CapsLock : bool
static member Clear : unit -> unit
static member CursorLeft : int with get, set
static member CursorSize : int with get, set
static member CursorTop : int with get, set
static member CursorVisible : bool with get, set
...
Console.WriteLine() : unit
(+0 other overloads)
Console.WriteLine(value: string) : unit
(+0 other overloads)
Console.WriteLine(value: obj) : unit
(+0 other overloads)
Console.WriteLine(value: uint64) : unit
(+0 other overloads)
Console.WriteLine(value: int64) : unit
(+0 other overloads)
Console.WriteLine(value: uint32) : unit
(+0 other overloads)
Console.WriteLine(value: int) : unit
(+0 other overloads)
Console.WriteLine(value: float32) : unit
(+0 other overloads)
Console.WriteLine(value: float) : unit
(+0 other overloads)
Console.WriteLine(value: decimal) : unit
(+0 other overloads)
val str : (string * bool * float) []
static member Async.RunSynchronously : computation:Async<'T> * ?timeout:int * ?cancellationToken:CancellationToken -> 'T
val sm : (string * bool * float) []
val filter : predicate:('T -> bool) -> array:'T [] -> 'T []
val y : bool
val z : float
val sortBy : projection:('T -> 'Key) -> array:'T [] -> 'T [] (requires comparison)
val iter : action:('T -> unit) -> array:'T [] -> unit
val results : string
Multiple items
type String =
new : value:char[] -> string + 8 overloads
member Chars : int -> char
member Clone : unit -> obj
member CompareTo : value:obj -> int + 1 overload
member Contains : value:string -> bool + 3 overloads
member CopyTo : sourceIndex:int * destination:char[] * destinationIndex:int * count:int -> unit
member EndsWith : value:string -> bool + 3 overloads
member EnumerateRunes : unit -> StringRuneEnumerator
member Equals : obj:obj -> bool + 2 overloads
member GetEnumerator : unit -> CharEnumerator
...
--------------------
String(value: char []) : String
String(value: nativeptr<char>) : String
String(value: nativeptr<sbyte>) : String
String(value: ReadOnlySpan<char>) : String
String(c: char, count: int) : String
String(value: char [], startIndex: int, length: int) : String
String(value: nativeptr<char>, startIndex: int, length: int) : String
String(value: nativeptr<sbyte>, startIndex: int, length: int) : String
String(value: nativeptr<sbyte>, startIndex: int, length: int, enc: Encoding) : String
val concat : sep:string -> strings:seq<string> -> string
val count : int
Multiple items
val int : value:'T -> int (requires member op_Explicit)
--------------------
type int = int32
--------------------
type int<'Measure> = int
val length : array:'T [] -> int
Console.ReadLine() : string
val ignore : value:'T -> unit
More information