0 people like it.

Download Yahoo! Finance data

Routine for downloading Yahoo! Finance data as a tuple consisting of a symbol name and data matrix.

 1: 
 2: 
 3: 
 4: 
 5: 
 6: 
 7: 
 8: 
 9: 
10: 
11: 
12: 
13: 
14: 
15: 
16: 
17: 
18: 
19: 
20: 
21: 
22: 
23: 
24: 
25: 
26: 
27: 
28: 
29: 
30: 
31: 
32: 
33: 
34: 
35: 
36: 
37: 
38: 
39: 
40: 
41: 
42: 
43: 
44: 
45: 
46: 
47: 
48: 
49: 
50: 
51: 
52: 
53: 
54: 
55: 
56: 
57: 
58: 
59: 
60: 
61: 
62: 
63: 
64: 
65: 
66: 
67: 
68: 
69: 
70: 
71: 
72: 
73: 
74: 
75: 
76: 
77: 
78: 
79: 
80: 
81: 
82: 
83: 
84: 
85: 
86: 
87: 
88: 

open System
open System.IO
open System.Xml
open System.Text
open System.Net
open System.Globalization
open System.Text.RegularExpressions

let UnixTimestampToDateTime unixTimeStamp =
    DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc).AddSeconds(unixTimeStamp).ToLocalTime()

let DateTimeToUnixTimestamp (dateTime: DateTime) =
    (dateTime.ToUniversalTime() - DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc)).TotalSeconds

let GetHistoricalData symbol (startDate: DateTime) (endDate: DateTime) =
    let url1 = new Uri("https://finance.yahoo.com/quote/" + symbol + "/history?p=" + symbol)
    let request = WebRequest.Create (url1) :?> HttpWebRequest
    request.CookieContainer <- new CookieContainer()
    request.Method <- "GET"
    let response = request.GetResponse() :?> HttpWebResponse
    let cookie = response.GetResponseHeader("Set-Cookie").Split(';').[0]
    use stream = response.GetResponseStream()
    use reader = new StreamReader(stream)
    let html = reader.ReadToEnd()
    let regexCrumb = new Regex("CrumbStore\":{\"crumb\":\"(?<crumb>.+?)\"}", 
                        RegexOptions.CultureInvariant ||| RegexOptions.Compiled, TimeSpan.FromSeconds(5.0))
    let matches = regexCrumb.Matches(html)
    
    let crumb = 
        match matches.Count with
        | 0 -> None
        | _ -> Some (matches.Item(0).Groups.["crumb"].Value)

    let url2 = "https://query1.finance.yahoo.com/v7/finance/download/" + symbol +
                "?period1=" + Math.Round(DateTimeToUnixTimestamp(startDate), 0).ToString() + "&period2=" +
                Math.Round(DateTimeToUnixTimestamp(endDate), 0).ToString() + 
                "&interval=1d&events=history&crumb=" + crumb.Value

    use wc = new WebClient()
    wc.Headers.Add(HttpRequestHeader.Cookie, cookie)
    wc.DownloadString(url2).Split('\n')
    |> Array.skip 1

let AsyncGetHistData symbol (startDate: DateTime) (endDate: DateTime) =
    async {
        printfn "Downloading symbol %s..." symbol
        let mutable data = Array.empty

        while data.Length = 0 do
            let url1 = new Uri("https://finance.yahoo.com/quote/" + symbol + "/history?p=" + symbol)
            let request = WebRequest.Create (url1) :?> HttpWebRequest
            request.CookieContainer <- new CookieContainer()
            request.Method <- "GET"
            use! response = request.AsyncGetResponse()
            let resp = response :?> HttpWebResponse
            let cookie = resp.GetResponseHeader("Set-Cookie").Split(';').[0]
            use stream = resp.GetResponseStream()
            use reader = new StreamReader(stream)
            let html = reader.ReadToEnd()
            let regexCrumb = new Regex("CrumbStore\":{\"crumb\":\"(?<crumb>.+?)\"}", 
                                RegexOptions.CultureInvariant ||| RegexOptions.Compiled, TimeSpan.FromSeconds(5.0))
            let matches = regexCrumb.Matches(html)
    
            let crumb = 
                match matches.Count with
                | 0 -> None
                | _ -> Some (matches.Item(0).Groups.["crumb"].Value)

            let url2 = "https://query1.finance.yahoo.com/v7/finance/download/" + symbol +
                        "?period1=" + Math.Round(DateTimeToUnixTimestamp(startDate), 0).ToString() + "&period2=" +
                        Math.Round(DateTimeToUnixTimestamp(endDate), 0).ToString() + 
                        "&interval=1d&events=history&crumb=" + crumb.Value

            use wc = new WebClient()
            wc.Headers.Add(HttpRequestHeader.Cookie, cookie)

            data <- 
                try
                    wc.DownloadString(url2).Split('\n')
                    |> Array.skip 1
                with
                | _ -> 
                    printfn "Symbol %s download failed, retrying..." symbol
                    Array.empty
        
        printfn "Symbol download %s succeeded." symbol
        return (symbol, data)
    }
namespace System
namespace System.IO
namespace System.Xml
namespace System.Text
namespace System.Net
namespace System.Globalization
namespace System.Text.RegularExpressions
val UnixTimestampToDateTime : unixTimeStamp:float -> DateTime

Full name: Script.UnixTimestampToDateTime
val unixTimeStamp : float
Multiple items
type DateTime =
  struct
    new : ticks:int64 -> DateTime + 10 overloads
    member Add : value:TimeSpan -> DateTime
    member AddDays : value:float -> DateTime
    member AddHours : value:float -> DateTime
    member AddMilliseconds : value:float -> DateTime
    member AddMinutes : value:float -> DateTime
    member AddMonths : months:int -> DateTime
    member AddSeconds : value:float -> DateTime
    member AddTicks : value:int64 -> DateTime
    member AddYears : value:int -> DateTime
    ...
  end

Full name: System.DateTime

--------------------
DateTime()
   (+0 other overloads)
DateTime(ticks: int64) : unit
   (+0 other overloads)
DateTime(ticks: int64, kind: DateTimeKind) : unit
   (+0 other overloads)
DateTime(year: int, month: int, day: int) : unit
   (+0 other overloads)
DateTime(year: int, month: int, day: int, calendar: Calendar) : unit
   (+0 other overloads)
DateTime(year: int, month: int, day: int, hour: int, minute: int, second: int) : unit
   (+0 other overloads)
DateTime(year: int, month: int, day: int, hour: int, minute: int, second: int, kind: DateTimeKind) : unit
   (+0 other overloads)
DateTime(year: int, month: int, day: int, hour: int, minute: int, second: int, calendar: Calendar) : unit
   (+0 other overloads)
DateTime(year: int, month: int, day: int, hour: int, minute: int, second: int, millisecond: int) : unit
   (+0 other overloads)
DateTime(year: int, month: int, day: int, hour: int, minute: int, second: int, millisecond: int, kind: DateTimeKind) : unit
   (+0 other overloads)
type DateTimeKind =
  | Unspecified = 0
  | Utc = 1
  | Local = 2

Full name: System.DateTimeKind
field DateTimeKind.Utc = 1
val DateTimeToUnixTimestamp : dateTime:DateTime -> float

Full name: Script.DateTimeToUnixTimestamp
val dateTime : DateTime
DateTime.ToUniversalTime() : DateTime
val GetHistoricalData : symbol:string -> startDate:DateTime -> endDate:DateTime -> string []

Full name: Script.GetHistoricalData
val symbol : string
val startDate : DateTime
val endDate : DateTime
val url1 : Uri
Multiple items
type Uri =
  new : uriString:string -> Uri + 5 overloads
  member AbsolutePath : string
  member AbsoluteUri : string
  member Authority : string
  member DnsSafeHost : string
  member Equals : comparand:obj -> bool
  member Fragment : string
  member GetComponents : components:UriComponents * format:UriFormat -> string
  member GetHashCode : unit -> int
  member GetLeftPart : part:UriPartial -> string
  ...

Full name: System.Uri

--------------------
Uri(uriString: string) : unit
Uri(uriString: string, uriKind: UriKind) : unit
Uri(baseUri: Uri, relativeUri: string) : unit
Uri(baseUri: Uri, relativeUri: Uri) : unit
val request : HttpWebRequest
type WebRequest =
  inherit MarshalByRefObject
  member Abort : unit -> unit
  member AuthenticationLevel : AuthenticationLevel with get, set
  member BeginGetRequestStream : callback:AsyncCallback * state:obj -> IAsyncResult
  member BeginGetResponse : callback:AsyncCallback * state:obj -> IAsyncResult
  member CachePolicy : RequestCachePolicy with get, set
  member ConnectionGroupName : string with get, set
  member ContentLength : int64 with get, set
  member ContentType : string with get, set
  member Credentials : ICredentials with get, set
  member EndGetRequestStream : asyncResult:IAsyncResult -> Stream
  ...

Full name: System.Net.WebRequest
WebRequest.Create(requestUri: Uri) : WebRequest
WebRequest.Create(requestUriString: string) : WebRequest
type HttpWebRequest =
  inherit WebRequest
  member Abort : unit -> unit
  member Accept : string with get, set
  member AddRange : range:int -> unit + 7 overloads
  member Address : Uri
  member AllowAutoRedirect : bool with get, set
  member AllowWriteStreamBuffering : bool with get, set
  member AutomaticDecompression : DecompressionMethods with get, set
  member BeginGetRequestStream : callback:AsyncCallback * state:obj -> IAsyncResult
  member BeginGetResponse : callback:AsyncCallback * state:obj -> IAsyncResult
  member ClientCertificates : X509CertificateCollection with get, set
  ...

Full name: System.Net.HttpWebRequest
property HttpWebRequest.CookieContainer: CookieContainer
Multiple items
type CookieContainer =
  new : unit -> CookieContainer + 2 overloads
  member Add : cookie:Cookie -> unit + 3 overloads
  member Capacity : int with get, set
  member Count : int
  member GetCookieHeader : uri:Uri -> string
  member GetCookies : uri:Uri -> CookieCollection
  member MaxCookieSize : int with get, set
  member PerDomainCapacity : int with get, set
  member SetCookies : uri:Uri * cookieHeader:string -> unit
  static val DefaultCookieLimit : int
  ...

Full name: System.Net.CookieContainer

--------------------
CookieContainer() : unit
CookieContainer(capacity: int) : unit
CookieContainer(capacity: int, perDomainCapacity: int, maxCookieSize: int) : unit
property HttpWebRequest.Method: string
val response : HttpWebResponse
HttpWebRequest.GetResponse() : WebResponse
type HttpWebResponse =
  inherit WebResponse
  member CharacterSet : string
  member Close : unit -> unit
  member ContentEncoding : string
  member ContentLength : int64
  member ContentType : string
  member Cookies : CookieCollection with get, set
  member GetResponseHeader : headerName:string -> string
  member GetResponseStream : unit -> Stream
  member Headers : WebHeaderCollection
  member IsMutuallyAuthenticated : bool
  ...

Full name: System.Net.HttpWebResponse
val cookie : string
HttpWebResponse.GetResponseHeader(headerName: string) : string
val stream : Stream
HttpWebResponse.GetResponseStream() : Stream
val reader : StreamReader
Multiple items
type StreamReader =
  inherit TextReader
  new : stream:Stream -> StreamReader + 9 overloads
  member BaseStream : Stream
  member Close : unit -> unit
  member CurrentEncoding : Encoding
  member DiscardBufferedData : unit -> unit
  member EndOfStream : bool
  member Peek : unit -> int
  member Read : unit -> int + 1 overload
  member ReadLine : unit -> string
  member ReadToEnd : unit -> string
  ...

Full name: System.IO.StreamReader

--------------------
StreamReader(stream: Stream) : unit
StreamReader(path: string) : unit
StreamReader(stream: Stream, detectEncodingFromByteOrderMarks: bool) : unit
StreamReader(stream: Stream, encoding: Encoding) : unit
StreamReader(path: string, detectEncodingFromByteOrderMarks: bool) : unit
StreamReader(path: string, encoding: Encoding) : unit
StreamReader(stream: Stream, encoding: Encoding, detectEncodingFromByteOrderMarks: bool) : unit
StreamReader(path: string, encoding: Encoding, detectEncodingFromByteOrderMarks: bool) : unit
StreamReader(stream: Stream, encoding: Encoding, detectEncodingFromByteOrderMarks: bool, bufferSize: int) : unit
StreamReader(path: string, encoding: Encoding, detectEncodingFromByteOrderMarks: bool, bufferSize: int) : unit
val html : string
StreamReader.ReadToEnd() : string
val regexCrumb : Regex
Multiple items
type Regex =
  new : pattern:string -> Regex + 1 overload
  member GetGroupNames : unit -> string[]
  member GetGroupNumbers : unit -> int[]
  member GroupNameFromNumber : i:int -> string
  member GroupNumberFromName : name:string -> int
  member IsMatch : input:string -> bool + 1 overload
  member Match : input:string -> Match + 2 overloads
  member Matches : input:string -> MatchCollection + 1 overload
  member Options : RegexOptions
  member Replace : input:string * replacement:string -> string + 5 overloads
  ...

Full name: System.Text.RegularExpressions.Regex

--------------------
Regex(pattern: string) : unit
Regex(pattern: string, options: RegexOptions) : unit
type RegexOptions =
  | None = 0
  | IgnoreCase = 1
  | Multiline = 2
  | ExplicitCapture = 4
  | Compiled = 8
  | Singleline = 16
  | IgnorePatternWhitespace = 32
  | RightToLeft = 64
  | ECMAScript = 256
  | CultureInvariant = 512

Full name: System.Text.RegularExpressions.RegexOptions
field RegexOptions.CultureInvariant = 512
field RegexOptions.Compiled = 8
Multiple items
type TimeSpan =
  struct
    new : ticks:int64 -> TimeSpan + 3 overloads
    member Add : ts:TimeSpan -> TimeSpan
    member CompareTo : value:obj -> int + 1 overload
    member Days : int
    member Duration : unit -> TimeSpan
    member Equals : value:obj -> bool + 1 overload
    member GetHashCode : unit -> int
    member Hours : int
    member Milliseconds : int
    member Minutes : int
    ...
  end

Full name: System.TimeSpan

--------------------
TimeSpan()
TimeSpan(ticks: int64) : unit
TimeSpan(hours: int, minutes: int, seconds: int) : unit
TimeSpan(days: int, hours: int, minutes: int, seconds: int) : unit
TimeSpan(days: int, hours: int, minutes: int, seconds: int, milliseconds: int) : unit
TimeSpan.FromSeconds(value: float) : TimeSpan
val matches : MatchCollection
Regex.Matches(input: string) : MatchCollection
Regex.Matches(input: string, startat: int) : MatchCollection
val crumb : string option
property MatchCollection.Count: int
union case Option.None: Option<'T>
union case Option.Some: Value: 'T -> Option<'T>
property MatchCollection.Item: int -> Match
val url2 : string
type Math =
  static val PI : float
  static val E : float
  static member Abs : value:sbyte -> sbyte + 6 overloads
  static member Acos : d:float -> float
  static member Asin : d:float -> float
  static member Atan : d:float -> float
  static member Atan2 : y:float * x:float -> float
  static member BigMul : a:int * b:int -> int64
  static member Ceiling : d:decimal -> decimal + 1 overload
  static member Cos : d:float -> float
  ...

Full name: System.Math
Math.Round(d: decimal) : decimal
Math.Round(a: float) : float
Math.Round(d: decimal, mode: MidpointRounding) : decimal
Math.Round(d: decimal, decimals: int) : decimal
Math.Round(value: float, mode: MidpointRounding) : float
Math.Round(value: float, digits: int) : float
Math.Round(d: decimal, decimals: int, mode: MidpointRounding) : decimal
Math.Round(value: float, digits: int, mode: MidpointRounding) : float
property Option.Value: string
val wc : WebClient
Multiple items
type WebClient =
  inherit Component
  new : unit -> WebClient
  member BaseAddress : string with get, set
  member CachePolicy : RequestCachePolicy with get, set
  member CancelAsync : unit -> unit
  member Credentials : ICredentials with get, set
  member DownloadData : address:string -> byte[] + 1 overload
  member DownloadDataAsync : address:Uri -> unit + 1 overload
  member DownloadFile : address:string * fileName:string -> unit + 1 overload
  member DownloadFileAsync : address:Uri * fileName:string -> unit + 1 overload
  member DownloadString : address:string -> string + 1 overload
  ...

Full name: System.Net.WebClient

--------------------
WebClient() : unit
property WebClient.Headers: WebHeaderCollection
Collections.Specialized.NameValueCollection.Add(c: Collections.Specialized.NameValueCollection) : unit
WebHeaderCollection.Add(header: string) : unit
WebHeaderCollection.Add(name: string, value: string) : unit
WebHeaderCollection.Add(header: HttpResponseHeader, value: string) : unit
WebHeaderCollection.Add(header: HttpRequestHeader, value: string) : unit
type HttpRequestHeader =
  | CacheControl = 0
  | Connection = 1
  | Date = 2
  | KeepAlive = 3
  | Pragma = 4
  | Trailer = 5
  | TransferEncoding = 6
  | Upgrade = 7
  | Via = 8
  | Warning = 9
  ...

Full name: System.Net.HttpRequestHeader
field HttpRequestHeader.Cookie = 25
WebClient.DownloadString(address: Uri) : string
WebClient.DownloadString(address: string) : string
type Array =
  member Clone : unit -> obj
  member CopyTo : array:Array * index:int -> unit + 1 overload
  member GetEnumerator : unit -> IEnumerator
  member GetLength : dimension:int -> int
  member GetLongLength : dimension:int -> int64
  member GetLowerBound : dimension:int -> int
  member GetUpperBound : dimension:int -> int
  member GetValue : [<ParamArray>] indices:int[] -> obj + 7 overloads
  member Initialize : unit -> unit
  member IsFixedSize : bool
  ...

Full name: System.Array
val skip : count:int -> array:'T [] -> 'T []

Full name: Microsoft.FSharp.Collections.Array.skip
val AsyncGetHistData : symbol:string -> startDate:DateTime -> endDate:DateTime -> Async<string * string []>

Full name: Script.AsyncGetHistData
val async : AsyncBuilder

Full name: Microsoft.FSharp.Core.ExtraTopLevelOperators.async
val printfn : format:Printf.TextWriterFormat<'T> -> 'T

Full name: Microsoft.FSharp.Core.ExtraTopLevelOperators.printfn
val mutable data : string []
val empty<'T> : 'T []

Full name: Microsoft.FSharp.Collections.Array.empty
property Array.Length: int
val response : WebResponse
member WebRequest.AsyncGetResponse : unit -> Async<WebResponse>
val resp : HttpWebResponse
Raw view Test code New version

More information

Link:http://fssnip.net/7WA
Posted:5 years ago
Author:Dr. Martin Lockstrom
Tags: #yahoofinance , #download , #financialdata

This web site is created using F# and Suave web server. It is hosted on Azure and the source code is on GitHub. Contributions are welcome!

The first version of fssnip.net has been created by @tomaspetricek back in 2010. This web site is a new, open-source and contribution-friendly version.