Fix url slashes problem for urls used in the type provider's sample parameter (fixes #247)

FSharp.Data.sln

@@ -1,7 +1,7 @@
 
 Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio 2013
-VisualStudioVersion = 12.0.20827.3
+VisualStudioVersion = 12.0.21005.1
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{36E72EB1-5847-4B38-8993-B951648CB0D9}"
 	ProjectSection(SolutionItems) = preProject

RELEASE_NOTES.md

@@ -97,3 +97,4 @@
 * Fix generated code doing repeated work.
 * Windows Phone 7 no longer supported.
 * CsvInference is now part of the runtime so it can be reused.
+* Fixed problem when using uri's with encoded slashes (%2F) in the sample parameter of CsvProvider, JsonProvider & XmlProvider

src/CommonRuntime/IO.fs

@@ -8,6 +8,7 @@ module FSharp.Data.Runtime.IO
 open System
 open System.IO
 open System.Net
+open FSharp.Net
 
 type internal UriResolutionType =
     | DesignTime
@@ -60,23 +61,6 @@ type internal UriResolver =
         Uri(Path.Combine(root, uri.OriginalString), UriKind.Absolute), false
 #endif
 
-/// consumes a stream asynchronously until the end
-/// and returns a memory stream with the full content
-let internal asyncRead (stream:Stream) = async {
-    // Allocate 4kb buffer for downloading data
-    let buffer = Array.zeroCreate (4 * 1024)
-    let output = new MemoryStream()
-    let reading = ref true
-
-    while reading.Value do
-      // Download one (at most) 4kb chunk and copy it
-      let! count = stream.AsyncRead(buffer, 0, buffer.Length)
-      output.Write(buffer, 0, count)
-      reading := count > 0
-
-    output.Seek(0L, SeekOrigin.Begin) |> ignore 
-    return output 
-}
 
 #if FX_NO_LOCAL_FILESYSTEM
 #else
@@ -100,16 +84,9 @@ let private watchForChanges (uri:Uri) (invalidate, addDisposer:IDisposable->unit
 /// and sets up a filesystem watcher that calls the invalidate function whenever the file changes
 let internal asyncOpenStream (invalidate:((unit->unit)*(IDisposable->unit)) option) (uriResolver:UriResolver) (uri:Uri) = async {
   let uri, isWeb = uriResolver.Resolve uri
-  if isWeb then 
-    let req = WebRequest.Create(uri) :?> HttpWebRequest
-#if FX_NO_WEBREQUEST_AUTOMATICDECOMPRESSION
-#else
-    req.AutomaticDecompression <- DecompressionMethods.Deflate ||| DecompressionMethods.GZip
-#endif
-    use! resp = Async.FromBeginEnd(req.BeginGetResponse, req.EndGetResponse)
-    use stream = resp.GetResponseStream()
-    let! memoryStream = asyncRead stream
-    return memoryStream :> Stream
+  if isWeb then
+    let! stream = Http.InnerRequest(uri.OriginalString, fun _ _ _ _ _ stream -> stream)
+    return stream :> Stream
   else
 #if FX_NO_LOCAL_FILESYSTEM
     return failwith "Only web locations are supported"

src/FSharp.Data.DesignTime.Silverlight.fsproj

@@ -48,6 +48,7 @@
     <DocumentationFile>bin\sl5-compiler\Release\FSharp.Data.DesignTime.xml</DocumentationFile>
   </PropertyGroup>
   <ItemGroup>
+    <Compile Include="Net\Http.fs" />
     <Compile Include="CommonRuntime\IO.fs" />
     <Compile Include="CommonRuntime\Caching.fs" />
     <Compile Include="CommonRuntime\HttpUtils.fs" />
@@ -65,7 +66,6 @@
     <Compile Include="CommonProviderImplementation\Helpers.fs" />
     <Compile Include="CommonProviderImplementation\ConversionsGenerator.fs" />
     <Compile Include="CommonProviderImplementation\Debug.fs" />
-    <Compile Include="Net\Http.fs" />
     <Compile Include="Json\JsonValue.fs" />
     <Compile Include="Json\JsonConversions.fs" />
     <Compile Include="Json\JsonExtensions.fs" />

src/FSharp.Data.DesignTime.fsproj

@@ -38,6 +38,7 @@
     <DocumentationFile>bin\Release\FSharp.Data.DesignTime.xml</DocumentationFile>
   </PropertyGroup>
   <ItemGroup>
+    <Compile Include="Net\Http.fs" />
     <Compile Include="CommonRuntime\IO.fs" />
     <Compile Include="CommonRuntime\Caching.fs" />
     <Compile Include="CommonRuntime\HttpUtils.fs" />
@@ -55,7 +56,6 @@
     <Compile Include="CommonProviderImplementation\Helpers.fs" />
     <Compile Include="CommonProviderImplementation\ConversionsGenerator.fs" />
     <Compile Include="CommonProviderImplementation\Debug.fs" />
-    <Compile Include="Net\Http.fs" />
     <Compile Include="Json\JsonValue.fs" />
     <Compile Include="Json\JsonConversions.fs" />
     <Compile Include="Json\JsonExtensions.fs" />

src/FSharp.Data.Experimental.DesignTime.Silverlight.fsproj

@@ -48,6 +48,7 @@
     <DocumentationFile>bin\sl5-compiler\Release\FSharp.Data.Experimental.DesignTime.xml</DocumentationFile>
   </PropertyGroup>
   <ItemGroup>
+    <Compile Include="Net\Http.fs" />
     <Compile Include="CommonRuntime\IO.fs" />
     <Compile Include="CommonRuntime\Caching.fs" />
     <Compile Include="CommonRuntime\HttpUtils.fs" />
@@ -65,7 +66,6 @@
     <Compile Include="CommonProviderImplementation\Helpers.fs" />
     <Compile Include="CommonProviderImplementation\ConversionsGenerator.fs" />
     <Compile Include="CommonProviderImplementation\Debug.fs" />
-    <Compile Include="Net\Http.fs" />
     <Compile Include="Json\JsonValue.fs" />
     <Compile Include="Json\JsonConversions.fs" />
     <Compile Include="Json\JsonExtensions.fs" />

src/FSharp.Data.Experimental.DesignTime.fsproj

@@ -38,6 +38,7 @@
     <DocumentationFile>bin\Release\FSharp.Data.Experimental.DesignTime.xml</DocumentationFile>
   </PropertyGroup>
   <ItemGroup>
+    <Compile Include="Net\Http.fs" />
     <Compile Include="CommonRuntime\IO.fs" />
     <Compile Include="CommonRuntime\Caching.fs" />
     <Compile Include="CommonRuntime\HttpUtils.fs" />
@@ -55,7 +56,6 @@
     <Compile Include="CommonProviderImplementation\Helpers.fs" />
     <Compile Include="CommonProviderImplementation\ConversionsGenerator.fs" />
     <Compile Include="CommonProviderImplementation\Debug.fs" />
-    <Compile Include="Net\Http.fs" />
     <Compile Include="Json\JsonValue.fs" />
     <Compile Include="Json\JsonConversions.fs" />
     <Compile Include="Json\JsonExtensions.fs" />

src/FSharp.Data.Experimental.Portable.fsproj

@@ -37,14 +37,14 @@
     <DocumentationFile>bin\portable\Release\FSharp.Data.Experimental.xml</DocumentationFile>
   </PropertyGroup>
   <ItemGroup>
+    <Compile Include="Net\Http.fs" />
     <Compile Include="CommonRuntime\IO.fs" />
     <Compile Include="CommonRuntime\Caching.fs" />
     <Compile Include="CommonRuntime\HttpUtils.fs" />
     <Compile Include="CommonRuntime\TextConversions.fs" />
     <Compile Include="CommonRuntime\TextRuntime.fs" />
     <Compile Include="CommonRuntime\StructuralTypes.fs" />
     <Compile Include="CommonRuntime\StructuralInference.fs" />
-    <Compile Include="Net\Http.fs" />
     <Compile Include="Json\JsonValue.fs" />
     <Compile Include="Json\JsonConversions.fs" />
     <Compile Include="Json\JsonExtensions.fs" />

src/FSharp.Data.Experimental.fsproj

@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
   <PropertyGroup>
@@ -38,14 +38,14 @@
     <DocumentationFile>bin\Release\FSharp.Data.Experimental.xml</DocumentationFile>
   </PropertyGroup>
   <ItemGroup>
+    <Compile Include="Net\Http.fs" />
     <Compile Include="CommonRuntime\IO.fs" />
     <Compile Include="CommonRuntime\Caching.fs" />
     <Compile Include="CommonRuntime\HttpUtils.fs" />
     <Compile Include="CommonRuntime\TextConversions.fs" />
     <Compile Include="CommonRuntime\TextRuntime.fs" />
     <Compile Include="CommonRuntime\StructuralTypes.fs" />
     <Compile Include="CommonRuntime\StructuralInference.fs" />
-    <Compile Include="Net\Http.fs" />
     <Compile Include="Json\JsonValue.fs" />
     <Compile Include="Json\JsonConversions.fs" />
     <Compile Include="Json\JsonExtensions.fs" />
@@ -73,10 +73,10 @@
   </Target>  -->
   <Target Name="AfterBuild">
     <Copy SourceFiles="$(ProjectDir)$(OutDir)$(TargetName)$(TargetExt)" DestinationFolder="$(ProjectDir)..\bin\v40" />
-    <Copy SourceFiles="$(ProjectDir)$(OutDir)$(TargetName).pdb" DestinationFolder="$(ProjectDir)..\bin\v40" Condition="Exists('$(ProjectDir)$(OutDir)$(TargetName).pdb')"  />
+    <Copy SourceFiles="$(ProjectDir)$(OutDir)$(TargetName).pdb" DestinationFolder="$(ProjectDir)..\bin\v40" Condition="Exists('$(ProjectDir)$(OutDir)$(TargetName).pdb')" />
     <Copy SourceFiles="$(ProjectDir)$(OutDir)$(TargetName).xml" DestinationFolder="$(ProjectDir)..\bin\v40" />
     <Copy SourceFiles="$(ProjectDir)$(OutDir)$(TargetName)$(TargetExt)" DestinationFolder="$(ProjectDir)..\bin" />
     <Copy SourceFiles="$(ProjectDir)$(OutDir)$(TargetName).pdb" DestinationFolder="$(ProjectDir)..\bin" Condition="Exists('$(ProjectDir)$(OutDir)$(TargetName).pdb')" />
     <Copy SourceFiles="$(ProjectDir)$(OutDir)$(TargetName).xml" DestinationFolder="$(ProjectDir)..\bin" />
   </Target>
-</Project>
+</Project>

src/FSharp.Data.Portable.fsproj

@@ -37,6 +37,7 @@
     <DocumentationFile>bin\portable\Release\FSharp.Data.xml</DocumentationFile>
   </PropertyGroup>
   <ItemGroup>
+    <Compile Include="Net\Http.fs" />
     <Compile Include="CommonRuntime\IO.fs" />
     <Compile Include="CommonRuntime\Caching.fs" />
     <Compile Include="CommonRuntime\HttpUtils.fs" />
@@ -46,7 +47,6 @@
     <Compile Include="CommonRuntime\StructuralInference.fs" />
     <Compile Include="CommonRuntime\Pluralizer.fs" />
     <Compile Include="CommonRuntime\NameUtils.fs" />
-    <Compile Include="Net\Http.fs" />
     <Compile Include="Json\JsonValue.fs" />
     <Compile Include="Json\JsonConversions.fs" />
     <Compile Include="Json\JsonExtensions.fs" />

src/FSharp.Data.Silverlight.fsproj

@@ -48,6 +48,7 @@
     <DocumentationFile>bin\sl5\Release\FSharp.Data.xml</DocumentationFile>
   </PropertyGroup>
   <ItemGroup>
+    <Compile Include="Net\Http.fs" />
     <Compile Include="CommonRuntime\IO.fs" />
     <Compile Include="CommonRuntime\Caching.fs" />
     <Compile Include="CommonRuntime\HttpUtils.fs" />
@@ -57,7 +58,6 @@
     <Compile Include="CommonRuntime\StructuralInference.fs" />
     <Compile Include="CommonRuntime\Pluralizer.fs" />
     <Compile Include="CommonRuntime\NameUtils.fs" />
-    <Compile Include="Net\Http.fs" />
     <Compile Include="Json\JsonValue.fs" />
     <Compile Include="Json\JsonConversions.fs" />
     <Compile Include="Json\JsonExtensions.fs" />

src/FSharp.Data.fsproj

@@ -38,6 +38,7 @@
     <DocumentationFile>bin\Release\FSharp.Data.xml</DocumentationFile>
   </PropertyGroup>
   <ItemGroup>
+    <Compile Include="Net\Http.fs" />
     <Compile Include="CommonRuntime\IO.fs" />
     <Compile Include="CommonRuntime\Caching.fs" />
     <Compile Include="CommonRuntime\HttpUtils.fs" />
@@ -47,7 +48,6 @@
     <Compile Include="CommonRuntime\StructuralInference.fs" />
     <Compile Include="CommonRuntime\Pluralizer.fs" />
     <Compile Include="CommonRuntime\NameUtils.fs" />
-    <Compile Include="Net\Http.fs" />
     <Compile Include="Json\JsonValue.fs" />
     <Compile Include="Json\JsonConversions.fs" />
     <Compile Include="Json\JsonExtensions.fs" />

src/Net/Http.fs

@@ -9,7 +9,6 @@ open System.IO
 open System.Net
 open System.Text
 open System.Reflection
-open FSharp.Data.Runtime.IO
 
 [<RequireQualifiedAccess>]
 /// The body to send in an HTTP request
@@ -62,6 +61,24 @@ type Http private() =
 #endif
     uri
 
+  /// consumes a stream asynchronously until the end
+  /// and returns a memory stream with the full content
+  static let asyncRead (stream:Stream) = async {
+      // Allocate 4kb buffer for downloading data
+      let buffer = Array.zeroCreate (4 * 1024)
+      let output = new MemoryStream()
+      let reading = ref true
+
+      while reading.Value do
+        // Download one (at most) 4kb chunk and copy it
+        let! count = stream.AsyncRead(buffer, 0, buffer.Length)
+        output.Write(buffer, 0, count)
+        reading := count > 0
+
+      output.Seek(0L, SeekOrigin.Begin) |> ignore 
+      return output 
+  }
+
   static let writeBody (req:HttpWebRequest) (postBytes:byte[]) = async { 
 #if FX_NO_WEBREQUEST_CONTENTLENGTH
 #else
@@ -106,10 +123,39 @@ type Http private() =
         return Unchecked.defaultof<_>
   }
 
+  static let toHttpResponse forceText responseUrl contentType statusCode cookies headers (memoryStream:MemoryStream) =
+
+    let isText (mimeType:string) =
+      let isText (mimeType:string) =
+        let mimeType = mimeType.Trim()
+        mimeType.StartsWith "text/" || 
+        mimeType = "application/json" || 
+        mimeType = "application/xml" ||
+        mimeType = "application/javascript" ||
+        mimeType = "application/ecmascript" ||
+        mimeType = "application/xml-dtd" ||
+        mimeType.StartsWith "application/" && mimeType.EndsWith "+xml"
+      mimeType.Split([| ';' |], StringSplitOptions.RemoveEmptyEntries)
+      |> Array.exists isText
+
+    let respBody = 
+      use memoryStream = memoryStream
+      if forceText || (isText contentType) then
+          use sr = new StreamReader(memoryStream)
+          sr.ReadToEnd() |> ResponseBody.Text
+      else
+          memoryStream.ToArray() |> ResponseBody.Binary
+
+    { Body = respBody
+      Headers = headers
+      ResponseUrl = responseUrl
+      Cookies = cookies
+      StatusCode = statusCode }
+
 #if FX_NO_WEBREQUEST_CLIENTCERTIFICATES
-  static member private InnerRequest(url:string, forceText, ?query, ?headers, ?meth, ?body, ?cookies, ?cookieContainer) = async {
+  static member internal InnerRequest(url:string, toHttpResponse, ?query, ?headers, ?meth, ?body, ?cookies, ?cookieContainer) = async {
 #else
-  static member private InnerRequest(url:string, forceText, ?query, ?headers, ?meth, ?body, ?cookies, ?cookieContainer, ?certificate) = async {
+  static member internal InnerRequest(url:string, toHttpResponse, ?query, ?headers, ?meth, ?body, ?cookies, ?cookieContainer, ?certificate) = async {
 #endif
     // Format query parameters
     let url = 
@@ -190,41 +236,18 @@ type Http private() =
         do! writeBody req bytes
     | None -> ()
 
-    let isText (mimeType:string) =
-        let isText (mimeType:string) =
-            let mimeType = mimeType.Trim()
-            mimeType.StartsWith "text/" || 
-            mimeType = "application/json" || 
-            mimeType = "application/xml" ||
-            mimeType = "application/javascript" ||
-            mimeType = "application/ecmascript" ||
-            mimeType = "application/xml-dtd" ||
-            mimeType.StartsWith "application/" && mimeType.EndsWith "+xml"
-        mimeType.Split([| ';' |], StringSplitOptions.RemoveEmptyEntries)
-        |> Array.exists isText
-
     // Send the request and get the response
     return! augmentWebExceptionsWithDetails <| fun () -> async {
       use! resp = Async.FromBeginEnd(req.BeginGetResponse, req.EndGetResponse)
-      use networkStream = resp.GetResponseStream()
-      use! memoryStream = asyncRead networkStream
-      let respBody = 
-        if forceText || (isText resp.ContentType) then
-            use sr = new StreamReader(memoryStream)
-            sr.ReadToEnd() |> ResponseBody.Text
-        else
-            memoryStream.ToArray() |> ResponseBody.Binary
+      let statusCode = 
+          match resp with
+          | :? HttpWebResponse as resp -> int resp.StatusCode
+          | _ -> 0
       let cookies = Map.ofList [ for cookie in cookieContainer.GetCookies uri |> Seq.cast<Cookie> -> cookie.Name, cookie.Value ]  
       let headers = Map.ofList [ for header in resp.Headers.AllKeys -> header, resp.Headers.[header] ]
-      let statusCode = 
-        match resp with
-        | :? HttpWebResponse as resp -> int resp.StatusCode
-        | _ -> 0
-      return { Body = respBody
-               Headers = headers
-               ResponseUrl = resp.ResponseUri.OriginalString
-               Cookies = cookies
-               StatusCode = statusCode } }
+      use networkStream = resp.GetResponseStream()
+      let! memoryStream = asyncRead networkStream
+      return toHttpResponse resp.ResponseUri.OriginalString resp.ContentType statusCode cookies headers memoryStream }
   }
 
   /// Download an HTTP web resource from the specified URL asynchronously
@@ -234,10 +257,10 @@ type Http private() =
   /// that will be encoded, and the method will automatically be set if not specified
 #if FX_NO_WEBREQUEST_CLIENTCERTIFICATES
   static member AsyncRequest(url, ?query, ?headers, ?meth, ?body, ?cookies, ?cookieContainer) = 
-    Http.InnerRequest(url, false, ?query=query, ?headers=headers, ?meth=meth, ?body=body, ?cookies=cookies, ?cookieContainer=cookieContainer)
+    Http.InnerRequest(url, toHttpResponse false, ?query=query, ?headers=headers, ?meth=meth, ?body=body, ?cookies=cookies, ?cookieContainer=cookieContainer)
 #else
   static member AsyncRequest(url, ?query, ?headers, ?meth, ?body, ?cookies, ?cookieContainer, ?certificate) = 
-    Http.InnerRequest(url, false, ?query=query, ?headers=headers, ?meth=meth, ?body=body, ?cookies=cookies, ?cookieContainer=cookieContainer, ?certificate=certificate)
+    Http.InnerRequest(url, toHttpResponse false, ?query=query, ?headers=headers, ?meth=meth, ?body=body, ?cookies=cookies, ?cookieContainer=cookieContainer, ?certificate=certificate)
 #endif
 
   /// Download an HTTP web resource from the specified URL asynchronously
@@ -247,10 +270,10 @@ type Http private() =
   /// that will be encoded, and the method will automatically be set if not specified
 #if FX_NO_WEBREQUEST_CLIENTCERTIFICATES
   static member AsyncRequestString(url, ?query, ?headers, ?meth, ?body, ?cookies, ?cookieContainer) = async {
-    let! response = Http.InnerRequest(url, true, ?query=query, ?headers=headers, ?meth=meth, ?body=body, ?cookies=cookies, ?cookieContainer=cookieContainer)
+    let! response = Http.InnerRequest(url, toHttpResponse true, ?query=query, ?headers=headers, ?meth=meth, ?body=body, ?cookies=cookies, ?cookieContainer=cookieContainer)
 #else
   static member AsyncRequestString(url, ?query, ?headers, ?meth, ?body, ?cookies, ?cookieContainer, ?certificate)  = async {
-    let! response = Http.InnerRequest(url, true, ?query=query, ?headers=headers, ?meth=meth, ?body=body, ?cookies=cookies, ?cookieContainer=cookieContainer, ?certificate=certificate)
+    let! response = Http.InnerRequest(url, toHttpResponse true, ?query=query, ?headers=headers, ?meth=meth, ?body=body, ?cookies=cookies, ?cookieContainer=cookieContainer, ?certificate=certificate)
 #endif
     return
         match response.Body with