diff --git a/src/Csv/CsvRuntime.fs b/src/Csv/CsvRuntime.fs index e62249ee0..489251cc8 100644 --- a/src/Csv/CsvRuntime.fs +++ b/src/Csv/CsvRuntime.fs @@ -49,8 +49,8 @@ module internal CsvReader = /// Reads a line with data that are separated using specified separators /// and may be quoted. Ends with newline or end of input. - let rec readLine data (chars: StringBuilder) = - match reader.Read() with + let rec readLine data (chars: StringBuilder) current = + match current with | -1 | Char '\r' | Char '\n' -> @@ -58,22 +58,20 @@ module internal CsvReader = item :: data | Separator -> let item = chars.ToString() - readLine (item :: data) (StringBuilder()) - | Quote -> readLine data (readString chars) - | Char c -> readLine data (chars.Append c) + readLine (item :: data) (StringBuilder()) (reader.Read()) + | Quote -> readLine data (readString chars) (reader.Read()) + | Char c -> readLine data (chars.Append c) (reader.Read()) /// Reads multiple lines from the input, skipping newline characters let rec readLines lineNumber = seq { - match reader.Peek() with + match reader.Read() with | -1 -> () | Char '\r' - | Char '\n' -> - reader.Read() |> ignore - yield! readLines lineNumber - | _ -> + | Char '\n' -> yield! readLines lineNumber + | current -> yield - readLine [] (StringBuilder()) + readLine [] (StringBuilder()) current |> List.rev |> Array.ofList, lineNumber diff --git a/tests/FSharp.Data.Core.Tests/CsvReader.fs b/tests/FSharp.Data.Core.Tests/CsvReader.fs index 2c714357c..bbd1760a1 100644 --- a/tests/FSharp.Data.Core.Tests/CsvReader.fs +++ b/tests/FSharp.Data.Core.Tests/CsvReader.fs @@ -72,3 +72,46 @@ let ``Quoted strings parsed correctly`` () = let expected = [|[|"12"; "a\n\rb"|]; [|"123"; "\"hello\" world"|]|] actual |> should equal expected + +[] +let ``Read all rows from non seekable slow network stream`` () = + + let data = """ABC;1 +DEF;2 +GHI;3 +"QUOTED";4 +;5""" + + let encoding = System.Text.Encoding.UTF8 + let bytes = data |> encoding.GetBytes + use memoryStream = new MemoryStream(bytes) + + use fakeNetworkStream = + { new System.IO.Stream () with + override _.CanRead: bool = memoryStream.CanRead + override _.CanSeek: bool = false + override _.CanWrite: bool = false + override _.Flush (): unit = memoryStream.Flush( ) + override _.Length: int64 = raise (System.NotSupportedException()) + override _.Position + with get (): int64 = memoryStream.Position + and set (v: int64): unit = raise (System.NotSupportedException ()) + override _.Read(buffer: byte[], offset: int, _: int): int = + memoryStream.Read (buffer, offset, 1 (* Ignores the count parameter and reads one byte only to simulate a slow network stream *)) + override _.ReadByte(): int = memoryStream.ReadByte () + override _.Seek(offset: int64, origin: SeekOrigin): int64 = raise (System.NotSupportedException ()) + override _.SetLength(value: int64): unit = raise (System.NotSupportedException ()) + override _.Write(buffer: byte[], offset: int, count: int): unit = raise (System.NotSupportedException ()) + override _.WriteByte(value: byte): unit = raise (System.NotSupportedException ()) } + + let reader = new StreamReader(fakeNetworkStream, encoding) + + let actual = readCsvFile reader ";" '"' |> Seq.map fst |> Array.ofSeq + let expected = + [| [| "ABC"; "1" |] + [| "DEF"; "2" |] + [| "GHI"; "3" |] + [| "QUOTED"; "4" |] + [| ""; "5" |] |] + + actual |> should equal expected \ No newline at end of file