diff --git a/apps/csv2sql/lib/csv2sql/data_transfer.ex b/apps/csv2sql/lib/csv2sql/data_transfer.ex index a488fd7..444d322 100644 --- a/apps/csv2sql/lib/csv2sql/data_transfer.ex +++ b/apps/csv2sql/lib/csv2sql/data_transfer.ex @@ -11,9 +11,11 @@ defmodule Csv2sql.DataTransfer do Helpers.print_msg("Begin data tranfer for file: " <> Path.basename(file)) insertion_chunk_size = Application.get_env(:csv2sql, Csv2sql.get_repo())[:insertion_chunk_size] + bom = :unicode.encoding_to_bom(:utf8) file |> File.stream!([:trim_bom]) + |> Stream.map(&String.replace_prefix(&1, bom, "")) |> CSV.parse_stream() |> Stream.chunk_every(insertion_chunk_size) |> Enum.each(fn data_chunk -> diff --git a/apps/csv2sql/lib/csv2sql/database.ex b/apps/csv2sql/lib/csv2sql/database.ex index da61727..df03f1d 100644 --- a/apps/csv2sql/lib/csv2sql/database.ex +++ b/apps/csv2sql/lib/csv2sql/database.ex @@ -94,7 +94,7 @@ defmodule Csv2sql.Database do Enum.map(data_chunk, fn chunk -> Enum.map(chunk, fn {col, val} -> val = - if val == "" do + if String.trim(val) == "" do nil else case(types[col]) do diff --git a/apps/csv2sql/lib/csv2sql/import_validator.ex b/apps/csv2sql/lib/csv2sql/import_validator.ex index bc210a0..6d2ba6e 100644 --- a/apps/csv2sql/lib/csv2sql/import_validator.ex +++ b/apps/csv2sql/lib/csv2sql/import_validator.ex @@ -51,8 +51,11 @@ defmodule Csv2sql.ImportValidator do Get row count in csv file """ def get_count_from_csv(file) do + bom = :unicode.encoding_to_bom(:utf8) + file |> File.stream!([:trim_bom]) + |> Stream.map(&String.replace_prefix(&1, bom, "")) |> CSV.parse_stream() |> Enum.count() end diff --git a/apps/csv2sql/lib/csv2sql/schema_maker.ex b/apps/csv2sql/lib/csv2sql/schema_maker.ex index 48779b4..e0a2547 100644 --- a/apps/csv2sql/lib/csv2sql/schema_maker.ex +++ b/apps/csv2sql/lib/csv2sql/schema_maker.ex @@ -103,10 +103,12 @@ defmodule Csv2sql.SchemaMaker do Application.get_env(:csv2sql, Csv2sql.SchemaMaker)[:schema_infer_chunk_size] db_type = Csv2sql.get_db_type() + bom = :unicode.encoding_to_bom(:utf8) types = path |> File.stream!([:trim_bom]) + |> Stream.map(&String.replace_prefix(&1, bom, "")) |> CSV.parse_stream() |> Stream.chunk_every(schema_infer_chunk_size) |> Task.async_stream(__MODULE__, :infer_type, [headers_type_list], @@ -193,7 +195,11 @@ defmodule Csv2sql.SchemaMaker do defp header_map_to_list(header_map, headers) do Enum.reduce(headers, [], fn header, acc -> - acc ++ [{header, header_map[header]}] + if not is_nil(header_map[header]) do + acc ++ [{header, header_map[header]}] + else + acc + end end) end diff --git a/apps/csv2sql/test/support/csv_src/test.csv b/apps/csv2sql/test/support/csv_src/test.csv new file mode 100644 index 0000000..ad32354 --- /dev/null +++ b/apps/csv2sql/test/support/csv_src/test.csv @@ -0,0 +1,6 @@ +id,name,description,salary,permanent,created_at +102,Arpan,a veryyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy veryyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy veryyyyyyyyyyyyyy longgggggg description,100.0,true,2020-01-01 22:15:45 +240,Sam,Hi I am sam,66.00,1,09-11-2001 7am-15 +353,Bob,My name is bob,88.0,false,09/11/20 7**PM/45/15 +378,Sally,My name is Sally,657.96,0,2015-01-30 22:44:45 +509,Heisenberg,Breaking bad,10880.82,1,2020-01-01 22:15:45