Fix BOM issue while parsing a file.

apps/csv2sql/lib/csv2sql/data_transfer.ex

@@ -11,9 +11,11 @@ defmodule Csv2sql.DataTransfer do
     Helpers.print_msg("Begin data tranfer for file: " <> Path.basename(file))
 
     insertion_chunk_size = Application.get_env(:csv2sql, Csv2sql.get_repo())[:insertion_chunk_size]
+    bom = :unicode.encoding_to_bom(:utf8)
 
     file
     |> File.stream!([:trim_bom])
+    |> Stream.map(&String.replace_prefix(&1, bom, ""))
     |> CSV.parse_stream()
     |> Stream.chunk_every(insertion_chunk_size)
     |> Enum.each(fn data_chunk ->

apps/csv2sql/lib/csv2sql/database.ex

@@ -94,7 +94,7 @@ defmodule Csv2sql.Database do
       Enum.map(data_chunk, fn chunk ->
         Enum.map(chunk, fn {col, val} ->
           val =
-            if val == "" do
+            if String.trim(val) == "" do
               nil
             else
               case(types[col]) do

apps/csv2sql/lib/csv2sql/import_validator.ex

@@ -51,8 +51,11 @@ defmodule Csv2sql.ImportValidator do
   Get row count in csv file
   """
   def get_count_from_csv(file) do
+    bom = :unicode.encoding_to_bom(:utf8)
+
     file
     |> File.stream!([:trim_bom])
+    |> Stream.map(&String.replace_prefix(&1, bom, ""))
     |> CSV.parse_stream()
     |> Enum.count()
   end

apps/csv2sql/lib/csv2sql/schema_maker.ex

@@ -103,10 +103,12 @@ defmodule Csv2sql.SchemaMaker do
       Application.get_env(:csv2sql, Csv2sql.SchemaMaker)[:schema_infer_chunk_size]
 
     db_type = Csv2sql.get_db_type()
+    bom = :unicode.encoding_to_bom(:utf8)
 
     types =
       path
       |> File.stream!([:trim_bom])
+      |> Stream.map(&String.replace_prefix(&1, bom, ""))
       |> CSV.parse_stream()
       |> Stream.chunk_every(schema_infer_chunk_size)
       |> Task.async_stream(__MODULE__, :infer_type, [headers_type_list],
@@ -193,7 +195,11 @@ defmodule Csv2sql.SchemaMaker do
 
   defp header_map_to_list(header_map, headers) do
     Enum.reduce(headers, [], fn header, acc ->
-      acc ++ [{header, header_map[header]}]
+      if not is_nil(header_map[header]) do
+        acc ++ [{header, header_map[header]}]
+      else
+        acc
+      end
     end)
   end
 

apps/csv2sql/test/support/csv_src/test.csv

@@ -0,0 +1,6 @@
+id,name,description,salary,permanent,created_at
+102,Arpan,a veryyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy veryyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy veryyyyyyyyyyyyyy longgggggg description,100.0,true,2020-01-01 22:15:45
+240,Sam,Hi I am sam,66.00,1,09-11-2001 7am-15
+353,Bob,My name is bob,88.0,false,09/11/20 7**PM/45/15
+378,Sally,My name is Sally,657.96,0,2015-01-30 22:44:45
+509,Heisenberg,Breaking bad,10880.82,1,2020-01-01 22:15:45