diff --git a/apps/csv2sql/.env.test.sample b/apps/csv2sql/.env.test.sample deleted file mode 100644 index 3520086..0000000 --- a/apps/csv2sql/.env.test.sample +++ /dev/null @@ -1,4 +0,0 @@ -export DB_URL="username:password@localhost/mysql" -export DB_TYPE="mysql" -export LOG_LEVEL="debug" -export SOURCE_DIRECTORY="priv/src/" diff --git a/apps/csv2sql/.formatter.exs b/apps/csv2sql/.formatter.exs deleted file mode 100644 index d2cda26..0000000 --- a/apps/csv2sql/.formatter.exs +++ /dev/null @@ -1,4 +0,0 @@ -# Used by "mix format" -[ - inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"] -] diff --git a/apps/csv2sql/.gitignore b/apps/csv2sql/.gitignore deleted file mode 100644 index 49a69b0..0000000 --- a/apps/csv2sql/.gitignore +++ /dev/null @@ -1,41 +0,0 @@ -# The directory Mix will write compiled artifacts to. -/_build/ - -# If you run "mix test --cover", coverage assets end up here. -/cover/ - -# The directory Mix downloads your dependencies sources to. -/deps/ - -# Where third-party dependencies like ExDoc output generated docs. -/doc/ - -# Ignore .fetch files in case you like to edit your project deps locally. -/.fetch - -# If the VM crashes, it generates a dump, let's ignore it too. -erl_crash.dump - -# Also ignore archive artifacts (built via "mix archive.build"). -*.ez - -# Ignore package tarball (built via "mix hex.build"). -csv2sql-*.tar - -# linter -/.elixir_ls/ - -# schema file -schema.sql - -# Formatting file -.formatter.exs - -# Env files -.env* - -# Keep sample files in version control -!.env*.sample - -# Test csv files -test/support/csv_src/* diff --git a/apps/csv2sql/.iex.exs b/apps/csv2sql/.iex.exs deleted file mode 100644 index 33f37fb..0000000 --- a/apps/csv2sql/.iex.exs +++ /dev/null @@ -1,26 +0,0 @@ -measure2 = fn fn1, fn2 -> - Benchee.run( - %{ - "func_1" => fn1, - "func_2" => fn2 - }, - warmup: 1, - time: 5, - memory_time: 2, - reduction_time: 2 -) -end - -measure1 = fn func -> - Benchee.run( - %{ - "func" => func - } -) -end - -# iex(3)> x="/home/arpan/dev/csv2sql/apps/csv2sql/priv/src/test.csv" -# "/home/arpan/dev/csv2sql/apps/csv2sql/priv/src/test.csv" -# iex(4)> measure2.(fn -> Csv2sql.SchemaMaker.get_types end, fn -> Csv2sql.TypeDeducer.get_types(x) end) - -# measure2.(fn -> Csv2sql.SchemaMaker.get_types end, fn -> Csv2sql.TypeDeducer.get_types end) diff --git a/apps/csv2sql/mix.exs b/apps/csv2sql/mix.exs deleted file mode 100644 index 0c2669e..0000000 --- a/apps/csv2sql/mix.exs +++ /dev/null @@ -1,87 +0,0 @@ -defmodule Csv2sql.MixProject do - use Mix.Project - - @app :csv2sql - - def project do - [ - app: :csv2sql, - version: "0.1.0", - elixir: "~> 1.13", - # directories to find source files - elixirc_path: elixirc_paths(Mix.env()), - start_permanent: Mix.env() == :prod, - deps: deps(), - aliases: aliases(), - compilers: Mix.compilers(), - test_coverage: [tool: ExCoveralls], - releases: [{@app, release()}] - ] - end - - # Run "mix help compile.app" to learn about applications. - def application do - [ - extra_applications: [:logger], - mod: {Csv2sql.Application, []} - # mod: {Csv2sql, []} - ] - end - - # Run "mix help deps" to learn about dependencies. - defp deps do - [ - {:nimble_csv, "~> 1.2"}, - {:myxql, "~> 0.6.4"}, - {:postgrex, "~> 0.17.5"}, - {:ecto, "~> 3.11.2"}, - {:ecto_sql, "~> 3.11.1"}, - {:sizeable, "~> 1.0"}, - {:timex, "~> 3.7.11"}, - {:typed_struct, "~> 0.3.0"}, - {:flow, "~> 1.2.4"}, - {:stream_split, "~> 0.1.7"}, - {:codepagex, "~> 0.1.6"}, - {:bakeware, "~> 0.2.4"}, - - # For dev and/or test - {:dotenv, github: "avdi/dotenv_elixir", only: [:test]}, - {:benchee, "~> 1.3.0", only: :dev}, - {:excoveralls, "~> 0.18.1", only: [:dev, :test]}, - {:credo, "~> 1.7.2", only: [:dev, :test], runtime: false}, - {:dialyxir, "~> 1.4.3", only: [:dev, :test], runtime: false}, - {:mix_unused, "~> 0.4.1", only: [:dev, :test], runtime: false} - ] - end - - defp elixirc_paths(:test), do: ["lib", "test/support"] - defp elixirc_paths(_), do: ["lib"] - - defp aliases do - [ - ctest: &custom_test_command/1 - ] - end - - def custom_test_command(_) do - # System.shell("source .env.test && MIX_ENV=test mix test --trace --warnings-as-errors --cover", into: IO.stream()) - System.cmd("mix", ["coveralls.html", "--trace", "--warnings-as-errors"], - env: [{"MIX_ENV", "test"}], - into: IO.stream() - ) - end - - def release do - [ - # keeps only significant chunks necessary for the VM operation - strip_beams: Mix.env() == :prod, - overwrite: true, - steps: [:assemble, &Bakeware.assemble/1], - bakeware: [ - # Highest compression level - compression_level: 1, - start_command: "start" - ] - ] - end -end diff --git a/apps/csv2sql/mix.lock b/apps/csv2sql/mix.lock deleted file mode 100644 index 5ab53fd..0000000 --- a/apps/csv2sql/mix.lock +++ /dev/null @@ -1,48 +0,0 @@ -%{ - "bakeware": {:hex, :bakeware, "0.2.4", "0aaf49b34f4bab2aa433f9ff1485d9401e421603160abd6d269c469fc7b65212", [:make, :mix], [{:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "7b97bcf6fbeee53bb32441d6c495bf478d26f9575633cfef6831e421e86ada6d"}, - "benchee": {:hex, :benchee, "1.3.0", "f64e3b64ad3563fa9838146ddefb2d2f94cf5b473bdfd63f5ca4d0657bf96694", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}, {:statistex, "~> 1.0", [hex: :statistex, repo: "hexpm", optional: false]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "34f4294068c11b2bd2ebf2c59aac9c7da26ffa0068afdf3419f1b176e16c5f81"}, - "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"}, - "certifi": {:hex, :certifi, "2.12.0", "2d1cca2ec95f59643862af91f001478c9863c2ac9cb6e2f89780bfd8de987329", [:rebar3], [], "hexpm", "ee68d85df22e554040cdb4be100f33873ac6051387baf6a8f6ce82272340ff1c"}, - "cli_spinners": {:hex, :cli_spinners, "0.1.0", "08d89c6f1840a8927daed48c675ecb2f20c05c855dc2b4b58c2933d393d5e0c9", [:mix], [], "hexpm", "3b8ccad722e518309d8b92230960ca4775a761164514f78a89c4f04d5a25c97f"}, - "codepagex": {:hex, :codepagex, "0.1.6", "49110d09a25ee336a983281a48ef883da4c6190481e0b063afe2db481af6117e", [:mix], [], "hexpm", "1521461097dde281edf084062f525a4edc6a5e49f4fd1f5ec41c9c4955d5bd59"}, - "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"}, - "connection": {:hex, :connection, "1.1.0", "ff2a49c4b75b6fb3e674bfc5536451607270aac754ffd1bdfe175abe4a6d7a68", [:mix], [], "hexpm", "722c1eb0a418fbe91ba7bd59a47e28008a189d47e37e0e7bb85585a016b2869c"}, - "credo": {:hex, :credo, "1.7.5", "643213503b1c766ec0496d828c90c424471ea54da77c8a168c725686377b9545", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "f799e9b5cd1891577d8c773d245668aa74a2fcd15eb277f51a0131690ebfb3fd"}, - "db_connection": {:hex, :db_connection, "2.6.0", "77d835c472b5b67fc4f29556dee74bf511bbafecdcaf98c27d27fa5918152086", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c2f992d15725e721ec7fbc1189d4ecdb8afef76648c746a8e1cad35e3b8a35f3"}, - "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"}, - "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"}, - "dialyxir": {:hex, :dialyxir, "1.4.3", "edd0124f358f0b9e95bfe53a9fcf806d615d8f838e2202a9f430d59566b6b53b", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "bf2cfb75cd5c5006bec30141b131663299c661a864ec7fbbc72dfa557487a986"}, - "dir_walker": {:hex, :dir_walker, "0.0.8", "5332225074e4887e6e60ca0242af490215f296511ded4df18d554ae25394f727", [:mix], [], "hexpm", "2f4fb16e6427523700df9eb12eece5679ad4459aaefb1ca3cb580184bfc8d173"}, - "dotenv": {:git, "https://github.com/avdi/dotenv_elixir.git", "d6fd3f327173fe18a455203987da95ef9f6cd4c5", []}, - "ecto": {:hex, :ecto, "3.11.2", "e1d26be989db350a633667c5cda9c3d115ae779b66da567c68c80cfb26a8c9ee", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3c38bca2c6f8d8023f2145326cc8a80100c3ffe4dcbd9842ff867f7fc6156c65"}, - "ecto_sql": {:hex, :ecto_sql, "3.11.1", "e9abf28ae27ef3916b43545f9578b4750956ccea444853606472089e7d169470", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.11.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16.0 or ~> 0.17.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ce14063ab3514424276e7e360108ad6c2308f6d88164a076aac8a387e1fea634"}, - "elixir_make": {:hex, :elixir_make, "0.6.3", "bc07d53221216838d79e03a8019d0839786703129599e9619f4ab74c8c096eac", [:mix], [], "hexpm", "f5cbd651c5678bcaabdbb7857658ee106b12509cd976c2c2fca99688e1daf716"}, - "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, - "excoveralls": {:hex, :excoveralls, "0.18.1", "a6f547570c6b24ec13f122a5634833a063aec49218f6fff27de9df693a15588c", [:mix], [{:castore, "~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "d65f79db146bb20399f23046015974de0079668b9abb2f5aac074d078da60b8d"}, - "expo": {:hex, :expo, "0.5.2", "beba786aab8e3c5431813d7a44b828e7b922bfa431d6bfbada0904535342efe2", [:mix], [], "hexpm", "8c9bfa06ca017c9cb4020fabe980bc7fdb1aaec059fd004c2ab3bff03b1c599c"}, - "file_system": {:hex, :file_system, "1.0.0", "b689cc7dcee665f774de94b5a832e578bd7963c8e637ef940cd44327db7de2cd", [:mix], [], "hexpm", "6752092d66aec5a10e662aefeed8ddb9531d79db0bc145bb8c40325ca1d8536d"}, - "flow": {:hex, :flow, "1.2.4", "1dd58918287eb286656008777cb32714b5123d3855956f29aa141ebae456922d", [:mix], [{:gen_stage, "~> 1.0", [hex: :gen_stage, repo: "hexpm", optional: false]}], "hexpm", "874adde96368e71870f3510b91e35bc31652291858c86c0e75359cbdd35eb211"}, - "gen_stage": {:hex, :gen_stage, "1.2.1", "19d8b5e9a5996d813b8245338a28246307fd8b9c99d1237de199d21efc4c76a1", [:mix], [], "hexpm", "83e8be657fa05b992ffa6ac1e3af6d57aa50aace8f691fcf696ff02f8335b001"}, - "gettext": {:hex, :gettext, "0.24.0", "6f4d90ac5f3111673cbefc4ebee96fe5f37a114861ab8c7b7d5b30a1108ce6d8", [:mix], [{:expo, "~> 0.5.1", [hex: :expo, repo: "hexpm", optional: false]}], "hexpm", "bdf75cdfcbe9e4622dd18e034b227d77dd17f0f133853a1c73b97b3d6c770e8b"}, - "hackney": {:hex, :hackney, "1.20.1", "8d97aec62ddddd757d128bfd1df6c5861093419f8f7a4223823537bad5d064e2", [:rebar3], [{:certifi, "~> 2.12.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "fe9094e5f1a2a2c0a7d10918fee36bfec0ec2a979994cff8cfe8058cd9af38e3"}, - "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, - "jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"}, - "libgraph": {:hex, :libgraph, "0.16.0", "3936f3eca6ef826e08880230f806bfea13193e49bf153f93edcf0239d4fd1d07", [:mix], [], "hexpm", "41ca92240e8a4138c30a7e06466acc709b0cbb795c643e9e17174a178982d6bf"}, - "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, - "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"}, - "mix_unused": {:hex, :mix_unused, "0.4.1", "9f8d759a300a79d2077d6baf617f3a5af6935d50b0f113c09295b265afc3e411", [:mix], [{:libgraph, ">= 0.0.0", [hex: :libgraph, repo: "hexpm", optional: false]}], "hexpm", "fa21f688a88e0710e3d96ac1c8e5a6181aea8a75c8a4214f0edcfeb069b831a3"}, - "myxql": {:hex, :myxql, "0.6.4", "1502ea37ee23c31b79725b95d4cc3553693c2bda7421b1febc50722fd988c918", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:geo, "~> 3.4", [hex: :geo, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "a3307f4671f3009d3708283649adf205bfe280f7e036fc8ef7f16dbf821ab8e9"}, - "nimble_csv": {:hex, :nimble_csv, "1.2.0", "4e26385d260c61eba9d4412c71cea34421f296d5353f914afe3f2e71cce97722", [:mix], [], "hexpm", "d0628117fcc2148178b034044c55359b26966c6eaa8e2ce15777be3bbc91b12a"}, - "parse_trans": {:hex, :parse_trans, "3.4.1", "6e6aa8167cb44cc8f39441d05193be6e6f4e7c2946cb2759f015f8c56b76e5ff", [:rebar3], [], "hexpm", "620a406ce75dada827b82e453c19cf06776be266f5a67cff34e1ef2cbb60e49a"}, - "postgrex": {:hex, :postgrex, "0.17.5", "0483d054938a8dc069b21bdd636bf56c487404c241ce6c319c1f43588246b281", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "50b8b11afbb2c4095a3ba675b4f055c416d0f3d7de6633a595fc131a828a67eb"}, - "shorter_maps": {:git, "https://github.com/boyzwj/shorter_maps.git", "787c3447e48c74cf3224f35aba03c84f4c6a3c13", []}, - "sizeable": {:hex, :sizeable, "1.0.2", "625fe06a5dad188b52121a140286f1a6ae1adf350a942cf419499ecd8a11ee29", [:mix], [], "hexpm", "4bab548e6dfba777b400ca50830a9e3a4128e73df77ab1582540cf5860601762"}, - "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, - "statistex": {:hex, :statistex, "1.0.0", "f3dc93f3c0c6c92e5f291704cf62b99b553253d7969e9a5fa713e5481cd858a5", [:mix], [], "hexpm", "ff9d8bee7035028ab4742ff52fc80a2aa35cece833cf5319009b52f1b5a86c27"}, - "stream_split": {:hex, :stream_split, "0.1.7", "2d3fd1fd21697da7f91926768d65f79409086052c9ec7ae593987388f52425f8", [:mix], [], "hexpm", "1dc072ff507a64404a0ad7af90df97096183fee8eeac7b300320cea7c4679147"}, - "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"}, - "timex": {:hex, :timex, "3.7.11", "bb95cb4eb1d06e27346325de506bcc6c30f9c6dea40d1ebe390b262fad1862d1", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.20", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 1.1", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "8b9024f7efbabaf9bd7aa04f65cf8dcd7c9818ca5737677c7b76acbc6a94d1aa"}, - "typed_struct": {:hex, :typed_struct, "0.3.0", "939789e3c1dca39d7170c87f729127469d1315dcf99fee8e152bb774b17e7ff7", [:mix], [], "hexpm", "c50bd5c3a61fe4e198a8504f939be3d3c85903b382bde4865579bc23111d1b6d"}, - "tzdata": {:hex, :tzdata, "1.1.1", "20c8043476dfda8504952d00adac41c6eda23912278add38edc140ae0c5bcc46", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "a69cec8352eafcd2e198dea28a34113b60fdc6cb57eb5ad65c10292a6ba89787"}, - "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"}, -} diff --git a/apps/csv2sql/todos.md b/apps/csv2sql/todos.md deleted file mode 100644 index f984dd2..0000000 --- a/apps/csv2sql/todos.md +++ /dev/null @@ -1,93 +0,0 @@ -SG DB load times: "2022-06-11 05:52:10.243225Z analyzing files" -> "2022-06-11 06:03:39.411448 - -gracefully termination of producers -benchmarking against previous solution and "load in file" (previous point is pre-requisite) - test with and without preload in producer -db connect required? -observer module -validator -error tracker module and gracefull termination in case of various type of errors -fix csv - dup col names, invalid col names, encoding issue, etc -tests, docs, comments, dialyzer - -Benchmarking results: - Old vs new csv2sql: - Single file: faster 1.5x - Bulk test: 5~7 sec faster - preloading in producer had no affect; removed it - - Comparison with load in file query like: - set session sql_mode = ''; - LOAD DATA INFILE "/var/lib/mysql-files/test2.csv" INTO TABLE test2 COLUMNS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '"' ESCAPED BY '"' LINES TERMINATED BY '\n' IGNORE 1 LINES; - - For a single 818MB file - - CSV2SQL: - Analyze - ~2min 10sec - Parse and Insert - ~8min 38sec - - Load infile query, only insert - 10 min 54.89 sec - Considering only insertion time csv2sql was ~20.7% faster than a load infile query - even if the analysis time is considered csv2sql is still faster. - This will especially be more noticeable with large file sizes and multiple files - -# Todo List - -* Move the config parsing into a separate testable module, setup proper default values - -* INFER SCHEMA Stage: - - After kick off use Flow to read ahead chunks of data and feed it into a flow stage to parse csv string. - - Send csv string to a flow stage to map the type for every row - - Finally send a flow stream to reduce to get the final type - -* INSERT INTO DB GenStage pipeline: - - Prepare a genStage pipeline on application start. - - Pipeline has 1 producer for every csv file. - - Producers are sitting idly - - After inferring schema kick-off the respective producer by sending it the schema. - - Before kickoff table is created. - - Producer reads data from file stream in chunks and send it to the next consumer-producer stage - - A DynamicConsumerSupervisor in-between produces multiple consumer-producers subscribed to every producer. - - Each consumer-producer receives a chunk and encodes it in the format suitable for db insertion and sends it to the next consumer stage. - - A DynamicConsumerSupervisor produces min(cpu-cores * 2, db pool) number of consumers, which insert a chunk into the correct table - -* Validation - - A separate GenStage/Flow pipeline to validate the data. - - -Tag test with DB and test these tests for both postgres and mysql. - - -Done: - -* Config Management -Better config parsing, use separate config structs, better code organization in modules, typespecs, 90%+ test coverage - -* TODO configs: - * Auto drop table if exists - -* Csv Type inference - Better type inference and cpu utilization(10% increased cpu usage), using Flow - For 818 MB csv with 34959673 rows with older apporoach it took 126 seconds vs 120 seconds with new approach. - Introduced flow and parallalized reduction of types maps for different chunks - better code organization, typespecs and 90%+ test coverage. - - The little improvement in peroformance becomes significant for larger csv files - - Details: - Name ips average deviation median 99th % - func_2 0.00831 2.01 min ±0.00% 2.01 min 2.01 min - func_1 0.00788 2.12 min ±0.00% 2.12 min 2.12 min - - Comparison: - func_2 0.00831 - func_1 0.00788 - 1.06x slower +0.110 min - - Memory usage statistics: - - Name Memory usage - func_2 0.00001 GB - func_1 38.00 GB - 3284106.23x memory usage +38.00 GB - -Windows terminal with nmake: - -from run: cmd /K "C:\Program Files (x86)\Microsoft Visual Studio\2017\BuildTools\VC\Auxiliary\Build\vcvarsall.bat" amd64 diff --git a/apps/dashboard/.formatter.exs b/apps/dashboard/.formatter.exs deleted file mode 100644 index 858bbbb..0000000 --- a/apps/dashboard/.formatter.exs +++ /dev/null @@ -1,5 +0,0 @@ -[ - plugins: [Phoenix.LiveView.HTMLFormatter], - import_deps: [:phoenix], - inputs: ["*.{heex,ex,exs}", "{config,lib,test}/**/*.{heex,ex,exs}"] -] diff --git a/apps/dashboard/.gitignore b/apps/dashboard/.gitignore deleted file mode 100644 index 6bdcf87..0000000 --- a/apps/dashboard/.gitignore +++ /dev/null @@ -1,34 +0,0 @@ -# The directory Mix will write compiled artifacts to. -/_build/ - -# If you run "mix test --cover", coverage assets end up here. -/cover/ - -# The directory Mix downloads your dependencies sources to. -/deps/ - -# Where 3rd-party dependencies like ExDoc output generated docs. -/doc/ - -# Ignore .fetch files in case you like to edit your project deps locally. -/.fetch - -# If the VM crashes, it generates a dump, let's ignore it too. -erl_crash.dump - -# Also ignore archive artifacts (built via "mix archive.build"). -*.ez - -# Ignore package tarball (built via "mix hex.build"). -dashboard-*.tar - -# Ignore assets that are produced by build tools. -/priv/static/assets/ - -# Ignore digested assets cache. -/priv/static/cache_manifest.json - -# In case you use Node.js/npm, you want to ignore these. -npm-debug.log -/assets/node_modules/ - diff --git a/apps/dashboard/README.md b/apps/dashboard/README.md deleted file mode 100644 index af2a1af..0000000 --- a/apps/dashboard/README.md +++ /dev/null @@ -1,11 +0,0 @@ -* Refs - -Windows installer creation: https://docs.microsoft.com/en-us/windows/msix/app-installer/how-to-create-appinstaller-file - -Live book installers: https://github.com/livebook-dev/livebook/tree/main/app_bundler/lib/app_bundler - -CI setup: -https://curiosum.com/blog/mastering-elixir-ci-pipeline?utm_medium=email&utm_source=elixir-radar -https://github.com/livebook-dev/livebook/blob/main/.github/workflows/test.yaml - -Docker file refs: https://github.com/livebook-dev/livebook/blob/main/Dockerfile diff --git a/apps/dashboard/assets/css/about_live.scss b/apps/dashboard/assets/css/about_live.scss deleted file mode 100644 index ffc1cc1..0000000 --- a/apps/dashboard/assets/css/about_live.scss +++ /dev/null @@ -1,40 +0,0 @@ -a { - text-decoration: none; -} - -.about-container { - display: flex; - justify-content: center; - margin-top: 80px; - flex-direction: column; - align-items: center; -} - -.header-wrapper { - text-align: center; - h1 { - color: #fff; - font-size: 3em; - font-family: "ubuntu"; - text-transform: uppercase; - font-weight: 700; - font-family: "Josefin Sans", sans-serif; - background: linear-gradient(to right, #e72a3a 10%, #fff 50%, #5bdaff 60%); - background-size: auto auto; - background-clip: border-box; - background-size: 200% auto; - color: #fff; - background-clip: text; - -webkit-background-clip: text; - -webkit-text-fill-color: transparent; - animation: textclip 1.5s linear infinite; - display: inline-block; - font-style: italic; - } -} - -@keyframes textclip { - to { - background-position: 200% center; - } -} \ No newline at end of file diff --git a/apps/dashboard/assets/css/app.scss b/apps/dashboard/assets/css/app.scss deleted file mode 100644 index cdaea2c..0000000 --- a/apps/dashboard/assets/css/app.scss +++ /dev/null @@ -1,5 +0,0 @@ -@import '../node_modules/bootstrap/scss/bootstrap'; -@import 'live_view'; -@import 'custom'; -@import 'start_live'; -@import 'about_live'; \ No newline at end of file diff --git a/apps/dashboard/assets/css/custom.scss b/apps/dashboard/assets/css/custom.scss deleted file mode 100644 index 95e4acd..0000000 --- a/apps/dashboard/assets/css/custom.scss +++ /dev/null @@ -1,181 +0,0 @@ -// Varaibles -$light-grey: #f0f0f0; - -.bg-light-grey { - background-color: $light-grey !important; -} - -// Disable horizontal scrollbar -html, -body { - overflow-x: hidden; - background: blanchedalmond; - - // Custom scrollbar styles - * { - scrollbar-width: thin; - } - - ::-webkit-scrollbar-thumb { - border-radius: 10px !important; - } -} - -.csv2sql-logo { - display: block; - min-width: 80px; - margin-top: 1rem; - - img { - height: 35px; - width: auto; - display: block; - } -} - -.top-navbar { - background-color: #4a4473; - display: flex; - justify-content: center; - height: 70px; - - .nav-header { - color: white; - font-size: 1rem; - font-variant: small-caps; - font-weight: bold; - margin-top: 1.3rem; - margin-left: 0.2rem; - } -} - -.custom-navbar { - max-width: 100%; - font-variant: small-caps; - font-weight: bold; - font-style: italic; -} - -.grow { - transition: all 0.04s ease-in-out; -} - -.grow:hover { - transform: scale(1.05); -} - -.tool-tip-wrapper { - width: fit-content; - position: relative; - top: -4px; - padding-right: 2px; - cursor: pointer; -} - -.mysql-label { - position: relative; - top: -14px; -} - -.postgres-label { - svg { - display: block; - position: relative; - left: 30%; - } - - div { - font-family: "Roboto", sans-serif; - font-size: 1.2rem; - - .inner-text { - color: #336791; - } - } -} - -.add-link { - cursor: pointer; - margin-top: 20px; - - span { - position: relative; - top: -4px; - color: darkgreen; - font-weight: bold; - font-style: italic; - text-decoration: underline; - } -} - -.opacity-transition { - transition: opacity 0.25s ease-in-out; -} - -.live-modal { - position: absolute; - width: 900px; - top: 50%; - left: 50%; - transform: translate(-50%, -50%); - z-index: 9999; - margin: auto; -} - -.db-connect-check { - height: 72px; -} - -.db-url-text { - width: 500px; - overflow: hidden; - white-space: nowrap; - text-overflow: ellipsis; -} - -// Custom scroll bar implementation -.scrollable-container { - padding: 10px; - overflow-y: auto; - scroll-behavior: smooth; - scrollbar-gutter: stable; // Reserve space for scrollbar to avoid UI shifting when scrollbar appears - scrollbar-color: #d4aa70 #e4e4e4; // Works for non-chromium (tested on firfox) - - // Custom Scrollbar css works only for chromium based browser - &::-webkit-scrollbar { - width: 10px; - } - - &::-webkit-scrollbar-track { - background-color: #e4e4e4; - border-radius: 100px; - } - - &::-webkit-scrollbar-thumb { - border-radius: 100px; - background-color: #d4aa70; - box-shadow: inset 2px 2px 5px 0 rgba(#fff, 0.5); - } -} - -#db-attrs-container { - height: 300px; - - li { - display: list-item; - } - - .db-attr-index { - padding: 2px; - } -} - -#date-patterns-container, #date-time-patterns-container { - height: 250px; -} - -// Matched Custom Date/DateTime pattern input glow -.matched-input { - border-color: green; - box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075), 0 0 8px green; -} diff --git a/apps/dashboard/assets/css/live_view.scss b/apps/dashboard/assets/css/live_view.scss deleted file mode 100644 index dc46ad1..0000000 --- a/apps/dashboard/assets/css/live_view.scss +++ /dev/null @@ -1,102 +0,0 @@ - -/* LiveView specific classes for your customizations */ -.invalid-feedback { - color: #a94442; - display: block; -} - -/* - Any DOM container with the phx-feedback-for attribute will receive a phx-no-feedback class in cases where the - form fields has yet to receive user input/focus, in such cases we hide the error. - - In our case if still show the errors always -*/ -// .phx-no-feedback.invalid-feedback, .phx-no-feedback .invalid-feedback { -// display: none; -// } - -.phx-click-loading { - opacity: 0.5; - transition: opacity 1s ease-out; -} - -.phx-disconnected{ - cursor: wait; -} - -.phx-disconnected *{ - pointer-events: none; -} - -.phx-modal { - opacity: 1!important; - position: fixed; - z-index: 1; - left: 0; - top: 0; - width: 100%; - height: 100%; - overflow: auto; - background-color: rgb(0,0,0); - background-color: rgba(0,0,0,0.4); -} - -.phx-modal-content { - background-color: #fefefe; - margin: 15% auto; - padding: 20px; - border: 1px solid #888; - width: 80%; -} - -.phx-modal-close { - color: #aaa; - float: right; - font-size: 28px; - font-weight: bold; -} - -.phx-modal-close:hover, -.phx-modal-close:focus { - color: black; - text-decoration: none; - cursor: pointer; -} - -/* Alerts and form errors */ -.alert { - padding: 15px; - margin-bottom: 20px; - border: 1px solid transparent; - border-radius: 4px; -} - -.alert-info { - color: #31708f; - background-color: #d9edf7; - border-color: #bce8f1; -} - -.alert-warning { - color: #8a6d3b; - background-color: #fcf8e3; - border-color: #faebcc; -} - -.alert-danger { - color: #a94442; - background-color: #f2dede; - border-color: #ebccd1; -} - -.alert p { - margin-bottom: 0; -} - -.alert:empty { - display: none; -} - -.config-page { - background-color: #f8f9fa; -} \ No newline at end of file diff --git a/apps/dashboard/assets/css/start_live.scss b/apps/dashboard/assets/css/start_live.scss deleted file mode 100644 index ed998e9..0000000 --- a/apps/dashboard/assets/css/start_live.scss +++ /dev/null @@ -1,189 +0,0 @@ -spinner.loading { - display: none; - text-align: center; - position: relative; - display: flex; - justify-content: center; - align-items: center; - width: 106px; - height: 106px; -} -.spinner.loading:before { - content: ""; - height: 120px; - width: 120px; - position: absolute; - top: 0; - left: 0; - border-width: 8px; - border-style: solid; - border-color: #770505 #ccc #ccc; - border-radius: 100%; - animation: rotation .7s infinite linear; -} -@keyframes rotation { - from { - transform: rotate(0deg); - } - to { - transform: rotate(359deg); - } -} - -.required-config{ - color: red; - font-size: 1em; -} - -#spinnerText{ - font-size: 1em; - font-weight: bolder; - font-variant: small-caps; - padding-top: 35%; - text-align: center; - vertical-align: middle; -} - -#error_stage { - font-size: 0.7em; -} - - -.main-footer { - height: 70px; - background-color: rebeccapurple; - opacity: 0.8; -} - -.main-footer .container { - height: 120px; - width: 120px; - position: absolute; - bottom: 5px; - border-radius: 50%; - background-image: linear-gradient(red, yellow); - border: 1px solid rgb(168, 159, 159); - box-shadow: 0 2px 5px rgba(201,209,230,.5), 0 7px 20px rgba(201,209,230,.75); - // center item - left: 0; - right: 0; - margin-left: auto; - margin-right: auto; -} - -.button-enabled{ - cursor: pointer; - opacity: 1; -} - -.button-disabled{ - cursor: not-allowed; - opacity: 0.8; -} - -.current-stats{ - display: flex; - flex-wrap: wrap; - border-radius: 10px; - width: 90%; - min-height: 150px; - box-shadow: 0 2px 5px rgba(201,209,230,.5), 0 7px 20px rgba(201,209,230,.75); - margin: auto; - margin-bottom: 50px; - margin-top: 20px; - padding: 10px; - font-size: 1em; - background-color: #f8f9fa; -} - -.current-stats > div { - width: 50%; -} - -.finish-check { - color: #cc00ff; -} - -.file-list { - margin: auto; - width: 90%; - box-shadow: 0 2px 5px rgba(201,209,230,.5), 0 7px 20px rgba(201,209,230,.75); - margin-bottom: 100px; -} - -.file-list-item { - min-height: 100px; - font-size: 1em; -} - -.file-name, .file-path, .file-size, .row_count, .records_inserted, .status { - display: block; -} - -.progress { - height: 25px !important; -} - -.progress-percentage{ - color: black; - font-size: 1em; - font-weight: bold; -} - -.stage_pending, -.stage_infer_schema, -.stage_loading_schema, -.stage_insert_data, -.stage_finished { - border-radius: 10px; - border: 1px solid black; - padding: 5px; - font-weight: bold; -} - -.stage_pending { - background-color: #FFA07A; -} - -.stage_infer_schema { - background-color: #8A2BE2; - color: white; -} - -.stage_loading_schema { - background-color: #BDB76B; -} - -.stage_insert_data { - background-color: #0000FF; - color: white; -} - -.stage_finished { -background-color: #3CB371; -} - -.status { - margin-top: 10px; - margin-bottom: 10px; -} - -.overall-status, .validation-status { - text-transform: capitalize; - border: 1px solid #FAEBD7; - background-color: #FAEBD7; - border-radius: 15px; - padding: 5px; - font-variant: small-caps; - font-weight: bold; - font-style: italic; - box-shadow: rgba(17, 17, 26, 0.1) 0px 8px 24px, rgba(17, 17, 26, 0.1) 0px 16px 56px, rgba(17, 17, 26, 0.1) 0px 24px 80px; -} - -.validation-status { - background-color: #3CB371; -} - -.error-status { - background-color: rgb(214, 29, 29); -} diff --git a/apps/dashboard/assets/js/app.js b/apps/dashboard/assets/js/app.js deleted file mode 100644 index 2c06761..0000000 --- a/apps/dashboard/assets/js/app.js +++ /dev/null @@ -1,55 +0,0 @@ -import * as bootstrap from 'bootstrap'; -window.bootstrap = bootstrap; -import "phoenix_html"; -// Establish Phoenix Socket and LiveView configuration. -import { Socket } from "phoenix"; -import { LiveSocket } from "phoenix_live_view"; - -let csrfToken = document.querySelector("meta[name='csrf-token']").getAttribute("content"); -let hooks = { - // When ever a new tooltip is mounted, initialize it - initTooltipPopup: { - mounted() { - // Initialize bootstrap tooltips - // Ref: https://getbootstrap.com/docs/5.2/components/tooltips/#enable-tooltips - new bootstrap.Tooltip(this.el); - - this.handleEvent("save-config", ({ config }) => { - localStorage.setItem("config", JSON.stringify(config)); - } - ) - } - } -}; - -let liveSocket = new LiveSocket( - "/live", - Socket, - { - hooks, - params: { - _csrf_token: csrfToken, - localConfig: (JSON.parse(localStorage.getItem('config') || '{}')) - } - }); - -liveSocket.connect(); -window.liveSocket = liveSocket; - -// Listen for phx "scroll-to-bottom" events and scroll to end of given element id -window.addEventListener( - "phx:scroll-to-bottom", - e => { - const element = document.getElementById(e.detail.id); - element.scrollTop = element.scrollHeight; - } -) - -// Listen for phx "scroll-into-view" events and scroll the given element id into view -window.addEventListener( - "phx:scroll-into-view", - e => { - const element = document.getElementById(e.detail.id); - element.scrollIntoView({ behavior: "smooth", block: "nearest" }); - } -) diff --git a/apps/dashboard/assets/package.json b/apps/dashboard/assets/package.json deleted file mode 100644 index 0e0cd1c..0000000 --- a/apps/dashboard/assets/package.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "devDependencies": { - "@popperjs/core": "^2.11.5", - "bootstrap": "5.2.0-beta1" - } -} diff --git a/apps/dashboard/assets/yarn.lock b/apps/dashboard/assets/yarn.lock deleted file mode 100644 index ecf7889..0000000 --- a/apps/dashboard/assets/yarn.lock +++ /dev/null @@ -1,13 +0,0 @@ -# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. -# yarn lockfile v1 - - -"@popperjs/core@^2.11.5": - version "2.11.5" - resolved "https://registry.npmjs.org/@popperjs/core/-/core-2.11.5.tgz" - integrity sha512-9X2obfABZuDVLCgPK9aX0a/x4jaOEweTTWE2+9sr0Qqqevj2Uv5XorvusThmc9XGYpS9yI+fhh8RTafBtGposw== - -bootstrap@5.2.0-beta1: - version "5.2.0-beta1" - resolved "https://registry.yarnpkg.com/bootstrap/-/bootstrap-5.2.0-beta1.tgz#75647ff7327c2cac9b8e1cb6728a790a5be4cefc" - integrity sha512-6qbgs177WZEFY4SLQUq3tEHayYG80nfDmyTpdKi0MJqRMdS+HAoq24+YKfx6wf+nHY0rx8zrh477J1lFu4WzOA== diff --git a/apps/dashboard/config/config.exs b/apps/dashboard/config/config.exs deleted file mode 100644 index 66a06ef..0000000 --- a/apps/dashboard/config/config.exs +++ /dev/null @@ -1,37 +0,0 @@ -import Config - -# Configures the endpoint -config :dashboard, DashboardWeb.Endpoint, - url: [host: "localhost"], - render_errors: [view: DashboardWeb.ErrorView, accepts: ~w(html json), layout: false], - pubsub_server: Dashboard.PubSub, - live_view: [signing_salt: "m3U44H+L"] - -# Configure esbuild (the version is required) -config :esbuild, - version: "0.14.29", - default: [ - args: - ~w(js/app.js --bundle --target=es2017 --outdir=../priv/static/assets --external:/fonts/* --external:/images/*), - cd: Path.expand("../assets", __DIR__), - env: %{"NODE_PATH" => Path.expand("../../../deps", __DIR__)} - ] - -config :dart_sass, - version: "1.53.0", - default: [ - args: ~w(css/app.scss ../priv/static/assets/app.css), - cd: Path.expand("../assets", __DIR__) - ] - -# Configures Elixir's Logger -config :logger, :console, - format: "$time $metadata[$level] $message\n", - metadata: [:request_id] - -# Use Jason for JSON parsing in Phoenix -config :phoenix, :json_library, Jason - -# Import environment specific config. This must remain at the bottom -# of this file so it overrides the configuration defined above. -import_config "#{config_env()}.exs" diff --git a/apps/dashboard/config/dev.exs b/apps/dashboard/config/dev.exs deleted file mode 100644 index 3d53953..0000000 --- a/apps/dashboard/config/dev.exs +++ /dev/null @@ -1,74 +0,0 @@ -import Config - -# For development, we disable any cache and enable -# debugging and code reloading. -# -# The watchers configuration can be used to run external -# watchers to your application. For example, we use it -# with esbuild to bundle .js and .css sources. -config :dashboard, DashboardWeb.Endpoint, - # Binding to loopback ipv4 address prevents access from other machines. - # Change to `ip: {0, 0, 0, 0}` to allow access from other machines. - http: [ip: {127, 0, 0, 1}, port: 4000], - check_origin: false, - code_reloader: true, - debug_errors: true, - secret_key_base: "SB0LWzWbkJdBpnbm7sH3yIaAZO1oUSIhxIi2zNTpJVq+aht7BUJXAyCc2l8plWvg", - watchers: [ - # Start the esbuild watcher by calling Esbuild.install_and_run(:default, args) - esbuild: {Esbuild, :install_and_run, [:default, ~w(--sourcemap=inline --watch)]}, - sass: { - DartSass, - :install_and_run, - [:default, ~w(--embed-source-map --source-map-urls=absolute --watch)] - } - ] - -# ## SSL Support -# -# In order to use HTTPS in development, a self-signed -# certificate can be generated by running the following -# Mix task: -# -# mix phx.gen.cert -# -# Note that this task requires Erlang/OTP 20 or later. -# Run `mix help phx.gen.cert` for more information. -# -# The `http:` config above can be replaced with: -# -# https: [ -# port: 4001, -# cipher_suite: :strong, -# keyfile: "priv/cert/selfsigned_key.pem", -# certfile: "priv/cert/selfsigned.pem" -# ], -# -# If desired, both `http:` and `https:` keys can be -# configured to run both http and https servers on -# different ports. - -# Watch static and templates for browser reloading. -config :dashboard, DashboardWeb.Endpoint, - live_reload: [ - patterns: [ - ~r"priv/static/.*(js|css|png|jpeg|jpg|gif|svg)$", - ~r"lib/dashboard_web/(live|views)/.*(ex)$", - ~r"lib/dashboard_web/templates/.*(eex)$" - ] - ], - sass: { - DartSass, - :install_and_run, - [:default, ~w(--embed-source-map --source-map-urls=absolute --watch)] - } - -# Do not include metadata nor timestamps in development logs -config :logger, :console, format: "[$level] $message\n" - -# Set a higher stacktrace during development. Avoid configuring such -# in production as building large stacktraces may be expensive. -config :phoenix, :stacktrace_depth, 20 - -# Initialize plugs at runtime for faster development compilation -config :phoenix, :plug_init_mode, :runtime diff --git a/apps/dashboard/config/prod.exs b/apps/dashboard/config/prod.exs deleted file mode 100644 index d5f5070..0000000 --- a/apps/dashboard/config/prod.exs +++ /dev/null @@ -1,49 +0,0 @@ -import Config - -# For production, don't forget to configure the url host -# to something meaningful, Phoenix uses this information -# when generating URLs. -# -# Note we also include the path to a cache manifest -# containing the digested version of static files. This -# manifest is generated by the `mix phx.digest` task, -# which you should run after static files are built and -# before starting your production server. -config :dashboard, DashboardWeb.Endpoint, cache_static_manifest: "priv/static/cache_manifest.json" - -# Do not print debug messages in production -config :logger, level: :info - -# ## SSL Support -# -# To get SSL working, you will need to add the `https` key -# to the previous section and set your `:url` port to 443: -# -# config :dashboard, DashboardWeb.Endpoint, -# ..., -# url: [host: "example.com", port: 443], -# https: [ -# ..., -# port: 443, -# cipher_suite: :strong, -# keyfile: System.get_env("SOME_APP_SSL_KEY_PATH"), -# certfile: System.get_env("SOME_APP_SSL_CERT_PATH") -# ] -# -# The `cipher_suite` is set to `:strong` to support only the -# latest and more secure SSL ciphers. This means old browsers -# and clients may not be supported. You can set it to -# `:compatible` for wider support. -# -# `:keyfile` and `:certfile` expect an absolute path to the key -# and cert in disk or a relative path inside priv, for example -# "priv/ssl/server.key". For all supported SSL configuration -# options, see https://hexdocs.pm/plug/Plug.SSL.html#configure/1 -# -# We also recommend setting `force_ssl` in your endpoint, ensuring -# no data is ever sent via http, always redirecting to https: -# -# config :dashboard, DashboardWeb.Endpoint, -# force_ssl: [hsts: true] -# -# Check `Plug.SSL` for all available options in `force_ssl`. diff --git a/apps/dashboard/config/runtime.exs b/apps/dashboard/config/runtime.exs deleted file mode 100644 index 4866b4e..0000000 --- a/apps/dashboard/config/runtime.exs +++ /dev/null @@ -1,50 +0,0 @@ -import Config - -# config/runtime.exs is executed for all environments, including -# during releases. It is executed after compilation and before the -# system starts, so it is typically used to load production configuration -# and secrets from environment variables or elsewhere. Do not define -# any compile-time configuration in here, as it won't be applied. -# The block below contains prod specific runtime configuration. - -# ## Using releases -# -# If you use `mix release`, you need to explicitly enable the server -# by passing the PHX_SERVER=true when you start it: -# -# PHX_SERVER=true bin/dashboard start -# -# Alternatively, you can use `mix phx.gen.release` to generate a `bin/server` -# script that automatically sets the env var above. -if System.get_env("PHX_SERVER") do - config :dashboard, DashboardWeb.Endpoint, server: true -end - -if config_env() == :prod do - # The secret key base is used to sign/encrypt cookies and other secrets. - # A default value is used in config/dev.exs and config/test.exs but you - # want to use a different value for prod and you most likely don't want - # to check this value into version control, so we use an environment - # variable instead. - secret_key_base = - System.get_env("SECRET_KEY_BASE") || - raise """ - environment variable SECRET_KEY_BASE is missing. - You can generate one by calling: mix phx.gen.secret - """ - - host = System.get_env("PHX_HOST") || "example.com" - port = String.to_integer(System.get_env("PORT") || "4000") - - config :dashboard, DashboardWeb.Endpoint, - url: [host: host, port: 443, scheme: "https"], - http: [ - # Enable IPv6 and bind on all interfaces. - # Set it to {0, 0, 0, 0, 0, 0, 0, 1} for local network only access. - # See the documentation on https://hexdocs.pm/plug_cowboy/Plug.Cowboy.html - # for details about using IPv6 vs IPv4 and loopback vs public addresses. - ip: {0, 0, 0, 0, 0, 0, 0, 0}, - port: port - ], - secret_key_base: secret_key_base -end diff --git a/apps/dashboard/config/test.exs b/apps/dashboard/config/test.exs deleted file mode 100644 index c87f630..0000000 --- a/apps/dashboard/config/test.exs +++ /dev/null @@ -1,14 +0,0 @@ -import Config - -# We don't run a server during test. If one is required, -# you can enable the server option below. -config :dashboard, DashboardWeb.Endpoint, - http: [ip: {127, 0, 0, 1}, port: 4002], - secret_key_base: "pYynG2vS41aV9IF+H/nexIqwuWGvQ5q6OtGkfyB7XLpX7oqj5RW0ClgL3G4sHw++", - server: false - -# Print only warnings and errors during test -config :logger, level: :warn - -# Initialize plugs at runtime for faster test compilation -config :phoenix, :plug_init_mode, :runtime diff --git a/apps/dashboard/lib/dashboard.ex b/apps/dashboard/lib/dashboard.ex deleted file mode 100644 index b1fe868..0000000 --- a/apps/dashboard/lib/dashboard.ex +++ /dev/null @@ -1,9 +0,0 @@ -defmodule Dashboard do - @moduledoc """ - Dashboard keeps the contexts that define your domain - and business logic. - - Contexts are also responsible for managing your data, regardless - if it comes from the database, an external API or others. - """ -end diff --git a/apps/dashboard/lib/dashboard/application.ex b/apps/dashboard/lib/dashboard/application.ex deleted file mode 100644 index fe31661..0000000 --- a/apps/dashboard/lib/dashboard/application.ex +++ /dev/null @@ -1,23 +0,0 @@ -defmodule Dashboard.Application do - @moduledoc false - - use Application - - @impl true - def start(_type, _args) do - children = [ - DashboardWeb.Telemetry, - {Phoenix.PubSub, name: Dashboard.PubSub}, - DashboardWeb.Endpoint - ] - - opts = [strategy: :one_for_one, name: Dashboard.Supervisor] - Supervisor.start_link(children, opts) - end - - @impl true - def config_change(changed, _new, removed) do - DashboardWeb.Endpoint.config_change(changed, removed) - :ok - end -end diff --git a/apps/dashboard/lib/dashboard/config.ex b/apps/dashboard/lib/dashboard/config.ex deleted file mode 100644 index 9b0217f..0000000 --- a/apps/dashboard/lib/dashboard/config.ex +++ /dev/null @@ -1,154 +0,0 @@ -# This is required so that changesets can be serialized and stored in browser local storage -# We are using only changes on a changeset -defimpl Jason.Encoder, for: Ecto.Changeset do - def encode(changeset, opts) do - Jason.Encode.map(changeset.changes, opts) - end -end - -defmodule DashBoard.Config do - use Ecto.Schema - import Ecto.Changeset - - @primary_key false - embedded_schema do - field(:source_directory) - field(:schema_path, :string) - field(:insert_schema, :boolean) - field(:insert_data, :boolean) - field(:ordered, :boolean) - field(:schema_infer_chunk_size, :integer) - field(:worker_count, :integer) - field(:parse_datetime, :boolean) - field(:remove_illegal_characters, :boolean) - - # Database related configs - field(:db_type, Ecto.Enum, values: [:mysql, :postgres]) - field(:db_name, :string) - field(:db_username, :string) - field(:db_password, :string) - field(:db_host, :string) - - # Other Database configs - field(:drop_existing_tables, :boolean) - field(:varchar_limit, :integer) - field(:db_worker_count, :integer) - field(:insertion_chunk_size, :integer) - field(:log, :boolean) - - # Misc - field(:csv_count, :integer) - field(:date_time_trial, :string) - - embeds_many(:db_attrs, DashBoard.DbAttribute, on_replace: :delete) - embeds_many(:date_patterns, DashBoard.DatePattern, on_replace: :delete) - embeds_many(:date_time_patterns, DashBoard.DateTimePattern, on_replace: :delete) - end - - def get_defaults do - Csv2sql.Config.Loader.get_defaults() - |> Map.merge(%{ - source_directory: get_source_directory(), - worker_count: System.schedulers_online() - }) - end - - def changeset(params), do: changeset(%__MODULE__{}, params) - - def changeset(%__MODULE__{} = config, params) do - attrs_to_cast = - :fields - |> __MODULE__.__schema__() - # Reject nested attributes from cast, they will be loaded via cast_embed/2 - |> Enum.reject(fn - field when field in ~w[db_attrs date_patterns date_time_patterns]a -> true - _field -> false - end) - - changeset = - config - |> cast(params, attrs_to_cast) - |> validate_limit_constraints(Csv2sql.Config.Loader.get_constraints()) - |> validate_source_directory() - |> validate_path(:schema_path, true) - |> cast_embed(:db_attrs) - |> cast_embed(:date_patterns) - |> cast_embed(:date_time_patterns) - - # Phoenix uses the value of changeset.action to decide if errors should be shown or not on a given form - %{changeset | action: :insert} - end - - defp validate_limit_constraints(changeset, constants) do - Enum.reduce( - constants, - changeset, - fn {key, %{min: min, max: max}}, changeset -> - validate_number( - changeset, - key, - greater_than_or_equal_to: min, - less_than_or_equal_to: max - ) - end - ) - end - - defp validate_source_directory(changeset) do - changeset - |> validate_path(:source_directory) - |> add_csv_count() - end - - defp validate_path(changeset, field, allow_blank \\ false) do - value = get_change(changeset, field, "") - - if allow_blank && value == "" do - changeset - else - value - |> File.dir?() - |> if( - do: changeset, - else: add_error(changeset, field, "Invalid path, enter a path to a valid directory") - ) - end - end - - defp add_csv_count(changeset) do - source_directory = get_change(changeset, :source_directory) - - if File.dir?(source_directory) do - csv_count = - source_directory - |> File.ls!() - |> Enum.filter(&is_csv?/1) - |> Enum.count() - - if csv_count == 0, - do: add_error(changeset, :source_directory, "No CSVs found at path"), - else: put_change(changeset, :csv_count, csv_count) - else - changeset - end - end - - defp is_csv?(filepath) do - filepath - |> String.trim() - |> String.slice(-4..-1) - |> String.downcase() == ".csv" - end - - defp get_source_directory() do - case :os.type() do - {:win32, _} -> - username = System.shell("echo %USERNAME%") |> elem(0) |> String.trim() - "C:/Users/#{username}/Desktop" - - {:unix, _} -> - {op, _exit_code} = System.shell("eval echo \"~$USER\"") - String.trim(op) - end - end -end diff --git a/apps/dashboard/lib/dashboard/date_pattern.ex b/apps/dashboard/lib/dashboard/date_pattern.ex deleted file mode 100644 index 0619463..0000000 --- a/apps/dashboard/lib/dashboard/date_pattern.ex +++ /dev/null @@ -1,18 +0,0 @@ -defmodule DashBoard.DatePattern do - use Ecto.Schema - import Ecto.Changeset - - @primary_key false - embedded_schema do - field(:id, :string) - field(:pattern, :string) - end - - def changeset(params), do: changeset(%__MODULE__{}, params) - - def changeset(struct, params) do - struct - |> cast(params, [:id, :pattern]) - |> validate_required([:id]) - end -end diff --git a/apps/dashboard/lib/dashboard/date_time_pattern.ex b/apps/dashboard/lib/dashboard/date_time_pattern.ex deleted file mode 100644 index a9d3677..0000000 --- a/apps/dashboard/lib/dashboard/date_time_pattern.ex +++ /dev/null @@ -1,18 +0,0 @@ -defmodule DashBoard.DateTimePattern do - use Ecto.Schema - import Ecto.Changeset - - @primary_key false - embedded_schema do - field(:id, :string) - field(:pattern, :string) - end - - def changeset(params), do: changeset(%__MODULE__{}, params) - - def changeset(struct, params) do - struct - |> cast(params, [:id, :pattern]) - |> validate_required([:id]) - end -end diff --git a/apps/dashboard/lib/dashboard/db_attribute.ex b/apps/dashboard/lib/dashboard/db_attribute.ex deleted file mode 100644 index 333e178..0000000 --- a/apps/dashboard/lib/dashboard/db_attribute.ex +++ /dev/null @@ -1,17 +0,0 @@ -defmodule DashBoard.DbAttribute do - use Ecto.Schema - import Ecto.Changeset - - @primary_key false - embedded_schema do - field(:id, :string) - field(:name, :string) - field(:value, :string) - end - - def changeset(struct, params) do - struct - |> cast(params, [:id, :name, :value]) - |> validate_required([:id]) - end -end diff --git a/apps/dashboard/lib/dashboard/helpers.ex b/apps/dashboard/lib/dashboard/helpers.ex deleted file mode 100644 index 4b9d4dd..0000000 --- a/apps/dashboard/lib/dashboard/helpers.ex +++ /dev/null @@ -1,69 +0,0 @@ -defmodule Dashboard.Helpers do - defguard present?(value) when value != nil and value != "" - - def match_date_time(changeset, date_time_sample) do - date_index = - changeset - |> Ecto.Changeset.get_field(:date_patterns, []) - |> get_matching_pattern_index(date_time_sample) - - if is_nil(date_index) do - date_time_index = - changeset - |> Ecto.Changeset.get_field(:date_time_patterns, []) - |> get_matching_pattern_index(date_time_sample) - - if is_nil(date_time_index), do: false, else: {:date_time, date_time_index} - else - if is_nil(date_index), do: false, else: {:date, date_index} - end - end - - def create_db_url(configs, opts \\ []) - - def create_db_url( - %{db_username: db_username, db_password: db_password, db_host: db_host, db_name: db_name} = - config, - opts - ) - when present?(db_username) and present?(db_password) and - present?(db_name) and present?(db_host) do - hide_password? = Keyword.get(opts, :hide_password, false) - password = if hide_password?, do: hide_password(db_password), else: db_password - - "ecto://#{db_username}:#{password}@#{db_host}/#{db_name}?#{make_query_params(config)}" - |> String.trim_trailing("?") - end - - def create_db_url(_configs, _hide_password), do: "NA" - - # == Private helpers == - defp make_query_params(%{db_attrs: db_attrs}) when is_list(db_attrs) do - db_attrs - |> Enum.map(& &1.changes) - |> Enum.reject(fn - %value{name: name, value: value} when present?(name) and present?(value) -> false - _ -> true - end) - |> Enum.into(%{}, fn %{name: name, value: value} -> {name, value} end) - |> URI.encode_query() - end - - defp make_query_params(_config), do: "" - - defp hide_password(password) do - password - |> String.split("") - |> Enum.map(fn _ -> "•" end) - |> Enum.join("") - end - - defp get_matching_pattern_index(patterns, string) do - Enum.find_index(patterns, fn pattern_struct -> - case Timex.parse(string, pattern_struct.pattern) do - {:ok, _} -> true - {:error, _} -> false - end - end) - end -end diff --git a/apps/dashboard/lib/dashboard_web.ex b/apps/dashboard/lib/dashboard_web.ex deleted file mode 100644 index 9e1524f..0000000 --- a/apps/dashboard/lib/dashboard_web.ex +++ /dev/null @@ -1,120 +0,0 @@ -defmodule DashboardWeb do - @moduledoc """ - The entrypoint for defining your web interface, such - as controllers, views, channels and so on. - - This can be used in your application as: - - use DashboardWeb, :controller - use DashboardWeb, :view - - The definitions below will be executed for every view, - controller, etc, so keep them short and clean, focused - on imports, uses and aliases. - - Do NOT define functions inside the quoted expressions - below. Instead, define any helper function in modules - and import those modules here. - """ - - def controller do - quote do - use Phoenix.Controller, namespace: DashboardWeb - - import Plug.Conn - alias DashboardWeb.Router.Helpers, as: Routes - end - end - - def view do - quote do - use Phoenix.View, - root: "lib/dashboard_web/templates", - namespace: DashboardWeb - - import Phoenix.Component - - # Import convenience functions from controllers - import Phoenix.Controller, - only: [get_flash: 1, get_flash: 2, view_module: 1, view_template: 1] - - # Include shared imports and aliases for views - unquote(view_helpers()) - end - end - - def live_view do - quote do - use Phoenix.LiveView, - layout: {DashboardWeb.LayoutView, :live} - - unquote(view_helpers()) - - import DashboardWeb.LiveHelpers - import DashboardWeb.Live.UI - alias DashboardWeb.Live.IconSvg - alias Phoenix.LiveView.JS - end - end - - def live_component do - quote do - use Phoenix.LiveComponent - - unquote(view_helpers()) - - import DashboardWeb.LiveHelpers - import DashboardWeb.Live.UI - alias DashboardWeb.Live.IconSvg - alias Phoenix.LiveView.JS - end - end - - def component do - quote do - use Phoenix.Component - - unquote(view_helpers()) - - import DashboardWeb.LiveHelpers - import DashboardWeb.Live.UI - alias DashboardWeb.Live.IconSvg - alias Phoenix.LiveView.JS - end - end - - def router do - quote do - use Phoenix.Router - - import Plug.Conn - import Phoenix.Controller - import Phoenix.LiveView.Router - end - end - - def channel do - quote do - use Phoenix.Channel - end - end - - defp view_helpers do - quote do - # Use all HTML functionality (forms, tags, etc) - use Phoenix.HTML - - # Import LiveView and .heex helpers (live_render, live_patch, <.form>, etc) - import Phoenix.LiveView.Helpers - - alias DashboardWeb.Router.Helpers, as: Routes - end - end - - @doc """ - When used, dispatch to the appropriate controller/view/etc. - """ - defmacro __using__(which) when is_atom(which) do - apply(__MODULE__, which, []) - end -end diff --git a/apps/dashboard/lib/dashboard_web/controllers/page_controller.ex b/apps/dashboard/lib/dashboard_web/controllers/page_controller.ex deleted file mode 100644 index a0cf65e..0000000 --- a/apps/dashboard/lib/dashboard_web/controllers/page_controller.ex +++ /dev/null @@ -1,7 +0,0 @@ -defmodule DashboardWeb.PageController do - use DashboardWeb, :controller - - def index(conn, _params) do - render(conn, "index.html") - end -end diff --git a/apps/dashboard/lib/dashboard_web/endpoint.ex b/apps/dashboard/lib/dashboard_web/endpoint.ex deleted file mode 100644 index 5248224..0000000 --- a/apps/dashboard/lib/dashboard_web/endpoint.ex +++ /dev/null @@ -1,45 +0,0 @@ -defmodule DashboardWeb.Endpoint do - use Phoenix.Endpoint, otp_app: :dashboard - - # The session will be stored in the cookie and signed, - # this means its contents can be read but not tampered with. - # Set :encryption_salt if you would also like to encrypt it. - @session_options [ - store: :cookie, - key: "_dashboard_key", - signing_salt: "nrBwdLbK" - ] - - socket "/live", Phoenix.LiveView.Socket, websocket: [connect_info: [session: @session_options]] - - # Serve at "/" the static files from "priv/static" directory. - # - # You should set gzip to true if you are running phx.digest - # when deploying your static files in production. - plug Plug.Static, - at: "/", - from: :dashboard, - gzip: false, - only: ~w(assets fonts images favicon.ico robots.txt) - - # Code reloading can be explicitly enabled under the - # :code_reloader configuration of your endpoint. - if code_reloading? do - socket "/phoenix/live_reload/socket", Phoenix.LiveReloader.Socket - plug Phoenix.LiveReloader - plug Phoenix.CodeReloader - end - - plug Plug.RequestId - plug Plug.Telemetry, event_prefix: [:phoenix, :endpoint] - - plug Plug.Parsers, - parsers: [:urlencoded, :multipart, :json], - pass: ["*/*"], - json_decoder: Phoenix.json_library() - - plug Plug.MethodOverride - plug Plug.Head - plug Plug.Session, @session_options - plug DashboardWeb.Router -end diff --git a/apps/dashboard/lib/dashboard_web/live/about_live.ex b/apps/dashboard/lib/dashboard_web/live/about_live.ex deleted file mode 100644 index bf024a0..0000000 --- a/apps/dashboard/lib/dashboard_web/live/about_live.ex +++ /dev/null @@ -1,12 +0,0 @@ -defmodule DashboardWeb.Live.AboutLive do - use DashboardWeb, :live_view - - def about_page(assigns) do - ~H""" -
-

csv2sql blazing fast csv to sql loader

-
Want to know more check out the project on github!
-
- """ - end -end diff --git a/apps/dashboard/lib/dashboard_web/live/config_live.ex b/apps/dashboard/lib/dashboard_web/live/config_live.ex deleted file mode 100644 index d0a662d..0000000 --- a/apps/dashboard/lib/dashboard_web/live/config_live.ex +++ /dev/null @@ -1,392 +0,0 @@ -defmodule DashboardWeb.Live.ConfigLive do - use DashboardWeb, :component - import DashboardWeb.Live.Modal.DbAttributesModal - import DashboardWeb.Live.Modal.DateTimePatternsModal - - # TODO: Break this in simple components and remove any unneeded classes and cleanup the markup - def config_page(assigns) do - assigns = assign(assigns, :form, to_form(assigns.changeset)) - - ~H""" - <.form for={@form} phx-change="validate-and-save"> - <.flow_card form={@form} /> - -
-
- Configurations marked with * are required -
- -
- <.config_item - form={@form} - id="source_directory_config" - title="Source directory" - tooltip="The source directory where the csvs are located, Example: /home/user/Desktop/csvs" - required_input={true} - placeholder="CSV source directory path" - > - <:input let={@form}> - <%= text_input(@form, :source_directory, - "phx-debounce": "1000", - class: "form-control", - placeholder: "CSV source directory path" - ) %> - <%= if (csv_count = Ecto.Changeset.get_field(@changeset, :csv_count)) && !is_nil(csv_count) && csv_count > 0 do %> - Found <%= csv_count %> csvs at given path - <% end %> - <%= error_tag(@form, :source_directory) %> - - - - <.config_item - form={@form} - id="schema_path_config" - title="Schema file path" - tooltip="The path were the generated schema file will be created, defaults to the csv source directory, Example: /home/user/Desktop/schema" - placeholder="Schema file path" - > - <:input let={@form}> - <%= text_input(@form, :schema_path, - "phx-debounce": "1000", - class: "form-control", - placeholder: "Schema file path" - ) %> - <%= error_tag(@form, :schema_path) %> - - - - <.config_item - form={@form} - id="worker_count_config" - title="Worker Count" - tooltip="The number of workers used. Increasing worker count may result in better performance but may lead to errors, decreasing worker count to 1 means, csv2sql will process one file at a time. The number of workers is directly related to the number of files processed simultaneously." - placeholder="Worker count" - > - <:input let={@form}> - <%= text_input(@form, :worker_count, - type: "number", - class: "form-control", - placeholder: "Worker count", - max: @constraints.worker_count.max, - min: @constraints.worker_count.min - ) %> - <%= error_tag(@form, :worker_count) %> - - - - <.config_item - form={@form} - id="schema_infer_chunk_size" - title="Chunk Size" - tooltip="The chunk size to use when the schema for a csv will be infered parallely . - For example: A chunk size 100 means the csv will be read 100 rows at a time - and separate processes will be used to infer the schema of each 100 row chunk" - placeholder="Chunk Size" - > - <:input let={@form}> - <%= text_input(@form, :schema_infer_chunk_size, - type: "number", - class: "form-control", - placeholder: "Chunk Size", - max: @constraints.schema_infer_chunk_size.max, - min: @constraints.schema_infer_chunk_size.min - ) %> - <%= error_tag(@form, :schema_infer_chunk_size) %> - - - -
- <.config_item - form={@form} - checkbox_input={true} - id="parse_datetime_config" - class="pb-2" - title="Parse Datetime" - tooltip="TODO: popup title" - > - <:input let={@form}> - <%= checkbox(@form, :parse_datetime, class: "form-check-input") %> - - - - <.config_item - form={@form} - checkbox_input={true} - id="remove_illegal_characters_config" - class="pt-0" - title="Remove illegal characters" - tooltip="TODO: popup title" - > - <:input let={@form}> - <%= checkbox(@form, :remove_illegal_characters, class: "form-check-input") %> - - - - <%= if Ecto.Changeset.get_field(@changeset, :parse_datetime) do %> -
- - - Add date time patterns - -
- <% end %> -
-
-
- -
-
- - Database Configurations -
- -
-
- - <.config_item - form={@form} - id="db_type_config" - class="pb-0" - title="DB Type" - required_input={true} - tooltip="Set wether to use MySQL or PostgreSQL" - > - <:input let={@form}> -
-
- <%= radio_button(@form, :db_type, :mysql, - class: "form-check-input mt-4", - id: "mysql_db_type", - checked: "checked", - disable: !db_needed(@changeset) - ) %> - -
-
- <%= radio_button(@form, :db_type, "postgres", - class: "form-check-input mt-4", - id: "postgres_db_type" - ) %> - -
-
-
- - - -
- <.config_item - form={@form} - checkbox_input={true} - id="drop_existing_tables_config" - class="pb-2" - title="Drop existing tables" - tooltip="TODO: popup title" - > - <:input let={@form}> - <%= checkbox(@form, :drop_existing_tables, class: "form-check-input") %> - - - - <.config_item - form={@form} - checkbox_input={true} - id="log_config_config" - class="pt-0" - title="Verbose Logging" - tooltip="TODO: popup title" - > - <:input let={@form}><%= checkbox(@form, :log, class: "form-check-input") %> - -
-
- -
- <.db_connection_check - changeset={@changeset} - db_connection_established={@db_connection_established} - /> - -
- <.config_item - form={@form} - id="db_name_config" - class="pb-0" - title="Database name" - tooltip="This is the name of the database which will be created (if not present already), required field if database access is required" - required_input={true} - placeholder="Database name" - > - <:input let={@form}> - <%= text_input(@form, :db_name, class: "form-control", placeholder: "Database name") %> - - - - <.config_item - form={@form} - id="db_host_config" - title="Database Host" - tooltip="database host, required field if database access is required" - required_input={true} - placeholder="Database host" - > - <:input let={@form}> - <%= text_input(@form, :db_host, class: "form-control", placeholder: "Database host") %> - - -
- -
- <.config_item - form={@form} - id="db_username_config" - title="Database username" - tooltip="database username, required field if database access is required" - required_input={true} - placeholder="Database username" - > - <:input let={@form}> - <%= text_input(@form, :db_username, - class: "form-control", - placeholder: "Database username" - ) %> - - - - <.config_item - form={@form} - id="db_password_config" - title="Database password" - tooltip="database password, required field if database access is required" - required_input={true} - placeholder="Database password" - > - <:input let={@form}> - <%= text_input(@form, :db_password, - type: "password", - class: "form-control", - placeholder: "Database password" - ) %> - - -
- -
- - - Add more database configurations - -
-
- -
- <.config_item - form={@form} - id="varchar_limit_config" - class="pb-0" - title="Varchar size" - tooltip="The character limit after which a db field becomes a text from a varchar" - placeholder="Varchar size" - > - <:input let={@form}> - <%= text_input(@form, :varchar_limit, - type: "number", - class: "form-control", - placeholder: "Varchar size", - max: @constraints.varchar_limit.max, - min: @constraints.varchar_limit.min - ) %> - <%= error_tag(@form, :varchar_limit) %> - - - - <.config_item - form={@form} - id="db_worker_count_config" - class="pb-0" - title="DB Worker Count" - tooltip="The number of database workers, the max number of processes writing to the database at once" - placeholder="DB Worker count" - > - <:input let={@form}> - <%= text_input(@form, :db_worker_count, - type: "number", - class: "form-control", - placeholder: "DB Worker count", - max: @constraints.db_worker_count.max, - min: @constraints.db_worker_count.min - ) %> - <%= error_tag(@form, :db_worker_count) %> - - - - <.config_item - form={@form} - id="insertion_chunk_size_config" - title="Insertion Chunk Size" - tooltip="number of records to insert into the database at once Increasing this may result in database errors for too many placeholders" - placeholder="Insertion Chunk Size" - > - <:input let={@form}> - <%= text_input(@form, :insertion_chunk_size, - type: "number", - class: "form-control", - placeholder: "Insertion Chunk size", - max: @constraints.insertion_chunk_size.max, - min: @constraints.insertion_chunk_size.min - ) %> - <%= error_tag(@form, :insertion_chunk_size) %> - - -
-
-
- - <%= if @modal != "add-more-db-attrs" do %> - <%= inputs_for @form, :db_attrs, fn assoc_form -> %> - <%= hidden_input(assoc_form, :id) %> - <%= hidden_input(assoc_form, :name) %> - <%= hidden_input(assoc_form, :value) %> - <% end %> - <% end %> - - <%= if @modal != "add-date-time-patterns" do %> - <%= inputs_for @form, :date_time_patterns, fn assoc_form -> %> - <%= hidden_input(assoc_form, :id) %> - <%= hidden_input(assoc_form, :pattern) %> - <% end %> - - <%= inputs_for @form, :date_patterns, fn assoc_form -> %> - <%= hidden_input(assoc_form, :id) %> - <%= hidden_input(assoc_form, :pattern) %> - <% end %> - <% end %> - - <%= case @modal do %> - <% "add-more-db-attrs" -> %> - <.db_attrs_modal id="db_attrs_modal" form={@form} changeset={@changeset} /> - <% "add-date-time-patterns" -> %> - <.date_time_patterns_modal - id="date_time_patterns_modal" - form={@form} - changeset={@changeset} - matching_date_time={@matching_date_time} - /> - <% _ -> %> - <% end %> - - """ - end - - defp db_needed(changeset) do - Ecto.Changeset.get_field(changeset, :insert_schema) || - Ecto.Changeset.get_field(changeset, :insert_data) - end -end diff --git a/apps/dashboard/lib/dashboard_web/live/helpers.ex b/apps/dashboard/lib/dashboard_web/live/helpers.ex deleted file mode 100644 index c4dfd4a..0000000 --- a/apps/dashboard/lib/dashboard_web/live/helpers.ex +++ /dev/null @@ -1,15 +0,0 @@ -defmodule DashboardWeb.LiveHelpers do - # import Phoenix.LiveView - # import Phoenix.LiveView.Helpers - # alias Phoenix.LiveView.JS - - def get_nav_item_attrs(%{page: page}, curr_page) when page == curr_page, - do: %{class: "nav-link active", "aria-current": "page"} - - def get_nav_item_attrs(_assigns, _curr_page), do: %{class: "nav-link bg-light-grey"} - - def get_config_item_attrs(status) when status not in [:working, :imported, :validating], - do: %{"phx-click" => "page-change", "phx-value-page" => "config"} - - def get_config_item_attrs(_status), do: %{} -end diff --git a/apps/dashboard/lib/dashboard_web/live/icon_svg.ex b/apps/dashboard/lib/dashboard_web/live/icon_svg.ex deleted file mode 100644 index 398cb21..0000000 --- a/apps/dashboard/lib/dashboard_web/live/icon_svg.ex +++ /dev/null @@ -1,455 +0,0 @@ -defmodule DashboardWeb.Live.IconSvg do - use DashboardWeb, :component - # https://www.reshot.com/free-svg-icons/ - - @dimensions %{width: 33, height: 33} - - def args(assigns) do - %{ - class: Map.get(assigns, :class, ""), - width: Map.get(assigns, :width, @dimensions[:width]), - height: Map.get(assigns, :height, @dimensions[:height]) - } - end - - def settings(assigns \\ %{}) do - ~H""" - - - - - - - - - - - - - - """ - end - - def start(assigns \\ %{}) do - ~H""" - - - - - - - - - - - - """ - end - - def about(assigns \\ %{}) do - ~H""" - - - - - - - - - - - - - """ - end - - def infer_schema(assigns \\ %{}) do - ~H""" - - - - - - - - - - - - - """ - end - - def insert_schema(assigns \\ %{}) do - ~H""" - - - - - - - - - - - """ - end - - def insert_data(assigns \\ %{}) do - ~H""" - - - - - - - - - - - """ - end - - def db_settings(assigns \\ %{}) do - ~H""" - - - - - - - - - - - - """ - end - - def bulb(assigns \\ %{}) do - ~H""" - - - - - - """ - end - - def right_arrow(assigns \\ %{}) do - ~H""" - - - - """ - end - - def mysql_icon(assigns \\ %{}) do - ~H""" - - - - - - - - """ - end - - def postgresql_icon(assigns \\ %{}) do - ~H""" - - - - - - - - - - - - - - - - - - - """ - end - - def add_icon(assigns \\ %{}) do - ~H""" - - - - """ - end - - def check_icon(assigns \\ %{}) do - ~H""" - - - - - - - - - """ - end - - def warn_icon(assigns \\ %{}) do - ~H""" - - - - - """ - end - - def remove_icon(assigns \\ %{}) do - ~H""" - - - - """ - end - - def empty_icon(assigns \\ %{}) do - ~H""" - - - - """ - end -end diff --git a/apps/dashboard/lib/dashboard_web/live/main_live.ex b/apps/dashboard/lib/dashboard_web/live/main_live.ex deleted file mode 100644 index 07440f1..0000000 --- a/apps/dashboard/lib/dashboard_web/live/main_live.ex +++ /dev/null @@ -1,327 +0,0 @@ -defmodule DashboardWeb.Live.MainLive do - use DashboardWeb, :live_view - import Dashboard.Helpers - alias DashBoard.Config - alias DashBoard.DbAttribute - alias Csv2sql.Database.ConnectionTest - alias DashboardWeb.Live.{ConfigLive, StartLive, AboutLive} - - @debounce_time 1000 - - @impl true - def mount(_params, _session, socket) do - local_storage_config = (get_connect_params(socket) || %{}) |> Map.get("localConfig", %{}) - - local_storage_config = - for {key, val} <- local_storage_config, into: %{}, do: {String.to_atom(key), val} - - # Check for DB connection on config load from local storage - timer_ref = Process.send_after(self(), :check_db_connection, @debounce_time) - - {:ok, - assign(socket, - page: "config", - modal: false, - path_validator_debouncer: nil, - db_connection_debouncer: timer_ref, - db_connection_established: false, - changeset: Config.get_defaults() |> Map.merge(local_storage_config) |> Config.changeset(), - matching_date_time: nil, - constraints: Csv2sql.Config.Loader.get_constraints(), - time_spend: 0, - state: %Csv2sql.ProgressTracker.State{ - status: :init, - start_time: nil, - validation_status: nil - }, - memory_usage: 0, - cpu_usage: 0 - )} - end - - @impl true - def handle_event("start", _unsigned_params, %{assigns: assigns} = socket) do - socket_state = assigns.state - - cond do - assigns.changeset.valid? and socket_state.status == :init -> - Csv2sql.ProgressTracker.add_subscriber() - - Task.start(fn -> - socket.assigns.changeset - |> prepare_args() - |> Csv2sql.Config.Loader.load() - - Csv2sql.Stages.Analyze.analyze_files() - end) - - send(self(), :update_state) - - {:noreply, socket} - - socket_state.status in [:working, :imported, :validating] -> - {:noreply, socket} - - true -> - Csv2sql.ProgressTracker.reset_state() - - {:noreply, - assign(socket, - state: %Csv2sql.ProgressTracker.State{ - status: :init, - start_time: nil, - validation_status: nil - }, - time_spend: 0, - memory_usage: 0, - cpu_usage: 0 - )} - end - end - - @impl true - def handle_event("page-change", %{"page" => page}, socket) do - {:noreply, assign(socket, %{page: page})} - end - - @impl true - def handle_event("open-modal", %{"modal" => modal}, socket) do - {:noreply, assign(socket, :modal, modal)} - end - - @impl true - def handle_event("close-modal", _attrs, socket) do - {:noreply, assign(socket, :modal, false)} - end - - @impl true - def handle_event("validate-and-save", attrs, socket) do - args = Map.get(attrs, "config", %{}) - - socket = - socket - |> assign( - page: "config", - changeset: Config.changeset(args) - ) - # DB connection checker is expensive and returns result to caller process with delay - # so we don't do this validation on changeset level - |> db_connection_checker(args) - |> update_matching_date_time(attrs) - - {:noreply, socket |> push_event("save-config", %{config: socket.assigns.changeset})} - end - - @impl true - def handle_event("add-new-" <> field, _attrs, %{assigns: assigns} = socket) - when field in ~w[db-attr date-pattern date-time-pattern] do - new_field = - field - |> String.replace("-", "_") - |> String.to_atom() - |> case do - :db_attr -> %DbAttribute{id: Nanoid.generate(), name: "", value: ""} - :date_pattern -> %DashBoard.DatePattern{id: Nanoid.generate()} - :date_time_pattern -> %DashBoard.DateTimePattern{id: Nanoid.generate()} - end - - association = "#{field}s" |> String.replace("-", "_") |> String.to_atom() - - updated_association = - assigns.changeset - |> Ecto.Changeset.get_field(association, []) - |> Enum.concat([new_field]) - - updated_changeset = - Ecto.Changeset.put_embed(assigns.changeset, association, updated_association) - - {:noreply, - socket - |> assign(changeset: updated_changeset) - |> push_event("scroll-to-bottom", %{id: "#{field}s-container"})} - end - - @impl true - def handle_event("remove-" <> field, %{"attrid" => attrid}, %{assigns: assigns} = socket) - when field in ~w[db-attr date-pattern date-time-pattern] do - association = "#{field}s" |> String.replace("-", "_") |> String.to_atom() - - updated_association = - assigns.changeset - # For relations get_change/2 return the original changeset data with changes applied, fetch_change!/2 returns raw db_config changesets - |> Ecto.Changeset.fetch_change!(association) - |> Enum.reject(fn embed_changeset -> - Ecto.Changeset.get_field(embed_changeset, :id) == attrid - end) - - updated_changeset = - Ecto.Changeset.put_embed(assigns.changeset, association, updated_association) - - {:noreply, - socket - |> assign(changeset: updated_changeset) - |> update_matching_date_time()} - end - - @impl true - def handle_info(:finish, socket) do - {:noreply, assign(socket, state: Csv2sql.ProgressTracker.get_state())} - end - - @impl true - def handle_info(:update_state, socket) do - state = socket.assigns.state - - time_taken = - if is_nil(state.start_time) do - 0 - else - DateTime.utc_now() - |> Time.diff(state.start_time, :millisecond) - |> Kernel./(1000) - |> Float.round() - end - - if state.status in [:init, :working, :imported, :validating] do - Process.send_after(self(), :update_state, 200) - end - - {:noreply, - assign(socket, - state: Csv2sql.ProgressTracker.get_state(), - time_spend: time_taken, - cpu_usage: :cpu_sup.util() |> Float.round(2), - memory_usage: :erlang.memory(:total) |> Sizeable.filesize() - )} - end - - @impl true - def handle_info(:check_db_connection, %{assigns: assigns} = socket) do - with( - db_url = create_db_url(assigns.changeset.changes, hide_password: false), - true <- not ("NA" == db_url), - db_type <- Ecto.Changeset.get_field(assigns.changeset, :db_type), - false <- is_nil(db_type), - args = %{db_type: db_type, db_url: db_url}, - resp = ConnectionTest.check_db_connection(self(), args), - :ok <- resp - ) do - socket - else - {:error, :on_going} -> - Process.send_after(self(), :check_db_connection, @debounce_time) - socket - - _ -> - assign(socket, db_connection_established: false) - end - - {:noreply, socket} - end - - # DB connection callbacks - @impl true - def handle_info({:connected, _}, socket), - do: {:noreply, assign(socket, db_connection_established: true)} - - @impl true - def handle_info({:error, _}, %{assigns: assigns} = socket) do - {:noreply, - assign( - socket, - changeset: - Ecto.Changeset.add_error(assigns.changeset, :db_url, "Could not connect to database"), - db_connection_established: false - )} - end - - @impl true - def handle_info({:report_error, reason}, socket) do - IO.inspect("Reported Error #{inspect(reason)}") - {:noreply, socket} - end - - defp render_page(assigns) do - case assigns.page do - "config" -> - ConfigLive.config_page(assigns) - - "start" -> - StartLive.start_page(assigns) - - "about" -> - AboutLive.about_page(assigns) - end - end - - defp db_connection_checker(%{assigns: assigns} = socket, args) do - if db_config_updated?(assigns, args) do - if assigns.db_connection_debouncer, - do: Process.cancel_timer(assigns.db_connection_debouncer) - - timer_ref = Process.send_after(self(), :check_db_connection, @debounce_time) - assign(socket, :db_connection_debouncer, timer_ref) - else - socket - end - end - - defp db_config_updated?(%{changeset: changeset}, args) do - # TODO: Take into account custom db params - Ecto.Changeset.get_field(changeset, :db_type) != Map.get(args, "db_type") || - Ecto.Changeset.get_field(changeset, :db_username) != Map.get(args, "db_username") || - Ecto.Changeset.get_field(changeset, :db_password) != Map.get(args, "db_password") || - Ecto.Changeset.get_field(changeset, :db_host) != Map.get(args, "db_host") || - Ecto.Changeset.get_field(changeset, :db_name) != Map.get(args, "db_name") - end - - defp update_matching_date_time(%{assigns: assigns} = socket, attrs \\ %{}) do - date_time_sample = - get_in(attrs, ["config", "date_time_trial"]) || - Ecto.Changeset.get_field(assigns.changeset, :date_time_trial) - - case match_date_time(assigns.changeset, date_time_sample) do - {type, index} -> - socket - |> assign(matching_date_time: {type, index}) - |> push_event("scroll-into-view", %{ - id: "config_#{type}_patterns_#{index}_pattern" - }) - - false -> - assign(socket, matching_date_time: nil) - end - end - - defp prepare_args(changeset) do - config = Ecto.Changeset.apply_changes(changeset) - - %{ - source_directory: config.source_directory, - schema_path: config.schema_path, - insert_schema: config.insert_schema, - insert_data: config.insert_data, - ordered: config.ordered, - date_patterns: prepare_date_patterns(config.date_patterns), - datetime_patterns: prepare_date_patterns(config.date_time_patterns), - schema_infer_chunk_size: config.schema_infer_chunk_size, - worker_count: config.worker_count, - parse_datetime: config.parse_datetime, - remove_illegal_characters: config.remove_illegal_characters, - db_type: config.db_type, - db_name: config.db_name, - db_url: create_db_url(changeset.changes), - drop_existing_tables: config.drop_existing_tables, - varchar_limit: config.varchar_limit, - db_worker_count: config.db_worker_count, - insertion_chunk_size: config.insertion_chunk_size, - log: config.log - } - end - - defp prepare_date_patterns(date_patterns) do - date_patterns - |> Enum.reject(&is_nil(&1.pattern)) - |> Enum.map(&%{id: &1.id, pattern: &1.pattern}) - end -end diff --git a/apps/dashboard/lib/dashboard_web/live/main_live.html.heex b/apps/dashboard/lib/dashboard_web/live/main_live.html.heex deleted file mode 100644 index 4d099d2..0000000 --- a/apps/dashboard/lib/dashboard_web/live/main_live.html.heex +++ /dev/null @@ -1,37 +0,0 @@ -
-
-
- -
-
CSV2SQL
-
- -
- -
- -<.render_page {assigns} /> diff --git a/apps/dashboard/lib/dashboard_web/live/modals/date_time_patterns_modal.ex b/apps/dashboard/lib/dashboard_web/live/modals/date_time_patterns_modal.ex deleted file mode 100644 index d0c23f5..0000000 --- a/apps/dashboard/lib/dashboard_web/live/modals/date_time_patterns_modal.ex +++ /dev/null @@ -1,187 +0,0 @@ -defmodule DashboardWeb.Live.Modal.DateTimePatternsModal do - use DashboardWeb, :component - import Dashboard.Helpers - - @date_time_pattern_hints [ - {"{YYYY}", "full year number (0..9999)"}, - {"{YY}", "the last two digits of the year number (0.99)"}, - {"{M}", "month number (1..12)"}, - {"{Mshort}", "abbreviated month name (Jan..Dec, no padding)"}, - {"{Mfull}", "full month name (January..December, no padding)"}, - {"{D}", "day number (1..31)"}, - {"{Dord}", "ordinal day of the year (1..366)"}, - {"{h24}", "hour of the day (00..23)"}, - {"{h12}", "hour of the day (1..12)"}, - {"{m}", "minutes of the hour (00..59)"}, - {"{s}", "seconds of the minute (00..60)"}, - {"{am}", "lowercase am or pm (no padding)"}, - {"{AM}", "uppercase AM or PM (no padding)"} - ] - - def date_time_patterns_modal(assigns) do - custom_date_time_patterns_added = - assigns.changeset |> Ecto.Changeset.get_field(:date_time_patterns, []) |> length() != 0 - - custom_date_patterns_added = - assigns.changeset |> Ecto.Changeset.get_field(:date_patterns, []) |> length() != 0 - - assigns = - assigns - |> assign(:custom_date_patterns_added, custom_date_patterns_added) - |> assign(:custom_date_time_patterns_added, custom_date_time_patterns_added) - - ~H""" - -
-
- Add date patterns -
- - - - -
-
-
-
-
- <%= if @custom_date_patterns_added do %> - <%= for {date_patterns_form, index} <- Enum.with_index(inputs_for(@form, :date_patterns), 1) do %> -
-
- <%= index %>. - <%= text_input(date_patterns_form, :pattern, - disabled: index == 1, - class: input_class(index, :date, @matching_date_time), - placeholder: "Pattern", - list: "date-patterns-suggestions" - ) %> -
- - <%= hidden_input(date_patterns_form, :id) %> - - <%= unless index == 1 do %> -
- -
- <% end %> -
- <% end %> - <% else %> -
-
-
- No custom date patterns added! -
-
- <% end %> -
-
- - Add datetime pattern -
-
-
-
- <%= if @custom_date_time_patterns_added do %> - <%= for {date_time_patterns_form, index} <- Enum.with_index(inputs_for(@form, :date_time_patterns), 1) do %> -
-
- <%= index %>. - <%= text_input(date_time_patterns_form, :pattern, - disabled: index == 1, - class: input_class(index, :date_time, @matching_date_time), - placeholder: "Pattern", - list: "date-time-patterns-suggestions" - ) %> -
- - <%= hidden_input(date_time_patterns_form, :id) %> - - <%= unless index == 1 do %> -
- -
- <% end %> -
- <% end %> - <% else %> -
-
-
- No custom date time patterns added! -
-
- <% end %> -
-
- - Add datetime pattern -
-
-
-
-

- Add custom date time patterns - Some commonly used pattern symbols are given below: -

-
- <%= for {pattern, description} <- date_time_pattern_hints() do %> -
-
<%= pattern %>
-
-
-
<%= description %>
-
- <% end %> -
-

*Fractional seconds and timezones are not supported.

- -
-
- - <.popup - title="Enter a sample date or date time to check if it matches any added custom patterns" - id="date-time-try-it-hint" - /> - -
- <%= if present?(Ecto.Changeset.get_field(@changeset, :date_time_trial)) && is_nil(@matching_date_time) do %> - No matching patterns found - <% end %> - <%= text_input(@form, :date_time_trial, - class: "form-control", - placeholder: "Pattern", - id: "date-time-trial" - ) %> -
-
-
-
-
- - """ - end - - defp date_time_pattern_hints, do: @date_time_pattern_hints - - defp input_class(index, type, {match_type, match_index}) - when type == match_type and index == match_index + 1, - do: "form-control matched-input" - - defp input_class(_index, _type, _), do: "form-control" -end diff --git a/apps/dashboard/lib/dashboard_web/live/modals/db_attrs_modal.ex b/apps/dashboard/lib/dashboard_web/live/modals/db_attrs_modal.ex deleted file mode 100644 index 98a15bf..0000000 --- a/apps/dashboard/lib/dashboard_web/live/modals/db_attrs_modal.ex +++ /dev/null @@ -1,76 +0,0 @@ -defmodule DashboardWeb.Live.Modal.DbAttributesModal do - use DashboardWeb, :component - - def db_attrs_modal(assigns) do - db_attrs_added = - assigns.changeset |> Ecto.Changeset.get_field(:db_attrs, []) |> length() != 0 - - assigns = assign(assigns, :db_attrs_added, db_attrs_added) - - ~H""" - -
-
- Add additional database attributes, click here to - learn - more -
- (Duplicate and empty attributes will be ignored) -
-
- - - - -
- <%= if @db_attrs_added do %> - <%= for {db_attrs_form, index} <- Enum.with_index(inputs_for(@form, :db_attrs), 1) do %> -
-
- <%= index %>. - <%= text_input(db_attrs_form, :name, - class: "form-control", - placeholder: "Attribute name", - list: "db-attrs-suggestions" - ) %> - = - <%= text_input(db_attrs_form, :value, - class: "form-control", - placeholder: "Attribute value" - ) %> -
- - <%= hidden_input(db_attrs_form, :id) %> - -
- -
-
- <% end %> - <% else %> -
-
-
- No additional database configs added! -
-
- <% end %> -
- -
- - Add more database configurations -
-
- - """ - end -end diff --git a/apps/dashboard/lib/dashboard_web/live/modals/modal.ex b/apps/dashboard/lib/dashboard_web/live/modals/modal.ex deleted file mode 100644 index 716e35b..0000000 --- a/apps/dashboard/lib/dashboard_web/live/modals/modal.ex +++ /dev/null @@ -1,40 +0,0 @@ -defmodule DashboardWeb.Live.Modals do - use DashboardWeb, :component - - def modal(assigns) do - ~H""" - -
-
-
-
<%= @title %>
- -
- -
-

<%= render_slot(@inner_block) %>

-
-
-
- """ - end - - defp hide_modal(js \\ %JS{}) do - js - |> JS.hide(to: ".live-modal", transition: "opacity-0") - |> JS.hide(to: ".modal-backdrop", transition: "opacity-0") - |> JS.push("close-modal") - end -end diff --git a/apps/dashboard/lib/dashboard_web/live/start_live.ex b/apps/dashboard/lib/dashboard_web/live/start_live.ex deleted file mode 100644 index 71908e0..0000000 --- a/apps/dashboard/lib/dashboard_web/live/start_live.ex +++ /dev/null @@ -1,124 +0,0 @@ -defmodule DashboardWeb.Live.StartLive do - use DashboardWeb, :live_view - - def start_page(assigns) do - ~H""" -
-
-
- Status: - - <%= cond do %> - <% not @changeset.valid? -> %> - Invalid Configurations! - - <% is_tuple(@state.status) -> %> - Error! check logs - - <% true -> %> - <%= show_status(@state.status) %> - <% end %> - -
- <%= if not is_nil(@state.validation_status) do %> -
- Validation Status: - <%= @state.validation_status %>! -
- <% end %> - <%= if @state.status != :init do %> -
Total Files: <%= Enum.count(Map.values(@state.files)) %>
-
Files Imported: <%= Enum.count(Map.values(@state.files), fn %{status: status} -> status == :done end) %>
-
CPU Usage: <%= @cpu_usage %>%
-
Memory Usage of Application: <%= @memory_usage %>
-
Time Elapsed: <%= @time_spend %> seconds
- <% end %> -
- -
- - <%= Enum.map Map.values(@state.files), fn %{name: name, path: path, size: size, row_count: row_count, rows_processed: rows_processed, status: status} -> %> -
- Name: - <%= name %> - - Path: - <%= path %> - - Size: - <%= size %> - - Total Number of Records: - <%= row_count %> - - - Status: - <%= case status do %> - <% :pending -> %> Pending - - <% :analyze -> %> Infering Schema - - <% :loading -> %> - Inserting Data - Record Inserted: - <%= rows_processed %> - -
- <% percentage_progress=if(row_count==0, do: 100, else: (rows_processed / row_count) * 100) %> -
- - <%= Float.round(percentage_progress * 1.0 , 2) %>% - -
-
- - <% :done -> %> Finished - <% end %> - -
- <% end %> - -
- -
- - """ - end - - defp item_success_class(status) do - if status == :done, do: "list-group-item-success", else: "" - end - - defp error_status_class(status) do - if is_tuple(status) or status == :failed, do: "error-status", else: "" - end - - defp spinner_loading_class(status) do - if status in [:working, :imported, :validating], - do: "spinner loading", - else: "" - end - - defp show_status(:imported), do: :working - - defp show_status(status), do: status - - defp button_class(changeset) do - if not changeset.valid?, do: "button-disabled", else: "button-enabled" - end -end diff --git a/apps/dashboard/lib/dashboard_web/live/ui.ex b/apps/dashboard/lib/dashboard_web/live/ui.ex deleted file mode 100644 index 1c960d7..0000000 --- a/apps/dashboard/lib/dashboard_web/live/ui.ex +++ /dev/null @@ -1,138 +0,0 @@ -defmodule DashboardWeb.Live.UI do - import Phoenix.HTML.Form - import Phoenix.Component - alias DashboardWeb.Live.IconSvg - - def db_connection_check(assigns \\ %{}) do - db_url = Dashboard.Helpers.create_db_url(assigns.changeset.changes) - assigns = assign(assigns, :db_url, db_url) - - # TODO: Maybe add loader for indicating on going db connect check - ~H""" -
-
- <%= if @db_connection_established do %> - - CAN CONNECT TO DATABASE! - <% else %> - - - COULD NOT CONNECT TO DATABASE! - - <% end %> -
- -
-

DATABASE URL: <%= @db_url %>

-
-
- """ - end - - def popup(assigns \\ %{}) do - ~H""" -
- -
- """ - end - - def config_item(assigns \\ %{}) do - ~H""" -
-
- <.popup title={@tooltip} id={@id} /> - - <%= if assigns[:checkbox_input] do %> -
- <%= render_slot(@input, @form) %> -
- <% end %> - -
- <%= if assigns[:required_input] do %> - * - <% end %> - <%= @title %> -
-
- - <%= if !assigns[:checkbox_input] do %> -
<%= render_slot(@input, @form) %>
- <% end %> -
- """ - end - - def flow_card(assigns \\ %{}) do - ~H""" -
-
- -
- Infer Schema -
-
- -
- -
- -
- -
-
<.popup title="Set whether to create database schema" id="insert_schema_popup" />
-
- <%= checkbox(@form, :insert_schema, class: "form-check-input") %> - <%= label(@form, :insert_schema, "Create Tables", class: "form-check-label") %> -
-
-
- -
- -
- -
- -
-
<.popup title="Set whether to insert the CSVs into the database" id="insert_data_popup" />
-
- <%= checkbox(@form, :insert_data, class: "form-check-input") %> - <%= label(@form, :insert_data, class: "form-check-label") %> -
-
-
-
- """ - end - - @doc """ - Generates tag for inlined form input errors. - """ - def error_tag(form, field) do - Enum.map(Keyword.get_values(form.errors, field), fn error -> - Phoenix.HTML.Tag.content_tag(:span, translate_error(error), - class: "invalid-feedback", - phx_feedback_for: input_name(form, field) - ) - end) - end - - @doc """ - Translates an error message. - """ - def translate_error({msg, opts}) do - Enum.reduce(opts, msg, fn {key, value}, acc -> - String.replace(acc, "%{#{key}}", fn _ -> to_string(value) end) - end) - end -end diff --git a/apps/dashboard/lib/dashboard_web/router.ex b/apps/dashboard/lib/dashboard_web/router.ex deleted file mode 100644 index 6d9dacf..0000000 --- a/apps/dashboard/lib/dashboard_web/router.ex +++ /dev/null @@ -1,18 +0,0 @@ -defmodule DashboardWeb.Router do - use DashboardWeb, :router - - pipeline :browser do - plug :accepts, ["html"] - plug :fetch_session - plug :fetch_live_flash - plug :put_root_layout, {DashboardWeb.LayoutView, :root} - plug :protect_from_forgery - plug :put_secure_browser_headers - end - - scope "/", DashboardWeb do - pipe_through :browser - - live "/", Live.MainLive, :index - end -end diff --git a/apps/dashboard/lib/dashboard_web/telemetry.ex b/apps/dashboard/lib/dashboard_web/telemetry.ex deleted file mode 100644 index 59cb1b2..0000000 --- a/apps/dashboard/lib/dashboard_web/telemetry.ex +++ /dev/null @@ -1,48 +0,0 @@ -defmodule DashboardWeb.Telemetry do - use Supervisor - import Telemetry.Metrics - - def start_link(arg) do - Supervisor.start_link(__MODULE__, arg, name: __MODULE__) - end - - @impl true - def init(_arg) do - children = [ - # Telemetry poller will execute the given period measurements - # every 10_000ms. Learn more here: https://hexdocs.pm/telemetry_metrics - {:telemetry_poller, measurements: periodic_measurements(), period: 10_000} - # Add reporters as children of your supervision tree. - # {Telemetry.Metrics.ConsoleReporter, metrics: metrics()} - ] - - Supervisor.init(children, strategy: :one_for_one) - end - - def metrics do - [ - # Phoenix Metrics - summary("phoenix.endpoint.stop.duration", - unit: {:native, :millisecond} - ), - summary("phoenix.router_dispatch.stop.duration", - tags: [:route], - unit: {:native, :millisecond} - ), - - # VM Metrics - summary("vm.memory.total", unit: {:byte, :kilobyte}), - summary("vm.total_run_queue_lengths.total"), - summary("vm.total_run_queue_lengths.cpu"), - summary("vm.total_run_queue_lengths.io") - ] - end - - defp periodic_measurements do - [ - # A module, function and arguments to be invoked periodically. - # This function must call :telemetry.execute/3 and a metric must be added above. - # {DashboardWeb, :count_users, []} - ] - end -end diff --git a/apps/dashboard/lib/dashboard_web/templates/layout/live.html.heex b/apps/dashboard/lib/dashboard_web/templates/layout/live.html.heex deleted file mode 100644 index 0543398..0000000 --- a/apps/dashboard/lib/dashboard_web/templates/layout/live.html.heex +++ /dev/null @@ -1 +0,0 @@ -<%= @inner_content %> diff --git a/apps/dashboard/lib/dashboard_web/templates/layout/root.html.heex b/apps/dashboard/lib/dashboard_web/templates/layout/root.html.heex deleted file mode 100644 index 2e1e612..0000000 --- a/apps/dashboard/lib/dashboard_web/templates/layout/root.html.heex +++ /dev/null @@ -1,21 +0,0 @@ - - - - - - - - <.live_title>CSV2SQL - Dashboard - - - - - <%= @inner_content %> - - diff --git a/apps/dashboard/lib/dashboard_web/views/error_view.ex b/apps/dashboard/lib/dashboard_web/views/error_view.ex deleted file mode 100644 index 0bd0648..0000000 --- a/apps/dashboard/lib/dashboard_web/views/error_view.ex +++ /dev/null @@ -1,7 +0,0 @@ -defmodule DashboardWeb.ErrorView do - use DashboardWeb, :view - - def template_not_found(template, _assigns) do - Phoenix.Controller.status_message_from_template(template) - end -end diff --git a/apps/dashboard/lib/dashboard_web/views/layout_view.ex b/apps/dashboard/lib/dashboard_web/views/layout_view.ex deleted file mode 100644 index 5104427..0000000 --- a/apps/dashboard/lib/dashboard_web/views/layout_view.ex +++ /dev/null @@ -1,3 +0,0 @@ -defmodule DashboardWeb.LayoutView do - use DashboardWeb, :view -end diff --git a/apps/dashboard/lib/dashboard_web/views/page_view.ex b/apps/dashboard/lib/dashboard_web/views/page_view.ex deleted file mode 100644 index 07e4875..0000000 --- a/apps/dashboard/lib/dashboard_web/views/page_view.ex +++ /dev/null @@ -1,3 +0,0 @@ -defmodule DashboardWeb.PageView do - use DashboardWeb, :view -end diff --git a/apps/dashboard/mix.exs b/apps/dashboard/mix.exs deleted file mode 100644 index 19fdc85..0000000 --- a/apps/dashboard/mix.exs +++ /dev/null @@ -1,71 +0,0 @@ -defmodule Dashboard.MixProject do - use Mix.Project - - def project do - [ - app: :dashboard, - version: "0.1.0", - build_path: "../../_build", - config_path: "../../config/config.exs", - deps_path: "../../deps", - lockfile: "../../mix.lock", - elixir: "~> 1.12", - elixirc_paths: elixirc_paths(Mix.env()), - compilers: Mix.compilers(), - start_permanent: Mix.env() == :prod, - aliases: aliases(), - deps: deps() - ] - end - - # Configuration for the OTP application. - # - # Type `mix help compile.app` for more information. - def application do - [ - mod: {Dashboard.Application, []}, - extra_applications: [:logger, :runtime_tools, :os_mon] - ] - end - - # Specifies which paths to compile per environment. - defp elixirc_paths(:test), do: ["lib", "test/support"] - defp elixirc_paths(_), do: ["lib"] - - # Specifies your project dependencies. - # - # Type `mix help deps` for examples and options. - defp deps do - [ - {:phoenix, "~> 1.7"}, - {:phoenix_html, "~> 3.0"}, - {:phoenix_view, "~> 2.0"}, - {:phoenix_live_view, "~> 0.20.2"}, - {:telemetry_metrics, "~> 1.0"}, - {:telemetry_poller, "~> 1.1.0"}, - {:jason, "~> 1.2"}, - {:plug_cowboy, "~> 2.5"}, - {:ecto, "~> 3.11.2"}, - {:phoenix_ecto, "~> 4.4"}, - {:csv2sql, in_umbrella: true}, - {:nanoid, "~> 2.1.0"}, - - # Development Deps - {:phoenix_live_reload, "~> 1.5.3", only: :dev}, - {:dart_sass, "~> 0.2", runtime: Mix.env() == :dev}, - {:esbuild, "~> 0.8.1", runtime: Mix.env() == :dev}, - {:floki, ">= 0.36.1", only: :test} - ] - end - - defp aliases do - [ - setup: ["deps.get"], - "assets.deploy": [ - "esbuild default --minify", - "sass default --no-source-map --style=compressed", - "phx.digest" - ] - ] - end -end diff --git a/apps/dashboard/priv/static/favicon-a8ca4e3a2bb8fea46a9ee9e102e7d3eb.ico b/apps/dashboard/priv/static/favicon-a8ca4e3a2bb8fea46a9ee9e102e7d3eb.ico deleted file mode 100644 index 73de524..0000000 Binary files a/apps/dashboard/priv/static/favicon-a8ca4e3a2bb8fea46a9ee9e102e7d3eb.ico and /dev/null differ diff --git a/apps/dashboard/priv/static/favicon.ico b/apps/dashboard/priv/static/favicon.ico deleted file mode 100644 index 73de524..0000000 Binary files a/apps/dashboard/priv/static/favicon.ico and /dev/null differ diff --git a/apps/dashboard/priv/static/images/logo.png b/apps/dashboard/priv/static/images/logo.png deleted file mode 100644 index 362c694..0000000 Binary files a/apps/dashboard/priv/static/images/logo.png and /dev/null differ diff --git a/apps/dashboard/priv/static/robots-9e2c81b0855bbff2baa8371bc4a78186.txt b/apps/dashboard/priv/static/robots-9e2c81b0855bbff2baa8371bc4a78186.txt deleted file mode 100644 index 26e06b5..0000000 --- a/apps/dashboard/priv/static/robots-9e2c81b0855bbff2baa8371bc4a78186.txt +++ /dev/null @@ -1,5 +0,0 @@ -# See https://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file -# -# To ban all spiders from the entire site uncomment the next two lines: -# User-agent: * -# Disallow: / diff --git a/apps/dashboard/priv/static/robots-9e2c81b0855bbff2baa8371bc4a78186.txt.gz b/apps/dashboard/priv/static/robots-9e2c81b0855bbff2baa8371bc4a78186.txt.gz deleted file mode 100644 index 24de2e1..0000000 Binary files a/apps/dashboard/priv/static/robots-9e2c81b0855bbff2baa8371bc4a78186.txt.gz and /dev/null differ diff --git a/apps/dashboard/priv/static/robots.txt b/apps/dashboard/priv/static/robots.txt deleted file mode 100644 index 26e06b5..0000000 --- a/apps/dashboard/priv/static/robots.txt +++ /dev/null @@ -1,5 +0,0 @@ -# See https://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file -# -# To ban all spiders from the entire site uncomment the next two lines: -# User-agent: * -# Disallow: / diff --git a/apps/dashboard/priv/static/robots.txt.gz b/apps/dashboard/priv/static/robots.txt.gz deleted file mode 100644 index 24de2e1..0000000 Binary files a/apps/dashboard/priv/static/robots.txt.gz and /dev/null differ diff --git a/apps/dashboard/test/dashboard_web/controllers/page_controller_test.exs b/apps/dashboard/test/dashboard_web/controllers/page_controller_test.exs deleted file mode 100644 index 76a419f..0000000 --- a/apps/dashboard/test/dashboard_web/controllers/page_controller_test.exs +++ /dev/null @@ -1,8 +0,0 @@ -defmodule DashboardWeb.PageControllerTest do - use DashboardWeb.ConnCase - - test "GET /", %{conn: conn} do - conn = get(conn, "/") - assert html_response(conn, 200) =~ "Welcome to Phoenix!" - end -end diff --git a/apps/dashboard/test/dashboard_web/views/error_view_test.exs b/apps/dashboard/test/dashboard_web/views/error_view_test.exs deleted file mode 100644 index ed1b1ce..0000000 --- a/apps/dashboard/test/dashboard_web/views/error_view_test.exs +++ /dev/null @@ -1,14 +0,0 @@ -defmodule DashboardWeb.ErrorViewTest do - use DashboardWeb.ConnCase, async: true - - # Bring render/3 and render_to_string/3 for testing custom views - import Phoenix.View - - test "renders 404.html" do - assert render_to_string(DashboardWeb.ErrorView, "404.html", []) == "Not Found" - end - - test "renders 500.html" do - assert render_to_string(DashboardWeb.ErrorView, "500.html", []) == "Internal Server Error" - end -end diff --git a/apps/dashboard/test/dashboard_web/views/layout_view_test.exs b/apps/dashboard/test/dashboard_web/views/layout_view_test.exs deleted file mode 100644 index a00b9dd..0000000 --- a/apps/dashboard/test/dashboard_web/views/layout_view_test.exs +++ /dev/null @@ -1,8 +0,0 @@ -defmodule DashboardWeb.LayoutViewTest do - use DashboardWeb.ConnCase, async: true - - # When testing helpers, you may want to import Phoenix.HTML and - # use functions such as safe_to_string() to convert the helper - # result into an HTML string. - # import Phoenix.HTML -end diff --git a/apps/dashboard/test/dashboard_web/views/page_view_test.exs b/apps/dashboard/test/dashboard_web/views/page_view_test.exs deleted file mode 100644 index 6e4c0ee..0000000 --- a/apps/dashboard/test/dashboard_web/views/page_view_test.exs +++ /dev/null @@ -1,3 +0,0 @@ -defmodule DashboardWeb.PageViewTest do - use DashboardWeb.ConnCase, async: true -end diff --git a/apps/dashboard/test/support/conn_case.ex b/apps/dashboard/test/support/conn_case.ex deleted file mode 100644 index 22e9fc1..0000000 --- a/apps/dashboard/test/support/conn_case.ex +++ /dev/null @@ -1,37 +0,0 @@ -defmodule DashboardWeb.ConnCase do - @moduledoc """ - This module defines the test case to be used by - tests that require setting up a connection. - - Such tests rely on `Phoenix.ConnTest` and also - import other functionality to make it easier - to build common data structures and query the data layer. - - Finally, if the test case interacts with the database, - we enable the SQL sandbox, so changes done to the database - are reverted at the end of every test. If you are using - PostgreSQL, you can even run database tests asynchronously - by setting `use DashboardWeb.ConnCase, async: true`, although - this option is not recommended for other databases. - """ - - use ExUnit.CaseTemplate - - using do - quote do - # Import conveniences for testing with connections - import Plug.Conn - import Phoenix.ConnTest - import DashboardWeb.ConnCase - - alias DashboardWeb.Router.Helpers, as: Routes - - # The default endpoint for testing - @endpoint DashboardWeb.Endpoint - end - end - - setup _tags do - {:ok, conn: Phoenix.ConnTest.build_conn()} - end -end diff --git a/apps/dashboard/test/test_helper.exs b/apps/dashboard/test/test_helper.exs deleted file mode 100644 index 869559e..0000000 --- a/apps/dashboard/test/test_helper.exs +++ /dev/null @@ -1 +0,0 @@ -ExUnit.start() diff --git a/config/config.exs b/config/config.exs deleted file mode 100644 index 6608f48..0000000 --- a/config/config.exs +++ /dev/null @@ -1,2 +0,0 @@ -import Config -import_config "../apps/dashboard/config/config.exs" diff --git a/apps/csv2sql/lib/csv2sql.ex b/lib/csv2sql.ex similarity index 100% rename from apps/csv2sql/lib/csv2sql.ex rename to lib/csv2sql.ex diff --git a/apps/csv2sql/lib/csv2sql/application.ex b/lib/csv2sql/application.ex similarity index 100% rename from apps/csv2sql/lib/csv2sql/application.ex rename to lib/csv2sql/application.ex diff --git a/apps/csv2sql/lib/csv2sql/config/config.ex b/lib/csv2sql/config/config.ex similarity index 100% rename from apps/csv2sql/lib/csv2sql/config/config.ex rename to lib/csv2sql/config/config.ex diff --git a/apps/csv2sql/lib/csv2sql/config/loader.ex b/lib/csv2sql/config/loader.ex similarity index 100% rename from apps/csv2sql/lib/csv2sql/config/loader.ex rename to lib/csv2sql/config/loader.ex diff --git a/apps/csv2sql/lib/csv2sql/database/connection_test.ex b/lib/csv2sql/database/connection_test.ex similarity index 100% rename from apps/csv2sql/lib/csv2sql/database/connection_test.ex rename to lib/csv2sql/database/connection_test.ex diff --git a/apps/csv2sql/lib/csv2sql/database/database.ex b/lib/csv2sql/database/database.ex similarity index 100% rename from apps/csv2sql/lib/csv2sql/database/database.ex rename to lib/csv2sql/database/database.ex diff --git a/apps/csv2sql/lib/csv2sql/database/mysql.ex b/lib/csv2sql/database/mysql.ex similarity index 100% rename from apps/csv2sql/lib/csv2sql/database/mysql.ex rename to lib/csv2sql/database/mysql.ex diff --git a/apps/csv2sql/lib/csv2sql/database/postgres.ex b/lib/csv2sql/database/postgres.ex similarity index 100% rename from apps/csv2sql/lib/csv2sql/database/postgres.ex rename to lib/csv2sql/database/postgres.ex diff --git a/apps/csv2sql/lib/csv2sql/db_loader/consumer.ex b/lib/csv2sql/db_loader/consumer.ex similarity index 100% rename from apps/csv2sql/lib/csv2sql/db_loader/consumer.ex rename to lib/csv2sql/db_loader/consumer.ex diff --git a/apps/csv2sql/lib/csv2sql/db_loader/consumer_supervisor.ex b/lib/csv2sql/db_loader/consumer_supervisor.ex similarity index 100% rename from apps/csv2sql/lib/csv2sql/db_loader/consumer_supervisor.ex rename to lib/csv2sql/db_loader/consumer_supervisor.ex diff --git a/apps/csv2sql/lib/csv2sql/db_loader/producer.ex b/lib/csv2sql/db_loader/producer.ex similarity index 100% rename from apps/csv2sql/lib/csv2sql/db_loader/producer.ex rename to lib/csv2sql/db_loader/producer.ex diff --git a/apps/csv2sql/lib/csv2sql/file.ex b/lib/csv2sql/file.ex similarity index 100% rename from apps/csv2sql/lib/csv2sql/file.ex rename to lib/csv2sql/file.ex diff --git a/apps/csv2sql/lib/csv2sql/helpers/helpers.ex b/lib/csv2sql/helpers/helpers.ex similarity index 100% rename from apps/csv2sql/lib/csv2sql/helpers/helpers.ex rename to lib/csv2sql/helpers/helpers.ex diff --git a/apps/csv2sql/lib/csv2sql/import_validator/import_validator.ex b/lib/csv2sql/import_validator/import_validator.ex similarity index 100% rename from apps/csv2sql/lib/csv2sql/import_validator/import_validator.ex rename to lib/csv2sql/import_validator/import_validator.ex diff --git a/apps/csv2sql/lib/csv2sql/progress_tracker/progress_tracker.ex b/lib/csv2sql/progress_tracker/progress_tracker.ex similarity index 100% rename from apps/csv2sql/lib/csv2sql/progress_tracker/progress_tracker.ex rename to lib/csv2sql/progress_tracker/progress_tracker.ex diff --git a/apps/csv2sql/lib/csv2sql/progress_tracker/state.ex b/lib/csv2sql/progress_tracker/state.ex similarity index 100% rename from apps/csv2sql/lib/csv2sql/progress_tracker/state.ex rename to lib/csv2sql/progress_tracker/state.ex diff --git a/apps/csv2sql/lib/csv2sql/repos.ex b/lib/csv2sql/repos.ex similarity index 100% rename from apps/csv2sql/lib/csv2sql/repos.ex rename to lib/csv2sql/repos.ex diff --git a/apps/csv2sql/lib/csv2sql/stages/analyze.ex b/lib/csv2sql/stages/analyze.ex similarity index 100% rename from apps/csv2sql/lib/csv2sql/stages/analyze.ex rename to lib/csv2sql/stages/analyze.ex diff --git a/apps/csv2sql/lib/csv2sql/type_deducer/type_checker.ex b/lib/csv2sql/type_deducer/type_checker.ex similarity index 100% rename from apps/csv2sql/lib/csv2sql/type_deducer/type_checker.ex rename to lib/csv2sql/type_deducer/type_checker.ex diff --git a/apps/csv2sql/lib/csv2sql/type_deducer/type_deducer.ex b/lib/csv2sql/type_deducer/type_deducer.ex similarity index 100% rename from apps/csv2sql/lib/csv2sql/type_deducer/type_deducer.ex rename to lib/csv2sql/type_deducer/type_deducer.ex diff --git a/apps/csv2sql/lib/csv2sql/types/types.ex b/lib/csv2sql/types/types.ex similarity index 100% rename from apps/csv2sql/lib/csv2sql/types/types.ex rename to lib/csv2sql/types/types.ex diff --git a/mix.exs b/mix.exs index 9e8f61f..0c2669e 100644 --- a/mix.exs +++ b/mix.exs @@ -1,49 +1,87 @@ -defmodule Csv2sql.Umbrella.MixProject do +defmodule Csv2sql.MixProject do use Mix.Project + @app :csv2sql + def project do [ - apps_path: "apps", - version: "4.0.0", - build_embedded: Mix.env() == :prod, + app: :csv2sql, + version: "0.1.0", + elixir: "~> 1.13", + # directories to find source files + elixirc_path: elixirc_paths(Mix.env()), start_permanent: Mix.env() == :prod, deps: deps(), - releases: [ - csv2sql_and_dashboard: [ - applications: [ - csv2sql: :permanent, - dashboard: :permanent - ] - ], - csv2sql_only: [ - applications: [csv2sql: :permanent] - ] - ], - name: "Csv2sql", - description: "Csv2Sql is a blazing fast fully automated tool to load huge CSV files into a RDBMS.", - package: package(), - source_url: "https://github.com/kreeti/csv2sql" + aliases: aliases(), + compilers: Mix.compilers(), + test_coverage: [tool: ExCoveralls], + releases: [{@app, release()}] ] end - defp package() do + # Run "mix help compile.app" to learn about applications. + def application do [ - name: "csv2sql", - licenses: ["MIT"], - links: %{ - "GitHub" => "https://github.com/kreeti/csv2sql" - } + extra_applications: [:logger], + mod: {Csv2sql.Application, []} + # mod: {Csv2sql, []} ] end - def application do - [extra_applications: [:logger]] + # Run "mix help deps" to learn about dependencies. + defp deps do + [ + {:nimble_csv, "~> 1.2"}, + {:myxql, "~> 0.6.4"}, + {:postgrex, "~> 0.17.5"}, + {:ecto, "~> 3.11.2"}, + {:ecto_sql, "~> 3.11.1"}, + {:sizeable, "~> 1.0"}, + {:timex, "~> 3.7.11"}, + {:typed_struct, "~> 0.3.0"}, + {:flow, "~> 1.2.4"}, + {:stream_split, "~> 0.1.7"}, + {:codepagex, "~> 0.1.6"}, + {:bakeware, "~> 0.2.4"}, + + # For dev and/or test + {:dotenv, github: "avdi/dotenv_elixir", only: [:test]}, + {:benchee, "~> 1.3.0", only: :dev}, + {:excoveralls, "~> 0.18.1", only: [:dev, :test]}, + {:credo, "~> 1.7.2", only: [:dev, :test], runtime: false}, + {:dialyxir, "~> 1.4.3", only: [:dev, :test], runtime: false}, + {:mix_unused, "~> 0.4.1", only: [:dev, :test], runtime: false} + ] end - defp deps do + defp elixirc_paths(:test), do: ["lib", "test/support"] + defp elixirc_paths(_), do: ["lib"] + + defp aliases do + [ + ctest: &custom_test_command/1 + ] + end + + def custom_test_command(_) do + # System.shell("source .env.test && MIX_ENV=test mix test --trace --warnings-as-errors --cover", into: IO.stream()) + System.cmd("mix", ["coveralls.html", "--trace", "--warnings-as-errors"], + env: [{"MIX_ENV", "test"}], + into: IO.stream() + ) + end + + def release do [ - {:credo, "~> 1.7"}, - {:ex_doc, "~> 0.33.0"} + # keeps only significant chunks necessary for the VM operation + strip_beams: Mix.env() == :prod, + overwrite: true, + steps: [:assemble, &Bakeware.assemble/1], + bakeware: [ + # Highest compression level + compression_level: 1, + start_command: "start" + ] ] end end diff --git a/mix.lock b/mix.lock index b0889e9..5ab53fd 100644 --- a/mix.lock +++ b/mix.lock @@ -2,84 +2,47 @@ "bakeware": {:hex, :bakeware, "0.2.4", "0aaf49b34f4bab2aa433f9ff1485d9401e421603160abd6d269c469fc7b65212", [:make, :mix], [{:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "7b97bcf6fbeee53bb32441d6c495bf478d26f9575633cfef6831e421e86ada6d"}, "benchee": {:hex, :benchee, "1.3.0", "f64e3b64ad3563fa9838146ddefb2d2f94cf5b473bdfd63f5ca4d0657bf96694", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}, {:statistex, "~> 1.0", [hex: :statistex, repo: "hexpm", optional: false]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "34f4294068c11b2bd2ebf2c59aac9c7da26ffa0068afdf3419f1b176e16c5f81"}, "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"}, - "cachex": {:hex, :cachex, "3.4.0", "868b2959ea4aeb328c6b60ff66c8d5123c083466ad3c33d3d8b5f142e13101fb", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm", "370123b1ab4fba4d2965fb18f87fd758325709787c8c5fce35b3fe80645ccbe5"}, - "castore": {:hex, :castore, "1.0.6", "ffc42f110ebfdafab0ea159cd43d31365fa0af0ce4a02ecebf1707ae619ee727", [:mix], [], "hexpm", "374c6e7ca752296be3d6780a6d5b922854ffcc74123da90f2f328996b962d33a"}, "certifi": {:hex, :certifi, "2.12.0", "2d1cca2ec95f59643862af91f001478c9863c2ac9cb6e2f89780bfd8de987329", [:rebar3], [], "hexpm", "ee68d85df22e554040cdb4be100f33873ac6051387baf6a8f6ce82272340ff1c"}, "cli_spinners": {:hex, :cli_spinners, "0.1.0", "08d89c6f1840a8927daed48c675ecb2f20c05c855dc2b4b58c2933d393d5e0c9", [:mix], [], "hexpm", "3b8ccad722e518309d8b92230960ca4775a761164514f78a89c4f04d5a25c97f"}, "codepagex": {:hex, :codepagex, "0.1.6", "49110d09a25ee336a983281a48ef883da4c6190481e0b063afe2db481af6117e", [:mix], [], "hexpm", "1521461097dde281edf084062f525a4edc6a5e49f4fd1f5ec41c9c4955d5bd59"}, "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"}, "connection": {:hex, :connection, "1.1.0", "ff2a49c4b75b6fb3e674bfc5536451607270aac754ffd1bdfe175abe4a6d7a68", [:mix], [], "hexpm", "722c1eb0a418fbe91ba7bd59a47e28008a189d47e37e0e7bb85585a016b2869c"}, - "cowboy": {:hex, :cowboy, "2.12.0", "f276d521a1ff88b2b9b4c54d0e753da6c66dd7be6c9fca3d9418b561828a3731", [:make, :rebar3], [{:cowlib, "2.13.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "8a7abe6d183372ceb21caa2709bec928ab2b72e18a3911aa1771639bef82651e"}, - "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"}, - "cowlib": {:hex, :cowlib, "2.13.0", "db8f7505d8332d98ef50a3ef34b34c1afddec7506e4ee4dd4a3a266285d282ca", [:make, :rebar3], [], "hexpm", "e1e1284dc3fc030a64b1ad0d8382ae7e99da46c3246b815318a4b848873800a4"}, - "credo": {:hex, :credo, "1.7.2", "fdee3a7cb553d8f2e773569181f0a4a2bb7d192e27e325404cc31b354f59d68c", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "dd15d6fbc280f6cf9b269f41df4e4992dee6615939653b164ef951f60afcb68e"}, - "dart_sass": {:hex, :dart_sass, "0.5.0", "c52ad951d9bf611399b6d5efbf404f58dc3a699753662081fbcd4752c6ddeed0", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}], "hexpm", "bb76938174af8e047e855d3bb83e84ceca17e9acbf1ac6429f9d18171da4a911"}, + "credo": {:hex, :credo, "1.7.5", "643213503b1c766ec0496d828c90c424471ea54da77c8a168c725686377b9545", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "f799e9b5cd1891577d8c773d245668aa74a2fcd15eb277f51a0131690ebfb3fd"}, "db_connection": {:hex, :db_connection, "2.6.0", "77d835c472b5b67fc4f29556dee74bf511bbafecdcaf98c27d27fa5918152086", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c2f992d15725e721ec7fbc1189d4ecdb8afef76648c746a8e1cad35e3b8a35f3"}, "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"}, "dialyxir": {:hex, :dialyxir, "1.4.3", "edd0124f358f0b9e95bfe53a9fcf806d615d8f838e2202a9f430d59566b6b53b", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "bf2cfb75cd5c5006bec30141b131663299c661a864ec7fbbc72dfa557487a986"}, + "dir_walker": {:hex, :dir_walker, "0.0.8", "5332225074e4887e6e60ca0242af490215f296511ded4df18d554ae25394f727", [:mix], [], "hexpm", "2f4fb16e6427523700df9eb12eece5679ad4459aaefb1ca3cb580184bfc8d173"}, "dotenv": {:git, "https://github.com/avdi/dotenv_elixir.git", "d6fd3f327173fe18a455203987da95ef9f6cd4c5", []}, - "earmark_parser": {:hex, :earmark_parser, "1.4.39", "424642f8335b05bb9eb611aa1564c148a8ee35c9c8a8bba6e129d51a3e3c6769", [:mix], [], "hexpm", "06553a88d1f1846da9ef066b87b57c6f605552cfbe40d20bd8d59cc6bde41944"}, "ecto": {:hex, :ecto, "3.11.2", "e1d26be989db350a633667c5cda9c3d115ae779b66da567c68c80cfb26a8c9ee", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3c38bca2c6f8d8023f2145326cc8a80100c3ffe4dcbd9842ff867f7fc6156c65"}, "ecto_sql": {:hex, :ecto_sql, "3.11.1", "e9abf28ae27ef3916b43545f9578b4750956ccea444853606472089e7d169470", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.11.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16.0 or ~> 0.17.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ce14063ab3514424276e7e360108ad6c2308f6d88164a076aac8a387e1fea634"}, - "elixir_make": {:hex, :elixir_make, "0.7.7", "7128c60c2476019ed978210c245badf08b03dbec4f24d05790ef791da11aa17c", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}], "hexpm", "5bc19fff950fad52bbe5f211b12db9ec82c6b34a9647da0c2224b8b8464c7e6c"}, + "elixir_make": {:hex, :elixir_make, "0.6.3", "bc07d53221216838d79e03a8019d0839786703129599e9619f4ab74c8c096eac", [:mix], [], "hexpm", "f5cbd651c5678bcaabdbb7857658ee106b12509cd976c2c2fca99688e1daf716"}, "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, - "esbuild": {:hex, :esbuild, "0.8.1", "0cbf919f0eccb136d2eeef0df49c4acf55336de864e63594adcea3814f3edf41", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "25fc876a67c13cb0a776e7b5d7974851556baeda2085296c14ab48555ea7560f"}, - "eternal": {:hex, :eternal, "1.2.2", "d1641c86368de99375b98d183042dd6c2b234262b8d08dfd72b9eeaafc2a1abd", [:mix], [], "hexpm", "2c9fe32b9c3726703ba5e1d43a1d255a4f3f2d8f8f9bc19f094c7cb1a7a9e782"}, - "ex_doc": {:hex, :ex_doc, "0.33.0", "690562b153153c7e4d455dc21dab86e445f66ceba718defe64b0ef6f0bd83ba0", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "3f69adc28274cb51be37d09b03e4565232862a4b10288a3894587b0131412124"}, "excoveralls": {:hex, :excoveralls, "0.18.1", "a6f547570c6b24ec13f122a5634833a063aec49218f6fff27de9df693a15588c", [:mix], [{:castore, "~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "d65f79db146bb20399f23046015974de0079668b9abb2f5aac074d078da60b8d"}, "expo": {:hex, :expo, "0.5.2", "beba786aab8e3c5431813d7a44b828e7b922bfa431d6bfbada0904535342efe2", [:mix], [], "hexpm", "8c9bfa06ca017c9cb4020fabe980bc7fdb1aaec059fd004c2ab3bff03b1c599c"}, "file_system": {:hex, :file_system, "1.0.0", "b689cc7dcee665f774de94b5a832e578bd7963c8e637ef940cd44327db7de2cd", [:mix], [], "hexpm", "6752092d66aec5a10e662aefeed8ddb9531d79db0bc145bb8c40325ca1d8536d"}, - "floki": {:hex, :floki, "0.36.1", "712b7f2ba19a4d5a47dfe3e74d81876c95bbcbee44fe551f0af3d2a388abb3da", [:mix], [], "hexpm", "21ba57abb8204bcc70c439b423fc0dd9f0286de67dc82773a14b0200ada0995f"}, "flow": {:hex, :flow, "1.2.4", "1dd58918287eb286656008777cb32714b5123d3855956f29aa141ebae456922d", [:mix], [{:gen_stage, "~> 1.0", [hex: :gen_stage, repo: "hexpm", optional: false]}], "hexpm", "874adde96368e71870f3510b91e35bc31652291858c86c0e75359cbdd35eb211"}, "gen_stage": {:hex, :gen_stage, "1.2.1", "19d8b5e9a5996d813b8245338a28246307fd8b9c99d1237de199d21efc4c76a1", [:mix], [], "hexpm", "83e8be657fa05b992ffa6ac1e3af6d57aa50aace8f691fcf696ff02f8335b001"}, "gettext": {:hex, :gettext, "0.24.0", "6f4d90ac5f3111673cbefc4ebee96fe5f37a114861ab8c7b7d5b30a1108ce6d8", [:mix], [{:expo, "~> 0.5.1", [hex: :expo, repo: "hexpm", optional: false]}], "hexpm", "bdf75cdfcbe9e4622dd18e034b227d77dd17f0f133853a1c73b97b3d6c770e8b"}, "hackney": {:hex, :hackney, "1.20.1", "8d97aec62ddddd757d128bfd1df6c5861093419f8f7a4223823537bad5d064e2", [:rebar3], [{:certifi, "~> 2.12.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "fe9094e5f1a2a2c0a7d10918fee36bfec0ec2a979994cff8cfe8058cd9af38e3"}, - "html_entities": {:hex, :html_entities, "0.5.2", "9e47e70598da7de2a9ff6af8758399251db6dbb7eebe2b013f2bbd2515895c3c", [:mix], [], "hexpm", "c53ba390403485615623b9531e97696f076ed415e8d8058b1dbaa28181f4fdcc"}, "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, "jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"}, - "jumper": {:hex, :jumper, "1.0.1", "3c00542ef1a83532b72269fab9f0f0c82bf23a35e27d278bfd9ed0865cecabff", [:mix], [], "hexpm", "318c59078ac220e966d27af3646026db9b5a5e6703cb2aa3e26bcfaba65b7433"}, "libgraph": {:hex, :libgraph, "0.16.0", "3936f3eca6ef826e08880230f806bfea13193e49bf153f93edcf0239d4fd1d07", [:mix], [], "hexpm", "41ca92240e8a4138c30a7e06466acc709b0cbb795c643e9e17174a178982d6bf"}, - "makeup": {:hex, :makeup, "1.1.2", "9ba8837913bdf757787e71c1581c21f9d2455f4dd04cfca785c70bbfff1a76a3", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cce1566b81fbcbd21eca8ffe808f33b221f9eee2cbc7a1706fc3da9ff18e6cac"}, - "makeup_elixir": {:hex, :makeup_elixir, "0.16.2", "627e84b8e8bf22e60a2579dad15067c755531fea049ae26ef1020cad58fe9578", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "41193978704763f6bbe6cc2758b84909e62984c7752b3784bd3c218bb341706b"}, - "makeup_erlang": {:hex, :makeup_erlang, "1.0.0", "6f0eff9c9c489f26b69b61440bf1b238d95badae49adac77973cbacae87e3c2e", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "ea7a9307de9d1548d2a72d299058d1fd2339e3d398560a0e46c27dab4891e4d2"}, "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, - "mime": {:hex, :mime, "2.0.5", "dc34c8efd439abe6ae0343edbb8556f4d63f178594894720607772a041b04b02", [:mix], [], "hexpm", "da0d64a365c45bc9935cc5c8a7fc5e49a0e0f9932a761c55d6c52b142780a05c"}, "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"}, "mix_unused": {:hex, :mix_unused, "0.4.1", "9f8d759a300a79d2077d6baf617f3a5af6935d50b0f113c09295b265afc3e411", [:mix], [{:libgraph, ">= 0.0.0", [hex: :libgraph, repo: "hexpm", optional: false]}], "hexpm", "fa21f688a88e0710e3d96ac1c8e5a6181aea8a75c8a4214f0edcfeb069b831a3"}, "myxql": {:hex, :myxql, "0.6.4", "1502ea37ee23c31b79725b95d4cc3553693c2bda7421b1febc50722fd988c918", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:geo, "~> 3.4", [hex: :geo, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "a3307f4671f3009d3708283649adf205bfe280f7e036fc8ef7f16dbf821ab8e9"}, - "nanoid": {:hex, :nanoid, "2.1.0", "d192a5bf1d774258bc49762b480fca0e3128178fa6d35a464af2a738526607fd", [:mix], [], "hexpm", "ebc7a342d02d213534a7f93a091d569b9fea7f26fcd3a638dc655060fc1f76ac"}, "nimble_csv": {:hex, :nimble_csv, "1.2.0", "4e26385d260c61eba9d4412c71cea34421f296d5353f914afe3f2e71cce97722", [:mix], [], "hexpm", "d0628117fcc2148178b034044c55359b26966c6eaa8e2ce15777be3bbc91b12a"}, - "nimble_parsec": {:hex, :nimble_parsec, "1.4.0", "51f9b613ea62cfa97b25ccc2c1b4216e81df970acd8e16e8d1bdc58fef21370d", [:mix], [], "hexpm", "9c565862810fb383e9838c1dd2d7d2c437b3d13b267414ba6af33e50d2d1cf28"}, "parse_trans": {:hex, :parse_trans, "3.4.1", "6e6aa8167cb44cc8f39441d05193be6e6f4e7c2946cb2759f015f8c56b76e5ff", [:rebar3], [], "hexpm", "620a406ce75dada827b82e453c19cf06776be266f5a67cff34e1ef2cbb60e49a"}, - "phoenix": {:hex, :phoenix, "1.7.12", "1cc589e0eab99f593a8aa38ec45f15d25297dd6187ee801c8de8947090b5a9d3", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.7", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "d646192fbade9f485b01bc9920c139bfdd19d0f8df3d73fd8eaf2dfbe0d2837c"}, - "phoenix_ecto": {:hex, :phoenix_ecto, "4.4.0", "0672ed4e4808b3fbed494dded89958e22fb882de47a97634c0b13e7b0b5f7720", [:mix], [{:ecto, "~> 3.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "09864e558ed31ee00bd48fcc1d4fc58ae9678c9e81649075431e69dbabb43cc1"}, - "phoenix_html": {:hex, :phoenix_html, "3.3.3", "380b8fb45912b5638d2f1d925a3771b4516b9a78587249cabe394e0a5d579dc9", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "923ebe6fec6e2e3b3e569dfbdc6560de932cd54b000ada0208b5f45024bdd76c"}, - "phoenix_live_dashboard": {:hex, :phoenix_live_dashboard, "0.6.5", "1495bb014be12c9a9252eca04b9af54246f6b5c1e4cd1f30210cd00ec540cf8e", [:mix], [{:ecto, "~> 3.6.2 or ~> 3.7", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_mysql_extras, "~> 0.3", [hex: :ecto_mysql_extras, repo: "hexpm", optional: true]}, {:ecto_psql_extras, "~> 0.7", [hex: :ecto_psql_extras, repo: "hexpm", optional: true]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 0.17.7", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "ef4fa50dd78364409039c99cf6f98ab5209b4c5f8796c17f4db118324f0db852"}, - "phoenix_live_reload": {:hex, :phoenix_live_reload, "1.5.3", "f2161c207fda0e4fb55165f650f7f8db23f02b29e3bff00ff7ef161d6ac1f09d", [:mix], [{:file_system, "~> 0.3 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}], "hexpm", "b4ec9cd73cb01ff1bd1cac92e045d13e7030330b74164297d1aee3907b54803c"}, - "phoenix_live_view": {:hex, :phoenix_live_view, "0.20.2", "025391424257d6c1ed2fab5c9fb11b1e5b21a8b47a416659b9a36492217dd5cb", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.6.15 or ~> 1.7.0", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 3.3 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.15", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "381c7f8a5702be838a6bf440a59ea010b458018e1a85cc432e8628641c00f07a"}, - "phoenix_pubsub": {:hex, :phoenix_pubsub, "2.1.3", "3168d78ba41835aecad272d5e8cd51aa87a7ac9eb836eabc42f6e57538e3731d", [:mix], [], "hexpm", "bba06bc1dcfd8cb086759f0edc94a8ba2bc8896d5331a1e2c2902bf8e36ee502"}, - "phoenix_template": {:hex, :phoenix_template, "1.0.4", "e2092c132f3b5e5b2d49c96695342eb36d0ed514c5b252a77048d5969330d639", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "2c0c81f0e5c6753faf5cca2f229c9709919aba34fab866d3bc05060c9c444206"}, - "phoenix_view": {:hex, :phoenix_view, "2.0.3", "4d32c4817fce933693741deeb99ef1392619f942633dde834a5163124813aad3", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}], "hexpm", "cd34049af41be2c627df99cd4eaa71fc52a328c0c3d8e7d4aa28f880c30e7f64"}, - "plug": {:hex, :plug, "1.15.3", "712976f504418f6dff0a3e554c40d705a9bcf89a7ccef92fc6a5ef8f16a30a97", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "cc4365a3c010a56af402e0809208873d113e9c38c401cabd88027ef4f5c01fd2"}, - "plug_cowboy": {:hex, :plug_cowboy, "2.7.1", "87677ffe3b765bc96a89be7960f81703223fe2e21efa42c125fcd0127dd9d6b2", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "02dbd5f9ab571b864ae39418db7811618506256f6d13b4a45037e5fe78dc5de3"}, - "plug_crypto": {:hex, :plug_crypto, "2.0.0", "77515cc10af06645abbfb5e6ad7a3e9714f805ae118fa1a70205f80d2d70fe73", [:mix], [], "hexpm", "53695bae57cc4e54566d993eb01074e4d894b65a3766f1c43e2c61a1b0f45ea9"}, "postgrex": {:hex, :postgrex, "0.17.5", "0483d054938a8dc069b21bdd636bf56c487404c241ce6c319c1f43588246b281", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "50b8b11afbb2c4095a3ba675b4f055c416d0f3d7de6633a595fc131a828a67eb"}, - "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, "shorter_maps": {:git, "https://github.com/boyzwj/shorter_maps.git", "787c3447e48c74cf3224f35aba03c84f4c6a3c13", []}, "sizeable": {:hex, :sizeable, "1.0.2", "625fe06a5dad188b52121a140286f1a6ae1adf350a942cf419499ecd8a11ee29", [:mix], [], "hexpm", "4bab548e6dfba777b400ca50830a9e3a4128e73df77ab1582540cf5860601762"}, - "sleeplocks": {:hex, :sleeplocks, "1.1.1", "3d462a0639a6ef36cc75d6038b7393ae537ab394641beb59830a1b8271faeed3", [:rebar3], [], "hexpm", "84ee37aeff4d0d92b290fff986d6a95ac5eedf9b383fadfd1d88e9b84a1c02e1"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, "statistex": {:hex, :statistex, "1.0.0", "f3dc93f3c0c6c92e5f291704cf62b99b553253d7969e9a5fa713e5481cd858a5", [:mix], [], "hexpm", "ff9d8bee7035028ab4742ff52fc80a2aa35cece833cf5319009b52f1b5a86c27"}, "stream_split": {:hex, :stream_split, "0.1.7", "2d3fd1fd21697da7f91926768d65f79409086052c9ec7ae593987388f52425f8", [:mix], [], "hexpm", "1dc072ff507a64404a0ad7af90df97096183fee8eeac7b300320cea7c4679147"}, "telemetry": {:hex, :telemetry, "1.2.1", "68fdfe8d8f05a8428483a97d7aab2f268aaff24b49e0f599faa091f1d4e7f61c", [:rebar3], [], "hexpm", "dad9ce9d8effc621708f99eac538ef1cbe05d6a874dd741de2e689c47feafed5"}, - "telemetry_metrics": {:hex, :telemetry_metrics, "1.0.0", "29f5f84991ca98b8eb02fc208b2e6de7c95f8bb2294ef244a176675adc7775df", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f23713b3847286a534e005126d4c959ebcca68ae9582118ce436b521d1d47d5d"}, - "telemetry_poller": {:hex, :telemetry_poller, "1.1.0", "58fa7c216257291caaf8d05678c8d01bd45f4bdbc1286838a28c4bb62ef32999", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "9eb9d9cbfd81cbd7cdd24682f8711b6e2b691289a0de6826e58452f28c103c8f"}, "timex": {:hex, :timex, "3.7.11", "bb95cb4eb1d06e27346325de506bcc6c30f9c6dea40d1ebe390b262fad1862d1", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.20", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 1.1", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "8b9024f7efbabaf9bd7aa04f65cf8dcd7c9818ca5737677c7b76acbc6a94d1aa"}, "typed_struct": {:hex, :typed_struct, "0.3.0", "939789e3c1dca39d7170c87f729127469d1315dcf99fee8e152bb774b17e7ff7", [:mix], [], "hexpm", "c50bd5c3a61fe4e198a8504f939be3d3c85903b382bde4865579bc23111d1b6d"}, "tzdata": {:hex, :tzdata, "1.1.1", "20c8043476dfda8504952d00adac41c6eda23912278add38edc140ae0c5bcc46", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "a69cec8352eafcd2e198dea28a34113b60fdc6cb57eb5ad65c10292a6ba89787"}, "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"}, - "unsafe": {:hex, :unsafe, "1.0.1", "a27e1874f72ee49312e0a9ec2e0b27924214a05e3ddac90e91727bc76f8613d8", [:mix], [], "hexpm", "6c7729a2d214806450d29766abc2afaa7a2cbecf415be64f36a6691afebb50e5"}, - "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"}, - "websock_adapter": {:hex, :websock_adapter, "0.5.6", "0437fe56e093fd4ac422de33bf8fc89f7bc1416a3f2d732d8b2c8fd54792fe60", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "e04378d26b0af627817ae84c92083b7e97aca3121196679b73c73b99d0d133ea"}, } diff --git a/apps/csv2sql/priv/src/.keep b/priv/src/.keep similarity index 100% rename from apps/csv2sql/priv/src/.keep rename to priv/src/.keep diff --git a/apps/csv2sql/test/config/loader_test.exs b/test/config/loader_test.exs similarity index 100% rename from apps/csv2sql/test/config/loader_test.exs rename to test/config/loader_test.exs diff --git a/apps/csv2sql/test/database/connection_test_test.exs b/test/database/connection_test_test.exs similarity index 100% rename from apps/csv2sql/test/database/connection_test_test.exs rename to test/database/connection_test_test.exs diff --git a/apps/csv2sql/test/database/database_test.exs b/test/database/database_test.exs similarity index 100% rename from apps/csv2sql/test/database/database_test.exs rename to test/database/database_test.exs diff --git a/apps/csv2sql/test/helpers/helpers_test.exs b/test/helpers/helpers_test.exs similarity index 100% rename from apps/csv2sql/test/helpers/helpers_test.exs rename to test/helpers/helpers_test.exs diff --git a/apps/csv2sql/test/stages/analyze_test.exs b/test/stages/analyze_test.exs similarity index 100% rename from apps/csv2sql/test/stages/analyze_test.exs rename to test/stages/analyze_test.exs diff --git a/apps/csv2sql/test/support/csv_src/test.csv b/test/support/csv_src/test.csv similarity index 100% rename from apps/csv2sql/test/support/csv_src/test.csv rename to test/support/csv_src/test.csv diff --git a/apps/csv2sql/test/support/fixtures/duplicate_cols.csv b/test/support/fixtures/duplicate_cols.csv similarity index 100% rename from apps/csv2sql/test/support/fixtures/duplicate_cols.csv rename to test/support/fixtures/duplicate_cols.csv diff --git a/apps/csv2sql/test/support/fixtures/invalid.csv b/test/support/fixtures/invalid.csv similarity index 100% rename from apps/csv2sql/test/support/fixtures/invalid.csv rename to test/support/fixtures/invalid.csv diff --git a/apps/csv2sql/test/support/fixtures/invalid_column_names.csv b/test/support/fixtures/invalid_column_names.csv similarity index 100% rename from apps/csv2sql/test/support/fixtures/invalid_column_names.csv rename to test/support/fixtures/invalid_column_names.csv diff --git a/apps/csv2sql/test/support/fixtures/test.csv b/test/support/fixtures/test.csv similarity index 100% rename from apps/csv2sql/test/support/fixtures/test.csv rename to test/support/fixtures/test.csv diff --git a/apps/csv2sql/test/support/test_helper.exs b/test/support/test_helper.exs similarity index 100% rename from apps/csv2sql/test/support/test_helper.exs rename to test/support/test_helper.exs diff --git a/apps/csv2sql/test/test_helper.exs b/test/test_helper.exs similarity index 100% rename from apps/csv2sql/test/test_helper.exs rename to test/test_helper.exs diff --git a/apps/csv2sql/test/type_deducer/type_checker_test.exs b/test/type_deducer/type_checker_test.exs similarity index 100% rename from apps/csv2sql/test/type_deducer/type_checker_test.exs rename to test/type_deducer/type_checker_test.exs diff --git a/apps/csv2sql/test/type_deducer/type_deducer_test.exs b/test/type_deducer/type_deducer_test.exs similarity index 100% rename from apps/csv2sql/test/type_deducer/type_deducer_test.exs rename to test/type_deducer/type_deducer_test.exs