Add unit testing

.github/workflows/builds.yml

@@ -97,7 +97,7 @@ jobs:
         with:
           platform: ${{ matrix.platform }}
           target: ${{ matrix.target }}
-          sconsflags: arch=${{ matrix.arch }} build_ovdl_library=yes
+          sconsflags: arch=${{ matrix.arch }} build_ovdl_library=yes run_ovdl_tests=yes
 
       - name: Delete compilation files
         if: ${{ matrix.platform == 'windows' }}

.gitignore

@@ -66,6 +66,8 @@ bin/*
 *.idb
 *.exp
 
+tests/bin/*
+
 # Build configuarion.
 /custom.py
 

.gitmodules

@@ -11,6 +11,12 @@
 [submodule "deps/fmt"]
 	path = deps/fmt
 	url = https://github.com/fmtlib/fmt
+	ignore = dirty
 [submodule "deps/range-v3"]
 	path = deps/range-v3
 	url = https://github.com/ericniebler/range-v3
+	ignore = dirty
+[submodule "tests/deps/snitch"]
+	path = tests/deps/snitch
+	url = https://github.com/snitch-org/snitch
+	ignore = dirty

SConstruct

@@ -14,7 +14,8 @@ env.PrependENVPath("PATH", os.getenv("PATH"))
 
 opts = env.SetupOptions()
 
-opts.Add(BoolVariable(key="build_ovdl_library", help="Build the openvic dataloader library.", default=env.get("build_ovdl_library", not env.is_standalone)))
+opts.Add(BoolVariable("run_ovdl_tests", "Build and run the openvic dataloader tests", env.is_standalone))
+opts.Add(BoolVariable("build_ovdl_library", "Build the openvic dataloader library.", env.get("build_ovdl_library", not env.is_standalone)))
 opts.Add(BoolVariable("build_ovdl_headless", "Build the openvic dataloader headless executable", env.is_standalone))
 
 env.FinalizeOptions()
@@ -56,6 +57,9 @@ library_name = "libopenvic-dataloader{}{}".format(suffix, env["LIBSUFFIX"])
 
 default_args = []
 
+if env["run_ovdl_tests"]:
+    env["build_ovdl_library"] = True
+
 if env["build_ovdl_library"]:
     library = env.StaticLibrary(target=os.path.join(BINDIR, library_name), source=sources)
     default_args += [library]
@@ -86,6 +90,9 @@ if env["build_ovdl_headless"]:
     )
     default_args += [headless_program]
 
+if env["run_ovdl_tests"]:
+    SConscript("tests/SCsub", "env")
+
 # Add compiledb if the option is set
 if env.get("compiledb", False):
     default_args += ["compiledb"]

include/openvic-dataloader/Error.hpp

@@ -117,11 +117,10 @@ namespace ovdl::error {
 	protected:
 		explicit AnnotatedError(dryad::node_ctor ctor, ErrorKind kind) : node_base(ctor, kind) {
 			insert_child_list_after(nullptr, AnnotationList {});
-			_last_annotation = nullptr;
 		}
 
 	private:
-		Annotation* _last_annotation;
+		Annotation* _last_annotation = nullptr;
 	};
 
 	struct ParseError : dryad::abstract_node_range<AnnotatedError, ErrorKind::FirstParseError, ErrorKind::LastParseError> {
@@ -207,7 +206,7 @@ namespace ovdl::error {
 
 	inline void AnnotatedError::push_back(AnnotationList p_annotations) {
 		if (p_annotations.empty()) return;
-		insert_child_list_after(annotations().end().deref(), p_annotations);
-		_last_annotation = *p_annotations.end();
+		insert_child_list_after(_last_annotation, p_annotations);
+		_last_annotation = p_annotations.back();
 	}
 }

include/openvic-dataloader/NodeLocation.hpp

@@ -11,10 +11,12 @@ namespace ovdl {
 		const char_type* _end = nullptr;
 
 		BasicNodeLocation() = default;
-		BasicNodeLocation(const char_type* pos) : _begin(pos),
-												  _end(pos) {}
-		BasicNodeLocation(const char_type* begin, const char_type* end) : _begin(begin),
-																		  _end(end) {}
+		BasicNodeLocation(const char_type* pos)
+			: _begin(pos),
+			  _end(pos) {}
+		BasicNodeLocation(const char_type* begin, const char_type* end)
+			: _begin(begin),
+			  _end(end) {}
 
 		BasicNodeLocation(const BasicNodeLocation&) noexcept = default;
 		BasicNodeLocation& operator=(const BasicNodeLocation&) = default;

include/openvic-dataloader/csv/LineObject.hpp

@@ -80,13 +80,31 @@ namespace ovdl::csv {
 
 		constexpr std::size_t value_count() const { return _suffix_end; }
 
+		struct SepTransformer {
+			const LineObject& line_object;
+			std::string_view separator;
+		};
+
+		constexpr SepTransformer use_sep(std::string_view seperator) const {
+			return { *this, seperator };
+		}
+
 	private:
 		// Should be position of first valid value on line
 		position_type _prefix_end = 0;
 		// Should be position after last value or position after last seperator
 		position_type _suffix_end = 0;
 	};
 
+	struct VectorSepTransformer {
+		const std::vector<LineObject>& vector;
+		std::string_view separator;
+	};
+
+	constexpr VectorSepTransformer use_sep(const std::vector<LineObject>& vector, std::string_view separator) {
+		return { vector, separator };
+	}
+
 	inline std::ostream& operator<<(std::ostream& stream, const LineObject& line) {
 		static constexpr char SEP = ';';
 		LineObject::position_type sep_index = 0;
@@ -110,4 +128,36 @@ namespace ovdl::csv {
 		}
 		return stream;
 	}
+
+	inline std::ostream& operator<<(std::ostream& stream, const LineObject::SepTransformer& transformer) {
+		auto quote_check = [&transformer, is_one = transformer.separator.size() == 1](const std::string_view str) {
+			if (is_one) {
+				char SEP = transformer.separator[0];
+				return std::any_of(str.begin(), str.end(), [SEP](char c) { return c == SEP || std::isspace(c); });
+			}
+			return std::any_of(str.begin(), str.end(), [](char c) { return std::isspace(c); }) ||
+				   str.find(transformer.separator) != std::string::npos;
+		};
+
+		LineObject::position_type sep_index = 0;
+		for (const auto& [pos, val] : transformer.line_object) {
+			while (sep_index < pos) {
+				stream << transformer.separator;
+				sep_index++;
+			}
+			if (quote_check(val)) {
+				stream << '"' << val << '"';
+			} else {
+				stream << val;
+			}
+		}
+		return stream;
+	}
+
+	inline std::ostream& operator<<(std::ostream& stream, const VectorSepTransformer& transformer) {
+		for (const LineObject& line : transformer.vector) {
+			stream << line.use_sep(transformer.separator) << '\n';
+		}
+		return stream;
+	}
 }

include/openvic-dataloader/csv/Parser.hpp

@@ -25,9 +25,9 @@ namespace ovdl::csv {
 		static Parser from_file(const char* path, std::optional<detail::Encoding> encoding_fallback = std::nullopt);
 		static Parser from_file(const std::filesystem::path& path, std::optional<detail::Encoding> encoding_fallback = std::nullopt);
 
-		constexpr Parser& load_from_buffer(const char* data, std::size_t size, std::optional<detail::Encoding> encoding_fallback = std::nullopt);
-		constexpr Parser& load_from_buffer(const char* start, const char* end, std::optional<detail::Encoding> encoding_fallback = std::nullopt);
-		constexpr Parser& load_from_string(const std::string_view string, std::optional<detail::Encoding> encoding_fallback = std::nullopt);
+		Parser& load_from_buffer(const char* data, std::size_t size, std::optional<detail::Encoding> encoding_fallback = std::nullopt);
+		Parser& load_from_buffer(const char* start, const char* end, std::optional<detail::Encoding> encoding_fallback = std::nullopt);
+		Parser& load_from_string(const std::string_view string, std::optional<detail::Encoding> encoding_fallback = std::nullopt);
 		Parser& load_from_file(const char* path, std::optional<detail::Encoding> encoding_fallback = std::nullopt);
 		Parser& load_from_file(const std::filesystem::path& path, std::optional<detail::Encoding> encoding_fallback = std::nullopt);
 

include/openvic-dataloader/v2script/AbstractSyntaxTree.hpp

@@ -108,9 +108,7 @@ namespace ovdl::v2script::ast {
 	struct ListValue : dryad::basic_node<NodeKind::ListValue, dryad::container_node<Value>> {
 		explicit ListValue(dryad::node_ctor ctor, StatementList statements);
 		explicit ListValue(dryad::node_ctor ctor, AssignStatementList statements);
-
-		explicit ListValue(dryad::node_ctor ctor) : ListValue(ctor, StatementList {}) {
-		}
+		explicit ListValue(dryad::node_ctor ctor);
 
 		DRYAD_CHILD_NODE_RANGE_GETTER(Statement, statements, nullptr, this->node_after(_last_statement));
 
@@ -167,8 +165,7 @@ namespace ovdl::v2script::ast {
 	struct FileTree : dryad::basic_node<NodeKind::FileTree, dryad::container_node<Node>> {
 		explicit FileTree(dryad::node_ctor ctor, StatementList statements);
 		explicit FileTree(dryad::node_ctor ctor, AssignStatementList statements);
-		explicit FileTree(dryad::node_ctor ctor) : FileTree(ctor, StatementList {}) {
-		}
+		explicit FileTree(dryad::node_ctor ctor);
 
 		DRYAD_CHILD_NODE_RANGE_GETTER(Statement, statements, nullptr, this->node_after(_last_node));
 

include/openvic-dataloader/v2script/Parser.hpp

@@ -33,9 +33,9 @@ namespace ovdl::v2script {
 		static Parser from_file(const char* path, std::optional<detail::Encoding> encoding_fallback = std::nullopt);
 		static Parser from_file(const std::filesystem::path& path, std::optional<detail::Encoding> encoding_fallback = std::nullopt);
 
-		constexpr Parser& load_from_buffer(const char* data, std::size_t size, std::optional<detail::Encoding> encoding_fallback = std::nullopt);
-		constexpr Parser& load_from_buffer(const char* start, const char* end, std::optional<detail::Encoding> encoding_fallback = std::nullopt);
-		constexpr Parser& load_from_string(const std::string_view string, std::optional<detail::Encoding> encoding_fallback = std::nullopt);
+		Parser& load_from_buffer(const char* data, std::size_t size, std::optional<detail::Encoding> encoding_fallback = std::nullopt);
+		Parser& load_from_buffer(const char* start, const char* end, std::optional<detail::Encoding> encoding_fallback = std::nullopt);
+		Parser& load_from_string(const std::string_view string, std::optional<detail::Encoding> encoding_fallback = std::nullopt);
 		Parser& load_from_file(const char* path, std::optional<detail::Encoding> encoding_fallback = std::nullopt);
 		Parser& load_from_file(const std::filesystem::path& path, std::optional<detail::Encoding> encoding_fallback = std::nullopt);
 

src/openvic-dataloader/csv/Parser.cpp

@@ -3,6 +3,7 @@
 #include <type_traits>
 #include <vector>
 
+#include <openvic-dataloader/Error.hpp>
 #include <openvic-dataloader/NodeLocation.hpp>
 #include <openvic-dataloader/csv/LineObject.hpp>
 #include <openvic-dataloader/csv/Parser.hpp>
@@ -15,6 +16,8 @@
 #include <lexy/input/buffer.hpp>
 #include <lexy/input/file.hpp>
 
+#include <dryad/node.hpp>
+
 #include "CsvGrammar.hpp"
 #include "CsvParseState.hpp"
 #include "detail/NullBuff.hpp"
@@ -53,6 +56,13 @@ struct Parser::ParseHandler final : detail::BasicFileParseHandler<CsvParseState>
 		return _lines;
 	}
 
+	Parser::error_range get_errors() {
+		using iterator = typename decltype(std::declval<const error::Root*>()->children())::iterator;
+		if (!is_valid())
+			return dryad::make_node_range<error::Error>(iterator::from_ptr(nullptr), iterator::from_ptr(nullptr));
+		return parse_state().logger().get_errors();
+	}
+
 private:
 	std::vector<csv::LineObject> _lines;
 };
@@ -126,19 +136,19 @@ constexpr void Parser::_run_load_func(detail::LoadCallback<ParseHandler, Args...
 	}
 }
 
-constexpr Parser& Parser::load_from_buffer(const char* data, std::size_t size, std::optional<detail::Encoding> encoding_fallback) {
+Parser& Parser::load_from_buffer(const char* data, std::size_t size, std::optional<detail::Encoding> encoding_fallback) {
 	// Type can't be deduced?
 	_run_load_func(std::mem_fn(&ParseHandler::load_buffer_size), data, size, encoding_fallback);
 	return *this;
 }
 
-constexpr Parser& Parser::load_from_buffer(const char* start, const char* end, std::optional<detail::Encoding> encoding_fallback) {
+Parser& Parser::load_from_buffer(const char* start, const char* end, std::optional<detail::Encoding> encoding_fallback) {
 	// Type can't be deduced?
 	_run_load_func(std::mem_fn(&ParseHandler::load_buffer), start, end, encoding_fallback);
 	return *this;
 }
 
-constexpr Parser& Parser::load_from_string(const std::string_view string, std::optional<detail::Encoding> encoding_fallback) {
+Parser& Parser::load_from_string(const std::string_view string, std::optional<detail::Encoding> encoding_fallback) {
 	return load_from_buffer(string.data(), string.size(), encoding_fallback);
 }
 
@@ -166,7 +176,7 @@ bool Parser::parse_csv(bool handle_strings) {
 	}();
 	_has_error = _parse_handler->parse_state().logger().errored();
 	_has_warning = _parse_handler->parse_state().logger().warned();
-	if (!errors->empty()) {
+	if (errors && !errors->empty()) {
 		_has_error = true;
 		_has_fatal_error = true;
 		if (&_error_stream.get() != &detail::cnull) {
@@ -182,7 +192,7 @@ const std::vector<csv::LineObject>& Parser::get_lines() const {
 }
 
 typename Parser::error_range Parser::get_errors() const {
-	return _parse_handler->parse_state().logger().get_errors();
+	return _parse_handler->get_errors();
 }
 
 const FilePosition Parser::get_error_position(const error::Error* error) const {

src/openvic-dataloader/v2script/AbstractSyntaxTree.cpp

@@ -1,4 +1,4 @@
-#include "openvic-dataloader/v2script/AbstractSyntaxTree.hpp"
+#include <openvic-dataloader/v2script/AbstractSyntaxTree.hpp>
 
 #include <lexy/dsl/option.hpp>
 #include <lexy/encoding.hpp>
@@ -31,6 +31,10 @@ ListValue::ListValue(dryad::node_ctor ctor, AssignStatementList statements) : no
 	}
 }
 
+ListValue::ListValue(dryad::node_ctor ctor) : node_base(ctor) {
+	_last_statement = nullptr;
+}
+
 FileTree::FileTree(dryad::node_ctor ctor, StatementList statements) : node_base(ctor) {
 	insert_child_list_after(nullptr, statements);
 	if (statements.empty()) {
@@ -49,6 +53,10 @@ FileTree::FileTree(dryad::node_ctor ctor, AssignStatementList statements) : node
 	}
 }
 
+FileTree::FileTree(dryad::node_ctor ctor) : node_base(ctor) {
+	_last_node = nullptr;
+}
+
 std::string FileAbstractSyntaxTree::make_list_visualizer() const {
 	const int INDENT_SIZE = 2;
 

src/openvic-dataloader/v2script/Parser.cpp

@@ -76,6 +76,9 @@ struct Parser::ParseHandler final : detail::BasicStateParseHandler<v2script::ast
 	}
 
 	Parser::error_range get_errors() {
+		using iterator = typename decltype(std::declval<const error::Root*>()->children())::iterator;
+		if (!is_valid())
+			return dryad::make_node_range<error::Error>(iterator::from_ptr(nullptr), iterator::from_ptr(nullptr));
 		return parse_state().logger().get_errors();
 	}
 };
@@ -149,19 +152,19 @@ constexpr void Parser::_run_load_func(detail::LoadCallback<Parser::ParseHandler*
 	}
 }
 
-constexpr Parser& Parser::load_from_buffer(const char* data, std::size_t size, std::optional<detail::Encoding> encoding_fallback) {
+Parser& Parser::load_from_buffer(const char* data, std::size_t size, std::optional<detail::Encoding> encoding_fallback) {
 	// Type can't be deduced?
 	_run_load_func(std::mem_fn(&ParseHandler::load_buffer_size), data, size, encoding_fallback);
 	return *this;
 }
 
-constexpr Parser& Parser::load_from_buffer(const char* start, const char* end, std::optional<detail::Encoding> encoding_fallback) {
+Parser& Parser::load_from_buffer(const char* start, const char* end, std::optional<detail::Encoding> encoding_fallback) {
 	// Type can't be deduced?
 	_run_load_func(std::mem_fn(&ParseHandler::load_buffer), start, end, encoding_fallback);
 	return *this;
 }
 
-constexpr Parser& Parser::load_from_string(const std::string_view string, std::optional<detail::Encoding> encoding_fallback) {
+Parser& Parser::load_from_string(const std::string_view string, std::optional<detail::Encoding> encoding_fallback) {
 	return load_from_buffer(string.data(), string.size(), encoding_fallback);
 }