From d7cd7fbeb0046268da15ce86f04bd83f3542d5be Mon Sep 17 00:00:00 2001 From: Andreas Hehn Date: Tue, 19 Aug 2025 05:10:53 -0700 Subject: [PATCH 01/11] Added bzip2 decompression to mps_parser --- cpp/libmps_parser/CMakeLists.txt | 10 ++ .../include/mps_parser/parser.hpp | 9 +- cpp/libmps_parser/src/mps_parser.cpp | 99 ++++++++++++++++++ cpp/libmps_parser/src/mps_parser.hpp | 6 +- cpp/libmps_parser/src/utilities/error.hpp | 29 ++++- cpp/libmps_parser/tests/mps_parser_test.cpp | 36 +++++++ .../linear_programming/good-mps-1.mps.bz2 | Bin 0 -> 230 bytes thirdparty/THIRD_PARTY_LICENSES | 44 ++++++++ 8 files changed, 227 insertions(+), 6 deletions(-) create mode 100644 datasets/linear_programming/good-mps-1.mps.bz2 diff --git a/cpp/libmps_parser/CMakeLists.txt b/cpp/libmps_parser/CMakeLists.txt index 33f0a7b581..89c5e74277 100644 --- a/cpp/libmps_parser/CMakeLists.txt +++ b/cpp/libmps_parser/CMakeLists.txt @@ -38,6 +38,7 @@ rapids_cmake_build_type(Release) # ############################################################################# # - User Options ------------------------------------------------------------ option(BUILD_TESTS "Configure CMake to build tests" ON) +option(MPS_PARSER_WITH_BZIP2 "Build with bzip2 decompression" ON) message(VERBOSE "cuOpt: Build mps-parser unit-tests: ${BUILD_TESTS}") @@ -50,6 +51,11 @@ if(CMAKE_COMPILER_IS_GNUCXX) list(APPEND MPS_PARSER_CXX_FLAGS -Werror -Wno-error=deprecated-declarations) endif(CMAKE_COMPILER_IS_GNUCXX) +if(MPS_PARSER_WITH_BZIP2) + find_package(BZip2 REQUIRED) + add_definitions(-DMPS_PARSER_WITH_BZIP2) +endif(MPS_PARSER_WITH_BZIP2) + if(DEFINE_ASSERT) add_definitions(-DASSERT_MODE) endif(DEFINE_ASSERT) @@ -107,6 +113,10 @@ target_include_directories(mps_parser "$" ) +if(MPS_PARSER_WITH_BZIP2) + target_include_directories(mps_parser PRIVATE BZip2::BZip2) +endif(MPS_PARSER_WITH_BZIP2) + # ################################################################################################## # - generate tests -------------------------------------------------------------------------------- if(BUILD_TESTS) diff --git a/cpp/libmps_parser/include/mps_parser/parser.hpp b/cpp/libmps_parser/include/mps_parser/parser.hpp index 9d55aaac0e..56ac397d7d 100644 --- a/cpp/libmps_parser/include/mps_parser/parser.hpp +++ b/cpp/libmps_parser/include/mps_parser/parser.hpp @@ -22,12 +22,17 @@ namespace cuopt::mps_parser { /** - * @brief Reads the equation from the input text file which is MPS formatted + * @brief Reads the equation from an MPS file. + * + * The input file can be a plain text file in MPS-format or a bzip2-compressed MPS + * file (.mps.bz2). * * Read this link http://lpsolve.sourceforge.net/5.5/mps-format.htm for more * details on both free and fixed MPS format. * - * @param[in] mps_file_path Path to MPS formatted file. + * Note: bzip2-compressed files can only be read if libbzip2 is installed. + * + * @param[in] mps_file_path Path to MPS file. * @param[in] fixed_mps_format If MPS file should be parsed as fixed, false by default * @return mps_data_model_t A fully formed LP problem which represents the given MPS file */ diff --git a/cpp/libmps_parser/src/mps_parser.cpp b/cpp/libmps_parser/src/mps_parser.cpp index b9d3daaa37..a03b8b4b48 100644 --- a/cpp/libmps_parser/src/mps_parser.cpp +++ b/cpp/libmps_parser/src/mps_parser.cpp @@ -28,6 +28,99 @@ #include #include #include +#include + +#ifdef MPS_PARSER_WITH_BZIP2 +#include +#include +#endif // MPS_PARSER_WITH_BZIP2 + +namespace { +using cuopt::mps_parser::mps_parser_expects_fatal; +using cuopt::mps_parser::mps_parser_expects; +using cuopt::mps_parser::error_type_t; + +struct FcloseDeleter { + void operator()(FILE* fp) { + mps_parser_expects_fatal(fclose(fp) == 0, + error_type_t::ValidationError, + "Error closing MPS file!"); + } +}; +} // end namespace + +#ifdef MPS_PARSER_WITH_BZIP2 +namespace { +using BZ2_bzReadOpen_t = decltype(&BZ2_bzReadOpen); +using BZ2_bzReadClose_t = decltype(&BZ2_bzReadClose); +using BZ2_bzRead_t = decltype(&BZ2_bzRead); + +std::vector bz2_file_to_string(const std::string& file) +{ + struct DlCloseDeleter { + void operator()(void* fp) { + mps_parser_expects_fatal(dlclose(fp) == 0, + error_type_t::ValidationError, + "Error closing libbz2.so!"); + } + }; + struct BzReadCloseDeleter { + void operator()(void* f) { + int bzerror; + if(f != nullptr) + fptr(&bzerror, f); + mps_parser_expects_fatal(bzerror == BZ_OK, + error_type_t::ValidationError, + "Error closing bzip2 file!"); + } + BZ2_bzReadClose_t fptr = nullptr; + }; + + std::unique_ptr lbz2handle{dlopen("libbz2.so", RTLD_LAZY)}; + mps_parser_expects(lbz2handle != nullptr, + error_type_t::ValidationError, + "Could not open .mps.bz2 file since libbz2.so was not found. In order to open .mps.bz2 files directly, please ensure libbzip2 is installed. Alternatively, decompress the .mps.bz2 file manually and open the uncompressed .mps file. Given path: %s", + file.c_str()); + + BZ2_bzReadOpen_t BZ2_bzReadOpen = reinterpret_cast(dlsym(lbz2handle.get(), "BZ2_bzReadOpen")); + BZ2_bzReadClose_t BZ2_bzReadClose = reinterpret_cast(dlsym(lbz2handle.get(), "BZ2_bzReadClose")); + BZ2_bzRead_t BZ2_bzRead = reinterpret_cast(dlsym(lbz2handle.get(), "BZ2_bzRead")); + mps_parser_expects(BZ2_bzReadOpen != nullptr && BZ2_bzReadClose != nullptr && BZ2_bzRead != nullptr, + error_type_t::ValidationError, + "Error loading libbzip2! Library version might be incompatible. Please decompress the .mps.bz2 file manually and open the uncompressed .mps file. Given path: %s", + file.c_str()); + + + std::unique_ptr fp{fopen(file.c_str(), "rb")}; + mps_parser_expects(fp != nullptr, + error_type_t::ValidationError, + "Error opening MPS file! Given path: %s", + file.c_str()); + int bzerror = BZ_OK; + std::unique_ptr bzfile{BZ2_bzReadOpen(&bzerror, fp.get(), 0, 0, nullptr, 0), {BZ2_bzReadClose}}; + mps_parser_expects(bzerror == BZ_OK, + error_type_t::ValidationError, + "Could not open bzip2 compressed file! Given path: %s", + file.c_str()); + + std::vector buf; + const size_t readbufsize = 1ull << 24; // 16MiB - just a guess. + std::vector readbuf(readbufsize); + while (bzerror == BZ_OK) { + const size_t bytes_read = BZ2_bzRead(&bzerror, bzfile.get(), readbuf.data(), readbuf.size()); + if (bzerror == BZ_OK || bzerror == BZ_STREAM_END) { + buf.insert(buf.end(), begin(readbuf), begin(readbuf) + bytes_read); + } + } + buf.push_back('\0'); + mps_parser_expects(bzerror == BZ_STREAM_END, + error_type_t::ValidationError, + "Error in bzip2 decompression of MPS file! Given path: %s", + file.c_str()); + return buf; +} +} // end namespace +#endif // MPS_PARSER_WITH_BZIP2 namespace cuopt::mps_parser { @@ -271,6 +364,12 @@ std::vector mps_parser_t::file_to_string(const std::string& file { // raft::common::nvtx::range fun_scope("file to string"); +#ifdef MPS_PARSER_WITH_BZIP2 + if(file.size() > 4 && file.substr(file.size() - 4, 4) == ".bz2") { + return bz2_file_to_string(file); + } +#endif // MPS_PARSER_WITH_BZIP2 + // Faster than using C++ I/O FILE* fp = fopen(file.c_str(), "r"); mps_parser_expects(fp != nullptr, diff --git a/cpp/libmps_parser/src/mps_parser.hpp b/cpp/libmps_parser/src/mps_parser.hpp index afaf470f31..42d9b4fca9 100644 --- a/cpp/libmps_parser/src/mps_parser.hpp +++ b/cpp/libmps_parser/src/mps_parser.hpp @@ -139,10 +139,10 @@ class mps_parser_t { std::unordered_set bounds_defined_for_var_id{}; static constexpr f_t unset_range_value = std::numeric_limits::infinity(); - /* Reads the equation from the input text file which is MPS formatted + /* Reads the MPS input file into a buffer. * - * Read this link http://lpsolve.sourceforge.net/5.5/mps-format.htm for more - * details on this format. + * If the file has a .bz2 suffix and libbzip2 is installed, this function + * can directly read and decompress the compressed MPS file. */ std::vector file_to_string(const std::string& file); void fill_problem(mps_data_model_t& problem); diff --git a/cpp/libmps_parser/src/utilities/error.hpp b/cpp/libmps_parser/src/utilities/error.hpp index 3aafa6340a..e8f2939b95 100644 --- a/cpp/libmps_parser/src/utilities/error.hpp +++ b/cpp/libmps_parser/src/utilities/error.hpp @@ -45,7 +45,7 @@ inline std::string error_to_string(error_type_t error) } /** - * @brief Macro for checking (pre-)conditions that throws an exception when a + * @brief Function for checking (pre-)conditions that throws an exception when a * condition is false * * @param[bool] cond From expression that evaluates to true or false @@ -70,6 +70,33 @@ inline void mps_parser_expects(bool cond, error_type_t error_type, const char* f } } +/** + * @brief Function for checking (pre-)conditions that aborts the program when a + * condition is false + * + * @param[bool] cond From expression that evaluates to true or false + * @param[error_type_t] error enum error type + * @param[const char *] fmt String format for error message + * @param variable set of arguments used for fmt + * @throw std::logic_error if the condition evaluates to false. + */ +inline void mps_parser_expects_fatal(bool cond, error_type_t error_type, const char* fmt, ...) +{ + if (not cond) { + va_list args; + va_start(args, fmt); + + char msg[2048]; + va_start(args, fmt); + vsnprintf(msg, sizeof(msg), fmt, args); + va_end(args); + std::string error_string = error_to_string(error_type); + std::fprintf(stderr, "{\"MPS_PARSER_ERROR_TYPE\": \"%s\", \"msg\": \"%s\"}\n", error_to_string(error_type).c_str(), msg); + std::fflush(stderr); + std::abort(); + } +} + #define MPS_PARSER_SET_ERROR_MSG(msg, location_prefix, fmt, ...) \ do { \ char err_msg[2048]; /* NOLINT */ \ diff --git a/cpp/libmps_parser/tests/mps_parser_test.cpp b/cpp/libmps_parser/tests/mps_parser_test.cpp index b70e8ff290..f258f475c9 100644 --- a/cpp/libmps_parser/tests/mps_parser_test.cpp +++ b/cpp/libmps_parser/tests/mps_parser_test.cpp @@ -757,4 +757,40 @@ TEST(mps_parser, good_mps_file_partial_bounds) EXPECT_EQ(10.0, mps.variable_upper_bounds[1]); } +TEST(mps_parser, good_mps_file_bzip2_compressed) +{ + auto mps = read_from_mps("linear_programming/good-mps-1.mps.bz2"); + EXPECT_EQ("good-1", mps.problem_name); + ASSERT_EQ(int(2), mps.row_names.size()); + EXPECT_EQ("ROW1", mps.row_names[0]); + EXPECT_EQ("ROW2", mps.row_names[1]); + ASSERT_EQ(int(2), mps.row_types.size()); + EXPECT_EQ(LesserThanOrEqual, mps.row_types[0]); + EXPECT_EQ(LesserThanOrEqual, mps.row_types[1]); + EXPECT_EQ("COST", mps.objective_name); + ASSERT_EQ(int(2), mps.var_names.size()); + EXPECT_EQ("VAR1", mps.var_names[0]); + EXPECT_EQ("VAR2", mps.var_names[1]); + ASSERT_EQ(int(2), mps.A_indices.size()); + ASSERT_EQ(int(2), mps.A_indices[0].size()); + EXPECT_EQ(int(0), mps.A_indices[0][0]); + EXPECT_EQ(int(1), mps.A_indices[0][1]); + ASSERT_EQ(int(2), mps.A_indices[1].size()); + EXPECT_EQ(int(0), mps.A_indices[1][0]); + EXPECT_EQ(int(1), mps.A_indices[1][1]); + ASSERT_EQ(int(2), mps.A_values.size()); + ASSERT_EQ(int(2), mps.A_values[0].size()); + EXPECT_EQ(3., mps.A_values[0][0]); + EXPECT_EQ(4., mps.A_values[0][1]); + ASSERT_EQ(int(2), mps.A_values[1].size()); + EXPECT_EQ(2.7, mps.A_values[1][0]); + EXPECT_EQ(10.1, mps.A_values[1][1]); + ASSERT_EQ(int(2), mps.b_values.size()); + EXPECT_EQ(5.4, mps.b_values[0]); + EXPECT_EQ(4.9, mps.b_values[1]); + ASSERT_EQ(int(2), mps.c_values.size()); + EXPECT_EQ(0.2, mps.c_values[0]); + EXPECT_EQ(0.1, mps.c_values[1]); +} + } // namespace cuopt::mps_parser diff --git a/datasets/linear_programming/good-mps-1.mps.bz2 b/datasets/linear_programming/good-mps-1.mps.bz2 new file mode 100644 index 0000000000000000000000000000000000000000..ee96fb0558b3ca29366a11c17f8ee5c0581502d9 GIT binary patch literal 230 zcmVEV1c!MAVV8q9CG= gK>X=. + + +----------------------------------------------------------------------------------------- +== bzip2 + +Usage: libmps_parser uses libbzip2 + +This program, "bzip2", the associated library "libbzip2", and all +documentation, are copyright (C) 1996-2019 Julian R Seward. All +rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. The origin of this software must not be misrepresented; you must + not claim that you wrote the original software. If you use this + software in a product, an acknowledgment in the product + documentation would be appreciated but is not required. + +3. Altered source versions must be plainly marked as such, and must + not be misrepresented as being the original software. + +4. The name of the author may not be used to endorse or promote + products derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Julian Seward, jseward@acm.org +bzip2/libbzip2 version 1.0.8 of 13 July 2019 From dc64c30baec61b44f19a0c2ad036ec05695b18a5 Mon Sep 17 00:00:00 2001 From: Andreas Hehn Date: Tue, 19 Aug 2025 05:11:46 -0700 Subject: [PATCH 02/11] Added zlib decompression to mps_parser --- cpp/libmps_parser/CMakeLists.txt | 10 ++ .../include/mps_parser/parser.hpp | 7 +- cpp/libmps_parser/src/mps_parser.cpp | 89 +++++++++++++++++- cpp/libmps_parser/src/mps_parser.hpp | 6 +- cpp/libmps_parser/tests/mps_parser_test.cpp | 36 +++++++ datasets/linear_programming/good-mps-1.mps.gz | Bin 0 -> 219 bytes thirdparty/THIRD_PARTY_LICENSES | 29 ++++++ 7 files changed, 170 insertions(+), 7 deletions(-) create mode 100644 datasets/linear_programming/good-mps-1.mps.gz diff --git a/cpp/libmps_parser/CMakeLists.txt b/cpp/libmps_parser/CMakeLists.txt index 89c5e74277..7bbb9cf7f0 100644 --- a/cpp/libmps_parser/CMakeLists.txt +++ b/cpp/libmps_parser/CMakeLists.txt @@ -39,6 +39,7 @@ rapids_cmake_build_type(Release) # - User Options ------------------------------------------------------------ option(BUILD_TESTS "Configure CMake to build tests" ON) option(MPS_PARSER_WITH_BZIP2 "Build with bzip2 decompression" ON) +option(MPS_PARSER_WITH_ZLIB "Build with zlib decompression" ON) message(VERBOSE "cuOpt: Build mps-parser unit-tests: ${BUILD_TESTS}") @@ -56,6 +57,11 @@ if(MPS_PARSER_WITH_BZIP2) add_definitions(-DMPS_PARSER_WITH_BZIP2) endif(MPS_PARSER_WITH_BZIP2) +if(MPS_PARSER_WITH_ZLIB) + find_package(ZLIB REQUIRED) + add_definitions(-DMPS_PARSER_WITH_ZLIB) +endif(MPS_PARSER_WITH_ZLIB) + if(DEFINE_ASSERT) add_definitions(-DASSERT_MODE) endif(DEFINE_ASSERT) @@ -117,6 +123,10 @@ if(MPS_PARSER_WITH_BZIP2) target_include_directories(mps_parser PRIVATE BZip2::BZip2) endif(MPS_PARSER_WITH_BZIP2) +if(MPS_PARSER_WITH_ZLIB) + target_include_directories(mps_parser PRIVATE ZLIB::ZLIB) +endif(MPS_PARSER_WITH_ZLIB) + # ################################################################################################## # - generate tests -------------------------------------------------------------------------------- if(BUILD_TESTS) diff --git a/cpp/libmps_parser/include/mps_parser/parser.hpp b/cpp/libmps_parser/include/mps_parser/parser.hpp index 56ac397d7d..f0dced0329 100644 --- a/cpp/libmps_parser/include/mps_parser/parser.hpp +++ b/cpp/libmps_parser/include/mps_parser/parser.hpp @@ -24,13 +24,14 @@ namespace cuopt::mps_parser { /** * @brief Reads the equation from an MPS file. * - * The input file can be a plain text file in MPS-format or a bzip2-compressed MPS - * file (.mps.bz2). + * The input file can be a plain text file in MPS-format or a compressed MPS + * file (.mps.gz or .mps.bz2). * * Read this link http://lpsolve.sourceforge.net/5.5/mps-format.htm for more * details on both free and fixed MPS format. * - * Note: bzip2-compressed files can only be read if libbzip2 is installed. + * Note: Compressed MPS files .mps.gz, .mps.bz2 can only be read if the compression + * libraries zlib or libbzip2 are installed, respectively. * * @param[in] mps_file_path Path to MPS file. * @param[in] fixed_mps_format If MPS file should be parsed as fixed, false by default diff --git a/cpp/libmps_parser/src/mps_parser.cpp b/cpp/libmps_parser/src/mps_parser.cpp index a03b8b4b48..d553eafedc 100644 --- a/cpp/libmps_parser/src/mps_parser.cpp +++ b/cpp/libmps_parser/src/mps_parser.cpp @@ -32,9 +32,16 @@ #ifdef MPS_PARSER_WITH_BZIP2 #include -#include #endif // MPS_PARSER_WITH_BZIP2 +#ifdef MPS_PARSER_WITH_ZLIB +#include +#endif // MPS_PARSER_WITH_ZLIB + +#if defined(MPS_PARSER_WITH_BZIP2) || defined(MPS_PARSER_WITH_ZLIB) +#include +#endif // MPS_PARSER_WITH_BZIP2 || MPS_PARSER_WITH_ZLIB + namespace { using cuopt::mps_parser::mps_parser_expects_fatal; using cuopt::mps_parser::mps_parser_expects; @@ -122,6 +129,80 @@ std::vector bz2_file_to_string(const std::string& file) } // end namespace #endif // MPS_PARSER_WITH_BZIP2 +#ifdef MPS_PARSER_WITH_ZLIB +namespace { +using gzopen_t = decltype(&gzopen); +using gzclose_r_t = decltype(&gzclose_r); +using gzbuffer_t = decltype(&gzbuffer); +using gzread_t = decltype(&gzread); +using gzerror_t = decltype(&gzerror); +std::vector zlib_file_to_string(const std::string& file) +{ + struct DlCloseDeleter { + void operator()(void* fp) { + mps_parser_expects_fatal(dlclose(fp) == 0, + error_type_t::ValidationError, + "Error closing libbz2.so!"); + } + }; + struct GzCloseDeleter { + void operator()(gzFile_s* f) { + int err = fptr(f); + mps_parser_expects_fatal(err == Z_OK, + error_type_t::ValidationError, + "Error closing gz file!"); + } + gzclose_r_t fptr = nullptr; + }; + + std::unique_ptr lzhandle{dlopen("libz.so", RTLD_LAZY)}; + mps_parser_expects(lzhandle != nullptr, + error_type_t::ValidationError, + "Could not open .mps.gz file since libz.so was not found. In order to open .mps.gz files directly, please ensure zlib is installed. Alternatively, decompress the .mps.gz file manually and open the uncompressed .mps file. Given path: %s", + file.c_str()); + gzopen_t gzopen = reinterpret_cast(dlsym(lzhandle.get(), "gzopen")); + gzclose_r_t gzclose_r = reinterpret_cast(dlsym(lzhandle.get(), "gzclose_r")); + gzbuffer_t gzbuffer = reinterpret_cast(dlsym(lzhandle.get(), "gzbuffer")); + gzread_t gzread = reinterpret_cast(dlsym(lzhandle.get(), "gzread")); + gzerror_t gzerror = reinterpret_cast(dlsym(lzhandle.get(), "gzerror")); + mps_parser_expects(gzopen != nullptr && gzclose_r != nullptr && gzbuffer != nullptr && gzread != nullptr && gzerror != nullptr, + error_type_t::ValidationError, + "Error loading zlib! Library version might be incompatible. Please decompress the .mps.gz file manually and open the uncompressed .mps file. Given path: %s", + file.c_str()); + std::unique_ptr gzfp{gzopen(file.c_str(), "rb"), {gzclose_r}}; + mps_parser_expects(gzfp != nullptr, + error_type_t::ValidationError, + "Error opening compressed MPS file! Given path: %s", + file.c_str()); + int zlib_status = gzbuffer(gzfp.get(), 1 << 20); // 1 MiB + mps_parser_expects(zlib_status == Z_OK, + error_type_t::ValidationError, + "Could not set zlib internal buffer size for decompression! Given path: %s", + file.c_str()); + std::vector buf; + const size_t readbufsize = 1ull << 24; // 16MiB + std::vector readbuf(readbufsize); + int bytes_read = -1; + while (bytes_read != 0) { + bytes_read = gzread(gzfp.get(), readbuf.data(), readbuf.size()); + if (bytes_read > 0) { + buf.insert(buf.end(), begin(readbuf), begin(readbuf) + bytes_read); + } + if (bytes_read < 0) { + gzerror(gzfp.get(), &zlib_status); + break; + } + } + buf.push_back('\0'); + mps_parser_expects(zlib_status == Z_OK, + error_type_t::ValidationError, + "Error in zlib decompression of MPS file! Given path: %s", + file.c_str()); + return buf; +} +} // end namespace +#endif // MPS_PARSER_WITH_ZLIB + namespace cuopt::mps_parser { template @@ -370,6 +451,12 @@ std::vector mps_parser_t::file_to_string(const std::string& file } #endif // MPS_PARSER_WITH_BZIP2 +#ifdef MPS_PARSER_WITH_ZLIB + if(file.size() > 3 && file.substr(file.size() - 3, 3) == ".gz") { + return zlib_file_to_string(file); + } +#endif // MPS_PARSER_WITH_ZLIB + // Faster than using C++ I/O FILE* fp = fopen(file.c_str(), "r"); mps_parser_expects(fp != nullptr, diff --git a/cpp/libmps_parser/src/mps_parser.hpp b/cpp/libmps_parser/src/mps_parser.hpp index 42d9b4fca9..f0325435b8 100644 --- a/cpp/libmps_parser/src/mps_parser.hpp +++ b/cpp/libmps_parser/src/mps_parser.hpp @@ -139,10 +139,10 @@ class mps_parser_t { std::unordered_set bounds_defined_for_var_id{}; static constexpr f_t unset_range_value = std::numeric_limits::infinity(); - /* Reads the MPS input file into a buffer. + /* Reads an MPS input file into a buffer. * - * If the file has a .bz2 suffix and libbzip2 is installed, this function - * can directly read and decompress the compressed MPS file. + * If the file has a .gz or .bz2 suffix and zlib or libbzip2 are installed, respectively, + * the function directly reads and decompresses the compressed MPS file. */ std::vector file_to_string(const std::string& file); void fill_problem(mps_data_model_t& problem); diff --git a/cpp/libmps_parser/tests/mps_parser_test.cpp b/cpp/libmps_parser/tests/mps_parser_test.cpp index f258f475c9..8effa0eb05 100644 --- a/cpp/libmps_parser/tests/mps_parser_test.cpp +++ b/cpp/libmps_parser/tests/mps_parser_test.cpp @@ -793,4 +793,40 @@ TEST(mps_parser, good_mps_file_bzip2_compressed) EXPECT_EQ(0.1, mps.c_values[1]); } +TEST(mps_parser, good_mps_file_zlib_compressed) +{ + auto mps = read_from_mps("linear_programming/good-mps-1.mps.gz"); + EXPECT_EQ("good-1", mps.problem_name); + ASSERT_EQ(int(2), mps.row_names.size()); + EXPECT_EQ("ROW1", mps.row_names[0]); + EXPECT_EQ("ROW2", mps.row_names[1]); + ASSERT_EQ(int(2), mps.row_types.size()); + EXPECT_EQ(LesserThanOrEqual, mps.row_types[0]); + EXPECT_EQ(LesserThanOrEqual, mps.row_types[1]); + EXPECT_EQ("COST", mps.objective_name); + ASSERT_EQ(int(2), mps.var_names.size()); + EXPECT_EQ("VAR1", mps.var_names[0]); + EXPECT_EQ("VAR2", mps.var_names[1]); + ASSERT_EQ(int(2), mps.A_indices.size()); + ASSERT_EQ(int(2), mps.A_indices[0].size()); + EXPECT_EQ(int(0), mps.A_indices[0][0]); + EXPECT_EQ(int(1), mps.A_indices[0][1]); + ASSERT_EQ(int(2), mps.A_indices[1].size()); + EXPECT_EQ(int(0), mps.A_indices[1][0]); + EXPECT_EQ(int(1), mps.A_indices[1][1]); + ASSERT_EQ(int(2), mps.A_values.size()); + ASSERT_EQ(int(2), mps.A_values[0].size()); + EXPECT_EQ(3., mps.A_values[0][0]); + EXPECT_EQ(4., mps.A_values[0][1]); + ASSERT_EQ(int(2), mps.A_values[1].size()); + EXPECT_EQ(2.7, mps.A_values[1][0]); + EXPECT_EQ(10.1, mps.A_values[1][1]); + ASSERT_EQ(int(2), mps.b_values.size()); + EXPECT_EQ(5.4, mps.b_values[0]); + EXPECT_EQ(4.9, mps.b_values[1]); + ASSERT_EQ(int(2), mps.c_values.size()); + EXPECT_EQ(0.2, mps.c_values[0]); + EXPECT_EQ(0.1, mps.c_values[1]); +} + } // namespace cuopt::mps_parser diff --git a/datasets/linear_programming/good-mps-1.mps.gz b/datasets/linear_programming/good-mps-1.mps.gz new file mode 100644 index 0000000000000000000000000000000000000000..b9a10d173d836a045738a5d19d743bfe3d88ee37 GIT binary patch literal 219 zcmV<103`n(iwFqnr=4g317~kf<1&< z@X!aeKJL~c6(mCI#jkhMSc0wYC9|2?VRk*}N7vk&SIs=2(rt$j3ohZ|s0x7@HNrNM z6z%hTtEHgeyof{zB95KA5yR`QkOg|jh|fUjAm(ip&1>&slQB0rT384%^=o*@C~ zM+`u+C{GS)MiQ#tvo9mIh%%2jGjEZPuQGlXQN&?LP7TF~bB4h`L$f9;b~YxxlPGkX V-;?Gdakf=^#oqSTOHh;n003T%V(kC` literal 0 HcmV?d00001 diff --git a/thirdparty/THIRD_PARTY_LICENSES b/thirdparty/THIRD_PARTY_LICENSES index d08969e773..6bce42d313 100644 --- a/thirdparty/THIRD_PARTY_LICENSES +++ b/thirdparty/THIRD_PARTY_LICENSES @@ -922,3 +922,32 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. Julian Seward, jseward@acm.org bzip2/libbzip2 version 1.0.8 of 13 July 2019 + + +----------------------------------------------------------------------------------------- +== zlib + +Usage: libmps_parser uses zlib + +Copyright notice: + + (C) 1995-2024 Jean-loup Gailly and Mark Adler + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + Jean-loup Gailly Mark Adler + jloup@gzip.org madler@alumni.caltech.edu From ac62457725b15000706683bbfdac3ba019704a4a Mon Sep 17 00:00:00 2001 From: Andreas Hehn Date: Thu, 28 Aug 2025 07:50:55 -0700 Subject: [PATCH 03/11] Added zlib and bzip2 to the recipes. --- conda/recipes/libcuopt/recipe.yaml | 6 ++++++ dependencies.yaml | 11 +++++++++++ 2 files changed, 17 insertions(+) diff --git a/conda/recipes/libcuopt/recipe.yaml b/conda/recipes/libcuopt/recipe.yaml index 9924107d27..4595763a62 100644 --- a/conda/recipes/libcuopt/recipe.yaml +++ b/conda/recipes/libcuopt/recipe.yaml @@ -57,6 +57,8 @@ cache: - cuda-version =${{ cuda_version }} - cmake ${{ cmake_version }} - ninja + - zlib + - bzip2 host: - cpp-argparse - cuda-version =${{ cuda_version }} @@ -70,6 +72,8 @@ cache: - libcusparse-dev - cuda-cudart-dev - boost + - zlib + - bzip2 outputs: - package: @@ -90,6 +94,8 @@ outputs: build: - cmake ${{ cmake_version }} - ${{ stdlib("c") }} + - zlib + - bzip2 ignore_run_exports: by_name: - cuda-cudart diff --git a/dependencies.yaml b/dependencies.yaml index ae7c345a79..27bea437c1 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -41,6 +41,7 @@ files: - test_python_common - test_python_cuopt - test_python_cuopt_server + - libmps_common - depends_on_rmm - depends_on_cupy - depends_on_cudf @@ -104,6 +105,7 @@ files: - build_python_common - depends_on_rapids_logger - run_common + - libmps_common py_test_cuopt_mps_parser: output: pyproject pyproject_dir: python/cuopt/cuopt/linear_programming/ @@ -121,6 +123,7 @@ files: includes: - run_common - depends_on_rapids_logger + - libmps_common py_build_libcuopt: output: pyproject pyproject_dir: python/libcuopt @@ -313,6 +316,8 @@ dependencies: - cpp-argparse - librmm==25.10.* - libraft-headers==25.10.* + - zlib + - bzip2 test_cpp: common: - output_types: [conda] @@ -548,6 +553,12 @@ dependencies: # pip recognizes the index as a global option for the requirements.txt file # This index is needed for rapids_logger - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + libmps_common: + common: + - output_types: requrements + packages: + - zlib + - bzip2 depends_on_rmm: common: - output_types: conda From 0672b415c446144c7e25f89c859bc4496d13cdf5 Mon Sep 17 00:00:00 2001 From: ahehn-nv Date: Thu, 28 Aug 2025 19:11:48 +0200 Subject: [PATCH 04/11] Apply review comments for cpp/libmps_parser/CMakeLists.txt Co-authored-by: Kyle Edwards --- cpp/libmps_parser/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/libmps_parser/CMakeLists.txt b/cpp/libmps_parser/CMakeLists.txt index 7bbb9cf7f0..4a3cbb7699 100644 --- a/cpp/libmps_parser/CMakeLists.txt +++ b/cpp/libmps_parser/CMakeLists.txt @@ -54,12 +54,12 @@ endif(CMAKE_COMPILER_IS_GNUCXX) if(MPS_PARSER_WITH_BZIP2) find_package(BZip2 REQUIRED) - add_definitions(-DMPS_PARSER_WITH_BZIP2) + add_compile_definitions(-DMPS_PARSER_WITH_BZIP2) endif(MPS_PARSER_WITH_BZIP2) if(MPS_PARSER_WITH_ZLIB) find_package(ZLIB REQUIRED) - add_definitions(-DMPS_PARSER_WITH_ZLIB) + add_compile_definitions(-DMPS_PARSER_WITH_ZLIB) endif(MPS_PARSER_WITH_ZLIB) if(DEFINE_ASSERT) From 20de1b4e5a73009c403f5efc883d7d5435f5facf Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Thu, 28 Aug 2025 14:16:03 -0500 Subject: [PATCH 05/11] add skip libs and style checks --- ci/build_wheel_cuopt_mps_parser.sh | 6 + conda/recipes/libcuopt/recipe.yaml | 2 + cpp/libmps_parser/src/mps_parser.cpp | 288 +++++++++++----------- cpp/libmps_parser/src/utilities/error.hpp | 5 +- 4 files changed, 161 insertions(+), 140 deletions(-) diff --git a/ci/build_wheel_cuopt_mps_parser.sh b/ci/build_wheel_cuopt_mps_parser.sh index 826b229e5b..bb5e764c37 100755 --- a/ci/build_wheel_cuopt_mps_parser.sh +++ b/ci/build_wheel_cuopt_mps_parser.sh @@ -33,6 +33,12 @@ fi ci/build_wheel.sh cuopt_mps_parser ${package_dir} + +EXCLUDE_ARGS=( + --exclude "libzlib.so" + --exclude "libbz2.so" +) + # repair wheels and write to the location that artifact-uploading code expects to find them python -m auditwheel repair -w "${RAPIDS_WHEEL_BLD_OUTPUT_DIR}" ${package_dir}/dist/* diff --git a/conda/recipes/libcuopt/recipe.yaml b/conda/recipes/libcuopt/recipe.yaml index 4595763a62..fb1993f825 100644 --- a/conda/recipes/libcuopt/recipe.yaml +++ b/conda/recipes/libcuopt/recipe.yaml @@ -105,6 +105,8 @@ outputs: - libcurand - libcusparse - librmm + - libzlib + - libbz2 tests: - package_contents: files: diff --git a/cpp/libmps_parser/src/mps_parser.cpp b/cpp/libmps_parser/src/mps_parser.cpp index d553eafedc..18f6ea2378 100644 --- a/cpp/libmps_parser/src/mps_parser.cpp +++ b/cpp/libmps_parser/src/mps_parser.cpp @@ -26,35 +26,35 @@ #include #include #include +#include #include #include -#include #ifdef MPS_PARSER_WITH_BZIP2 #include -#endif // MPS_PARSER_WITH_BZIP2 +#endif // MPS_PARSER_WITH_BZIP2 #ifdef MPS_PARSER_WITH_ZLIB #include -#endif // MPS_PARSER_WITH_ZLIB +#endif // MPS_PARSER_WITH_ZLIB #if defined(MPS_PARSER_WITH_BZIP2) || defined(MPS_PARSER_WITH_ZLIB) #include -#endif // MPS_PARSER_WITH_BZIP2 || MPS_PARSER_WITH_ZLIB +#endif // MPS_PARSER_WITH_BZIP2 || MPS_PARSER_WITH_ZLIB namespace { -using cuopt::mps_parser::mps_parser_expects_fatal; -using cuopt::mps_parser::mps_parser_expects; using cuopt::mps_parser::error_type_t; +using cuopt::mps_parser::mps_parser_expects; +using cuopt::mps_parser::mps_parser_expects_fatal; struct FcloseDeleter { - void operator()(FILE* fp) { - mps_parser_expects_fatal(fclose(fp) == 0, - error_type_t::ValidationError, - "Error closing MPS file!"); - } + void operator()(FILE* fp) + { + mps_parser_expects_fatal( + fclose(fp) == 0, error_type_t::ValidationError, "Error closing MPS file!"); + } }; -} // end namespace +} // end namespace #ifdef MPS_PARSER_WITH_BZIP2 namespace { @@ -64,70 +64,76 @@ using BZ2_bzRead_t = decltype(&BZ2_bzRead); std::vector bz2_file_to_string(const std::string& file) { - struct DlCloseDeleter { - void operator()(void* fp) { - mps_parser_expects_fatal(dlclose(fp) == 0, - error_type_t::ValidationError, - "Error closing libbz2.so!"); - } - }; - struct BzReadCloseDeleter { - void operator()(void* f) { - int bzerror; - if(f != nullptr) - fptr(&bzerror, f); - mps_parser_expects_fatal(bzerror == BZ_OK, - error_type_t::ValidationError, - "Error closing bzip2 file!"); - } - BZ2_bzReadClose_t fptr = nullptr; - }; - - std::unique_ptr lbz2handle{dlopen("libbz2.so", RTLD_LAZY)}; - mps_parser_expects(lbz2handle != nullptr, - error_type_t::ValidationError, - "Could not open .mps.bz2 file since libbz2.so was not found. In order to open .mps.bz2 files directly, please ensure libbzip2 is installed. Alternatively, decompress the .mps.bz2 file manually and open the uncompressed .mps file. Given path: %s", - file.c_str()); - - BZ2_bzReadOpen_t BZ2_bzReadOpen = reinterpret_cast(dlsym(lbz2handle.get(), "BZ2_bzReadOpen")); - BZ2_bzReadClose_t BZ2_bzReadClose = reinterpret_cast(dlsym(lbz2handle.get(), "BZ2_bzReadClose")); - BZ2_bzRead_t BZ2_bzRead = reinterpret_cast(dlsym(lbz2handle.get(), "BZ2_bzRead")); - mps_parser_expects(BZ2_bzReadOpen != nullptr && BZ2_bzReadClose != nullptr && BZ2_bzRead != nullptr, - error_type_t::ValidationError, - "Error loading libbzip2! Library version might be incompatible. Please decompress the .mps.bz2 file manually and open the uncompressed .mps file. Given path: %s", - file.c_str()); + struct DlCloseDeleter { + void operator()(void* fp) + { + mps_parser_expects_fatal( + dlclose(fp) == 0, error_type_t::ValidationError, "Error closing libbz2.so!"); + } + }; + struct BzReadCloseDeleter { + void operator()(void* f) + { + int bzerror; + if (f != nullptr) fptr(&bzerror, f); + mps_parser_expects_fatal( + bzerror == BZ_OK, error_type_t::ValidationError, "Error closing bzip2 file!"); + } + BZ2_bzReadClose_t fptr = nullptr; + }; + std::unique_ptr lbz2handle{dlopen("libbz2.so", RTLD_LAZY)}; + mps_parser_expects( + lbz2handle != nullptr, + error_type_t::ValidationError, + "Could not open .mps.bz2 file since libbz2.so was not found. In order to open .mps.bz2 files " + "directly, please ensure libbzip2 is installed. Alternatively, decompress the .mps.bz2 file " + "manually and open the uncompressed .mps file. Given path: %s", + file.c_str()); + + BZ2_bzReadOpen_t BZ2_bzReadOpen = + reinterpret_cast(dlsym(lbz2handle.get(), "BZ2_bzReadOpen")); + BZ2_bzReadClose_t BZ2_bzReadClose = + reinterpret_cast(dlsym(lbz2handle.get(), "BZ2_bzReadClose")); + BZ2_bzRead_t BZ2_bzRead = reinterpret_cast(dlsym(lbz2handle.get(), "BZ2_bzRead")); + mps_parser_expects( + BZ2_bzReadOpen != nullptr && BZ2_bzReadClose != nullptr && BZ2_bzRead != nullptr, + error_type_t::ValidationError, + "Error loading libbzip2! Library version might be incompatible. Please decompress the .mps.bz2 " + "file manually and open the uncompressed .mps file. Given path: %s", + file.c_str()); - std::unique_ptr fp{fopen(file.c_str(), "rb")}; - mps_parser_expects(fp != nullptr, - error_type_t::ValidationError, - "Error opening MPS file! Given path: %s", - file.c_str()); - int bzerror = BZ_OK; - std::unique_ptr bzfile{BZ2_bzReadOpen(&bzerror, fp.get(), 0, 0, nullptr, 0), {BZ2_bzReadClose}}; - mps_parser_expects(bzerror == BZ_OK, - error_type_t::ValidationError, - "Could not open bzip2 compressed file! Given path: %s", - file.c_str()); - - std::vector buf; - const size_t readbufsize = 1ull << 24; // 16MiB - just a guess. - std::vector readbuf(readbufsize); - while (bzerror == BZ_OK) { - const size_t bytes_read = BZ2_bzRead(&bzerror, bzfile.get(), readbuf.data(), readbuf.size()); - if (bzerror == BZ_OK || bzerror == BZ_STREAM_END) { - buf.insert(buf.end(), begin(readbuf), begin(readbuf) + bytes_read); - } + std::unique_ptr fp{fopen(file.c_str(), "rb")}; + mps_parser_expects(fp != nullptr, + error_type_t::ValidationError, + "Error opening MPS file! Given path: %s", + file.c_str()); + int bzerror = BZ_OK; + std::unique_ptr bzfile{ + BZ2_bzReadOpen(&bzerror, fp.get(), 0, 0, nullptr, 0), {BZ2_bzReadClose}}; + mps_parser_expects(bzerror == BZ_OK, + error_type_t::ValidationError, + "Could not open bzip2 compressed file! Given path: %s", + file.c_str()); + + std::vector buf; + const size_t readbufsize = 1ull << 24; // 16MiB - just a guess. + std::vector readbuf(readbufsize); + while (bzerror == BZ_OK) { + const size_t bytes_read = BZ2_bzRead(&bzerror, bzfile.get(), readbuf.data(), readbuf.size()); + if (bzerror == BZ_OK || bzerror == BZ_STREAM_END) { + buf.insert(buf.end(), begin(readbuf), begin(readbuf) + bytes_read); } - buf.push_back('\0'); - mps_parser_expects(bzerror == BZ_STREAM_END, - error_type_t::ValidationError, - "Error in bzip2 decompression of MPS file! Given path: %s", - file.c_str()); - return buf; + } + buf.push_back('\0'); + mps_parser_expects(bzerror == BZ_STREAM_END, + error_type_t::ValidationError, + "Error in bzip2 decompression of MPS file! Given path: %s", + file.c_str()); + return buf; } -} // end namespace -#endif // MPS_PARSER_WITH_BZIP2 +} // end namespace +#endif // MPS_PARSER_WITH_BZIP2 #ifdef MPS_PARSER_WITH_ZLIB namespace { @@ -138,70 +144,74 @@ using gzread_t = decltype(&gzread); using gzerror_t = decltype(&gzerror); std::vector zlib_file_to_string(const std::string& file) { - struct DlCloseDeleter { - void operator()(void* fp) { - mps_parser_expects_fatal(dlclose(fp) == 0, - error_type_t::ValidationError, - "Error closing libbz2.so!"); - } - }; - struct GzCloseDeleter { - void operator()(gzFile_s* f) { - int err = fptr(f); - mps_parser_expects_fatal(err == Z_OK, - error_type_t::ValidationError, - "Error closing gz file!"); - } - gzclose_r_t fptr = nullptr; - }; + struct DlCloseDeleter { + void operator()(void* fp) + { + mps_parser_expects_fatal( + dlclose(fp) == 0, error_type_t::ValidationError, "Error closing libbz2.so!"); + } + }; + struct GzCloseDeleter { + void operator()(gzFile_s* f) + { + int err = fptr(f); + mps_parser_expects_fatal( + err == Z_OK, error_type_t::ValidationError, "Error closing gz file!"); + } + gzclose_r_t fptr = nullptr; + }; - std::unique_ptr lzhandle{dlopen("libz.so", RTLD_LAZY)}; - mps_parser_expects(lzhandle != nullptr, - error_type_t::ValidationError, - "Could not open .mps.gz file since libz.so was not found. In order to open .mps.gz files directly, please ensure zlib is installed. Alternatively, decompress the .mps.gz file manually and open the uncompressed .mps file. Given path: %s", - file.c_str()); - gzopen_t gzopen = reinterpret_cast(dlsym(lzhandle.get(), "gzopen")); - gzclose_r_t gzclose_r = reinterpret_cast(dlsym(lzhandle.get(), "gzclose_r")); - gzbuffer_t gzbuffer = reinterpret_cast(dlsym(lzhandle.get(), "gzbuffer")); - gzread_t gzread = reinterpret_cast(dlsym(lzhandle.get(), "gzread")); - gzerror_t gzerror = reinterpret_cast(dlsym(lzhandle.get(), "gzerror")); - mps_parser_expects(gzopen != nullptr && gzclose_r != nullptr && gzbuffer != nullptr && gzread != nullptr && gzerror != nullptr, - error_type_t::ValidationError, - "Error loading zlib! Library version might be incompatible. Please decompress the .mps.gz file manually and open the uncompressed .mps file. Given path: %s", - file.c_str()); - std::unique_ptr gzfp{gzopen(file.c_str(), "rb"), {gzclose_r}}; - mps_parser_expects(gzfp != nullptr, - error_type_t::ValidationError, - "Error opening compressed MPS file! Given path: %s", - file.c_str()); - int zlib_status = gzbuffer(gzfp.get(), 1 << 20); // 1 MiB - mps_parser_expects(zlib_status == Z_OK, - error_type_t::ValidationError, - "Could not set zlib internal buffer size for decompression! Given path: %s", - file.c_str()); - std::vector buf; - const size_t readbufsize = 1ull << 24; // 16MiB - std::vector readbuf(readbufsize); - int bytes_read = -1; - while (bytes_read != 0) { - bytes_read = gzread(gzfp.get(), readbuf.data(), readbuf.size()); - if (bytes_read > 0) { - buf.insert(buf.end(), begin(readbuf), begin(readbuf) + bytes_read); - } - if (bytes_read < 0) { - gzerror(gzfp.get(), &zlib_status); - break; - } + std::unique_ptr lzhandle{dlopen("libz.so", RTLD_LAZY)}; + mps_parser_expects( + lzhandle != nullptr, + error_type_t::ValidationError, + "Could not open .mps.gz file since libz.so was not found. In order to open .mps.gz files " + "directly, please ensure zlib is installed. Alternatively, decompress the .mps.gz file " + "manually and open the uncompressed .mps file. Given path: %s", + file.c_str()); + gzopen_t gzopen = reinterpret_cast(dlsym(lzhandle.get(), "gzopen")); + gzclose_r_t gzclose_r = reinterpret_cast(dlsym(lzhandle.get(), "gzclose_r")); + gzbuffer_t gzbuffer = reinterpret_cast(dlsym(lzhandle.get(), "gzbuffer")); + gzread_t gzread = reinterpret_cast(dlsym(lzhandle.get(), "gzread")); + gzerror_t gzerror = reinterpret_cast(dlsym(lzhandle.get(), "gzerror")); + mps_parser_expects( + gzopen != nullptr && gzclose_r != nullptr && gzbuffer != nullptr && gzread != nullptr && + gzerror != nullptr, + error_type_t::ValidationError, + "Error loading zlib! Library version might be incompatible. Please decompress the .mps.gz file " + "manually and open the uncompressed .mps file. Given path: %s", + file.c_str()); + std::unique_ptr gzfp{gzopen(file.c_str(), "rb"), {gzclose_r}}; + mps_parser_expects(gzfp != nullptr, + error_type_t::ValidationError, + "Error opening compressed MPS file! Given path: %s", + file.c_str()); + int zlib_status = gzbuffer(gzfp.get(), 1 << 20); // 1 MiB + mps_parser_expects(zlib_status == Z_OK, + error_type_t::ValidationError, + "Could not set zlib internal buffer size for decompression! Given path: %s", + file.c_str()); + std::vector buf; + const size_t readbufsize = 1ull << 24; // 16MiB + std::vector readbuf(readbufsize); + int bytes_read = -1; + while (bytes_read != 0) { + bytes_read = gzread(gzfp.get(), readbuf.data(), readbuf.size()); + if (bytes_read > 0) { buf.insert(buf.end(), begin(readbuf), begin(readbuf) + bytes_read); } + if (bytes_read < 0) { + gzerror(gzfp.get(), &zlib_status); + break; } - buf.push_back('\0'); - mps_parser_expects(zlib_status == Z_OK, - error_type_t::ValidationError, - "Error in zlib decompression of MPS file! Given path: %s", - file.c_str()); - return buf; + } + buf.push_back('\0'); + mps_parser_expects(zlib_status == Z_OK, + error_type_t::ValidationError, + "Error in zlib decompression of MPS file! Given path: %s", + file.c_str()); + return buf; } -} // end namespace -#endif // MPS_PARSER_WITH_ZLIB +} // end namespace +#endif // MPS_PARSER_WITH_ZLIB namespace cuopt::mps_parser { @@ -446,16 +456,16 @@ std::vector mps_parser_t::file_to_string(const std::string& file // raft::common::nvtx::range fun_scope("file to string"); #ifdef MPS_PARSER_WITH_BZIP2 - if(file.size() > 4 && file.substr(file.size() - 4, 4) == ".bz2") { - return bz2_file_to_string(file); + if (file.size() > 4 && file.substr(file.size() - 4, 4) == ".bz2") { + return bz2_file_to_string(file); } -#endif // MPS_PARSER_WITH_BZIP2 +#endif // MPS_PARSER_WITH_BZIP2 #ifdef MPS_PARSER_WITH_ZLIB - if(file.size() > 3 && file.substr(file.size() - 3, 3) == ".gz") { - return zlib_file_to_string(file); + if (file.size() > 3 && file.substr(file.size() - 3, 3) == ".gz") { + return zlib_file_to_string(file); } -#endif // MPS_PARSER_WITH_ZLIB +#endif // MPS_PARSER_WITH_ZLIB // Faster than using C++ I/O FILE* fp = fopen(file.c_str(), "r"); diff --git a/cpp/libmps_parser/src/utilities/error.hpp b/cpp/libmps_parser/src/utilities/error.hpp index e8f2939b95..17518d8daf 100644 --- a/cpp/libmps_parser/src/utilities/error.hpp +++ b/cpp/libmps_parser/src/utilities/error.hpp @@ -91,7 +91,10 @@ inline void mps_parser_expects_fatal(bool cond, error_type_t error_type, const c vsnprintf(msg, sizeof(msg), fmt, args); va_end(args); std::string error_string = error_to_string(error_type); - std::fprintf(stderr, "{\"MPS_PARSER_ERROR_TYPE\": \"%s\", \"msg\": \"%s\"}\n", error_to_string(error_type).c_str(), msg); + std::fprintf(stderr, + "{\"MPS_PARSER_ERROR_TYPE\": \"%s\", \"msg\": \"%s\"}\n", + error_to_string(error_type).c_str(), + msg); std::fflush(stderr); std::abort(); } From 4cc3a6ce9b03c4108f46c6366b5f3348004f5bbd Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Thu, 28 Aug 2025 14:19:26 -0500 Subject: [PATCH 06/11] Add deps --- ci/build_wheel_cuopt_mps_parser.sh | 2 +- conda/environments/all_cuda-129_arch-aarch64.yaml | 2 ++ conda/environments/all_cuda-129_arch-x86_64.yaml | 2 ++ dependencies.yaml | 2 +- python/cuopt/cuopt/linear_programming/pyproject.toml | 4 ++++ 5 files changed, 10 insertions(+), 2 deletions(-) diff --git a/ci/build_wheel_cuopt_mps_parser.sh b/ci/build_wheel_cuopt_mps_parser.sh index bb5e764c37..3ca2220dfd 100755 --- a/ci/build_wheel_cuopt_mps_parser.sh +++ b/ci/build_wheel_cuopt_mps_parser.sh @@ -40,6 +40,6 @@ EXCLUDE_ARGS=( ) # repair wheels and write to the location that artifact-uploading code expects to find them -python -m auditwheel repair -w "${RAPIDS_WHEEL_BLD_OUTPUT_DIR}" ${package_dir}/dist/* +python -m auditwheel repair "${EXCLUDE_ARGS[@]}" -w "${RAPIDS_WHEEL_BLD_OUTPUT_DIR}" ${package_dir}/dist/* ci/validate_wheel.sh "${package_dir}" "${RAPIDS_WHEEL_BLD_OUTPUT_DIR}" diff --git a/conda/environments/all_cuda-129_arch-aarch64.yaml b/conda/environments/all_cuda-129_arch-aarch64.yaml index 3565eae6a1..55f8aff0ab 100644 --- a/conda/environments/all_cuda-129_arch-aarch64.yaml +++ b/conda/environments/all_cuda-129_arch-aarch64.yaml @@ -7,6 +7,7 @@ channels: dependencies: - boost - breathe +- bzip2 - c-compiler - ccache - clang-tools=20.1.4 @@ -81,6 +82,7 @@ dependencies: - sphinxcontrib-websupport - sysroot_linux-aarch64==2.28 - uvicorn==0.34.* +- zlib - pip: - nvidia_sphinx_theme - swagger-plugin-for-sphinx diff --git a/conda/environments/all_cuda-129_arch-x86_64.yaml b/conda/environments/all_cuda-129_arch-x86_64.yaml index 04132f8c7a..9dbced79f8 100644 --- a/conda/environments/all_cuda-129_arch-x86_64.yaml +++ b/conda/environments/all_cuda-129_arch-x86_64.yaml @@ -7,6 +7,7 @@ channels: dependencies: - boost - breathe +- bzip2 - c-compiler - ccache - clang-tools=20.1.4 @@ -81,6 +82,7 @@ dependencies: - sphinxcontrib-websupport - sysroot_linux-64==2.28 - uvicorn==0.34.* +- zlib - pip: - nvidia_sphinx_theme - swagger-plugin-for-sphinx diff --git a/dependencies.yaml b/dependencies.yaml index 27bea437c1..314cfea3d2 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -555,7 +555,7 @@ dependencies: - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple libmps_common: common: - - output_types: requrements + - output_types: [requirements, pyproject, conda] packages: - zlib - bzip2 diff --git a/python/cuopt/cuopt/linear_programming/pyproject.toml b/python/cuopt/cuopt/linear_programming/pyproject.toml index 6bd26656f5..0e2c5d7688 100644 --- a/python/cuopt/cuopt/linear_programming/pyproject.toml +++ b/python/cuopt/cuopt/linear_programming/pyproject.toml @@ -19,8 +19,10 @@ authors = [ license = { text = "Apache-2.0" } requires-python = ">=3.10" dependencies = [ + "bzip2", "numpy>=1.23.5,<3.0a0", "rapids-logger==0.1.*,>=0.0.0a0", + "zlib", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ "Intended Audience :: Developers", @@ -78,9 +80,11 @@ commit-files = [ ] disable-cuda = true requires = [ + "bzip2", "cmake>=3.30.4", "cython>=3.0.3", "ninja", "numpy>=1.23.5,<3.0a0", "rapids-logger==0.1.*,>=0.0.0a0", + "zlib", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../../../dependencies.yaml and run `rapids-dependency-file-generator`. From 476e97e0db47b6805804b1eb90a0b6eb9caa2a3e Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Fri, 29 Aug 2025 11:58:19 -0500 Subject: [PATCH 07/11] fix so file name --- cpp/libmps_parser/src/mps_parser.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/libmps_parser/src/mps_parser.cpp b/cpp/libmps_parser/src/mps_parser.cpp index 18f6ea2378..4d58947aa1 100644 --- a/cpp/libmps_parser/src/mps_parser.cpp +++ b/cpp/libmps_parser/src/mps_parser.cpp @@ -161,7 +161,7 @@ std::vector zlib_file_to_string(const std::string& file) gzclose_r_t fptr = nullptr; }; - std::unique_ptr lzhandle{dlopen("libz.so", RTLD_LAZY)}; + std::unique_ptr lzhandle{dlopen("libz.so.1", RTLD_LAZY)}; mps_parser_expects( lzhandle != nullptr, error_type_t::ValidationError, From 6d54acf0536ab8eb9d19b7a56c56efcb771edafa Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Fri, 29 Aug 2025 12:03:37 -0500 Subject: [PATCH 08/11] remove deps and update so file name --- dependencies.yaml | 9 --------- python/cuopt/cuopt/linear_programming/pyproject.toml | 4 ---- 2 files changed, 13 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 314cfea3d2..ed3f9ede80 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -41,7 +41,6 @@ files: - test_python_common - test_python_cuopt - test_python_cuopt_server - - libmps_common - depends_on_rmm - depends_on_cupy - depends_on_cudf @@ -105,7 +104,6 @@ files: - build_python_common - depends_on_rapids_logger - run_common - - libmps_common py_test_cuopt_mps_parser: output: pyproject pyproject_dir: python/cuopt/cuopt/linear_programming/ @@ -123,7 +121,6 @@ files: includes: - run_common - depends_on_rapids_logger - - libmps_common py_build_libcuopt: output: pyproject pyproject_dir: python/libcuopt @@ -553,12 +550,6 @@ dependencies: # pip recognizes the index as a global option for the requirements.txt file # This index is needed for rapids_logger - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple - libmps_common: - common: - - output_types: [requirements, pyproject, conda] - packages: - - zlib - - bzip2 depends_on_rmm: common: - output_types: conda diff --git a/python/cuopt/cuopt/linear_programming/pyproject.toml b/python/cuopt/cuopt/linear_programming/pyproject.toml index 0e2c5d7688..6bd26656f5 100644 --- a/python/cuopt/cuopt/linear_programming/pyproject.toml +++ b/python/cuopt/cuopt/linear_programming/pyproject.toml @@ -19,10 +19,8 @@ authors = [ license = { text = "Apache-2.0" } requires-python = ">=3.10" dependencies = [ - "bzip2", "numpy>=1.23.5,<3.0a0", "rapids-logger==0.1.*,>=0.0.0a0", - "zlib", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ "Intended Audience :: Developers", @@ -80,11 +78,9 @@ commit-files = [ ] disable-cuda = true requires = [ - "bzip2", "cmake>=3.30.4", "cython>=3.0.3", "ninja", "numpy>=1.23.5,<3.0a0", "rapids-logger==0.1.*,>=0.0.0a0", - "zlib", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../../../dependencies.yaml and run `rapids-dependency-file-generator`. From f0adaaf881cc28f79936eaecf2fed4728fe80b51 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Fri, 29 Aug 2025 14:03:05 -0500 Subject: [PATCH 09/11] Add deps --- conda/recipes/mps-parser/recipe.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/conda/recipes/mps-parser/recipe.yaml b/conda/recipes/mps-parser/recipe.yaml index c9e10c8a87..3fcec0a06f 100644 --- a/conda/recipes/mps-parser/recipe.yaml +++ b/conda/recipes/mps-parser/recipe.yaml @@ -42,9 +42,13 @@ requirements: - python =${{ py_version }} - rapids-build-backend >=0.4.0,<0.5.0.dev0 - scikit-build-core >=0.10.0 + - bzip2 + - zlib run: - numpy >=1.23,<3.0a0 - python + - bzip2 + - zlib tests: - python: From d050010c2eaeda61df0685730d487b7a504b2184 Mon Sep 17 00:00:00 2001 From: Ramakrishna Prabhu Date: Fri, 29 Aug 2025 14:15:15 -0500 Subject: [PATCH 10/11] add deps --- conda/recipes/libcuopt/recipe.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/conda/recipes/libcuopt/recipe.yaml b/conda/recipes/libcuopt/recipe.yaml index fb1993f825..176e296ddf 100644 --- a/conda/recipes/libcuopt/recipe.yaml +++ b/conda/recipes/libcuopt/recipe.yaml @@ -96,6 +96,12 @@ outputs: - ${{ stdlib("c") }} - zlib - bzip2 + host: + - zlib + - bzip2 + run: + - zlib + - bzip2 ignore_run_exports: by_name: - cuda-cudart From 50abd299090b88b6e72154280286a5b93e7ca98b Mon Sep 17 00:00:00 2001 From: Ramakrishnap <42624703+rgsl888prabhu@users.noreply.github.com> Date: Fri, 29 Aug 2025 15:57:07 -0500 Subject: [PATCH 11/11] Update recipe.yaml --- conda/recipes/mps-parser/recipe.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/conda/recipes/mps-parser/recipe.yaml b/conda/recipes/mps-parser/recipe.yaml index 3fcec0a06f..bf3666f089 100644 --- a/conda/recipes/mps-parser/recipe.yaml +++ b/conda/recipes/mps-parser/recipe.yaml @@ -32,6 +32,7 @@ requirements: build: - cmake ${{ cmake_version }} - ninja + - libmps-parser =${{ version }} - ${{ compiler("c") }} - ${{ compiler("cxx") }} - ${{ stdlib("c") }} @@ -42,13 +43,9 @@ requirements: - python =${{ py_version }} - rapids-build-backend >=0.4.0,<0.5.0.dev0 - scikit-build-core >=0.10.0 - - bzip2 - - zlib run: - numpy >=1.23,<3.0a0 - python - - bzip2 - - zlib tests: - python: