diff --git a/apps/CMakeLists.txt b/apps/CMakeLists.txt index 97935a92..48f65c5e 100644 --- a/apps/CMakeLists.txt +++ b/apps/CMakeLists.txt @@ -51,6 +51,7 @@ endif() if (COPT_FOUND) _add_executable( ilp_bsp_scheduler ) _add_executable( ilp_hypergraph_partitioner ) +_add_executable( ilp_partitioning_test_suite ) endif() endif() diff --git a/apps/config/bsp_test_suite_config.json b/apps/config/bsp_test_suite_config.json index 76f77640..52d325d3 100644 --- a/apps/config/bsp_test_suite_config.json +++ b/apps/config/bsp_test_suite_config.json @@ -18,16 +18,30 @@ "parameters": {} }, { - "name": "GreedyRecomputer", + "name": "GreedyRecomputerMask7", + "id": "GreedyRecomputer", "run" : true, - "name_suffix" : "Locking", + "name_suffix" : "GreedyBsp", "parameters": { + "methodmask" : 7, "scheduler": { - "name": "BspLocking", + "name": "GreedyBspHC", + "id": "LocalSearch", + "run" : true, + "name_suffix" : "GreedyBspHC", "parameters": { - "max_percent_idle_processors": 0.4, - "increase_parallelism_in_new_superstep": true - } + "improver": { + "name" : "hill_climb" + }, + "scheduler": { + "name": "GreedyBsp", + "id": "GreedyBsp", + "parameters": { + "max_percent_idle_processors": 0.2, + "increase_parallelism_in_new_superstep": true + } + } + } } } }, diff --git a/apps/config/partitioning_test_suite_config.json b/apps/config/partitioning_test_suite_config.json new file mode 100644 index 00000000..5edd80e3 --- /dev/null +++ b/apps/config/partitioning_test_suite_config.json @@ -0,0 +1,30 @@ +{ + "globalParameters": { + "timeLimit": 3600, + "graphDirectory": "graphs/", + "archDirectory": "machines/", + "outputStatsFile": "all_run_stats.csv", + "outputLogFile": "log.txt", + "outputSchedule": false + }, + "algorithms": [ + { + "name": "ILP", + "id" : "ILP", + "run" : true, + "parameters": {} + }, + { + "name": "ILP_dupl", + "id" : "ILP_dupl", + "run" : true, + "parameters": {} + }, + { + "name": "ILP_repl", + "id" : "ILP_repl", + "run" : true, + "parameters": {} + } + ] +} \ No newline at end of file diff --git a/apps/ilp_partitioning_test_suite.cpp b/apps/ilp_partitioning_test_suite.cpp new file mode 100644 index 00000000..4a311189 --- /dev/null +++ b/apps/ilp_partitioning_test_suite.cpp @@ -0,0 +1,27 @@ +/* +Copyright 2024 Huawei Technologies Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +@author Toni Boehnlein, Benjamin Lozes, Pal Andras Papp, Raphael S. Steiner +*/ + +#include "osp/graph_implementations/adj_list_impl/computational_dag_edge_idx_vector_impl.hpp" +#include "test_suite_runner/PartitioningTestSuiteRunner.hpp" + +using GraphT = osp::ComputationalDagEdgeIdxVectorImplDefIntT; + +int main(int argc, char *argv[]) { + osp::PartitioningTestSuiteRunner runner; + return runner.Run(argc, argv); +} \ No newline at end of file diff --git a/apps/test_suite_runner/AbstractTestSuiteRunner.hpp b/apps/test_suite_runner/AbstractTestSuiteRunner.hpp index 7b4f249c..99f84eaf 100644 --- a/apps/test_suite_runner/AbstractTestSuiteRunner.hpp +++ b/apps/test_suite_runner/AbstractTestSuiteRunner.hpp @@ -189,7 +189,7 @@ class AbstractTestSuiteRunner { } } - int Run(int argc, char *argv[]) { + int virtual Run(int argc, char *argv[]) { try { parser_.ParseArgs(argc, argv); } catch (const std::exception &e) { @@ -312,6 +312,8 @@ class AbstractTestSuiteRunner { continue; } + bspInstance.SetAllOnesCompatibilityMatrix(); + for (auto &algorithmConfigPair : parser_.scheduler_) { const pt::ptree &algoConfig = algorithmConfigPair.second; diff --git a/apps/test_suite_runner/PartitioningTestSuiteRunner.hpp b/apps/test_suite_runner/PartitioningTestSuiteRunner.hpp new file mode 100644 index 00000000..3dbbad92 --- /dev/null +++ b/apps/test_suite_runner/PartitioningTestSuiteRunner.hpp @@ -0,0 +1,203 @@ +/* +Copyright 2024 Huawei Technologies Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +@author Toni Boehnlein, Benjamin Lozes, Pal Andras Papp, Raphael S. Steiner +*/ + +#pragma once + +#include "AbstractTestSuiteRunner.hpp" +#include "StringToScheduler/run_partitioner.hpp" +#include "osp/auxiliary/io/hmetis_hypergraph_file_reader.hpp" +#include "osp/auxiliary/io/mtx_hypergraph_file_reader.hpp" +#include "osp/bsp/model/BspSchedule.hpp" +#include "osp/graph_implementations/adj_list_impl/computational_dag_vector_impl.hpp" +#include "osp/partitioning/model/partitioning.hpp" +#include "osp/partitioning/model/partitioning_replication.hpp" + +namespace osp { + +class PartitioningStatsModule : public IStatisticModule> { + public: + std::vector GetMetricHeaders() const override { return {"Cost", "CutNet"}; } + + std::map RecordStatistics(const Partitioning &partitioning, + std::ofstream & /*log_stream*/) const override { + std::map stats; + stats["Cost"] = std::to_string(partitioning.ComputeConnectivityCost()); + stats["CutNet"] = std::to_string(partitioning.ComputeCutNetCost()); + return stats; + } +}; + +template +class PartitioningTestSuiteRunner : public AbstractTestSuiteRunner, GraphType> { + private: + protected: + ReturnStatus ComputeTargetObjectImpl(const BspInstance &instance, + std::unique_ptr> &targetObject, + const pt::ptree &algoConfig, + long long &computationTimeMs) override { + return ReturnStatus::ERROR; // unused + } + + void CreateAndRegisterStatisticModules(const std::string &moduleName) override { + if (moduleName == "PartitioningStats") { + this->activeStatsModules_.push_back(std::make_unique()); + } + } + + public: + PartitioningTestSuiteRunner() : AbstractTestSuiteRunner, GraphType>() {} + + int virtual Run(int argc, char *argv[]) override; +}; + +template +int PartitioningTestSuiteRunner::Run(int argc, char *argv[]) { + using HypergraphT = HypergraphDefT; + try { + this->parser_.ParseArgs(argc, argv); + } catch (const std::exception &e) { + std::cerr << "Error parsing command line arguments: " << e.what() << std::endl; + return 1; + } + + if (!this->ParseCommonConfig()) { + return 1; + } + + this->SetupLogFile(); + + CreateAndRegisterStatisticModules("PartitioningStats"); + this->SetupStatisticsFile(); + + for (const auto &machineEntry : std::filesystem::recursive_directory_iterator(this->machineDirPath_)) { + if (std::filesystem::is_directory(machineEntry)) { + this->logStream_ << "Skipping directory " << machineEntry.path().string() << std::endl; + continue; + } + std::string filenameMachine = machineEntry.path().string(); + std::string nameMachine = filenameMachine.substr(filenameMachine.rfind('/') + 1); + if (nameMachine.rfind('.') != std::string::npos) { + nameMachine = nameMachine.substr(0, nameMachine.rfind('.')); + } + + // Temporary hack. Until there is no separate file format for partitioning problem parameters, we abuse + // bsp arch files: 1st number is number of parts, 2nd is imbalance allowed (percentage), rest is ignored + BspArchitecture arch; + if (!file_reader::ReadBspArchitecture(filenameMachine, arch)) { + this->logStream_ << "Reading architecture file " << filenameMachine << " failed." << std::endl; + continue; + } + this->logStream_ << "Start Machine: " + filenameMachine + "\n"; + std::cout << "Start Machine: " + filenameMachine + "\n"; + + for (const auto &graphEntry : std::filesystem::recursive_directory_iterator(this->graphDirPath_)) { + if (std::filesystem::is_directory(graphEntry)) { + this->logStream_ << "Skipping directory " << graphEntry.path().string() << std::endl; + continue; + } + std::string filenameGraph = graphEntry.path().string(); + std::string nameGraph = filenameGraph.substr(filenameGraph.rfind('/') + 1); + if (nameGraph.rfind('.') != std::string::npos) { + nameGraph = nameGraph.substr(0, nameGraph.rfind('.')); + } + this->logStream_ << "Start Hypergraph: " + filenameGraph + "\n"; + std::cout << "Start Hypergraph: " + filenameGraph + "\n"; + + bool graphStatus = false; + GraphType dag; + + std::string fileEnding = filenameGraph.substr(filenameGraph.rfind(".") + 1); + + PartitioningProblem + instance; //(ConvertFromCdagAsHyperdag(dag), arch.NumberOfProcessors()); + instance.SetNumberOfPartitions(arch.NumberOfProcessors()); + + if (fileEnding == "mtx") { + graphStatus = file_reader::ReadHypergraphMartixMarketFormat( + filenameGraph, instance.GetHypergraph(), file_reader::MatrixToHypergraphFormat::FINE_GRAINED); + + } else if (fileEnding == "mtx2") { + graphStatus = file_reader::ReadHypergraphMartixMarketFormat( + filenameGraph, instance.GetHypergraph(), file_reader::MatrixToHypergraphFormat::ROW_NET); + + } else if (fileEnding == "hmetis") { + graphStatus = file_reader::ReadHypergraphMetisFormat(filenameGraph, instance.GetHypergraph()); + + } else { + graphStatus = file_reader::ReadGraph(filenameGraph, dag); + instance.SetHypergraph(ConvertFromCdagAsHyperdag(dag)); + } + + if (!graphStatus) { + this->logStream_ << "Reading graph file " << filenameGraph << " failed." << std::endl; + continue; + } + + instance.SetMaxWorkWeightViaImbalanceFactor(static_cast(arch.CommunicationCosts()) / 100.0); + + for (auto &algorithmConfigPair : this->parser_.scheduler_) { + const pt::ptree &algoConfig = algorithmConfigPair.second; + + std::string currentAlgoName = algoConfig.get_child("name").get_value(); + this->logStream_ << "Start Algorithm " + currentAlgoName + "\n"; + std::cout << "Start Algorithm " + currentAlgoName + "\n"; + + long long computationTimeMs; + const auto startTime = std::chrono::high_resolution_clock::now(); + + std::pair cost; + ReturnStatus execStatus = RunPartitioner(this->parser_, algoConfig, instance, cost); + + const auto finishTime = std::chrono::high_resolution_clock::now(); + computationTimeMs = std::chrono::duration_cast(finishTime - startTime).count(); + + if (execStatus != ReturnStatus::OSP_SUCCESS && execStatus != ReturnStatus::BEST_FOUND) { + if (execStatus == ReturnStatus::ERROR) { + this->logStream_ << "Error computing with " << currentAlgoName << "." << std::endl; + } else if (execStatus == ReturnStatus::TIMEOUT) { + this->logStream_ << "Partitioner " << currentAlgoName << " timed out." << std::endl; + } + continue; + } + + // currently not writing output to file + + std::cout << "OSP-Cost: " << cost.first << std::endl << "OSP-CutNet: " << cost.second << std::endl; + + if (this->statsOutStream_.is_open()) { + std::map currentRowValues; + currentRowValues["Graph"] = nameGraph; + currentRowValues["Machine"] = nameMachine; + currentRowValues["Algorithm"] = currentAlgoName; + currentRowValues["TimeToCompute(ms)"] = std::to_string(computationTimeMs); + currentRowValues["Cost"] = std::to_string(cost.first); + currentRowValues["CutNet"] = std::to_string(cost.second); + + for (size_t i = 0; i < this->allCsvHeaders_.size(); ++i) { + this->statsOutStream_ << currentRowValues[this->allCsvHeaders_[i]] + << (i == this->allCsvHeaders_.size() - 1 ? "" : ","); + } + this->statsOutStream_ << "\n"; + } + } + } + } + return 0; +} + +} // namespace osp diff --git a/apps/test_suite_runner/StringToScheduler/run_bsp_recomp_scheduler.hpp b/apps/test_suite_runner/StringToScheduler/run_bsp_recomp_scheduler.hpp index c9f1cf06..03f9e2f4 100644 --- a/apps/test_suite_runner/StringToScheduler/run_bsp_recomp_scheduler.hpp +++ b/apps/test_suite_runner/StringToScheduler/run_bsp_recomp_scheduler.hpp @@ -45,9 +45,14 @@ ReturnStatus RunBspRecompScheduler(const ConfigParser &parser, std::cout << "Running algorithm: " << algorithm.get_child("name").get_value() << std::endl; - if (algorithm.get_child("name").get_value() == "GreedyRecomputer") { + if (algorithm.get_child("id").get_value() == "GreedyRecomputer") { BspSchedule bspSchedule(schedule.GetInstance()); + unsigned methodMask = algorithm.get_child("parameters").get_child("methodmask").get_value(); + if (methodMask > 7) { + return ReturnStatus::ERROR; + } + ReturnStatus status = RunBspScheduler(parser, algorithm.get_child("parameters").get_child("scheduler"), bspSchedule); BspScheduleCS initialSchedule(std::move(bspSchedule)); @@ -58,7 +63,11 @@ ReturnStatus RunBspRecompScheduler(const ConfigParser &parser, GreedyRecomputer scheduler; - return scheduler.ComputeRecompScheduleBasic(initialSchedule, schedule); + if (methodMask == 0) { + return scheduler.ComputeRecompScheduleBasic(initialSchedule, schedule); + } else { + return scheduler.ComputeRecompScheduleAdvanced(initialSchedule, schedule, methodMask); + } } else { throw std::invalid_argument("Parameter error: Unknown algorithm.\n"); diff --git a/apps/test_suite_runner/StringToScheduler/run_bsp_scheduler.hpp b/apps/test_suite_runner/StringToScheduler/run_bsp_scheduler.hpp index a4b56a7c..745f66fc 100644 --- a/apps/test_suite_runner/StringToScheduler/run_bsp_scheduler.hpp +++ b/apps/test_suite_runner/StringToScheduler/run_bsp_scheduler.hpp @@ -196,7 +196,16 @@ ReturnStatus RunBspScheduler(const ConfigParser &parser, std::unique_ptr> improver = GetBspImproverByName(parser, algorithm.get_child("parameters").get_child("improver")); - return improver->ImproveSchedule(schedule); + + const unsigned timeLimit = parser.globalParams_.get_optional("timeLimit").value_or(std::numeric_limits::max()); + improver->SetTimeLimitSeconds(timeLimit); + if (timeLimit == std::numeric_limits::max()) { + return improver->ImproveSchedule(schedule); + } else { + return improver->ImproveScheduleWithTimeLimit(schedule); + } + + #ifdef COPT } else if (id == "FullILP") { CoptFullScheduler scheduler; diff --git a/apps/test_suite_runner/StringToScheduler/run_partitioner.hpp b/apps/test_suite_runner/StringToScheduler/run_partitioner.hpp new file mode 100644 index 00000000..e5932977 --- /dev/null +++ b/apps/test_suite_runner/StringToScheduler/run_partitioner.hpp @@ -0,0 +1,75 @@ +/* +Copyright 2024 Huawei Technologies Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +@author Toni Boehnlein, Benjamin Lozes, Pal Andras Papp, Raphael S. Steiner +*/ + +#pragma once + +#include +#include +#include +#include +#include + +#include "../ConfigParser.hpp" +#include "osp/partitioning/partitioners/partitioning_ILP.hpp" +#include "osp/partitioning/partitioners/partitioning_ILP_replication.hpp" + +namespace osp { + +const std::set GetAvailablePartitionerNames() { return {"ILP", "ILP_dupl", "ILP_repl"}; } + + +template +ReturnStatus RunPartitioner(const ConfigParser &parser, + const boost::property_tree::ptree &algorithm, + const PartitioningProblem > &instance, + std::pair &cost) { + using Hgraph = Hypergraph; + + std::cout << "Running algorithm: " << algorithm.get_child("name").get_value() << std::endl; + + if (algorithm.get_child("name").get_value() == "ILP") { + HypergraphPartitioningILP partitioner; + Partitioning solution(instance); + + const unsigned timeLimit = parser.globalParams_.get_child("timeLimit").get_value(); + partitioner.SetTimeLimitSeconds(timeLimit); + ReturnStatus status = partitioner.ComputePartitioning(solution); + cost = {solution.ComputeConnectivityCost(), solution.ComputeCutNetCost()}; + return status; + + } else if (algorithm.get_child("name").get_value() == "ILP_dupl" + || algorithm.get_child("name").get_value() == "ILP_repl") { + HypergraphPartitioningILPWithReplication partitioner; + PartitioningWithReplication solution(instance); + + const unsigned timeLimit = parser.globalParams_.get_child("timeLimit").get_value(); + partitioner.SetTimeLimitSeconds(timeLimit); + if (algorithm.get_child("name").get_value() == "ILP_repl") { + partitioner.SetReplicationModel(HypergraphPartitioningILPWithReplication::ReplicationModelInIlp::GENERAL); + } + + ReturnStatus status = partitioner.ComputePartitioning(solution); + cost = {solution.ComputeConnectivityCost(), solution.ComputeCutNetCost()}; + return status; + + } else { + throw std::invalid_argument("Parameter error: Unknown algorithm.\n"); + } +} + +} // namespace osp \ No newline at end of file diff --git a/include/osp/auxiliary/io/hmetis_hypergraph_file_reader.hpp b/include/osp/auxiliary/io/hmetis_hypergraph_file_reader.hpp new file mode 100644 index 00000000..e3def15b --- /dev/null +++ b/include/osp/auxiliary/io/hmetis_hypergraph_file_reader.hpp @@ -0,0 +1,222 @@ +/* +Copyright 2024 Huawei Technologies Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +@author Toni Boehnlein, Christos Matzoros, Benjamin Lozes, Pal Andras Papp, Raphael S. Steiner +*/ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include "osp/auxiliary/io/filepath_checker.hpp" +#include "osp/partitioning/model/hypergraph.hpp" + +namespace osp { +namespace file_reader { + +// reads a Hypergraph from a file in hMetis format +template +bool ReadHypergraphMetisFormat(std::ifstream &infile, + Hypergraph &hgraph) { + std::string line; + + // Skip comments or empty lines (robustly) + while (std::getline(infile, line)) { + if (line.empty() || line[0] == '%') { + continue; + } + + // Null byte check + if (line.find('\0') != std::string::npos) { + std::cerr << "Error: Null byte detected in header line.\n"; + return false; + } + + if (line.size() > MAX_LINE_LENGTH) { + std::cerr << "Error: Line too long, possible malformed or malicious file.\n"; + return false; + } + break; // We found the actual header line + } + + if (infile.eof()) { + std::cerr << "Error: Unexpected end of file while reading header.\n"; + return false; + } + + IndexType numVertices, numHyperedges, weightCode = 0; + + std::istringstream headerStream(line); + if (!(headerStream >> numHyperedges >> numVertices) || numVertices <= 0 || numHyperedges <= 0) { + std::cerr << "Error: Invalid header.\n"; + return false; + } + + headerStream >> weightCode; + bool hasVertexWeights = false, hasHyperedgeWeights = false; + + switch(weightCode) { + case 0: + break; + case 1: + hasHyperedgeWeights = true; + break; + case 10: + hasVertexWeights = true; + break; + case 11: + hasHyperedgeWeights = true; + hasVertexWeights = true; + break; + default: + std::cerr << "Error: Invalid weight code in header (must be 0, 1, 10 or 11).\n"; + return false; + } + + hgraph.Reset(numVertices, 0); + + IndexType edgesRead = 0; + while (edgesRead < numHyperedges && std::getline(infile, line)) { + if (line.empty() || line[0] == '%') { + continue; + } + if (line.size() > MAX_LINE_LENGTH) { + std::cerr << "Error: Line too long.\n"; + return false; + } + + std::istringstream edgeStream(line); + + CommwType weight = 1; + if (hasHyperedgeWeights && !(edgeStream >> weight)) { + std::cerr << "Error: Malformed hyperedge row (weight).\n"; + return false; + } + + std::vector verticesInHyperedge; + IndexType vertex; + while (edgeStream >> vertex) { + --vertex; // Convert to 0-based + if (vertex < 0 || vertex >= numVertices) { + std::cerr << "Error: Malformed hyperedge row (vertex entry).\n"; + return false; + } + verticesInHyperedge.push_back(vertex); + } + + if (verticesInHyperedge.empty()) { + std::cerr << "Error: Empty hyperedge in file.\n"; + return false; + } + + std::sort(verticesInHyperedge.begin(), verticesInHyperedge.end()); + for (unsigned index = 0; index < verticesInHyperedge.size() - 1; ++index) { + if (verticesInHyperedge[index] == verticesInHyperedge[index + 1]) { + std::cerr << "Error: Malformed hyperedge row (same vertex appears multiple times).\n"; + return false; + } + } + + hgraph.AddHyperedge(verticesInHyperedge, weight); + ++edgesRead; + } + + if (edgesRead != numHyperedges) { + std::cerr << "Error: Incomplete hyperedge entries.\n"; + return false; + } + + if (hasVertexWeights) { + IndexType vertexWeightsRead = 0; + while (vertexWeightsRead < numVertices && std::getline(infile, line)) { + if (line.empty() || line[0] == '%') { + continue; + } + if (line.size() > MAX_LINE_LENGTH) { + std::cerr << "Error: Line too long.\n"; + return false; + } + + std::istringstream weightStream(line); + WorkwType weight = 1; + if (!(weightStream >> weight)) { + std::cerr << "Error: Malformed vertex weight row.\n"; + return false; + } + + hgraph.SetVertexWorkWeight(vertexWeightsRead, weight); + + ++vertexWeightsRead; + } + + if (vertexWeightsRead != numVertices) { + std::cerr << "Error: Incomplete vertex weight lines.\n"; + return false; + } + + } + + while (std::getline(infile, line)) { + if (!line.empty() && line[0] != '%') { + std::cerr << "Error: Extra data after hypergraph content.\n"; + return false; + } + } + + return true; +} + +template +bool ReadHypergraphMetisFormat(const std::string &filename, + Hypergraph &hgraph) { + // Ensure the file is .hmetis format + if (std::filesystem::path(filename).extension() != ".hmetis") { + std::cerr << "Error: Only .hmetis files are accepted.\n"; + return false; + } + + if (!IsPathSafe(filename)) { + std::cerr << "Error: Unsafe file path (potential traversal attack).\n"; + return false; + } + + if (std::filesystem::is_symlink(filename)) { + std::cerr << "Error: Symbolic links are not allowed.\n"; + return false; + } + + if (!std::filesystem::is_regular_file(filename)) { + std::cerr << "Error: Input is not a regular file.\n"; + return false; + } + + std::ifstream infile(filename); + if (!infile.is_open()) { + std::cerr << "Error: Failed to open file.\n"; + return false; + } + + return ReadHypergraphMetisFormat(infile, hgraph); +} + +} // namespace file_reader + +} // namespace osp diff --git a/include/osp/auxiliary/io/mtx_graph_file_reader.hpp b/include/osp/auxiliary/io/mtx_graph_file_reader.hpp index 63b20519..d565598b 100644 --- a/include/osp/auxiliary/io/mtx_graph_file_reader.hpp +++ b/include/osp/auxiliary/io/mtx_graph_file_reader.hpp @@ -102,6 +102,8 @@ bool ReadComputationalDagMartixMarketFormat(std::ifstream &infile, GraphT &graph return false; } + std::cout << "row: " << row << " col: " << col << " val: " << val << std::endl; + row -= 1; col -= 1; // Convert to 0-based @@ -152,7 +154,7 @@ bool ReadComputationalDagMartixMarketFormat(std::ifstream &infile, GraphT &graph template bool ReadComputationalDagMartixMarketFormat(const std::string &filename, GraphT &graph) { // Ensure the file is .mtx format - if (std::filesystem::path(filename).extension() != ".mtx") { + if (std::filesystem::path(filename).extension() != ".mtx" && std::filesystem::path(filename).extension() != ".mtx2") { std::cerr << "Error: Only .mtx files are accepted.\n"; return false; } diff --git a/include/osp/auxiliary/io/mtx_hypergraph_file_reader.hpp b/include/osp/auxiliary/io/mtx_hypergraph_file_reader.hpp index 42acbd8d..6e155b1c 100644 --- a/include/osp/auxiliary/io/mtx_hypergraph_file_reader.hpp +++ b/include/osp/auxiliary/io/mtx_hypergraph_file_reader.hpp @@ -32,9 +32,15 @@ limitations under the License. namespace osp { namespace file_reader { -// reads a matrix into Hypergraph format, where nonzeros are vertices, and rows/columns are hyperedges +enum class MatrixToHypergraphFormat { FINE_GRAINED, ROW_NET }; + +// reads a matrix into Hypergraph format, covering two different formats: +// - fine-grained: nonzeros are vertices, and rows/columns are hyperedges +// - row-net: columns are vertices, rows are hyperedges template -bool ReadHypergraphMartixMarketFormat(std::ifstream &infile, Hypergraph &hgraph) { +bool ReadHypergraphMartixMarketFormat(std::ifstream &infile, + Hypergraph &hgraph, + MatrixToHypergraphFormat format = MatrixToHypergraphFormat::FINE_GRAINED) { std::string line; // Skip comments or empty lines (robustly) @@ -69,16 +75,19 @@ bool ReadHypergraphMartixMarketFormat(std::ifstream &infile, Hypergraph(nEntries); + const IndexType numNodes = (format == MatrixToHypergraphFormat::FINE_GRAINED) ? static_cast(nEntries) + : static_cast(mCol); hgraph.Reset(numNodes, 0); - for (IndexType node = 0; node < numNodes; ++node) { - hgraph.SetVertexWorkWeight(node, static_cast(1)); - hgraph.SetVertexMemoryWeight(node, static_cast(1)); - } std::vector> rowHyperedges(static_cast(mRow)); - std::vector> columnHyperedges(static_cast(mCol)); + std::vector> columnHyperedges; + std::vector nrNonZerosInColumn; + if (format == MatrixToHypergraphFormat::FINE_GRAINED) { + columnHyperedges.resize(static_cast(mCol)); + } else { + nrNonZerosInColumn.resize(static_cast(mCol), 0); + } int entriesRead = 0; while (entriesRead < nEntries && std::getline(infile, line)) { @@ -92,9 +101,8 @@ bool ReadHypergraphMartixMarketFormat(std::ifstream &infile, Hypergraph> row >> col >> val)) { + if (!(entryStream >> row >> col)) { std::cerr << "Error: Malformed matrix entry.\n"; return false; } @@ -107,13 +115,19 @@ bool ReadHypergraphMartixMarketFormat(std::ifstream &infile, Hypergraph(row) >= numNodes || static_cast(col) >= numNodes) { + IndexType rowFormatLimit = (format == MatrixToHypergraphFormat::FINE_GRAINED) ? numNodes : static_cast(mRow); + if (static_cast(row) >= rowFormatLimit || static_cast(col) >= numNodes) { std::cerr << "Error: Index exceeds vertex type limit.\n"; return false; } - rowHyperedges[static_cast(row)].push_back(static_cast(entriesRead)); - columnHyperedges[static_cast(col)].push_back(static_cast(entriesRead)); + if (format == MatrixToHypergraphFormat::FINE_GRAINED) { + rowHyperedges[static_cast(row)].push_back(static_cast(entriesRead)); + columnHyperedges[static_cast(col)].push_back(static_cast(entriesRead)); + } else { + rowHyperedges[static_cast(row)].push_back(static_cast(col)); + ++nrNonZerosInColumn[static_cast(col)]; + } ++entriesRead; } @@ -136,9 +150,20 @@ bool ReadHypergraphMartixMarketFormat(std::ifstream &infile, Hypergraph(mCol); ++col) { - if (!columnHyperedges[col].empty()) { - hgraph.AddHyperedge(columnHyperedges[col]); + if (format == MatrixToHypergraphFormat::FINE_GRAINED) { + for (IndexType col = 0; col < static_cast(mCol); ++col) { + if (!columnHyperedges[col].empty()) { + hgraph.AddHyperedge(columnHyperedges[col]); + } + } + for (IndexType node = 0; node < numNodes; ++node) { + hgraph.SetVertexWorkWeight(node, static_cast(1)); + hgraph.SetVertexMemoryWeight(node, static_cast(1)); + } + } else { + for (IndexType node = 0; node < numNodes; ++node) { + hgraph.SetVertexWorkWeight(node, nrNonZerosInColumn[node]); + hgraph.SetVertexMemoryWeight(node, static_cast(1)); } } @@ -146,9 +171,11 @@ bool ReadHypergraphMartixMarketFormat(std::ifstream &infile, Hypergraph -bool ReadHypergraphMartixMarketFormat(const std::string &filename, Hypergraph &hgraph) { +bool ReadHypergraphMartixMarketFormat(const std::string &filename, + Hypergraph &hgraph, + MatrixToHypergraphFormat format = MatrixToHypergraphFormat::FINE_GRAINED) { // Ensure the file is .mtx format - if (std::filesystem::path(filename).extension() != ".mtx") { + if (std::filesystem::path(filename).extension() != ".mtx" && std::filesystem::path(filename).extension() != ".mtx2") { std::cerr << "Error: Only .mtx files are accepted.\n"; return false; } @@ -174,7 +201,7 @@ bool ReadHypergraphMartixMarketFormat(const std::string &filename, Hypergraph::SatisfiesConstraints() const { if (nodeFirstAvailableOnProc[node][fromProc] > val) { // std::cout << "Not a valid schedule: node " << node << " not yet available for sending from processor " - // << from_proc << " in superstep "<< val <<"." << std::endl; + // << fromProc << " in superstep "<< val <<"." << std::endl; return false; } } @@ -341,7 +341,7 @@ void BspScheduleRecomp::CleanSchedule() for (auto const &[key, val] : commSchedule_) { arrivesAt[std::get<0>(key)][std::get<2>(key)].insert(val); } - + // - computation steps for (const auto &node : instance_->GetComputationalDag().Vertices()) { for (unsigned index = 0; index < nodeToProcessorAndSupertepAssignment_[node].size(); ) { @@ -386,7 +386,7 @@ void BspScheduleRecomp::CleanSchedule() usedAt[std::get<0>(key)][std::get<1>(key)].insert(val); } - // - computation steps + // - computation steps for (const auto &node : instance_->GetComputationalDag().Vertices()) { for (unsigned index = 0; index < nodeToProcessorAndSupertepAssignment_[node].size(); ) { const auto &procAndStep = nodeToProcessorAndSupertepAssignment_[node][index]; diff --git a/include/osp/bsp/scheduler/GreedySchedulers/GreedyRecomputer.hpp b/include/osp/bsp/scheduler/GreedySchedulers/GreedyRecomputer.hpp index 1feeecf7..f61da61b 100644 --- a/include/osp/bsp/scheduler/GreedySchedulers/GreedyRecomputer.hpp +++ b/include/osp/bsp/scheduler/GreedySchedulers/GreedyRecomputer.hpp @@ -76,7 +76,7 @@ class GreedyRecomputer { ReturnStatus ComputeRecompScheduleBasic(BspScheduleCS &initialSchedule, BspScheduleRecomp &recompSchedule); - ReturnStatus ComputeRecompScheduleAdvanced(BspScheduleCS &initialSchedule, BspScheduleRecomp &recompSchedule); + ReturnStatus ComputeRecompScheduleAdvanced(BspScheduleCS &initialSchedule, BspScheduleRecomp &recompSchedule, unsigned methodMask = 7); }; template @@ -89,25 +89,37 @@ ReturnStatus GreedyRecomputer::ComputeRecompScheduleBasic(BspScheduleCS< } template -ReturnStatus GreedyRecomputer::ComputeRecompScheduleAdvanced(BspScheduleCS &initialSchedule, BspScheduleRecomp &recompSchedule) +ReturnStatus GreedyRecomputer::ComputeRecompScheduleAdvanced(BspScheduleCS &initialSchedule, BspScheduleRecomp &recompSchedule, unsigned methodMask) { recompSchedule = BspScheduleRecomp(initialSchedule); bool keepsImproving = true; while (keepsImproving) { - keepsImproving = BatchRemoveSteps(recompSchedule); // no need for greedyImprove if we use this more general one - recompSchedule.MergeSupersteps(); + keepsImproving = false; - keepsImproving = MergeEntireSupersteps(recompSchedule) || keepsImproving; - recompSchedule.CleanSchedule(); - recompSchedule.MergeSupersteps(); + if ((methodMask & (1U << 0)) == 0) { + keepsImproving = GreedyImprove(recompSchedule) || keepsImproving; + recompSchedule.MergeSupersteps(); + } else { + keepsImproving = BatchRemoveSteps(recompSchedule) || keepsImproving; // no need for greedyImprove if we use this more general one + recompSchedule.MergeSupersteps(); + } + + if ((methodMask & (1U << 1)) > 0) { + keepsImproving = MergeEntireSupersteps(recompSchedule) || keepsImproving; + recompSchedule.CleanSchedule(); + recompSchedule.MergeSupersteps(); + } - keepsImproving = RecomputeEntireSupersteps(recompSchedule) || keepsImproving; - recompSchedule.MergeSupersteps(); + if ((methodMask & (1U << 2)) > 0) { + keepsImproving = RecomputeEntireSupersteps(recompSchedule) || keepsImproving; + recompSchedule.MergeSupersteps(); + recompSchedule.CleanSchedule(); + } // add further methods, if desired } - + return ReturnStatus::OSP_SUCCESS; } @@ -182,7 +194,7 @@ bool GreedyRecomputer::GreedyImprove(BspScheduleRecomp &schedule CostType increase = workCost_[toProc][compStep] + G.VertexWorkWeight(node) > maxWork_[compStep] ? workCost_[toProc][compStep] + G.VertexWorkWeight(node) - maxWork_[compStep] : 0; - + if (increase < smallestIncrease) { bestStep = compStep; smallestIncrease = increase; @@ -242,7 +254,7 @@ bool GreedyRecomputer::MergeEntireSupersteps(BspScheduleRecomp & std::vector stepRemoved(schedule.NumberOfSupersteps(), false); const GraphT &G = schedule.GetInstance().GetComputationalDag(); - + unsigned previousStep = 0; for (unsigned step = 0; step < schedule.NumberOfSupersteps() - 1; ++step) { if (stepRemoved[step]) { @@ -250,7 +262,7 @@ bool GreedyRecomputer::MergeEntireSupersteps(BspScheduleRecomp & } for (unsigned nextStep = step + 1; nextStep < schedule.NumberOfSupersteps(); ++nextStep) { - + // TRY TO MERGE step AND nextStep std::set newCommStepsBefore, newCommStepsAfter; std::set > newWorkSteps; @@ -267,7 +279,7 @@ bool GreedyRecomputer::MergeEntireSupersteps(BspScheduleRecomp & newCommStepsAfter.insert(entry); continue; } - + if (step > 0 && firstPresent_[node][fromProc] <= previousStep) { newCommStepsBefore.insert(entry); } else { @@ -290,7 +302,7 @@ bool GreedyRecomputer::MergeEntireSupersteps(BspScheduleRecomp & if (firstPresent_[pred][proc] <= step) { continue; } - + unsigned sendFromProcBefore = std::numeric_limits::max(); for (unsigned procOffset = 0; procOffset < schedule.GetInstance().NumberOfProcessors(); ++procOffset) { unsigned fromProc = (proc + procOffset) % schedule.GetInstance().NumberOfProcessors(); @@ -320,9 +332,9 @@ bool GreedyRecomputer::MergeEntireSupersteps(BspScheduleRecomp & break; } - // EVALUATE COST + // EVALUATE COST int costChange = 0; - + // work cost in merged step std::vector newWorkCost(schedule.GetInstance().NumberOfProcessors()); for (unsigned proc = 0; proc < schedule.GetInstance().NumberOfProcessors(); ++proc) { @@ -337,7 +349,7 @@ bool GreedyRecomputer::MergeEntireSupersteps(BspScheduleRecomp & for (unsigned proc = 0; proc < schedule.GetInstance().NumberOfProcessors(); ++proc) { newMax = std::max(newMax, newWorkCost[proc]); } - + costChange += static_cast(newMax) - static_cast(maxWork_[step] + maxWork_[nextStep]); // comm cost before merged step @@ -348,12 +360,12 @@ bool GreedyRecomputer::MergeEntireSupersteps(BspScheduleRecomp & newRecCost[proc] = recCost_[proc][previousStep]; } for (const KeyTriple &newComm : newCommStepsBefore) { - CostType commCost = G.VertexCommWeight(std::get<0>(newComm)) * + CostType commCost = G.VertexCommWeight(std::get<0>(newComm)) * schedule.GetInstance().GetArchitecture().CommunicationCosts(std::get<1>(newComm), std::get<2>(newComm)); newSendCost[std::get<1>(newComm)] += commCost; newRecCost[std::get<2>(newComm)] += commCost; } - + newMax = 0; for (unsigned proc = 0; proc < schedule.GetInstance().NumberOfProcessors(); ++proc) { newMax = std::max(newMax, newSendCost[proc]); @@ -378,7 +390,7 @@ bool GreedyRecomputer::MergeEntireSupersteps(BspScheduleRecomp & newSendCost[std::get<1>(newComm)] += commCost; newRecCost[std::get<2>(newComm)] += commCost; } - + newMax = 0; for (unsigned proc = 0; proc < schedule.GetInstance().NumberOfProcessors(); ++proc) { newMax = std::max(newMax, newSendCost[proc]); @@ -424,11 +436,11 @@ bool GreedyRecomputer::MergeEntireSupersteps(BspScheduleRecomp & for (const KeyTriple &entry : commSteps_[step]) { neededOnProc_[std::get<0>(entry)][std::get<1>(entry)].erase(neededOnProc_[std::get<0>(entry)][std::get<1>(entry)].lower_bound(step)); } - + for (const KeyTriple &entry : commSteps_[nextStep]) { neededOnProc_[std::get<0>(entry)][std::get<1>(entry)].erase(neededOnProc_[std::get<0>(entry)][std::get<1>(entry)].lower_bound(nextStep)); } - + commSteps_[step].clear(); for (unsigned proc = 0; proc < schedule.GetInstance().NumberOfProcessors(); ++proc) { sendCost_[proc][step] = 0; @@ -447,7 +459,7 @@ bool GreedyRecomputer::MergeEntireSupersteps(BspScheduleRecomp & } maxComm_[nextStep] = 0; - + maxComm_[step] = 0; for (unsigned proc = 0; proc < schedule.GetInstance().NumberOfProcessors(); ++proc) { maxComm_[step] = std::max(maxComm_[step], sendCost_[proc][step]); @@ -459,7 +471,7 @@ bool GreedyRecomputer::MergeEntireSupersteps(BspScheduleRecomp & for (const KeyTriple &newComm : newCommStepsBefore) { AddCommStep(schedule, newComm, previousStep); } - + for (unsigned proc = 0; proc < schedule.GetInstance().NumberOfProcessors(); ++proc) { maxComm_[previousStep] = std::max(maxComm_[previousStep], sendCost_[proc][previousStep]); maxComm_[previousStep] = std::max(maxComm_[previousStep], recCost_[proc][previousStep]); @@ -499,7 +511,7 @@ bool GreedyRecomputer::RecomputeEntireSupersteps(BspScheduleRecomp(entry), std::get<2>(entry))].emplace_back(std::get<1>(entry), step); } } - + for (unsigned step = 0; step < schedule.NumberOfSupersteps(); ++step) { for (unsigned fromProc = 0; fromProc < schedule.GetInstance().NumberOfProcessors(); ++fromProc) { for (unsigned toProc = 0; toProc < schedule.GetInstance().NumberOfProcessors(); ++toProc) { @@ -533,7 +545,7 @@ bool GreedyRecomputer::RecomputeEntireSupersteps(BspScheduleRecomp checkIfDisposable; for (const VertexIdx node : mustReplicate) { if (internalOutDegree.at(node) == 0) { @@ -602,14 +614,14 @@ bool GreedyRecomputer::RecomputeEntireSupersteps(BspScheduleRecomp(newMax) - static_cast(maxWork_[step]); // comm cost before merged step @@ -626,7 +638,7 @@ bool GreedyRecomputer::RecomputeEntireSupersteps(BspScheduleRecomp(newComm)] += commCost; newRecCost += commCost; } - + newMax = std::max(maxComm_[step - 1], newRecCost); for (unsigned proc = 0; proc < schedule.GetInstance().NumberOfProcessors(); ++proc) { newMax = std::max(newMax, newSendCost[proc]); @@ -658,7 +670,7 @@ bool GreedyRecomputer::RecomputeEntireSupersteps(BspScheduleRecomp newSendCost(schedule.GetInstance().NumberOfProcessors()); CostType newRecCost = recCost_[toProc][stepChanged] - changingStep.second; for (unsigned proc = 0; proc < schedule.GetInstance().NumberOfProcessors(); ++proc) { @@ -667,7 +679,7 @@ bool GreedyRecomputer::RecomputeEntireSupersteps(BspScheduleRecomp::RecomputeEntireSupersteps(BspScheduleRecomp::RecomputeEntireSupersteps(BspScheduleRecomp::BatchRemoveSteps(BspScheduleRecomp &sched } } } - + for (unsigned step = 0; step < schedule.NumberOfSupersteps(); ++step) { - + bool canReduce = (maxComm_[step] > 0); while (canReduce) { @@ -790,10 +802,12 @@ bool GreedyRecomputer::BatchRemoveSteps(BspScheduleRecomp &sched CostType workIncrease = 0; CostType commDecrease = std::numeric_limits::max(); + std::map, unsigned> newFirstComputable, newFirstNeeded; + for (unsigned proc = 0; proc < schedule.GetInstance().NumberOfProcessors(); ++proc) { for (unsigned sendOrRec = 0; sendOrRec < 2; ++sendOrRec) { - - std::set *currentCommSteps; + + std::set *currentCommSteps; if (sendOrRec == 0) { if (!sendSaturated[proc]) { continue; @@ -820,7 +834,18 @@ bool GreedyRecomputer::BatchRemoveSteps(BspScheduleRecomp &sched continue; } - for (unsigned compStep = firstComputable[node][toProc]; compStep <= *neededOnProc_[node][toProc].begin(); ++compStep) { + unsigned lowerBound = firstComputable[node][toProc]; + auto newFirstItr = newFirstComputable.find({node, toProc}); + if (newFirstItr != newFirstComputable.end()) { + lowerBound = std::max(lowerBound, newFirstItr->second); + } + unsigned upperBound = *neededOnProc_[node][toProc].begin(); + newFirstItr = newFirstNeeded.find({node, toProc}); + if (newFirstItr != newFirstNeeded.end()) { + upperBound = std::min(upperBound, newFirstItr->second); + } + + for (unsigned compStep = lowerBound; compStep <= upperBound; ++compStep) { auto itr = workIncreased.find(std::make_pair(toProc, compStep)); CostType assignedExtra = (itr != workIncreased.end()) ? itr->second : 0; CostType increase = 0; @@ -857,6 +882,23 @@ bool GreedyRecomputer::BatchRemoveSteps(BspScheduleRecomp &sched * schedule.GetInstance().GetArchitecture().CommunicationCosts(fromProc, toProc); commDecrease = std::min(commDecrease, commCost); + for (const VertexIdx &succ : G.Children(node)) { + auto newFirstItr = newFirstComputable.find({succ, toProc}); + if (newFirstItr != newFirstComputable.end()) { + newFirstItr->second = std::max(newFirstItr->second, bestStepTarget); + } else { + newFirstComputable[{succ, toProc}] = bestStepTarget; + } + } + for (const VertexIdx &pred : G.Parents(node)) { + auto newFirstItr = newFirstNeeded.find({pred, toProc}); + if (newFirstItr != newFirstNeeded.end()) { + newFirstItr->second = std::min(newFirstItr->second, bestStepTarget); + } else { + newFirstNeeded[{pred, toProc}] = bestStepTarget; + } + } + } else { skipStep = true; } @@ -864,11 +906,11 @@ bool GreedyRecomputer::BatchRemoveSteps(BspScheduleRecomp &sched if (skipStep) { // weird edge case if all comm steps have weight 0 (can be removed?) break; - } + } } if (skipStep) { continue; - } + } if (maxComm_[step] > 0 && commSteps_[step].size() == removedCommSteps.size()) { commDecrease += schedule.GetInstance().GetArchitecture().SynchronisationCosts(); @@ -898,7 +940,7 @@ bool GreedyRecomputer::BatchRemoveSteps(BspScheduleRecomp &sched maxComm_[step] = std::max(maxComm_[step], sendCost_[proc][step]); maxComm_[step] = std::max(maxComm_[step], recCost_[proc][step]); } - + canReduce = true; improved = true; } @@ -936,17 +978,17 @@ void GreedyRecomputer::RefreshAuxData(const BspScheduleRecomp &s nodesPerProcAndStep_.clear(); nodesPerProcAndStep_.resize(P, std::vector >(S)); - + neededOnProc_.clear(); neededOnProc_.resize(N, std::vector >(P, {S})); - + maxWork_.clear(); maxComm_.clear(); maxWork_.resize(S, 0); maxComm_.resize(S, 0); commSteps_.clear(); - commSteps_.resize(S); + commSteps_.resize(S); for (VertexIdx node = 0; node < N; ++node) { for (const std::pair &procAndStep : schedule.Assignments(node)) { diff --git a/include/osp/partitioning/model/partitioning_problem.hpp b/include/osp/partitioning/model/partitioning_problem.hpp index 5e3b90cd..75728a4d 100644 --- a/include/osp/partitioning/model/partitioning_problem.hpp +++ b/include/osp/partitioning/model/partitioning_problem.hpp @@ -39,8 +39,8 @@ class PartitioningProblem { HypergraphT hgraph_; unsigned nrOfPartitions_; - WorkwType maxWorkWeightPerPartition_; - MemwType maxMemoryWeightPerPartition_; + WorkwType maxWorkWeightPerPartition_ = std::numeric_limits::max(); + MemwType maxMemoryWeightPerPartition_ = std::numeric_limits::max(); bool allowsReplication_ = false; diff --git a/include/osp/partitioning/partitioners/partitioning_ILP_base.hpp b/include/osp/partitioning/partitioners/partitioning_ILP_base.hpp index afc25af4..713fb2d4 100644 --- a/include/osp/partitioning/partitioners/partitioning_ILP_base.hpp +++ b/include/osp/partitioning/partitioners/partitioning_ILP_base.hpp @@ -55,20 +55,26 @@ class HypergraphPartitioningILPBase { template void HypergraphPartitioningILPBase::SolveIlp(Model &model) { - model.SetIntParam(COPT_INTPARAM_LOGTOCONSOLE, 0); + // model.SetIntParam(COPT_INTPARAM_LOGTOCONSOLE, 0); model.SetDblParam(COPT_DBLPARAM_TIMELIMIT, timeLimitSeconds_); model.SetIntParam(COPT_INTPARAM_THREADS, 128); - model.SetIntParam(COPT_INTPARAM_STRONGBRANCHING, 1); - model.SetIntParam(COPT_INTPARAM_LPMETHOD, 1); - model.SetIntParam(COPT_INTPARAM_ROUNDINGHEURLEVEL, 1); + // model.SetIntParam(COPT_INTPARAM_STRONGBRANCHING, 1); + // model.SetIntParam(COPT_INTPARAM_LPMETHOD, 1); + // model.SetIntParam(COPT_INTPARAM_ROUNDINGHEURLEVEL, 1); - model.SetIntParam(COPT_INTPARAM_SUBMIPHEURLEVEL, 1); - // model.SetIntParam(COPT_INTPARAM_PRESOLVE, 1); - // model.SetIntParam(COPT_INTPARAM_CUTLEVEL, 0); - model.SetIntParam(COPT_INTPARAM_TREECUTLEVEL, 2); - // model.SetIntParam(COPT_INTPARAM_DIVINGHEURLEVEL, 2); + model.SetIntParam(COPT_INTPARAM_PRESOLVE, 0); + model.SetIntParam(COPT_INTPARAM_LPMETHOD, 1); + model.SetIntParam(COPT_INTPARAM_CUTLEVEL, 1); + model.SetIntParam(COPT_INTPARAM_ROOTCUTLEVEL, 2); + model.SetIntParam(COPT_INTPARAM_TREECUTLEVEL, 1); + + // model.SetIntParam(COPT_INTPARAM_SUBMIPHEURLEVEL, 1); + // model.SetIntParam(COPT_INTPARAM_PRESOLVE, 1); + // model.SetIntParam(COPT_INTPARAM_CUTLEVEL, 0); + // model.SetIntParam(COPT_INTPARAM_TREECUTLEVEL, 2); + // model.SetIntParam(COPT_INTPARAM_DIVINGHEURLEVEL, 2); model.Solve(); } diff --git a/include/osp/partitioning/partitioners/partitioning_ILP_replication.hpp b/include/osp/partitioning/partitioners/partitioning_ILP_replication.hpp index ac9d04da..6ac32c62 100644 --- a/include/osp/partitioning/partitioners/partitioning_ILP_replication.hpp +++ b/include/osp/partitioning/partitioners/partitioning_ILP_replication.hpp @@ -89,6 +89,19 @@ void HypergraphPartitioningILPWithReplication::SetupExtraVariablesC const IndexType numberOfParts = instance.GetNumberOfPartitions(); const IndexType numberOfVertices = instance.GetHypergraph().NumVertices(); + IndexType heaviestNode = 0; + auto maxWeightFound = -1.0; + + for (IndexType node = 0; node < numberOfVertices; node++) { + auto w = instance.GetHypergraph().GetVertexWorkWeight(node); + if (w > maxWeightFound) { + maxWeightFound = w; + heaviestNode = node; + } + } + // symmetry breaking + model.AddConstr(this->nodeInPartition_[heaviestNode][0] == 1); + if (replicationModel_ == ReplicationModelInIlp::GENERAL) { // create variables for each pin+partition combination std::map, IndexType> pinIdMap; diff --git a/include/osp/pebbling/pebblers/pebblingILP/MultiProcessorPebbling.hpp b/include/osp/pebbling/pebblers/pebblingILP/MultiProcessorPebbling.hpp index b6b7948b..72ceabab 100644 --- a/include/osp/pebbling/pebblers/pebblingILP/MultiProcessorPebbling.hpp +++ b/include/osp/pebbling/pebblers/pebblingILP/MultiProcessorPebbling.hpp @@ -239,9 +239,9 @@ class MultiProcessorPebbling : public Scheduler { template void MultiProcessorPebbling::SolveIlp() { - if (!verbose_) { - model_.SetIntParam(COPT_INTPARAM_LOGTOCONSOLE, 0); - } + // if (!verbose_) { + // model_.SetIntParam(COPT_INTPARAM_LOGTOCONSOLE, 0); + // } model_.SetDblParam(COPT_DBLPARAM_TIMELIMIT, timeLimitSeconds_); model_.SetIntParam(COPT_INTPARAM_THREADS, 128); diff --git a/include/osp/pebbling/pebblers/pebblingILP/partialILP/AcyclicPartitioningILP.hpp b/include/osp/pebbling/pebblers/pebblingILP/partialILP/AcyclicPartitioningILP.hpp index e2bd1268..7c3088eb 100644 --- a/include/osp/pebbling/pebblers/pebblingILP/partialILP/AcyclicPartitioningILP.hpp +++ b/include/osp/pebbling/pebblers/pebblingILP/partialILP/AcyclicPartitioningILP.hpp @@ -156,7 +156,7 @@ class AcyclicPartitioningILP { template void AcyclicPartitioningILP::SolveIlp() { - model_.SetIntParam(COPT_INTPARAM_LOGTOCONSOLE, 0); + // model_.SetIntParam(COPT_INTPARAM_LOGTOCONSOLE, 0); model_.SetDblParam(COPT_DBLPARAM_TIMELIMIT, timeLimitSeconds_); model_.SetIntParam(COPT_INTPARAM_THREADS, 128); diff --git a/tests/hypergraph_and_partition.cpp b/tests/hypergraph_and_partition.cpp index 9e42f9b5..cfedc2ca 100644 --- a/tests/hypergraph_and_partition.cpp +++ b/tests/hypergraph_and_partition.cpp @@ -24,6 +24,7 @@ limitations under the License. #include "osp/auxiliary/io/hdag_graph_file_reader.hpp" #include "osp/auxiliary/io/mtx_hypergraph_file_reader.hpp" +#include "osp/auxiliary/io/hmetis_hypergraph_file_reader.hpp" #include "osp/auxiliary/io/partitioning_file_writer.hpp" #include "osp/graph_implementations/adj_list_impl/computational_dag_vector_impl.hpp" #include "osp/partitioning/model/hypergraph_utility.hpp" @@ -59,6 +60,21 @@ BOOST_AUTO_TEST_CASE(HypergraphAndPartitionTest) { BOOST_CHECK_EQUAL(hgraph.NumVertices(), 27); BOOST_CHECK_EQUAL(hgraph.NumHyperedges(), 16); + // Matrix format, columns are vertices, rows are hyperedges + status = file_reader::ReadHypergraphMartixMarketFormat((cwd / "data/mtx_tests/ErdosRenyi_8_19_A.mtx").string(), hgraph, + file_reader::MatrixToHypergraphFormat::ROW_NET); + BOOST_CHECK(status); + BOOST_CHECK_EQUAL(hgraph.NumVertices(), 8); + BOOST_CHECK_EQUAL(hgraph.NumHyperedges(), 8); + + // Hmetis format for hypergraphs + status = file_reader::ReadHypergraphMetisFormat((cwd / "data/hmetis/example01.hmetis").string(), hgraph); + BOOST_CHECK(status); + BOOST_CHECK_EQUAL(hgraph.NumVertices(), 8); + BOOST_CHECK_EQUAL(hgraph.NumHyperedges(), 6); + BOOST_CHECK_EQUAL(hgraph.GetVertexWorkWeight(0), 1); + BOOST_CHECK_EQUAL(hgraph.GetHyperedgeWeight(0), 1); + // DAG format, all hyperedges have size 2 hgraph = ConvertFromCdagAsDag(dag); BOOST_CHECK_EQUAL(dag.NumVertices(), hgraph.NumVertices());