From 3ae70f2761a9c2448f1275853f99fab2625a566e Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Thu, 22 Aug 2019 16:20:50 -0400 Subject: [PATCH 1/4] Rewrote flat traversal, removed min/max calc for zero sided contexts --- cpp/perspective/src/cpp/context_zero.cpp | 3 +- cpp/perspective/src/cpp/flat_traversal.cpp | 61 +++++++++++-------- cpp/perspective/src/cpp/storage.cpp | 2 + .../src/include/perspective/multi_sort.h | 3 + 4 files changed, 44 insertions(+), 25 deletions(-) diff --git a/cpp/perspective/src/cpp/context_zero.cpp b/cpp/perspective/src/cpp/context_zero.cpp index 9ab7046c36..c86d76368e 100644 --- a/cpp/perspective/src/cpp/context_zero.cpp +++ b/cpp/perspective/src/cpp/context_zero.cpp @@ -55,7 +55,7 @@ t_ctx0::step_end() { } m_traversal->step_end(); - +#ifndef BUILD_WASM t_uindex ncols = m_config.get_num_columns(); std::vector rval(ncols); @@ -84,6 +84,7 @@ t_ctx0::step_end() { #endif m_minmax = rval; +#endif } // ASGGrid data interface diff --git a/cpp/perspective/src/cpp/flat_traversal.cpp b/cpp/perspective/src/cpp/flat_traversal.cpp index 06df11d395..42cbb71063 100644 --- a/cpp/perspective/src/cpp/flat_traversal.cpp +++ b/cpp/perspective/src/cpp/flat_traversal.cpp @@ -248,38 +248,51 @@ t_ftrav::step_end() { auto new_index = std::make_shared>(); new_index->reserve(new_size); - t_pkeyidx_map added; + t_uindex i = 0; + t_multisorter sorter(get_sort_orders(m_sortby)); + std::vector new_rows; - for (t_uindex idx = 0, loop_end = m_index->size(); idx < loop_end; ++idx) { - t_mselem& elem = (*m_index)[idx]; - if (!elem.m_deleted) { - new_index->push_back(elem); - added[elem.m_pkey] = idx; + for (t_pkmselem_map::const_iterator pkelem_iter = m_new_elems.begin(); + pkelem_iter != m_new_elems.end(); ++pkelem_iter) { + new_rows.push_back(pkelem_iter->second); + } + std::sort(new_rows.begin(), new_rows.end(), sorter); + for (auto it = new_rows.begin(); it != new_rows.end(); ++it) { + const t_mselem& elem = *it; + if (!m_pkeyidx.contains(elem.m_pkey)) { + while (i < m_index->size()) { + const t_mselem& new_elem = (*m_index)[i]; + if (new_elem.m_deleted) { + i++; + m_pkeyidx.erase(new_elem.m_pkey); + } else if (sorter(new_elem, elem)) { + m_pkeyidx[new_elem.m_pkey] = new_index->size(); + new_index->push_back(new_elem); + i++; + } else { + break; + } + } + } else { + i++; } + + m_pkeyidx[elem.m_pkey] = new_index->size(); + new_index->push_back(elem); } - - for (t_pkmselem_map::const_iterator pkelem_iter = m_new_elems.begin(); - pkelem_iter != m_new_elems.end(); ++pkelem_iter) { - - const t_mselem& elem = pkelem_iter->second; - t_pkeyidx_map::const_iterator iter = added.find(pkelem_iter->first); - if (iter == added.end()) { - new_index->push_back(elem); + + while (i < m_index->size()) { + const t_mselem& new_elem = (*m_index)[i++]; + if (new_elem.m_deleted) { + m_pkeyidx.erase(new_elem.m_pkey); } else { - (*new_index)[iter->second] = elem; + m_pkeyidx[new_elem.m_pkey] = new_index->size(); + new_index->push_back(new_elem); } } - std::swap(new_index, m_index); - t_multisorter sorter(get_sort_orders(m_sortby)); - std::sort(m_index->begin(), m_index->end(), sorter); - - m_pkeyidx.clear(); + std::swap(new_index, m_index); m_new_elems.clear(); - - for (t_index idx = 0, loop_end = m_index->size(); idx < loop_end; ++idx) { - m_pkeyidx[(*m_index)[idx].m_pkey] = idx; - } } void diff --git a/cpp/perspective/src/cpp/storage.cpp b/cpp/perspective/src/cpp/storage.cpp index 917cfdd246..22e6c86f14 100644 --- a/cpp/perspective/src/cpp/storage.cpp +++ b/cpp/perspective/src/cpp/storage.cpp @@ -519,7 +519,9 @@ void t_lstore::clear() { PSP_TRACE_SENTINEL(); PSP_VERBOSE_ASSERT(m_init, "touching uninited object"); +#ifndef BUILD_WASM memset(m_base, 0, size_t(capacity())); +#endif { t_unlock_store tmp(this); m_size = 0; diff --git a/cpp/perspective/src/include/perspective/multi_sort.h b/cpp/perspective/src/include/perspective/multi_sort.h index 6ee3d389cb..08e1de68b2 100644 --- a/cpp/perspective/src/include/perspective/multi_sort.h +++ b/cpp/perspective/src/include/perspective/multi_sort.h @@ -96,6 +96,8 @@ cmp_mselem(const t_mselem& a, const t_mselem& b, const std::vector& const t_tscalar& second = b.m_row[idx]; t_sorttype order = sort_order[idx]; + +#ifndef BUILD_WASM t_nancmp nancmp = nan_compare(order, first, second); if (first.is_floating_point() && nancmp.m_active) { @@ -110,6 +112,7 @@ cmp_mselem(const t_mselem& a, const t_mselem& b, const std::vector& default: { continue; } break; } } +#endif if (first == second) continue; From f327a40172b036d0336db44cc6bd76b5c86aa79c Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Thu, 22 Aug 2019 16:55:16 -0400 Subject: [PATCH 2/4] Lint fix --- cpp/perspective/src/cpp/flat_traversal.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cpp/perspective/src/cpp/flat_traversal.cpp b/cpp/perspective/src/cpp/flat_traversal.cpp index 42cbb71063..93886c4780 100644 --- a/cpp/perspective/src/cpp/flat_traversal.cpp +++ b/cpp/perspective/src/cpp/flat_traversal.cpp @@ -253,7 +253,7 @@ t_ftrav::step_end() { std::vector new_rows; for (t_pkmselem_map::const_iterator pkelem_iter = m_new_elems.begin(); - pkelem_iter != m_new_elems.end(); ++pkelem_iter) { + pkelem_iter != m_new_elems.end(); ++pkelem_iter) { new_rows.push_back(pkelem_iter->second); } std::sort(new_rows.begin(), new_rows.end(), sorter); @@ -276,11 +276,11 @@ t_ftrav::step_end() { } else { i++; } - + m_pkeyidx[elem.m_pkey] = new_index->size(); new_index->push_back(elem); } - + while (i < m_index->size()) { const t_mselem& new_elem = (*m_index)[i++]; if (new_elem.m_deleted) { @@ -291,7 +291,7 @@ t_ftrav::step_end() { } } - std::swap(new_index, m_index); + std::swap(new_index, m_index); m_new_elems.clear(); } From ffa29a70224d55f3ee9ef7038781b7ce62622da9 Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Sun, 25 Aug 2019 18:10:31 -0400 Subject: [PATCH 3/4] Fixed misnamed define --- cpp/perspective/src/cpp/context_zero.cpp | 2 +- cpp/perspective/src/cpp/storage.cpp | 2 +- cpp/perspective/src/include/perspective/multi_sort.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp/perspective/src/cpp/context_zero.cpp b/cpp/perspective/src/cpp/context_zero.cpp index c86d76368e..fa41f56e2e 100644 --- a/cpp/perspective/src/cpp/context_zero.cpp +++ b/cpp/perspective/src/cpp/context_zero.cpp @@ -55,7 +55,7 @@ t_ctx0::step_end() { } m_traversal->step_end(); -#ifndef BUILD_WASM +#ifndef PSP_ENABLE_WASM t_uindex ncols = m_config.get_num_columns(); std::vector rval(ncols); diff --git a/cpp/perspective/src/cpp/storage.cpp b/cpp/perspective/src/cpp/storage.cpp index 22e6c86f14..4e15ac3809 100644 --- a/cpp/perspective/src/cpp/storage.cpp +++ b/cpp/perspective/src/cpp/storage.cpp @@ -519,7 +519,7 @@ void t_lstore::clear() { PSP_TRACE_SENTINEL(); PSP_VERBOSE_ASSERT(m_init, "touching uninited object"); -#ifndef BUILD_WASM +#ifndef PSP_ENABLE_WASM memset(m_base, 0, size_t(capacity())); #endif { diff --git a/cpp/perspective/src/include/perspective/multi_sort.h b/cpp/perspective/src/include/perspective/multi_sort.h index 08e1de68b2..4d20e3c72b 100644 --- a/cpp/perspective/src/include/perspective/multi_sort.h +++ b/cpp/perspective/src/include/perspective/multi_sort.h @@ -97,7 +97,7 @@ cmp_mselem(const t_mselem& a, const t_mselem& b, const std::vector& t_sorttype order = sort_order[idx]; -#ifndef BUILD_WASM +#ifndef PSP_ENABLE_WASM t_nancmp nancmp = nan_compare(order, first, second); if (first.is_floating_point() && nancmp.m_active) { From 1cd218925861a1fb92b8d25a27dc97f1a99fa8ad Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Mon, 26 Aug 2019 04:04:38 -0400 Subject: [PATCH 4/4] Fixed sort/update regression --- cpp/perspective/src/cpp/flat_traversal.cpp | 49 +++++++++---------- cpp/perspective/src/cpp/multi_sort.cpp | 16 ++++-- .../src/include/perspective/multi_sort.h | 1 + 3 files changed, 37 insertions(+), 29 deletions(-) diff --git a/cpp/perspective/src/cpp/flat_traversal.cpp b/cpp/perspective/src/cpp/flat_traversal.cpp index 93886c4780..06a8f601d6 100644 --- a/cpp/perspective/src/cpp/flat_traversal.cpp +++ b/cpp/perspective/src/cpp/flat_traversal.cpp @@ -258,36 +258,34 @@ t_ftrav::step_end() { } std::sort(new_rows.begin(), new_rows.end(), sorter); for (auto it = new_rows.begin(); it != new_rows.end(); ++it) { - const t_mselem& elem = *it; - if (!m_pkeyidx.contains(elem.m_pkey)) { - while (i < m_index->size()) { - const t_mselem& new_elem = (*m_index)[i]; - if (new_elem.m_deleted) { - i++; - m_pkeyidx.erase(new_elem.m_pkey); - } else if (sorter(new_elem, elem)) { - m_pkeyidx[new_elem.m_pkey] = new_index->size(); - new_index->push_back(new_elem); - i++; - } else { - break; - } + const t_mselem& new_elem = *it; + while (i < m_index->size()) { + const t_mselem& old_elem = (*m_index)[i]; + if (old_elem.m_deleted) { + i++; + m_pkeyidx.erase(old_elem.m_pkey); + } else if (old_elem.m_updated) { + i++; + } else if (sorter(old_elem, new_elem)) { + m_pkeyidx[old_elem.m_pkey] = new_index->size(); + new_index->push_back(old_elem); + i++; + } else { + break; } - } else { - i++; } - m_pkeyidx[elem.m_pkey] = new_index->size(); - new_index->push_back(elem); + m_pkeyidx[new_elem.m_pkey] = new_index->size(); + new_index->push_back(new_elem); } while (i < m_index->size()) { - const t_mselem& new_elem = (*m_index)[i++]; - if (new_elem.m_deleted) { - m_pkeyidx.erase(new_elem.m_pkey); - } else { - m_pkeyidx[new_elem.m_pkey] = new_index->size(); - new_index->push_back(new_elem); + const t_mselem& old_elem = (*m_index)[i++]; + if (old_elem.m_deleted) { + m_pkeyidx.erase(old_elem.m_pkey); + } else if (!old_elem.m_updated) { + m_pkeyidx[old_elem.m_pkey] = new_index->size(); + new_index->push_back(old_elem); } } @@ -316,7 +314,8 @@ t_ftrav::update_row( } t_mselem mselem; fill_sort_elem(state, config, pkey, mselem); - (*m_index)[pkiter->second] = mselem; + (*m_index)[pkiter->second].m_updated = true; + m_new_elems[pkey] = mselem; } void diff --git a/cpp/perspective/src/cpp/multi_sort.cpp b/cpp/perspective/src/cpp/multi_sort.cpp index da26112e31..550078a028 100644 --- a/cpp/perspective/src/cpp/multi_sort.cpp +++ b/cpp/perspective/src/cpp/multi_sort.cpp @@ -19,30 +19,35 @@ namespace perspective { t_mselem::t_mselem() : m_pkey(mknone()) , m_order(0) - , m_deleted(false) {} + , m_deleted(false) + , m_updated(false) {} t_mselem::t_mselem(const std::vector& row) : m_row(row) , m_pkey(mknone()) , m_order(0) - , m_deleted(false) {} + , m_deleted(false) + , m_updated(false) {} t_mselem::t_mselem(const std::vector& row, t_uindex order) : m_row(row) , m_pkey(mknone()) , m_order(order) - , m_deleted(false) {} + , m_deleted(false) + , m_updated(false) {} t_mselem::t_mselem(const t_tscalar& pkey, const std::vector& row) : m_row(row) , m_pkey(pkey) , m_order(0) - , m_deleted(false) {} + , m_deleted(false) + , m_updated(false) {} t_mselem::t_mselem(const t_mselem& other) { m_pkey = other.m_pkey; m_row = other.m_row; m_deleted = other.m_deleted; + m_updated = other.m_updated; m_order = other.m_order; } @@ -50,6 +55,7 @@ t_mselem::t_mselem(t_mselem&& other) { m_pkey = other.m_pkey; m_row = std::move(other.m_row); m_deleted = other.m_deleted; + m_updated = other.m_updated; m_order = other.m_order; } @@ -59,6 +65,7 @@ t_mselem::operator=(const t_mselem& other) { m_row = other.m_row; m_deleted = other.m_deleted; m_order = other.m_order; + m_updated = other.m_updated; return *this; } @@ -67,6 +74,7 @@ t_mselem::operator=(t_mselem&& other) { m_pkey = other.m_pkey; m_row = std::move(other.m_row); m_deleted = other.m_deleted; + m_updated = other.m_updated; m_order = other.m_order; return *this; } diff --git a/cpp/perspective/src/include/perspective/multi_sort.h b/cpp/perspective/src/include/perspective/multi_sort.h index 4d20e3c72b..b9d002876d 100644 --- a/cpp/perspective/src/include/perspective/multi_sort.h +++ b/cpp/perspective/src/include/perspective/multi_sort.h @@ -31,6 +31,7 @@ struct PERSPECTIVE_EXPORT t_mselem { t_tscalar m_pkey; t_uindex m_order; bool m_deleted; + bool m_updated; }; } // end namespace perspective