From b583e5370e97fbd2e8ca014046f64ff9974d06af Mon Sep 17 00:00:00 2001 From: mike31 Date: Tue, 5 Mar 2019 13:05:40 +0200 Subject: [PATCH 01/17] Enterprise subscriptions, working, not tested --- src/community/community.cpp | 16 +++++++++++++ src/community/community.h | 3 +++ src/rpc/rpcclient.cpp | 3 +++ src/rpc/rpchelp.cpp | 2 ++ src/rpc/rpclist.cpp | 1 + src/rpc/rpcserver.h | 1 + src/rpc/rpcstreams.cpp | 46 ++++++++++++++++++++++++++++++++++++- src/rpc/rpcutils.cpp | 29 +++++++++++++++++++++-- 8 files changed, 98 insertions(+), 3 deletions(-) diff --git a/src/community/community.cpp b/src/community/community.cpp index d07be507..7146523b 100644 --- a/src/community/community.cpp +++ b/src/community/community.cpp @@ -26,11 +26,21 @@ int mc_EnterpriseFeatures::STR_CreateSubscription(mc_TxEntity *entity,const std: return MC_ERR_FOUND; } +int mc_EnterpriseFeatures::STR_TrimSubscription(mc_TxEntity *entity,const std::string parameters) +{ + return MC_ERR_NOERROR; +} + int mc_EnterpriseFeatures::STR_IsIndexSkipped(mc_TxImport *import,mc_TxEntity *parent_entity,mc_TxEntity *entity) { return 0; } +int mc_EnterpriseFeatures::STR_NoRetrieve(mc_TxEntity *entity) +{ + return 0; +} + int mc_EnterpriseFeatures::STR_IsOutOfSync(mc_TxEntity *entity) { return 0; @@ -107,6 +117,12 @@ std::string mc_EnterpriseFeatures::ENT_TextConstant(const char* name) return ""; } +void mc_EnterpriseFeatures::ENT_InitRPCHelpMap() +{ + +} + + void mc_EnterpriseFeatures::LIC_RPCVerifyFeature(uint64_t feature) { throw JSONRPCError(RPC_NOT_SUPPORTED, "This feature is available only in Enterprise edition of MultiChain, please call \"help enterprise\" for details"); diff --git a/src/community/community.h b/src/community/community.h index 7c6aab8b..77552f33 100644 --- a/src/community/community.h +++ b/src/community/community.h @@ -37,7 +37,9 @@ typedef struct mc_EnterpriseFeatures uint32_t mode); // Unused int STR_CreateSubscription(mc_TxEntity *entity,const std::string parameters); + int STR_TrimSubscription(mc_TxEntity *entity,const std::string parameters); int STR_IsIndexSkipped(mc_TxImport *import,mc_TxEntity *parent_entity,mc_TxEntity *entity); + int STR_NoRetrieve(mc_TxEntity *entity); int STR_IsOutOfSync(mc_TxEntity *entity); int STR_SetSyncFlag(mc_TxEntity *entity,bool confirm); int STR_GetSubscriptions(mc_Buffer *subscriptions); @@ -55,6 +57,7 @@ typedef struct mc_EnterpriseFeatures int ENT_MinWalletDatVersion(); void ENT_RPCVerifyEdition(); std::string ENT_TextConstant(const char* name); + void ENT_InitRPCHelpMap(); void LIC_RPCVerifyFeature(uint64_t feature); bool LIC_VerifyFeature(uint64_t feature,std::string& reason); diff --git a/src/rpc/rpcclient.cpp b/src/rpc/rpcclient.cpp index 2efbfc95..85c5c3d5 100644 --- a/src/rpc/rpcclient.cpp +++ b/src/rpc/rpcclient.cpp @@ -208,6 +208,7 @@ static const std::string vAPINames[] = "storechunk", "submitblock", "subscribe", +"trimsubscribe", "teststreamfilter", "testtxfilter", "unsubscribe", @@ -381,6 +382,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "create", 3 }, { "subscribe", 0 }, { "subscribe", 1 }, + { "trimsubscribe", 0 }, { "unsubscribe", 0 }, { "unsubscribe", 1 }, { "listassettransactions", 1 }, @@ -578,6 +580,7 @@ static const CRPCConvertParamMayBeString vRPCConvertParamsMayBeString[] = { "importaddress", 0 }, { "importprivkey", 0 }, { "subscribe", 0 }, + { "trimsubscribe", 0 }, { "unsubscribe", 0 }, { "liststreamkeys", 1 }, { "liststreampublishers", 1 }, diff --git a/src/rpc/rpchelp.cpp b/src/rpc/rpchelp.cpp index fb5a5382..b71c8001 100644 --- a/src/rpc/rpchelp.cpp +++ b/src/rpc/rpchelp.cpp @@ -4757,6 +4757,8 @@ void mc_InitRPCHelpMap() mc_InitRPCHelpMap19(); mc_InitRPCHelpMap20(); + pEF->ENT_InitRPCHelpMap(); + mc_InitRPCLogParamCountMap(); mc_InitRPCAllowedWhenWaitingForUpgradeSet(); mc_InitRPCAllowedWhenOffline(); diff --git a/src/rpc/rpclist.cpp b/src/rpc/rpclist.cpp index fa3f5caf..af1b3c31 100644 --- a/src/rpc/rpclist.cpp +++ b/src/rpc/rpclist.cpp @@ -256,6 +256,7 @@ static const CRPCCommand vRPCCommands[] = { "wallet", "publishmultifrom", &publishmultifrom, false, false, true }, { "wallet", "subscribe", &subscribe, false, false, true }, { "wallet", "unsubscribe", &unsubscribe, false, false, true }, + { "wallet", "trimsubscribe", &trimsubscribe, false, false, true }, { "wallet", "listassettransactions", &listassettransactions, false, false, true }, { "wallet", "getassettransaction", &getassettransaction, false, false, true }, { "wallet", "getstreamitem", &getstreamitem, false, false, true }, diff --git a/src/rpc/rpcserver.h b/src/rpc/rpcserver.h index c44504d2..84d57a0e 100644 --- a/src/rpc/rpcserver.h +++ b/src/rpc/rpcserver.h @@ -268,6 +268,7 @@ extern json_spirit::Value publishmulti(const json_spirit::Array& params, bool fH extern json_spirit::Value publishmultifrom(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value subscribe(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value unsubscribe(const json_spirit::Array& params, bool fHelp); +extern json_spirit::Value trimsubscribe(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value listassettransactions(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value getassettransaction(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value getstreamitem(const json_spirit::Array& params, bool fHelp); diff --git a/src/rpc/rpcstreams.cpp b/src/rpc/rpcstreams.cpp index 917fcd1c..29edce1d 100644 --- a/src/rpc/rpcstreams.cpp +++ b/src/rpc/rpcstreams.cpp @@ -203,7 +203,7 @@ Value liststreams(const Array& params, bool fHelp) { if(paramtobool(params[1])) { - output_level=0x3E; + output_level=0xBE; if(mc_gState->m_Features->StreamFilters()) { output_level |= 0x40; @@ -1053,6 +1053,50 @@ Value publishfrom(const Array& params, bool fHelp) return wtx.GetHash().GetHex(); } +Value trimsubscribe(const Array& params, bool fHelp) +{ + if (fHelp || params.size() != 2) + throw runtime_error("Help message not found\n"); + + pEF->ENT_RPCVerifyEdition(); + + string indexes=params[1].get_str(); + + vector inputEntities; + vector inputStrings; + if(params[0].type() == str_type) + { + inputStrings.push_back(params[0].get_str()); + } + else + { + inputStrings=ParseStringList(params[0]); + } + + for(int is=0;is<(int)inputStrings.size();is++) + { + mc_EntityDetails entity_to_subscribe; + Value param=inputStrings[is]; + ParseEntityIdentifier(param,&entity_to_subscribe, MC_ENT_TYPE_STREAM); + inputEntities.push_back(entity_to_subscribe); + } + + for(int is=0;is<(int)inputStrings.size();is++) + { + mc_EntityDetails* lpEntity; + lpEntity=&inputEntities[is]; + + mc_TxEntity entity; + entity.Zero(); + memcpy(entity.m_EntityID,lpEntity->GetTxID()+MC_AST_SHORT_TXID_OFFSET,MC_AST_SHORT_TXID_SIZE); + entity.m_EntityType=MC_TET_STREAM | MC_TET_CHAINPOS; + pEF->STR_TrimSubscription(&entity,indexes); + } + + return Value::null; +} + + Value subscribe(const Array& params, bool fHelp) { if (fHelp || params.size() < 1 || params.size() > ((pEF->ENT_EditionNumeric() == 0) ? 2 : 3)) diff --git a/src/rpc/rpcutils.cpp b/src/rpc/rpcutils.cpp index f585308b..d1106c23 100644 --- a/src/rpc/rpcutils.cpp +++ b/src/rpc/rpcutils.cpp @@ -13,6 +13,7 @@ #include "utils/utilmoneystr.h" #include "wallet/wallettxs.h" #include "json/json_spirit_ubjson.h" +#include "community/community.h" #include @@ -797,6 +798,7 @@ Object StreamEntry(const unsigned char *txid,uint32_t output_level,mc_EntityDeta // 0x0010 stats // 0x0020 creators // 0x0040 filters +// 0x0080 subscription // 0x0800 skip name and ref Object entry; @@ -985,14 +987,37 @@ Object StreamEntry(const unsigned char *txid,uint32_t output_level,mc_EntityDeta if(output_level & 0x0008) { entry.push_back(Pair("subscribed",true)); - if(entStat.m_Flags & MC_EFL_NOT_IN_SYNC) - { + if( ((entStat.m_Flags & MC_EFL_NOT_IN_SYNC) != 0) || + (pEF->STR_IsOutOfSync(&(entStat.m_Entity)) != 0) ) + { entry.push_back(Pair("synchronized",false)); } else { entry.push_back(Pair("synchronized",true)); } + if(output_level & 0x0080) + { + entry.push_back(Pair("retrieve",(pEF->STR_NoRetrieve(&(entStat.m_Entity))==0))); + + mc_TxImport dummy_import; + dummy_import.m_ImportID=1; + vector> index_types; + index_types.push_back(pair("items",MC_TET_STREAM | MC_TET_CHAINPOS)); + index_types.push_back(pair("keys",MC_TET_STREAM_KEY | MC_TET_CHAINPOS)); + index_types.push_back(pair("publishers",MC_TET_STREAM_PUBLISHER | MC_TET_CHAINPOS)); + index_types.push_back(pair("items-local",MC_TET_STREAM | MC_TET_TIMERECEIVED)); + index_types.push_back(pair("keys-local",MC_TET_STREAM_KEY | MC_TET_TIMERECEIVED)); + index_types.push_back(pair("publishers-local",MC_TET_STREAM_PUBLISHER | MC_TET_TIMERECEIVED)); + + Object indexes; + for(unsigned int ind=0;indSTR_IsIndexSkipped(&dummy_import,NULL,&(entStat.m_Entity)) == 0))); + } + entry.push_back(Pair("indexes",indexes)); + } } if(output_level & 0x0010) { From 18b8b69d555ef122925e22226ebc4804b9d926e8 Mon Sep 17 00:00:00 2001 From: mike31 Date: Tue, 5 Mar 2019 13:30:59 +0200 Subject: [PATCH 02/17] Fixed enterprise subscription sunc flag and help for community --- src/rpc/rpcserver.cpp | 5 ++++- src/rpc/rpcutils.cpp | 35 +++++++++++++++++++++++++++-------- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/src/rpc/rpcserver.cpp b/src/rpc/rpcserver.cpp index 828dd49d..ac9b460f 100644 --- a/src/rpc/rpcserver.cpp +++ b/src/rpc/rpcserver.cpp @@ -251,7 +251,10 @@ string CRPCTable::help(string strCommand) const } catch (std::exception& e) { - strHelp = string(e.what()); + if(strCommand != "") + { + strHelp = string(e.what()); + } } } else diff --git a/src/rpc/rpcutils.cpp b/src/rpc/rpcutils.cpp index d1106c23..da2c979f 100644 --- a/src/rpc/rpcutils.cpp +++ b/src/rpc/rpcutils.cpp @@ -986,6 +986,13 @@ Object StreamEntry(const unsigned char *txid,uint32_t output_level,mc_EntityDeta { if(output_level & 0x0008) { + vector> index_types; + index_types.push_back(pair("items",MC_TET_STREAM | MC_TET_CHAINPOS)); + index_types.push_back(pair("keys",MC_TET_STREAM_KEY | MC_TET_CHAINPOS)); + index_types.push_back(pair("publishers",MC_TET_STREAM_PUBLISHER | MC_TET_CHAINPOS)); + index_types.push_back(pair("items-local",MC_TET_STREAM | MC_TET_TIMERECEIVED)); + index_types.push_back(pair("keys-local",MC_TET_STREAM_KEY | MC_TET_TIMERECEIVED)); + index_types.push_back(pair("publishers-local",MC_TET_STREAM_PUBLISHER | MC_TET_TIMERECEIVED)); entry.push_back(Pair("subscribed",true)); if( ((entStat.m_Flags & MC_EFL_NOT_IN_SYNC) != 0) || (pEF->STR_IsOutOfSync(&(entStat.m_Entity)) != 0) ) @@ -994,21 +1001,33 @@ Object StreamEntry(const unsigned char *txid,uint32_t output_level,mc_EntityDeta } else { - entry.push_back(Pair("synchronized",true)); + bool fSynchronized=true; + if(pEF->ENT_EditionNumeric() == 0) + { + for(unsigned int ind=1;indFindEntity(&entStat)) + { + if( (entStat.m_Flags & MC_EFL_NOT_IN_SYNC) != 0) + { + fSynchronized=false; + } + } + } + } + } + entry.push_back(Pair("synchronized",fSynchronized)); } if(output_level & 0x0080) { + entStat.m_Entity.m_EntityType=MC_TET_STREAM | MC_TET_CHAINPOS; entry.push_back(Pair("retrieve",(pEF->STR_NoRetrieve(&(entStat.m_Entity))==0))); mc_TxImport dummy_import; dummy_import.m_ImportID=1; - vector> index_types; - index_types.push_back(pair("items",MC_TET_STREAM | MC_TET_CHAINPOS)); - index_types.push_back(pair("keys",MC_TET_STREAM_KEY | MC_TET_CHAINPOS)); - index_types.push_back(pair("publishers",MC_TET_STREAM_PUBLISHER | MC_TET_CHAINPOS)); - index_types.push_back(pair("items-local",MC_TET_STREAM | MC_TET_TIMERECEIVED)); - index_types.push_back(pair("keys-local",MC_TET_STREAM_KEY | MC_TET_TIMERECEIVED)); - index_types.push_back(pair("publishers-local",MC_TET_STREAM_PUBLISHER | MC_TET_TIMERECEIVED)); Object indexes; for(unsigned int ind=0;ind Date: Fri, 8 Mar 2019 07:50:14 +0200 Subject: [PATCH 03/17] retrieve/purgestreamitems APIs --- src/community/community.cpp | 11 +++++++++++ src/community/community.h | 2 ++ src/rpc/rpcclient.cpp | 8 +++++++- src/rpc/rpclist.cpp | 2 ++ src/rpc/rpcserver.h | 2 ++ src/rpc/rpcstreams.cpp | 24 ++++++++++++++++++++++-- 6 files changed, 46 insertions(+), 3 deletions(-) diff --git a/src/community/community.cpp b/src/community/community.cpp index 7146523b..5b965b76 100644 --- a/src/community/community.cpp +++ b/src/community/community.cpp @@ -61,6 +61,17 @@ int mc_EnterpriseFeatures::STR_PutSubscriptions(mc_Buffer *subscriptions) return MC_ERR_NOERROR; } +Value mc_EnterpriseFeatures::STR_RPCRetrieveStreamItems(const Array& params) +{ + return Value::null; +} + +Value mc_EnterpriseFeatures::STR_RPCPurgeStreamItems(const Array& params) +{ + return Value::null; +} + + int mc_EnterpriseFeatures::WLT_CreateSubscription(mc_TxEntity *entity,uint32_t retrieve,uint32_t indexes,uint32_t *rescan_mode) { *rescan_mode=0; diff --git a/src/community/community.h b/src/community/community.h index 77552f33..2188b823 100644 --- a/src/community/community.h +++ b/src/community/community.h @@ -44,6 +44,8 @@ typedef struct mc_EnterpriseFeatures int STR_SetSyncFlag(mc_TxEntity *entity,bool confirm); int STR_GetSubscriptions(mc_Buffer *subscriptions); int STR_PutSubscriptions(mc_Buffer *subscriptions); + Value STR_RPCRetrieveStreamItems(const Array& params); + Value STR_RPCPurgeStreamItems(const Array& params); int WLT_CreateSubscription(mc_TxEntity *entity,uint32_t retrieve,uint32_t indexes,uint32_t *rescan_mode); int WLT_DeleteSubscription(mc_TxEntity *entity,uint32_t rescan_mode); diff --git a/src/rpc/rpcclient.cpp b/src/rpc/rpcclient.cpp index 85c5c3d5..c08f82bd 100644 --- a/src/rpc/rpcclient.cpp +++ b/src/rpc/rpcclient.cpp @@ -174,9 +174,11 @@ static const std::string vAPINames[] = "publishfrom", "publishmulti", "publishmultifrom", +"purgestreamitems", "reconsiderblock", "resendwallettransactions", "resume", +"retrievestreamitems", "revoke", "revokefrom", "runstreamfilter", @@ -208,9 +210,9 @@ static const std::string vAPINames[] = "storechunk", "submitblock", "subscribe", -"trimsubscribe", "teststreamfilter", "testtxfilter", +"trimsubscribe", "unsubscribe", "validateaddress", "verifychain", @@ -383,6 +385,8 @@ static const CRPCConvertParam vRPCConvertParams[] = { "subscribe", 0 }, { "subscribe", 1 }, { "trimsubscribe", 0 }, + { "retrievestreamitems", 1 }, + { "purgestreamitems", 1 }, { "unsubscribe", 0 }, { "unsubscribe", 1 }, { "listassettransactions", 1 }, @@ -581,6 +585,8 @@ static const CRPCConvertParamMayBeString vRPCConvertParamsMayBeString[] = { "importprivkey", 0 }, { "subscribe", 0 }, { "trimsubscribe", 0 }, + { "retrievestreamitems", 1 }, + { "purgestreamitems", 1 }, { "unsubscribe", 0 }, { "liststreamkeys", 1 }, { "liststreampublishers", 1 }, diff --git a/src/rpc/rpclist.cpp b/src/rpc/rpclist.cpp index af1b3c31..24070856 100644 --- a/src/rpc/rpclist.cpp +++ b/src/rpc/rpclist.cpp @@ -257,6 +257,8 @@ static const CRPCCommand vRPCCommands[] = { "wallet", "subscribe", &subscribe, false, false, true }, { "wallet", "unsubscribe", &unsubscribe, false, false, true }, { "wallet", "trimsubscribe", &trimsubscribe, false, false, true }, + { "wallet", "retrievestreamitems", &retrievestreamitems, false, false, true }, + { "wallet", "purgestreamitems", &purgestreamitems, false, false, true }, { "wallet", "listassettransactions", &listassettransactions, false, false, true }, { "wallet", "getassettransaction", &getassettransaction, false, false, true }, { "wallet", "getstreamitem", &getstreamitem, false, false, true }, diff --git a/src/rpc/rpcserver.h b/src/rpc/rpcserver.h index 84d57a0e..0165de5e 100644 --- a/src/rpc/rpcserver.h +++ b/src/rpc/rpcserver.h @@ -269,6 +269,8 @@ extern json_spirit::Value publishmultifrom(const json_spirit::Array& params, boo extern json_spirit::Value subscribe(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value unsubscribe(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value trimsubscribe(const json_spirit::Array& params, bool fHelp); +extern json_spirit::Value retrievestreamitems(const json_spirit::Array& params, bool fHelp); +extern json_spirit::Value purgestreamitems(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value listassettransactions(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value getassettransaction(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value getstreamitem(const json_spirit::Array& params, bool fHelp); diff --git a/src/rpc/rpcstreams.cpp b/src/rpc/rpcstreams.cpp index 29edce1d..e7c01fac 100644 --- a/src/rpc/rpcstreams.cpp +++ b/src/rpc/rpcstreams.cpp @@ -2635,7 +2635,7 @@ int GetAndQueryDirtyList(vector& conditions, mc_EntityDetails return dirty_count; } -void FillContitionsList(vector& conditions, Value param) +void FillConditionsList(vector& conditions, Value param) { bool key_found=false; bool publisher_found=false; @@ -2777,7 +2777,7 @@ Value liststreamqueryitems(const Array& params, bool fHelp) verbose=paramtobool(params[2]); } - FillContitionsList(conditions,params[1]); + FillConditionsList(conditions,params[1]); if(conditions.size() == 0) { @@ -2847,3 +2847,23 @@ Value liststreamqueryitems(const Array& params, bool fHelp) return retArray; } + +Value retrievestreamitems(const Array& params, bool fHelp) +{ + if (fHelp || params.size() != 2) + throw runtime_error("Help message not found\n"); + + pEF->ENT_RPCVerifyEdition(); + + return pEF->STR_RPCRetrieveStreamItems(params); +} + +Value purgestreamitems(const Array& params, bool fHelp) +{ + if (fHelp || params.size() != 2) + throw runtime_error("Help message not found\n"); + + pEF->ENT_RPCVerifyEdition(); + + return pEF->STR_RPCPurgeStreamItems(params); +} \ No newline at end of file From 1fe02df565e7f8f57b6e79cd28e1f46781891b70 Mon Sep 17 00:00:00 2001 From: mike31 Date: Sun, 10 Mar 2019 09:55:01 +0200 Subject: [PATCH 04/17] setruntime autosubscribe with empty string --- src/core/init.cpp | 2 +- src/rpc/rpcmisc.cpp | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/core/init.cpp b/src/core/init.cpp index eedef9c5..378d10e1 100644 --- a/src/core/init.cpp +++ b/src/core/init.cpp @@ -424,7 +424,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += " " + _("(more details and % substitutions online)") + "\n"; /* MCHN START */ strUsage += " -walletdbversion=2|3 " + _("Specify wallet version, 2 - Berkeley DB, 3 (default) - proprietary") + "\n"; - strUsage += " -autosubscribe=streams|assets|\"streams,assets\"|\"assets,streams\" " + _("Automatically subscribe to new streams and/or assets") + "\n"; + strUsage += " -autosubscribe=streams|assets|\"streams,assets\"|\"assets,streams\"|\"\" " + _("Automatically subscribe to new streams and/or assets") + "\n"; /* MCHN END */ strUsage += " -zapwallettxes= " + _("Delete all wallet transactions and only recover those parts of the blockchain through -rescan on startup") + "\n"; strUsage += " " + _("(1 = keep tx meta data e.g. account owner and payment request information, 2 = drop tx meta data)") + "\n"; diff --git a/src/rpc/rpcmisc.cpp b/src/rpc/rpcmisc.cpp index df827d76..dd370cdf 100644 --- a/src/rpc/rpcmisc.cpp +++ b/src/rpc/rpcmisc.cpp @@ -510,7 +510,11 @@ Value setruntimeparam(const json_spirit::Array& params, bool fHelp) mode |= MC_WMD_AUTOSUBSCRIBE_STREAMS; mode |= MC_WMD_AUTOSUBSCRIBE_ASSETS; found=true; - } + } + if( autosubscribe=="" ) + { + found=true; + } if(!found) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter value"); From 6e7ad86ac1744d48e473230a34e4f2b6fd4af3ac Mon Sep 17 00:00:00 2001 From: mike31 Date: Sun, 10 Mar 2019 18:41:31 +0200 Subject: [PATCH 05/17] retrievestreamitems API --- src/wallet/wallettxs.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/wallet/wallettxs.cpp b/src/wallet/wallettxs.cpp index e2f07a78..09534f94 100644 --- a/src/wallet/wallettxs.cpp +++ b/src/wallet/wallettxs.cpp @@ -5,6 +5,7 @@ #include "wallet/wallettxs.h" #include "utils/core_io.h" +#include "community/community.h" #include "json/json_spirit_utils.h" #include "json/json_spirit_value.h" @@ -2373,7 +2374,8 @@ int mc_WalletTxs::AddTx(mc_TxImport *import,const CWalletTx& tx,int block,CDiskT if(imp->FindEntity(&entity) >= 0) { - if(chunk_hashes) + if( (chunk_hashes != NULL) && + (pEF->STR_NoRetrieve(&entity) == 0) ) { mc_ChunkDBRow chunk_def; mc_TxEntity chunk_entity; From 271a253f512a8888b084d2363cd5ea6cefa54c25 Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 11 Mar 2019 10:22:42 +0200 Subject: [PATCH 06/17] Chunk counts returned on entity removal --- src/wallet/chunkdb.cpp | 25 ++++++++++++++++++++++--- src/wallet/chunkdb.h | 4 ++-- src/wallet/wallettxs.cpp | 2 +- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/src/wallet/chunkdb.cpp b/src/wallet/chunkdb.cpp index 6cb6110b..231b3db2 100644 --- a/src/wallet/chunkdb.cpp +++ b/src/wallet/chunkdb.cpp @@ -534,7 +534,7 @@ int mc_ChunkDB::SourceChunksRecovery() return err; } -int mc_ChunkDB::RemoveEntityInternal(mc_TxEntity *entity) +int mc_ChunkDB::RemoveEntityInternal(mc_TxEntity *entity,uint32_t *removed_chunks,uint64_t *removed_size) { int err; mc_SubscriptionDBRow subscription; @@ -552,6 +552,15 @@ int mc_ChunkDB::RemoveEntityInternal(mc_TxEntity *entity) err=MC_ERR_NOERROR; sprintf_hex(enthex,entity->m_EntityID,MC_TDB_ENTITY_ID_SIZE); + if(removed_chunks) + { + *removed_chunks=0; + } + if(removed_size) + { + *removed_size=0; + } + subscription.Zero(); subscription.m_RecordType=MC_CDB_TYPE_SUBSCRIPTION; memcpy(&subscription.m_Entity,entity,sizeof(mc_TxEntity)); @@ -693,9 +702,19 @@ int mc_ChunkDB::RemoveEntityInternal(mc_TxEntity *entity) { chunk_found=1; memcpy(chunk_def.m_Hash,buf+param_value_start,bytes); + if(removed_chunks) + { + *removed_chunks+=1; + } } } break; + case MC_ENT_SPRM_CHUNK_SIZE: + if(removed_size) + { + *removed_size+=(uint32_t)mc_GetLE(buf+param_value_start,bytes); + } + break; case MC_ENT_SPRM_ITEM_COUNT: if(bytes != sizeof(uint32_t)) { @@ -789,12 +808,12 @@ int mc_ChunkDB::RemoveEntityInternal(mc_TxEntity *entity) return err; } -int mc_ChunkDB::RemoveEntity(mc_TxEntity *entity) +int mc_ChunkDB::RemoveEntity(mc_TxEntity *entity,uint32_t *removed_chunks,uint64_t *removed_size) { int err; Lock(); - err=RemoveEntityInternal(entity); + err=RemoveEntityInternal(entity,removed_chunks,removed_size); UnLock(); return err; diff --git a/src/wallet/chunkdb.h b/src/wallet/chunkdb.h index c50849e3..e752bae4 100644 --- a/src/wallet/chunkdb.h +++ b/src/wallet/chunkdb.h @@ -159,8 +159,8 @@ typedef struct mc_ChunkDB int AddSubscription(mc_SubscriptionDBRow *subscription); int AddEntity(mc_TxEntity *entity,uint32_t flags); // Adds entity int AddEntityInternal(mc_TxEntity *entity,uint32_t flags); - int RemoveEntity(mc_TxEntity *entity); - int RemoveEntityInternal(mc_TxEntity *entity); + int RemoveEntity(mc_TxEntity *entity,uint32_t *removed_chunks,uint64_t *removed_size); + int RemoveEntityInternal(mc_TxEntity *entity,uint32_t *removed_chunks,uint64_t *removed_size); int SourceChunksRecovery(); mc_SubscriptionDBRow *FindSubscription(const mc_TxEntity *entity); // Finds subscription diff --git a/src/wallet/wallettxs.cpp b/src/wallet/wallettxs.cpp index 09534f94..f96ed461 100644 --- a/src/wallet/wallettxs.cpp +++ b/src/wallet/wallettxs.cpp @@ -1340,7 +1340,7 @@ int mc_WalletTxs::Unsubscribe(mc_Buffer* lpEntities,bool purge) { for(j=0;jGetCount();j++) { - m_ChunkDB->RemoveEntity((mc_TxEntity*)lpEntities->GetRow(j)); + m_ChunkDB->RemoveEntity((mc_TxEntity*)lpEntities->GetRow(j),NULL,NULL); } } } From b2a97106c286e46f6fb20888b7c0af964b0c0f61 Mon Sep 17 00:00:00 2001 From: mike31 Date: Tue, 12 Mar 2019 17:33:31 +0200 Subject: [PATCH 07/17] purgestreamitems API --- src/community/community.cpp | 10 ++++++++++ src/community/community.h | 4 +++- src/wallet/chunkdb.cpp | 4 ++++ src/wallet/chunkdb.h | 1 + src/wallet/wallettxs.cpp | 5 +++++ 5 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/community/community.cpp b/src/community/community.cpp index 5b965b76..228b428f 100644 --- a/src/community/community.cpp +++ b/src/community/community.cpp @@ -71,6 +71,16 @@ Value mc_EnterpriseFeatures::STR_RPCPurgeStreamItems(const Array& params) return Value::null; } +int mc_EnterpriseFeatures::STR_RestoreChunkIfNeeded(mc_ChunkDBRow *chunk_def) +{ + return MC_ERR_NOERROR; +} + +int mc_EnterpriseFeatures::STR_RemoveDataFromFile(int fHan, uint32_t from, uint32_t size, uint32_t mode) +{ + return MC_ERR_NOERROR; +} + int mc_EnterpriseFeatures::WLT_CreateSubscription(mc_TxEntity *entity,uint32_t retrieve,uint32_t indexes,uint32_t *rescan_mode) { diff --git a/src/community/community.h b/src/community/community.h index 2188b823..1bef5e80 100644 --- a/src/community/community.h +++ b/src/community/community.h @@ -45,7 +45,9 @@ typedef struct mc_EnterpriseFeatures int STR_GetSubscriptions(mc_Buffer *subscriptions); int STR_PutSubscriptions(mc_Buffer *subscriptions); Value STR_RPCRetrieveStreamItems(const Array& params); - Value STR_RPCPurgeStreamItems(const Array& params); + Value STR_RPCPurgeStreamItems(const Array& params); + int STR_RemoveDataFromFile(int fHan, uint32_t from, uint32_t size, uint32_t mode); + int STR_RestoreChunkIfNeeded(mc_ChunkDBRow *chunk_def); int WLT_CreateSubscription(mc_TxEntity *entity,uint32_t retrieve,uint32_t indexes,uint32_t *rescan_mode); int WLT_DeleteSubscription(mc_TxEntity *entity,uint32_t rescan_mode); diff --git a/src/wallet/chunkdb.cpp b/src/wallet/chunkdb.cpp index 231b3db2..cf9dd298 100644 --- a/src/wallet/chunkdb.cpp +++ b/src/wallet/chunkdb.cpp @@ -3,6 +3,7 @@ #include "multichain/multichain.h" #include "wallet/chunkdb.h" +#include "community/community.h" #define MC_CDB_TMP_FLAG_SHOULD_COMMIT 0x00000001 #define MC_CDB_FILE_PAGE_SIZE 0x00100000 @@ -756,6 +757,7 @@ int mc_ChunkDB::RemoveEntityInternal(mc_TxEntity *entity,uint32_t *removed_chunk file_offset=file_size; } } +// pEF->STR_RemoveDataFromFile(FileHan,0,file_size,0); } @@ -1284,10 +1286,12 @@ int mc_ChunkDB::GetChunkDefInternal( memcpy((char*)chunk_def+m_ValueOffset,ptr,m_ValueSize); // ptr=GetChunkInternal(chunk_def,-1,-1,&bytes); } +/* else { chunk_def->m_Pos=on_disk_items; } + */ } } err=MC_ERR_NOT_FOUND; diff --git a/src/wallet/chunkdb.h b/src/wallet/chunkdb.h index e752bae4..6f6b8b46 100644 --- a/src/wallet/chunkdb.h +++ b/src/wallet/chunkdb.h @@ -28,6 +28,7 @@ #define MC_CDB_FLUSH_MODE_DATASYNC 0x00000100 #define MC_CFL_STORAGE_FLUSHED 0x01000000 +#define MC_CFL_STORAGE_PURGED 0x02000000 /** File DB Row*/ diff --git a/src/wallet/wallettxs.cpp b/src/wallet/wallettxs.cpp index f96ed461..f8ceba26 100644 --- a/src/wallet/wallettxs.cpp +++ b/src/wallet/wallettxs.cpp @@ -2413,6 +2413,11 @@ int mc_WalletTxs::AddTx(mc_TxImport *import,const CWalletTx& tx,int block,CDiskT // Feeding async chunk retriever here } } + else + { + pEF->STR_RestoreChunkIfNeeded(&chunk_def); + } + chunk_hashes+=MC_CDB_CHUNK_HASH_SIZE; } } From 7e995a2f33bbb039ca8e4b75021a6672ec1653d7 Mon Sep 17 00:00:00 2001 From: mike31 Date: Wed, 13 Mar 2019 14:03:42 +0200 Subject: [PATCH 08/17] Restoring purged chunk in community --- src/community/community.cpp | 5 ----- src/community/community.h | 1 - src/wallet/chunkdb.cpp | 18 ++++++++++++++++++ src/wallet/chunkdb.h | 1 + src/wallet/wallettxs.cpp | 2 +- 5 files changed, 20 insertions(+), 7 deletions(-) diff --git a/src/community/community.cpp b/src/community/community.cpp index 228b428f..3c3ee0a9 100644 --- a/src/community/community.cpp +++ b/src/community/community.cpp @@ -71,11 +71,6 @@ Value mc_EnterpriseFeatures::STR_RPCPurgeStreamItems(const Array& params) return Value::null; } -int mc_EnterpriseFeatures::STR_RestoreChunkIfNeeded(mc_ChunkDBRow *chunk_def) -{ - return MC_ERR_NOERROR; -} - int mc_EnterpriseFeatures::STR_RemoveDataFromFile(int fHan, uint32_t from, uint32_t size, uint32_t mode) { return MC_ERR_NOERROR; diff --git a/src/community/community.h b/src/community/community.h index 1bef5e80..e8cefb10 100644 --- a/src/community/community.h +++ b/src/community/community.h @@ -47,7 +47,6 @@ typedef struct mc_EnterpriseFeatures Value STR_RPCRetrieveStreamItems(const Array& params); Value STR_RPCPurgeStreamItems(const Array& params); int STR_RemoveDataFromFile(int fHan, uint32_t from, uint32_t size, uint32_t mode); - int STR_RestoreChunkIfNeeded(mc_ChunkDBRow *chunk_def); int WLT_CreateSubscription(mc_TxEntity *entity,uint32_t retrieve,uint32_t indexes,uint32_t *rescan_mode); int WLT_DeleteSubscription(mc_TxEntity *entity,uint32_t rescan_mode); diff --git a/src/wallet/chunkdb.cpp b/src/wallet/chunkdb.cpp index cf9dd298..5afac632 100644 --- a/src/wallet/chunkdb.cpp +++ b/src/wallet/chunkdb.cpp @@ -2148,3 +2148,21 @@ int mc_ChunkDB::Commit(int block,uint32_t flush_mode) return err; } +int mc_ChunkDB::RestoreChunkIfNeeded(mc_ChunkDBRow *chunk_def) +{ + if( (chunk_def->m_StorageFlags & MC_CFL_STORAGE_PURGED) == 0 ) + { + return MC_ERR_NOERROR; + } + + int err=MC_ERR_NOERROR; + chunk_def->m_StorageFlags-=MC_CFL_STORAGE_PURGED; + + chunk_def->SwapPosBytes(); + err=m_DB->Write((char*)chunk_def+m_KeyOffset,m_KeySize, + (char*)chunk_def+m_ValueOffset,m_ValueSize,MC_OPT_DB_DATABASE_DEFAULT); + chunk_def->SwapPosBytes(); + + return err; +} + diff --git a/src/wallet/chunkdb.h b/src/wallet/chunkdb.h index 6f6b8b46..e9594538 100644 --- a/src/wallet/chunkdb.h +++ b/src/wallet/chunkdb.h @@ -223,6 +223,7 @@ typedef struct mc_ChunkDB uint32_t fileid, uint32_t flush_mode); + int RestoreChunkIfNeeded(mc_ChunkDBRow *chunk_def); int AddToFile(const void *chunk, uint32_t size, diff --git a/src/wallet/wallettxs.cpp b/src/wallet/wallettxs.cpp index f8ceba26..370c2876 100644 --- a/src/wallet/wallettxs.cpp +++ b/src/wallet/wallettxs.cpp @@ -2415,7 +2415,7 @@ int mc_WalletTxs::AddTx(mc_TxImport *import,const CWalletTx& tx,int block,CDiskT } else { - pEF->STR_RestoreChunkIfNeeded(&chunk_def); + m_ChunkDB->RestoreChunkIfNeeded(&chunk_def); } chunk_hashes+=MC_CDB_CHUNK_HASH_SIZE; From 947d9b4240f2191477e221186f379c8f9ecf518a Mon Sep 17 00:00:00 2001 From: mike31 Date: Wed, 13 Mar 2019 15:13:39 +0200 Subject: [PATCH 09/17] Storing chunk->next attempt map in chunk collector DB --- src/wallet/chunkcollector.cpp | 71 ++++++++++++++++++++++++++++++----- 1 file changed, 61 insertions(+), 10 deletions(-) diff --git a/src/wallet/chunkcollector.cpp b/src/wallet/chunkcollector.cpp index bd550947..f9cd2b06 100644 --- a/src/wallet/chunkcollector.cpp +++ b/src/wallet/chunkcollector.cpp @@ -4,6 +4,8 @@ #include "multichain/multichain.h" #include "wallet/chunkcollector.h" +#define MC_IMPOSSIBLE_NEXT_ATTEMPT 0xFFFFFFFF + void mc_ChunkEntityKey::Zero() { memset(this,0, sizeof(mc_ChunkEntityKey)); @@ -169,23 +171,57 @@ void mc_ChunkCollector::GetDBRow(mc_ChunkCollectorRow* collect_row) int mc_ChunkCollector::DeleteDBRow(mc_ChunkCollectorRow *collect_row) { + uint32_t next_attempt; SetDBRow(collect_row); + + next_attempt=m_DBRow.m_QueryNextAttempt; + if(next_attempt) + { + m_DBRow.m_QueryNextAttempt=MC_IMPOSSIBLE_NEXT_ATTEMPT; + m_DB->Delete((char*)&m_DBRow+m_KeyDBOffset,m_KeyDBSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); + m_DBRow.m_QueryNextAttempt=next_attempt; + } return m_DB->Delete((char*)&m_DBRow+m_KeyDBOffset,m_KeyDBSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); } int mc_ChunkCollector::InsertDBRow(mc_ChunkCollectorRow *collect_row) { + uint32_t next_attempt; + uint32_t query_attempts; collect_row->m_State.m_Status &= MC_CCF_ERROR_MASK; SetDBRow(collect_row); collect_row->m_State.m_Status |= MC_CCF_INSERTED; + + next_attempt=m_DBRow.m_QueryNextAttempt; + if(next_attempt) + { + query_attempts=m_DBRow.m_QueryAttempts; + m_DBRow.m_QueryNextAttempt=MC_IMPOSSIBLE_NEXT_ATTEMPT; + m_DBRow.m_QueryAttempts=next_attempt; + m_DB->Write((char*)&m_DBRow+m_KeyDBOffset,m_KeyDBSize,(char*)&m_DBRow+m_ValueDBOffset,m_ValueDBSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); + m_DBRow.m_QueryNextAttempt=next_attempt; + m_DBRow.m_QueryAttempts=query_attempts; + } + return m_DB->Write((char*)&m_DBRow+m_KeyDBOffset,m_KeyDBSize,(char*)&m_DBRow+m_ValueDBOffset,m_ValueDBSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); } int mc_ChunkCollector::UpdateDBRow(mc_ChunkCollectorRow *collect_row) { + uint32_t next_attempt; + uint32_t query_attempts; int err; collect_row->m_State.m_Status &= MC_CCF_ERROR_MASK; SetDBRow(collect_row); + + next_attempt=m_DBRow.m_QueryNextAttempt; + if(next_attempt) + { + m_DBRow.m_QueryNextAttempt=MC_IMPOSSIBLE_NEXT_ATTEMPT; + m_DB->Delete((char*)&m_DBRow+m_KeyDBOffset,m_KeyDBSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); + m_DBRow.m_QueryNextAttempt=next_attempt; + } + err=m_DB->Delete((char*)&m_DBRow+m_KeyDBOffset,m_KeyDBSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); if(err) { @@ -194,6 +230,18 @@ int mc_ChunkCollector::UpdateDBRow(mc_ChunkCollectorRow *collect_row) collect_row->m_DBNextAttempt=collect_row->m_State.m_QueryNextAttempt; m_DBRow.m_QueryNextAttempt=mc_SwapBytes32(collect_row->m_DBNextAttempt); collect_row->m_State.m_Status |= MC_CCF_INSERTED; + + next_attempt=m_DBRow.m_QueryNextAttempt; + if(next_attempt) + { + query_attempts=m_DBRow.m_QueryAttempts; + m_DBRow.m_QueryNextAttempt=MC_IMPOSSIBLE_NEXT_ATTEMPT; + m_DBRow.m_QueryAttempts=next_attempt; + m_DB->Write((char*)&m_DBRow+m_KeyDBOffset,m_KeyDBSize,(char*)&m_DBRow+m_ValueDBOffset,m_ValueDBSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); + m_DBRow.m_QueryNextAttempt=next_attempt; + m_DBRow.m_QueryAttempts=query_attempts; + } + return m_DB->Write((char*)&m_DBRow+m_KeyDBOffset,m_KeyDBSize,(char*)&m_DBRow+m_ValueDBOffset,m_ValueDBSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); } @@ -243,17 +291,20 @@ int mc_ChunkCollector::ReadFromDB(mc_Buffer *mempool,int rows) if(ptr) { memcpy((char*)&m_DBRow,ptr,m_TotalDBSize); - GetDBRow(&collect_row); - collect_row.m_State.m_Status |= MC_CCF_INSERTED; - if(m_ChunkDB->GetChunkDef(&chunk_def,collect_row.m_ChunkDef.m_Hash,&(collect_row.m_ChunkDef.m_Entity),collect_row.m_TxID,collect_row.m_Vout) == MC_ERR_NOERROR) - { - collect_row.m_State.m_Status |= MC_CCF_DELETED; - } - mprow=mempool->Seek(&collect_row); - if(mprow < 0) + if(m_DBRow.m_QueryNextAttempt != MC_IMPOSSIBLE_NEXT_ATTEMPT) { - mempool->Add(&collect_row); - row++; + GetDBRow(&collect_row); + collect_row.m_State.m_Status |= MC_CCF_INSERTED; + if(m_ChunkDB->GetChunkDef(&chunk_def,collect_row.m_ChunkDef.m_Hash,&(collect_row.m_ChunkDef.m_Entity),collect_row.m_TxID,collect_row.m_Vout) == MC_ERR_NOERROR) + { + collect_row.m_State.m_Status |= MC_CCF_DELETED; + } + mprow=mempool->Seek(&collect_row); + if(mprow < 0) + { + mempool->Add(&collect_row); + row++; + } } } else From 41adbdb6b1125e3874a9ffac6cf06dfbeefa57fc Mon Sep 17 00:00:00 2001 From: mike31 Date: Wed, 13 Mar 2019 15:26:52 +0200 Subject: [PATCH 10/17] Avoiding storing of collected chunks for purged items --- src/protocol/relay.cpp | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/protocol/relay.cpp b/src/protocol/relay.cpp index e56b38c1..99697ada 100644 --- a/src/protocol/relay.cpp +++ b/src/protocol/relay.cpp @@ -163,19 +163,22 @@ int MultichainProcessChunkResponse(const CRelayResponsePair *response_pair,map < strError="Chunk data hash mismatch"; goto exitlbl; } - chunk_err=pwalletTxsMain->m_ChunkDB->AddChunk(chunk->m_Hash,&(chunk->m_Entity),(unsigned char*)collect_row->m_TxID,collect_row->m_Vout,ptrOut,NULL,sizeOut,0,0); - if(chunk_err) + if( (collect_row->m_State.m_Status & MC_CCF_DELETED ) == 0 ) { - if(chunk_err != MC_ERR_FOUND) + chunk_err=pwalletTxsMain->m_ChunkDB->AddChunk(chunk->m_Hash,&(chunk->m_Entity),(unsigned char*)collect_row->m_TxID,collect_row->m_Vout,ptrOut,NULL,sizeOut,0,0); + if(chunk_err) { - strError=strprintf("Internal chunk DB error: %d",chunk_err); - goto exitlbl; + if(chunk_err != MC_ERR_FOUND) + { + strError=strprintf("Internal chunk DB error: %d",chunk_err); + goto exitlbl; + } + } + else + { + for(int k=0;k<2;k++)collector->m_StatTotal[k].m_Delivered+=k ? collect_row->m_ChunkDef.m_Size : 1; + LogPrint("chunks","Retrieved chunk %s\n",(*(uint256*)(chunk->m_Hash)).ToString().c_str()); } - } - else - { - for(int k=0;k<2;k++)collector->m_StatTotal[k].m_Delivered+=k ? collect_row->m_ChunkDef.m_Size : 1; - LogPrint("chunks","Retrieved chunk %s\n",(*(uint256*)(chunk->m_Hash)).ToString().c_str()); } collect_row->m_State.m_Status |= MC_CCF_DELETED; } From ce4dba349c041677d9e46f08fbde05994e9edc22 Mon Sep 17 00:00:00 2001 From: mike31 Date: Wed, 13 Mar 2019 17:27:03 +0200 Subject: [PATCH 11/17] Counting actually ourged chunks on unsubscribe --- src/wallet/chunkdb.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/wallet/chunkdb.cpp b/src/wallet/chunkdb.cpp index 5afac632..fc7c5a42 100644 --- a/src/wallet/chunkdb.cpp +++ b/src/wallet/chunkdb.cpp @@ -703,14 +703,14 @@ int mc_ChunkDB::RemoveEntityInternal(mc_TxEntity *entity,uint32_t *removed_chunk { chunk_found=1; memcpy(chunk_def.m_Hash,buf+param_value_start,bytes); - if(removed_chunks) - { - *removed_chunks+=1; - } } } break; case MC_ENT_SPRM_CHUNK_SIZE: + if(removed_chunks) + { + *removed_chunks+=1; + } if(removed_size) { *removed_size+=(uint32_t)mc_GetLE(buf+param_value_start,bytes); From ed49ca238d8184f59fac76067e1031a099c2ae1d Mon Sep 17 00:00:00 2001 From: mike31 Date: Thu, 14 Mar 2019 15:44:41 +0200 Subject: [PATCH 12/17] Setting not-in-sync flag for skipped indexes --- src/wallet/wallettxdb.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/wallet/wallettxdb.cpp b/src/wallet/wallettxdb.cpp index c4c35d5a..9148b99f 100644 --- a/src/wallet/wallettxdb.cpp +++ b/src/wallet/wallettxdb.cpp @@ -2611,7 +2611,9 @@ mc_TxImport *mc_TxDB::StartImport(mc_Buffer *lpEntities,int block,int *err) lpChainEntStat=m_Imports->GetEntity(row); if( (lpChainEntStat->m_Flags & MC_EFL_NOT_IN_SYNC) == 0 ) // in-sync rows ordered by timereceived should be copied { - if( (lpChainEntStat->m_Entity.m_EntityType & MC_TET_ORDERMASK) == MC_TET_TIMERECEIVED) + if( ((lpChainEntStat->m_Entity.m_EntityType & MC_TET_ORDERMASK) == MC_TET_TIMERECEIVED) && + (pEF->STR_IsIndexSkipped(NULL,NULL,&(lpChainEntStat->m_Entity)) == 0) && + (pEF->STR_IsIndexSkipped(m_Imports+slot,NULL,&(lpChainEntStat->m_Entity)) == 0) ) { lpEntStat->m_Flags -= MC_EFL_NOT_IN_SYNC; erow.Zero(); @@ -3018,7 +3020,10 @@ int mc_TxDB::CompleteImport(mc_TxImport *import,uint32_t flags) { if( (lpdel->m_Entity.IsSubscription() != 0) || ( (flags & MC_EFL_NOT_IN_SYNC_AFTER_IMPORT) == 0) ) { - lpdel->m_Flags-=MC_EFL_NOT_IN_SYNC; + if(pEF->STR_IsIndexSkipped(import,NULL,&(lpdel->m_Entity)) == 0) + { + lpdel->m_Flags-=MC_EFL_NOT_IN_SYNC; + } } } From 6e4eafcb173c04214d9dc792e48621db31f9129e Mon Sep 17 00:00:00 2001 From: mike31 Date: Thu, 14 Mar 2019 16:33:11 +0200 Subject: [PATCH 13/17] purgepublisheditems API --- src/community/community.cpp | 6 ++++++ src/community/community.h | 1 + src/rpc/rpcclient.cpp | 3 +++ src/rpc/rpclist.cpp | 1 + src/rpc/rpcserver.h | 1 + src/rpc/rpcstreams.cpp | 12 +++++++++++- 6 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/community/community.cpp b/src/community/community.cpp index 3c3ee0a9..63c7b5d9 100644 --- a/src/community/community.cpp +++ b/src/community/community.cpp @@ -71,6 +71,12 @@ Value mc_EnterpriseFeatures::STR_RPCPurgeStreamItems(const Array& params) return Value::null; } +Value mc_EnterpriseFeatures::STR_RPCPurgePublishedItems(const Array& params) +{ + return Value::null; +} + + int mc_EnterpriseFeatures::STR_RemoveDataFromFile(int fHan, uint32_t from, uint32_t size, uint32_t mode) { return MC_ERR_NOERROR; diff --git a/src/community/community.h b/src/community/community.h index e8cefb10..1540c53c 100644 --- a/src/community/community.h +++ b/src/community/community.h @@ -46,6 +46,7 @@ typedef struct mc_EnterpriseFeatures int STR_PutSubscriptions(mc_Buffer *subscriptions); Value STR_RPCRetrieveStreamItems(const Array& params); Value STR_RPCPurgeStreamItems(const Array& params); + Value STR_RPCPurgePublishedItems(const Array& params); int STR_RemoveDataFromFile(int fHan, uint32_t from, uint32_t size, uint32_t mode); int WLT_CreateSubscription(mc_TxEntity *entity,uint32_t retrieve,uint32_t indexes,uint32_t *rescan_mode); diff --git a/src/rpc/rpcclient.cpp b/src/rpc/rpcclient.cpp index c08f82bd..192fb11a 100644 --- a/src/rpc/rpcclient.cpp +++ b/src/rpc/rpcclient.cpp @@ -175,6 +175,7 @@ static const std::string vAPINames[] = "publishmulti", "publishmultifrom", "purgestreamitems", +"purgepublisheditems", "reconsiderblock", "resendwallettransactions", "resume", @@ -387,6 +388,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "trimsubscribe", 0 }, { "retrievestreamitems", 1 }, { "purgestreamitems", 1 }, + { "purgepublisheditems", 0 }, { "unsubscribe", 0 }, { "unsubscribe", 1 }, { "listassettransactions", 1 }, @@ -587,6 +589,7 @@ static const CRPCConvertParamMayBeString vRPCConvertParamsMayBeString[] = { "trimsubscribe", 0 }, { "retrievestreamitems", 1 }, { "purgestreamitems", 1 }, + { "purgepublisheditems", 0 }, { "unsubscribe", 0 }, { "liststreamkeys", 1 }, { "liststreampublishers", 1 }, diff --git a/src/rpc/rpclist.cpp b/src/rpc/rpclist.cpp index 24070856..aa5901e2 100644 --- a/src/rpc/rpclist.cpp +++ b/src/rpc/rpclist.cpp @@ -259,6 +259,7 @@ static const CRPCCommand vRPCCommands[] = { "wallet", "trimsubscribe", &trimsubscribe, false, false, true }, { "wallet", "retrievestreamitems", &retrievestreamitems, false, false, true }, { "wallet", "purgestreamitems", &purgestreamitems, false, false, true }, + { "wallet", "purgepublisheditems", &purgepublisheditems, false, false, true }, { "wallet", "listassettransactions", &listassettransactions, false, false, true }, { "wallet", "getassettransaction", &getassettransaction, false, false, true }, { "wallet", "getstreamitem", &getstreamitem, false, false, true }, diff --git a/src/rpc/rpcserver.h b/src/rpc/rpcserver.h index 0165de5e..53e68ae1 100644 --- a/src/rpc/rpcserver.h +++ b/src/rpc/rpcserver.h @@ -271,6 +271,7 @@ extern json_spirit::Value unsubscribe(const json_spirit::Array& params, bool fHe extern json_spirit::Value trimsubscribe(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value retrievestreamitems(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value purgestreamitems(const json_spirit::Array& params, bool fHelp); +extern json_spirit::Value purgepublisheditems(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value listassettransactions(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value getassettransaction(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value getstreamitem(const json_spirit::Array& params, bool fHelp); diff --git a/src/rpc/rpcstreams.cpp b/src/rpc/rpcstreams.cpp index e7c01fac..d8878ee2 100644 --- a/src/rpc/rpcstreams.cpp +++ b/src/rpc/rpcstreams.cpp @@ -2866,4 +2866,14 @@ Value purgestreamitems(const Array& params, bool fHelp) pEF->ENT_RPCVerifyEdition(); return pEF->STR_RPCPurgeStreamItems(params); -} \ No newline at end of file +} + +Value purgepublisheditems(const Array& params, bool fHelp) +{ + if (fHelp || params.size() != 1) + throw runtime_error("Help message not found\n"); + + pEF->ENT_RPCVerifyEdition(); + + return pEF->STR_RPCPurgePublishedItems(params); +} From 2faa07bcf511d6d99f9ef36c9a20b5f3b685dc3f Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 18 Mar 2019 14:31:26 +0200 Subject: [PATCH 14/17] Disabled Date.parse JS function and version 2.0 release --- src/chainparams/params.cpp | 27 +++++++++++++++++++++++++++ src/chainparams/state.h | 1 + src/filters/filter_win.cpp | 16 +++++++++++++++- src/v8/v8filter.cpp | 8 ++++++++ src/v8_win/v8_win.cpp | 8 ++++---- src/v8_win/v8_win.h | 10 +++++++--- src/v8_win/v8engine.cpp | 4 ++-- src/v8_win/v8engine.h | 2 +- src/v8_win/v8filter.cpp | 15 ++++++++++++--- src/v8_win/v8filter.h | 7 ++++++- src/version/version.cpp | 4 ++-- 11 files changed, 85 insertions(+), 17 deletions(-) diff --git a/src/chainparams/params.cpp b/src/chainparams/params.cpp index f162ab47..e53dcb9c 100644 --- a/src/chainparams/params.cpp +++ b/src/chainparams/params.cpp @@ -2272,5 +2272,32 @@ int mc_Features::FixedJSDateFunctions() return ret; } +int mc_Features::DisabledJSDateParse() +{ + int ret=0; + if(mc_gState->m_NetworkParams->IsProtocolMultichain() == 0) + { + return 0; + } + int protocol=mc_gState->m_NetworkParams->ProtocolVersion(); + + if(protocol) + { + if(protocol >= 20009) + { + ret=1; + } + else + { + if(Filters() == 0) + { + ret=1; + } + } + } + + return ret; +} + diff --git a/src/chainparams/state.h b/src/chainparams/state.h index a7a806cc..8f7c02fc 100644 --- a/src/chainparams/state.h +++ b/src/chainparams/state.h @@ -155,6 +155,7 @@ typedef struct mc_Features int ImplicitConnectPermission(); int LicenseTokens(); int FixedJSDateFunctions(); + int DisabledJSDateParse(); } mc_Features; typedef struct mc_BlockHeaderInfo diff --git a/src/filters/filter_win.cpp b/src/filters/filter_win.cpp index 42cdb29a..3cbca627 100644 --- a/src/filters/filter_win.cpp +++ b/src/filters/filter_win.cpp @@ -114,8 +114,22 @@ int mc_FilterEngine::CreateFilter(std::string script, std::string main_name, std const char **callbackNames = vec2cstrs(callback_names, n_callbackNames); auto v8filter = static_cast(filter->m_Impl); char result[RESULT_SIZE]; + uint32_t jsInjectionParams=0; + if(mc_gState->m_Features->FilterLimitedMathSet()) + { + jsInjectionParams |= MC_V8W_JS_INJECTION_LIMITED_MATH_SET; + } + if(mc_gState->m_Features->FixedJSDateFunctions()) + { + jsInjectionParams |= MC_V8W_JS_INJECTION_FIXED_DATE_FUNCTIONS; + } + if(mc_gState->m_Features->DisabledJSDateParse()) + { + jsInjectionParams |= MC_V8W_JS_INJECTION_DISABLED_DATE_PARSE; + } + retval = V8Engine_CreateFilter(v8engine, script.c_str(), main_name.c_str(), callbackNames, n_callbackNames, - v8filter, mc_gState->m_Features->FilterLimitedMathSet(), mc_gState->m_Features->FixedJSDateFunctions(), result); + v8filter, jsInjectionParams, result); delete [] callbackNames; if (fDebug) LogPrint("v8filter", "v8filter: retval=%d result=%s\n", retval, result); diff --git a/src/v8/v8filter.cpp b/src/v8/v8filter.cpp index 710a4462..18b5a990 100644 --- a/src/v8/v8filter.cpp +++ b/src/v8/v8filter.cpp @@ -171,6 +171,10 @@ for (var fn of Object.getOwnPropertyNames(Math)) { delete Date.now; )"; +static std::string jsDeleteDateParse = R"( +delete Date.parse; +)"; + V8Filter::~V8Filter() { if (m_isRunning) @@ -216,6 +220,10 @@ int V8Filter::Initialize(V8Engine *engine, std::string script, std::string funct { jsPreamble += jsLimitMathSet; } + if (mc_gState->m_Features->DisabledJSDateParse()) + { + jsPreamble += jsDeleteDateParse; + } int status = this->CompileAndLoadScript(jsPreamble, "", "preamble", strResult); if (status != MC_ERR_NOERROR || !strResult.empty()) diff --git a/src/v8_win/v8_win.cpp b/src/v8_win/v8_win.cpp index 91596c83..ae26b9d1 100644 --- a/src/v8_win/v8_win.cpp +++ b/src/v8_win/v8_win.cpp @@ -79,14 +79,14 @@ bool V8Filter_IsRunning(V8Filter_t *filter_) } int V8Filter_Initialize(V8Filter_t *filter_, V8Engine_t *engine_, const char *script_, const char *functionName_, - const char **callbackNames_, size_t nCallbackNames_, bool isFilterLimitedMathSet_, bool isFixedJSDateFunctions_, + const char **callbackNames_, size_t nCallbackNames_, uint32_t jsInjectionParams, char *strResult_) { auto filter = reinterpret_cast(filter_); auto engine = reinterpret_cast(engine_); std::vector callbackNames = cstrs2vec(callbackNames_, nCallbackNames_); std::string strResult; - int retval = filter->Initialize(engine, script_, functionName_, callbackNames, isFilterLimitedMathSet_, isFixedJSDateFunctions_, strResult); + int retval = filter->Initialize(engine, script_, functionName_, callbackNames, jsInjectionParams, strResult); strcpy_s(strResult_, RESULT_SIZE, strResult.c_str()); return retval; } @@ -127,13 +127,13 @@ int V8Engine_Initialize(V8Engine_t *engine_, IFilterCallback_t *filterCallback_, } int V8Engine_CreateFilter(V8Engine_t *engine_, const char *script_, const char *mainName_, const char **callbackNames_, - size_t nCallbackNames_, V8Filter_t *filter_, bool isFilterLimitedMathSet_, bool isFixedJSDateFunctions_, char *strResult_) + size_t nCallbackNames_, V8Filter_t *filter_, uint32_t jsInjectionParams, char *strResult_) { auto engine = reinterpret_cast(engine_); auto filter = reinterpret_cast(filter_); std::vector callbackNames = cstrs2vec(callbackNames_, nCallbackNames_); std::string strResult; - int retval = engine->CreateFilter(script_, mainName_, callbackNames, filter, isFilterLimitedMathSet_, isFixedJSDateFunctions_, strResult); + int retval = engine->CreateFilter(script_, mainName_, callbackNames, filter, jsInjectionParams, strResult); strcpy_s(strResult_, RESULT_SIZE, strResult.c_str()); return retval; } diff --git a/src/v8_win/v8_win.h b/src/v8_win/v8_win.h index 7c34fc53..f582894b 100644 --- a/src/v8_win/v8_win.h +++ b/src/v8_win/v8_win.h @@ -6,6 +6,11 @@ #include "declspec.h" +#define MC_V8W_JS_INJECTION_LIMITED_MATH_SET 0x00000001 +#define MC_V8W_JS_INJECTION_FIXED_DATE_FUNCTIONS 0x00000002 +#define MC_V8W_JS_INJECTION_DISABLED_DATE_PARSE 0x00000004 + + #ifdef __cplusplus extern "C" { #endif @@ -127,14 +132,13 @@ DLLEXPORT int V8Engine_Initialize(V8Engine_t* engine_, IFilterCallback_t* filter * If empty, register no callback functions. * @param nCallbackNames_ The number of callback names in @p callback_names_ * @param filter_ The filter object to initialize. - * @param isFilterLimitedMathSet_ @c true if JS Math functions have to be suppressed. - * @param isFixedJSDateFunctions_ @c true if limited set of JS Date functions is allowed. + * @param jsInjectionParams @c JC injection params. MC_V8W_JS_INJECTION constants * @param strResult_ Reason for failure if unsuccessful. * @return MC_ERR_INTERNAL_ERROR if the engine failed, MC_ERR_NOERROR otherwise. */ DLLEXPORT int V8Engine_CreateFilter(V8Engine_t* engine_, const char* script_, const char* mainName_, const char** callbackNames_, size_t nCallbackNames_, V8Filter_t* filter_, - bool isFilterLimitedMathSet_, bool FixedJSDateFunctions_, char* strResult_); + uint32_t jsInjectionParams, char* strResult_); /** * Run the filter function in the JS script. diff --git a/src/v8_win/v8engine.cpp b/src/v8_win/v8engine.cpp index 418e5917..0379a51a 100644 --- a/src/v8_win/v8engine.cpp +++ b/src/v8_win/v8engine.cpp @@ -49,11 +49,11 @@ int V8Engine::Initialize(IFilterCallback *filterCallback, std::string dataDir_, } int V8Engine::CreateFilter(std::string script, std::string mainName, const std::vector &callbackNames, - V8Filter *filter, bool isFilterLimitedMathSet, bool isFixedJSDateFunctions_, std::string &strResult) + V8Filter *filter, jsInjectionParams, std::string &strResult) { logger->debug("V8Engine::CreateFilter - enter"); strResult.clear(); - int retval = filter->Initialize(this, script, mainName, callbackNames, isFilterLimitedMathSet, isFixedJSDateFunctions_, strResult); + int retval = filter->Initialize(this, script, mainName, callbackNames, jsInjectionParams, strResult); logger->debug("V8Engine::CreateFilter - leave retval={} strResult='{}'", retval, strResult); return retval; } diff --git a/src/v8_win/v8engine.h b/src/v8_win/v8engine.h index 7fabe121..d2d2311c 100644 --- a/src/v8_win/v8engine.h +++ b/src/v8_win/v8engine.h @@ -49,7 +49,7 @@ class V8Engine * @return MC_ERR_INTERNAL_ERROR if the engine failed, MC_ERR_NOERROR otherwise. */ int CreateFilter(std::string script, std::string mainName, const std::vector &callbackNames, - V8Filter *filter, bool isFilterLimitedMathSet, bool isFixedJSDateFunctions_, std::string &strResult); + V8Filter *filter, uint32_t jsInjectionParams, std::string &strResult); /** * Run the filter function in the JS script. diff --git a/src/v8_win/v8filter.cpp b/src/v8_win/v8filter.cpp index 8d145cd2..cdded332 100644 --- a/src/v8_win/v8filter.cpp +++ b/src/v8_win/v8filter.cpp @@ -129,6 +129,11 @@ for (var fn of Object.getOwnPropertyNames(Math)) { } delete Date.now; )"; + +static std::string jsDeleteDateParse = R"( +delete Date.parse; +)"; + // clang-format on V8Filter::~V8Filter() @@ -142,7 +147,7 @@ V8Filter::~V8Filter() } int V8Filter::Initialize(V8Engine *engine, std::string script, std::string functionName, - const std::vector &callbackNames, bool isFilterLimitedMathSet, bool isFixedJSDateFunctions, + const std::vector &callbackNames, jsInjectionParams, std::string &strResult) { logger->debug("V8Filter::Initialize - enter"); @@ -174,14 +179,18 @@ int V8Filter::Initialize(V8Engine *engine, std::string script, std::string funct m_context.Reset(isolate, context); std::string jsPreamble = jsFixture; - if (isFixedJSDateFunctions) + if(jsInjectionParams & MC_V8W_JS_INJECTION_FIXED_DATE_FUNCTIONS) { jsPreamble=jsFixtureDateFunctions; } - if (isFilterLimitedMathSet) + if(jsInjectionParams & MC_V8W_JS_INJECTION_LIMITED_MATH_SET) { jsPreamble += jsLimitMathSet; } + if(jsInjectionParams & MC_V8W_JS_INJECTION_DISABLED_DATE_PARSE) + { + jsPreamble += jsDeleteDateParse; + } logger->debug(" Processing preamble"); int status = this->CompileAndLoadScript(jsPreamble, "", "preamble", strResult); diff --git a/src/v8_win/v8filter.h b/src/v8_win/v8filter.h index 8138bb44..c941ba79 100644 --- a/src/v8_win/v8filter.h +++ b/src/v8_win/v8filter.h @@ -7,6 +7,11 @@ #include "json/json_spirit.h" #include +#define MC_V8W_JS_INJECTION_LIMITED_MATH_SET 0x00000001 +#define MC_V8W_JS_INJECTION_FIXED_DATE_FUNCTIONS 0x00000002 +#define MC_V8W_JS_INJECTION_DISABLED_DATE_PARSE 0x00000004 + + namespace mc_v8 { class V8Engine; @@ -36,7 +41,7 @@ class V8Filter * MC_ERR_NOERROR otherwise. */ int Initialize(V8Engine *engine, std::string script, std::string functionName, - const std::vector &callbackNames, bool isFilterLimitedMathSet, bool isFixedJSDateFunctions, std::string &strResult); + const std::vector &callbackNames, uint32_t jsInjectionParams, std::string &strResult); /** * Run the filter function in the JS script. diff --git a/src/version/version.cpp b/src/version/version.cpp index 0363ebff..66a4eaed 100644 --- a/src/version/version.cpp +++ b/src/version/version.cpp @@ -13,8 +13,8 @@ int mc_State::VersionInfo(int version) return custom_version; } - int this_build=20000203; - int this_protocol=20008; + int this_build=20000901; + int this_protocol=20009; if(version < 0) { From d8a67678f43ba6f56750a58e69dd8310a3c1d340 Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 18 Mar 2019 14:47:44 +0200 Subject: [PATCH 15/17] Disabling Date.parse() for Windows --- src/v8_win/v8_win.cpp | 4 ++-- src/v8_win/v8_win.h | 4 ++-- src/v8_win/v8engine.cpp | 2 +- src/v8_win/v8engine.h | 2 +- src/v8_win/v8filter.cpp | 2 +- src/v8_win/v8filter.h | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/v8_win/v8_win.cpp b/src/v8_win/v8_win.cpp index ae26b9d1..de962eb0 100644 --- a/src/v8_win/v8_win.cpp +++ b/src/v8_win/v8_win.cpp @@ -79,7 +79,7 @@ bool V8Filter_IsRunning(V8Filter_t *filter_) } int V8Filter_Initialize(V8Filter_t *filter_, V8Engine_t *engine_, const char *script_, const char *functionName_, - const char **callbackNames_, size_t nCallbackNames_, uint32_t jsInjectionParams, + const char **callbackNames_, size_t nCallbackNames_, int jsInjectionParams, char *strResult_) { auto filter = reinterpret_cast(filter_); @@ -127,7 +127,7 @@ int V8Engine_Initialize(V8Engine_t *engine_, IFilterCallback_t *filterCallback_, } int V8Engine_CreateFilter(V8Engine_t *engine_, const char *script_, const char *mainName_, const char **callbackNames_, - size_t nCallbackNames_, V8Filter_t *filter_, uint32_t jsInjectionParams, char *strResult_) + size_t nCallbackNames_, V8Filter_t *filter_, int jsInjectionParams, char *strResult_) { auto engine = reinterpret_cast(engine_); auto filter = reinterpret_cast(filter_); diff --git a/src/v8_win/v8_win.h b/src/v8_win/v8_win.h index f582894b..8dc8b491 100644 --- a/src/v8_win/v8_win.h +++ b/src/v8_win/v8_win.h @@ -88,7 +88,7 @@ DLLEXPORT bool V8Filter_IsRunning(V8Filter_t* filter_); */ DLLEXPORT int V8Filter_Initialize(V8Filter_t* filter_, V8Engine_t* engine_, const char* script_, const char* functionName_, const char** callbackNames_, size_t nCallbackNames_, - bool isFilterLimitedMathSet_, bool isFixedJSDateFunctions_, char* strResult_); + int jsInjectionParams, char* strResult_); /** * Run the filter function in the JS script. @@ -138,7 +138,7 @@ DLLEXPORT int V8Engine_Initialize(V8Engine_t* engine_, IFilterCallback_t* filter */ DLLEXPORT int V8Engine_CreateFilter(V8Engine_t* engine_, const char* script_, const char* mainName_, const char** callbackNames_, size_t nCallbackNames_, V8Filter_t* filter_, - uint32_t jsInjectionParams, char* strResult_); + int jsInjectionParams, char* strResult_); /** * Run the filter function in the JS script. diff --git a/src/v8_win/v8engine.cpp b/src/v8_win/v8engine.cpp index 0379a51a..40fb0039 100644 --- a/src/v8_win/v8engine.cpp +++ b/src/v8_win/v8engine.cpp @@ -49,7 +49,7 @@ int V8Engine::Initialize(IFilterCallback *filterCallback, std::string dataDir_, } int V8Engine::CreateFilter(std::string script, std::string mainName, const std::vector &callbackNames, - V8Filter *filter, jsInjectionParams, std::string &strResult) + V8Filter *filter, int jsInjectionParams, std::string &strResult) { logger->debug("V8Engine::CreateFilter - enter"); strResult.clear(); diff --git a/src/v8_win/v8engine.h b/src/v8_win/v8engine.h index d2d2311c..bd14f7a2 100644 --- a/src/v8_win/v8engine.h +++ b/src/v8_win/v8engine.h @@ -49,7 +49,7 @@ class V8Engine * @return MC_ERR_INTERNAL_ERROR if the engine failed, MC_ERR_NOERROR otherwise. */ int CreateFilter(std::string script, std::string mainName, const std::vector &callbackNames, - V8Filter *filter, uint32_t jsInjectionParams, std::string &strResult); + V8Filter *filter, int jsInjectionParams, std::string &strResult); /** * Run the filter function in the JS script. diff --git a/src/v8_win/v8filter.cpp b/src/v8_win/v8filter.cpp index cdded332..89c394ff 100644 --- a/src/v8_win/v8filter.cpp +++ b/src/v8_win/v8filter.cpp @@ -147,7 +147,7 @@ V8Filter::~V8Filter() } int V8Filter::Initialize(V8Engine *engine, std::string script, std::string functionName, - const std::vector &callbackNames, jsInjectionParams, + const std::vector &callbackNames, int jsInjectionParams, std::string &strResult) { logger->debug("V8Filter::Initialize - enter"); diff --git a/src/v8_win/v8filter.h b/src/v8_win/v8filter.h index c941ba79..71e1f519 100644 --- a/src/v8_win/v8filter.h +++ b/src/v8_win/v8filter.h @@ -41,7 +41,7 @@ class V8Filter * MC_ERR_NOERROR otherwise. */ int Initialize(V8Engine *engine, std::string script, std::string functionName, - const std::vector &callbackNames, uint32_t jsInjectionParams, std::string &strResult); + const std::vector &callbackNames, int jsInjectionParams, std::string &strResult); /** * Run the filter function in the JS script. From 478bddbf7777f3a69bff20f8e8c7e46104db2145 Mon Sep 17 00:00:00 2001 From: mike31 Date: Tue, 19 Mar 2019 12:32:49 +0200 Subject: [PATCH 16/17] Adding multi-item chunk to the queue without checking --- src/wallet/chunkcollector.cpp | 9 ++-- src/wallet/chunkcollector.h | 1 + src/wallet/chunkdb.cpp | 88 ++++++++++++++++++++++------------- src/wallet/chunkdb.h | 11 ++++- 4 files changed, 71 insertions(+), 38 deletions(-) diff --git a/src/wallet/chunkcollector.cpp b/src/wallet/chunkcollector.cpp index f9cd2b06..2d0629ca 100644 --- a/src/wallet/chunkcollector.cpp +++ b/src/wallet/chunkcollector.cpp @@ -295,13 +295,14 @@ int mc_ChunkCollector::ReadFromDB(mc_Buffer *mempool,int rows) { GetDBRow(&collect_row); collect_row.m_State.m_Status |= MC_CCF_INSERTED; - if(m_ChunkDB->GetChunkDef(&chunk_def,collect_row.m_ChunkDef.m_Hash,&(collect_row.m_ChunkDef.m_Entity),collect_row.m_TxID,collect_row.m_Vout) == MC_ERR_NOERROR) - { - collect_row.m_State.m_Status |= MC_CCF_DELETED; - } mprow=mempool->Seek(&collect_row); if(mprow < 0) { + if(m_ChunkDB->GetChunkDefWithLimit(&chunk_def,collect_row.m_ChunkDef.m_Hash,&(collect_row.m_ChunkDef.m_Entity),collect_row.m_TxID,collect_row.m_Vout, + MC_CCW_MAX_ITEMS_PER_CHUNKFOR_CHECK) == MC_ERR_NOERROR) + { + collect_row.m_State.m_Status |= MC_CCF_DELETED; + } mempool->Add(&collect_row); row++; } diff --git a/src/wallet/chunkcollector.h b/src/wallet/chunkcollector.h index 49cbb953..a10dc971 100644 --- a/src/wallet/chunkcollector.h +++ b/src/wallet/chunkcollector.h @@ -29,6 +29,7 @@ #define MC_CCW_MAX_MBS_PER_SECOND 8 #define MC_CCW_MAX_DELAY_BETWEEN_COLLECTS 1000 #define MC_CCW_QUERY_SPLIT 4 +#define MC_CCW_MAX_ITEMS_PER_CHUNKFOR_CHECK 16 typedef struct mc_ChunkEntityKey diff --git a/src/wallet/chunkdb.cpp b/src/wallet/chunkdb.cpp index fc7c5a42..352bc6ab 100644 --- a/src/wallet/chunkdb.cpp +++ b/src/wallet/chunkdb.cpp @@ -1126,7 +1126,8 @@ int mc_ChunkDB::GetChunkDefInternal( const void *entity, const unsigned char *txid, const int vout, - int *mempool_row) + int *mempool_row, + int check_limit) { mc_SubscriptionDBRow *subscription; int err,value_len,mprow; @@ -1259,39 +1260,42 @@ int mc_ChunkDB::GetChunkDefInternal( { // ptr=GetChunkInternal(chunk_def,-1,-1,&bytes); total_items=chunk_def->m_ItemCount; - while(chunk_def->m_Pos < on_disk_items) + if( (check_limit == -1) || ((int)on_disk_items <= check_limit) ) { - if( (chunk_def->m_TxIDStart == 0) || (chunk_def->m_TxIDStart == (uint32_t)mc_GetLE((void*)txid,4))) + while(chunk_def->m_Pos < on_disk_items) { - ptr=GetChunkInternal(chunk_def,-1,-1,&bytes); - if(mc_ScriptMatchesTxIDAndVOut(ptr,bytes,txid,vout) == MC_ERR_NOERROR) + if( (chunk_def->m_TxIDStart == 0) || (chunk_def->m_TxIDStart == (uint32_t)mc_GetLE((void*)txid,4))) { - return MC_ERR_NOERROR; - } - } - chunk_def->m_Pos+=1; - - if(chunk_def->m_Pos < total_items) - { - chunk_def->SwapPosBytes(); - ptr=(unsigned char*)m_DB->Read((char*)chunk_def+m_KeyOffset,m_KeySize,&value_len,0,&err); - chunk_def->SwapPosBytes(); - if(err) - { - return err; - } - - if(ptr) - { - memcpy((char*)chunk_def+m_ValueOffset,ptr,m_ValueSize); -// ptr=GetChunkInternal(chunk_def,-1,-1,&bytes); + ptr=GetChunkInternal(chunk_def,-1,-1,&bytes); + if(mc_ScriptMatchesTxIDAndVOut(ptr,bytes,txid,vout) == MC_ERR_NOERROR) + { + return MC_ERR_NOERROR; + } } -/* - else + chunk_def->m_Pos+=1; + + if(chunk_def->m_Pos < total_items) { - chunk_def->m_Pos=on_disk_items; + chunk_def->SwapPosBytes(); + ptr=(unsigned char*)m_DB->Read((char*)chunk_def+m_KeyOffset,m_KeySize,&value_len,0,&err); + chunk_def->SwapPosBytes(); + if(err) + { + return err; + } + + if(ptr) + { + memcpy((char*)chunk_def+m_ValueOffset,ptr,m_ValueSize); + // ptr=GetChunkInternal(chunk_def,-1,-1,&bytes); + } + /* + else + { + chunk_def->m_Pos=on_disk_items; + } + */ } - */ } } err=MC_ERR_NOT_FOUND; @@ -1348,12 +1352,30 @@ int mc_ChunkDB::GetChunkDef( int err; Lock(); - err=GetChunkDefInternal(chunk_def,hash,entity,txid,vout,NULL); + err=GetChunkDefInternal(chunk_def,hash,entity,txid,vout,NULL,-1); + UnLock(); + + return err; +} + +int mc_ChunkDB::GetChunkDefWithLimit( + mc_ChunkDBRow *chunk_def, + const unsigned char *hash, + const void *entity, + const unsigned char *txid, + const int vout, + int check_limit) +{ + int err; + + Lock(); + err=GetChunkDefInternal(chunk_def,hash,entity,txid,vout,NULL,check_limit); UnLock(); return err; } + int mc_ChunkDB::AddChunkInternal( const unsigned char *hash, const mc_TxEntity *entity, @@ -1385,7 +1407,7 @@ int mc_ChunkDB::AddChunkInternal( chunk_def.Zero(); - err=GetChunkDefInternal(&chunk_def,hash,entity,txid,vout,NULL); + err=GetChunkDefInternal(&chunk_def,hash,entity,txid,vout,NULL,-1); if(err == MC_ERR_NOERROR) { return MC_ERR_FOUND; @@ -1421,7 +1443,7 @@ int mc_ChunkDB::AddChunkInternal( if(txid) { - err=GetChunkDefInternal(&entity_chunk_def,hash,entity,NULL,-1,&mempool_entity_row); + err=GetChunkDefInternal(&entity_chunk_def,hash,entity,NULL,-1,&mempool_entity_row,-1); if(err == MC_ERR_NOERROR) { total_items=entity_chunk_def.m_ItemCount; @@ -1448,7 +1470,7 @@ int mc_ChunkDB::AddChunkInternal( if(add_entity_row) { - err=GetChunkDefInternal(&null_chunk_def,hash,NULL,NULL,-1,&mempool_last_null_row); + err=GetChunkDefInternal(&null_chunk_def,hash,NULL,NULL,-1,&mempool_last_null_row,-1); if(err == MC_ERR_NOT_FOUND) { add_null_row=1; @@ -1655,7 +1677,7 @@ unsigned char *mc_ChunkDB::GetChunkInternal(mc_ChunkDBRow *chunk_def, } subscription=(mc_SubscriptionDBRow *)m_Subscriptions->GetRow(subscription_id); - if(GetChunkDefInternal(&chunk_def_zero,chunk_def->m_Hash,&(subscription->m_Entity),NULL,0,NULL) == MC_ERR_NOERROR) + if(GetChunkDefInternal(&chunk_def_zero,chunk_def->m_Hash,&(subscription->m_Entity),NULL,0,NULL,-1) == MC_ERR_NOERROR) { return GetChunkInternal(&chunk_def_zero,offset,len,bytes); } diff --git a/src/wallet/chunkdb.h b/src/wallet/chunkdb.h index e9594538..aae14d22 100644 --- a/src/wallet/chunkdb.h +++ b/src/wallet/chunkdb.h @@ -195,7 +195,8 @@ typedef struct mc_ChunkDB const void *entity, const unsigned char *txid, const int vout, - int *mempool_entity_row); + int *mempool_entity_row, + int check_limit); int GetChunkDef( @@ -205,6 +206,14 @@ typedef struct mc_ChunkDB const unsigned char *txid, const int vout); + int GetChunkDefWithLimit( + mc_ChunkDBRow *chunk_def, + const unsigned char *hash, // Chunk hash (before chopping) + const void *entity, + const unsigned char *txid, + const int vout, + int check_limit); + unsigned char *GetChunkInternal(mc_ChunkDBRow *chunk_def, int32_t offset, int32_t len, From cbcc4eca8493e7615db2ca0ebff45cad54bbd457 Mon Sep 17 00:00:00 2001 From: mike31 Date: Thu, 21 Mar 2019 09:41:17 +0200 Subject: [PATCH 17/17] Fixed old style stream open flag processing --- src/chainparams/params.cpp | 27 +++++++++++++++++++++++++++ src/chainparams/state.h | 1 + src/entities/asset.cpp | 24 ++++++++++++------------ 3 files changed, 40 insertions(+), 12 deletions(-) diff --git a/src/chainparams/params.cpp b/src/chainparams/params.cpp index e53dcb9c..a6f1e95e 100644 --- a/src/chainparams/params.cpp +++ b/src/chainparams/params.cpp @@ -2299,5 +2299,32 @@ int mc_Features::DisabledJSDateParse() return ret; } +int mc_Features::FixedLegacyPermissionRestrictionFlag() +{ + int ret=0; + if(mc_gState->m_NetworkParams->IsProtocolMultichain() == 0) + { + return 0; + } + int protocol=mc_gState->m_NetworkParams->ProtocolVersion(); + + if(protocol) + { + if(protocol >= 20009) + { + ret=1; + } + else + { + if(Filters() == 0) + { + ret=1; + } + } + } + + return ret; +} + diff --git a/src/chainparams/state.h b/src/chainparams/state.h index 8f7c02fc..a06a583d 100644 --- a/src/chainparams/state.h +++ b/src/chainparams/state.h @@ -156,6 +156,7 @@ typedef struct mc_Features int LicenseTokens(); int FixedJSDateFunctions(); int DisabledJSDateParse(); + int FixedLegacyPermissionRestrictionFlag(); } mc_Features; typedef struct mc_BlockHeaderInfo diff --git a/src/entities/asset.cpp b/src/entities/asset.cpp index 895c558f..5106cc4a 100644 --- a/src/entities/asset.cpp +++ b/src/entities/asset.cpp @@ -661,26 +661,26 @@ void mc_EntityDetails::Set(mc_EntityLedgerRow* row) value_offset=mc_FindSpecialParamInDetailsScript(m_LedgerRow.m_Script,m_LedgerRow.m_ScriptSize,MC_ENT_SPRM_PERMISSIONS,&value_size); if(value_offset <= m_LedgerRow.m_ScriptSize) { -/* - if(m_Permissions & MC_PTP_WRITE) + if( (value_offset != m_LedgerRow.m_ScriptSize) || (mc_gState->m_Features->FixedLegacyPermissionRestrictionFlag() == 0)) { - m_Permissions -= MC_PTP_WRITE; - } - */ - m_Permissions |= MC_PTP_SPECIFIED; - if((value_size>0) && (value_size<=4)) - { - m_ScriptPermissions=(uint32_t)mc_GetLE(m_LedgerRow.m_Script+value_offset,value_size); - m_Permissions |= m_ScriptPermissions; + m_Permissions |= MC_PTP_SPECIFIED; + if((value_size>0) && (value_size<=4)) + { + m_ScriptPermissions=(uint32_t)mc_GetLE(m_LedgerRow.m_Script+value_offset,value_size); + m_Permissions |= m_ScriptPermissions; + } } } value_offset=mc_FindSpecialParamInDetailsScript(m_LedgerRow.m_Script,m_LedgerRow.m_ScriptSize,MC_ENT_SPRM_RESTRICTIONS,&value_size); if(value_offset <= m_LedgerRow.m_ScriptSize) { - if((value_size>0) && (value_size<=4)) + if( (value_offset != m_LedgerRow.m_ScriptSize) || (mc_gState->m_Features->FixedLegacyPermissionRestrictionFlag() == 0)) { - m_Restrictions |= (uint32_t)mc_GetLE(m_LedgerRow.m_Script+value_offset,value_size); + if((value_size>0) && (value_size<=4)) + { + m_Restrictions |= (uint32_t)mc_GetLE(m_LedgerRow.m_Script+value_offset,value_size); + } } }