diff --git a/src/chainparams/params.cpp b/src/chainparams/params.cpp index f162ab47..a6f1e95e 100644 --- a/src/chainparams/params.cpp +++ b/src/chainparams/params.cpp @@ -2272,5 +2272,59 @@ int mc_Features::FixedJSDateFunctions() return ret; } +int mc_Features::DisabledJSDateParse() +{ + int ret=0; + if(mc_gState->m_NetworkParams->IsProtocolMultichain() == 0) + { + return 0; + } + int protocol=mc_gState->m_NetworkParams->ProtocolVersion(); + + if(protocol) + { + if(protocol >= 20009) + { + ret=1; + } + else + { + if(Filters() == 0) + { + ret=1; + } + } + } + + return ret; +} + +int mc_Features::FixedLegacyPermissionRestrictionFlag() +{ + int ret=0; + if(mc_gState->m_NetworkParams->IsProtocolMultichain() == 0) + { + return 0; + } + int protocol=mc_gState->m_NetworkParams->ProtocolVersion(); + + if(protocol) + { + if(protocol >= 20009) + { + ret=1; + } + else + { + if(Filters() == 0) + { + ret=1; + } + } + } + + return ret; +} + diff --git a/src/chainparams/state.h b/src/chainparams/state.h index a7a806cc..a06a583d 100644 --- a/src/chainparams/state.h +++ b/src/chainparams/state.h @@ -155,6 +155,8 @@ typedef struct mc_Features int ImplicitConnectPermission(); int LicenseTokens(); int FixedJSDateFunctions(); + int DisabledJSDateParse(); + int FixedLegacyPermissionRestrictionFlag(); } mc_Features; typedef struct mc_BlockHeaderInfo diff --git a/src/community/community.cpp b/src/community/community.cpp index d07be507..63c7b5d9 100644 --- a/src/community/community.cpp +++ b/src/community/community.cpp @@ -26,11 +26,21 @@ int mc_EnterpriseFeatures::STR_CreateSubscription(mc_TxEntity *entity,const std: return MC_ERR_FOUND; } +int mc_EnterpriseFeatures::STR_TrimSubscription(mc_TxEntity *entity,const std::string parameters) +{ + return MC_ERR_NOERROR; +} + int mc_EnterpriseFeatures::STR_IsIndexSkipped(mc_TxImport *import,mc_TxEntity *parent_entity,mc_TxEntity *entity) { return 0; } +int mc_EnterpriseFeatures::STR_NoRetrieve(mc_TxEntity *entity) +{ + return 0; +} + int mc_EnterpriseFeatures::STR_IsOutOfSync(mc_TxEntity *entity) { return 0; @@ -51,6 +61,28 @@ int mc_EnterpriseFeatures::STR_PutSubscriptions(mc_Buffer *subscriptions) return MC_ERR_NOERROR; } +Value mc_EnterpriseFeatures::STR_RPCRetrieveStreamItems(const Array& params) +{ + return Value::null; +} + +Value mc_EnterpriseFeatures::STR_RPCPurgeStreamItems(const Array& params) +{ + return Value::null; +} + +Value mc_EnterpriseFeatures::STR_RPCPurgePublishedItems(const Array& params) +{ + return Value::null; +} + + +int mc_EnterpriseFeatures::STR_RemoveDataFromFile(int fHan, uint32_t from, uint32_t size, uint32_t mode) +{ + return MC_ERR_NOERROR; +} + + int mc_EnterpriseFeatures::WLT_CreateSubscription(mc_TxEntity *entity,uint32_t retrieve,uint32_t indexes,uint32_t *rescan_mode) { *rescan_mode=0; @@ -107,6 +139,12 @@ std::string mc_EnterpriseFeatures::ENT_TextConstant(const char* name) return ""; } +void mc_EnterpriseFeatures::ENT_InitRPCHelpMap() +{ + +} + + void mc_EnterpriseFeatures::LIC_RPCVerifyFeature(uint64_t feature) { throw JSONRPCError(RPC_NOT_SUPPORTED, "This feature is available only in Enterprise edition of MultiChain, please call \"help enterprise\" for details"); diff --git a/src/community/community.h b/src/community/community.h index 7c6aab8b..1540c53c 100644 --- a/src/community/community.h +++ b/src/community/community.h @@ -37,11 +37,17 @@ typedef struct mc_EnterpriseFeatures uint32_t mode); // Unused int STR_CreateSubscription(mc_TxEntity *entity,const std::string parameters); + int STR_TrimSubscription(mc_TxEntity *entity,const std::string parameters); int STR_IsIndexSkipped(mc_TxImport *import,mc_TxEntity *parent_entity,mc_TxEntity *entity); + int STR_NoRetrieve(mc_TxEntity *entity); int STR_IsOutOfSync(mc_TxEntity *entity); int STR_SetSyncFlag(mc_TxEntity *entity,bool confirm); int STR_GetSubscriptions(mc_Buffer *subscriptions); int STR_PutSubscriptions(mc_Buffer *subscriptions); + Value STR_RPCRetrieveStreamItems(const Array& params); + Value STR_RPCPurgeStreamItems(const Array& params); + Value STR_RPCPurgePublishedItems(const Array& params); + int STR_RemoveDataFromFile(int fHan, uint32_t from, uint32_t size, uint32_t mode); int WLT_CreateSubscription(mc_TxEntity *entity,uint32_t retrieve,uint32_t indexes,uint32_t *rescan_mode); int WLT_DeleteSubscription(mc_TxEntity *entity,uint32_t rescan_mode); @@ -55,6 +61,7 @@ typedef struct mc_EnterpriseFeatures int ENT_MinWalletDatVersion(); void ENT_RPCVerifyEdition(); std::string ENT_TextConstant(const char* name); + void ENT_InitRPCHelpMap(); void LIC_RPCVerifyFeature(uint64_t feature); bool LIC_VerifyFeature(uint64_t feature,std::string& reason); diff --git a/src/core/init.cpp b/src/core/init.cpp index eedef9c5..378d10e1 100644 --- a/src/core/init.cpp +++ b/src/core/init.cpp @@ -424,7 +424,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += " " + _("(more details and % substitutions online)") + "\n"; /* MCHN START */ strUsage += " -walletdbversion=2|3 " + _("Specify wallet version, 2 - Berkeley DB, 3 (default) - proprietary") + "\n"; - strUsage += " -autosubscribe=streams|assets|\"streams,assets\"|\"assets,streams\" " + _("Automatically subscribe to new streams and/or assets") + "\n"; + strUsage += " -autosubscribe=streams|assets|\"streams,assets\"|\"assets,streams\"|\"\" " + _("Automatically subscribe to new streams and/or assets") + "\n"; /* MCHN END */ strUsage += " -zapwallettxes= " + _("Delete all wallet transactions and only recover those parts of the blockchain through -rescan on startup") + "\n"; strUsage += " " + _("(1 = keep tx meta data e.g. account owner and payment request information, 2 = drop tx meta data)") + "\n"; diff --git a/src/entities/asset.cpp b/src/entities/asset.cpp index 895c558f..5106cc4a 100644 --- a/src/entities/asset.cpp +++ b/src/entities/asset.cpp @@ -661,26 +661,26 @@ void mc_EntityDetails::Set(mc_EntityLedgerRow* row) value_offset=mc_FindSpecialParamInDetailsScript(m_LedgerRow.m_Script,m_LedgerRow.m_ScriptSize,MC_ENT_SPRM_PERMISSIONS,&value_size); if(value_offset <= m_LedgerRow.m_ScriptSize) { -/* - if(m_Permissions & MC_PTP_WRITE) + if( (value_offset != m_LedgerRow.m_ScriptSize) || (mc_gState->m_Features->FixedLegacyPermissionRestrictionFlag() == 0)) { - m_Permissions -= MC_PTP_WRITE; - } - */ - m_Permissions |= MC_PTP_SPECIFIED; - if((value_size>0) && (value_size<=4)) - { - m_ScriptPermissions=(uint32_t)mc_GetLE(m_LedgerRow.m_Script+value_offset,value_size); - m_Permissions |= m_ScriptPermissions; + m_Permissions |= MC_PTP_SPECIFIED; + if((value_size>0) && (value_size<=4)) + { + m_ScriptPermissions=(uint32_t)mc_GetLE(m_LedgerRow.m_Script+value_offset,value_size); + m_Permissions |= m_ScriptPermissions; + } } } value_offset=mc_FindSpecialParamInDetailsScript(m_LedgerRow.m_Script,m_LedgerRow.m_ScriptSize,MC_ENT_SPRM_RESTRICTIONS,&value_size); if(value_offset <= m_LedgerRow.m_ScriptSize) { - if((value_size>0) && (value_size<=4)) + if( (value_offset != m_LedgerRow.m_ScriptSize) || (mc_gState->m_Features->FixedLegacyPermissionRestrictionFlag() == 0)) { - m_Restrictions |= (uint32_t)mc_GetLE(m_LedgerRow.m_Script+value_offset,value_size); + if((value_size>0) && (value_size<=4)) + { + m_Restrictions |= (uint32_t)mc_GetLE(m_LedgerRow.m_Script+value_offset,value_size); + } } } diff --git a/src/filters/filter_win.cpp b/src/filters/filter_win.cpp index 42cdb29a..3cbca627 100644 --- a/src/filters/filter_win.cpp +++ b/src/filters/filter_win.cpp @@ -114,8 +114,22 @@ int mc_FilterEngine::CreateFilter(std::string script, std::string main_name, std const char **callbackNames = vec2cstrs(callback_names, n_callbackNames); auto v8filter = static_cast(filter->m_Impl); char result[RESULT_SIZE]; + uint32_t jsInjectionParams=0; + if(mc_gState->m_Features->FilterLimitedMathSet()) + { + jsInjectionParams |= MC_V8W_JS_INJECTION_LIMITED_MATH_SET; + } + if(mc_gState->m_Features->FixedJSDateFunctions()) + { + jsInjectionParams |= MC_V8W_JS_INJECTION_FIXED_DATE_FUNCTIONS; + } + if(mc_gState->m_Features->DisabledJSDateParse()) + { + jsInjectionParams |= MC_V8W_JS_INJECTION_DISABLED_DATE_PARSE; + } + retval = V8Engine_CreateFilter(v8engine, script.c_str(), main_name.c_str(), callbackNames, n_callbackNames, - v8filter, mc_gState->m_Features->FilterLimitedMathSet(), mc_gState->m_Features->FixedJSDateFunctions(), result); + v8filter, jsInjectionParams, result); delete [] callbackNames; if (fDebug) LogPrint("v8filter", "v8filter: retval=%d result=%s\n", retval, result); diff --git a/src/protocol/relay.cpp b/src/protocol/relay.cpp index e56b38c1..99697ada 100644 --- a/src/protocol/relay.cpp +++ b/src/protocol/relay.cpp @@ -163,19 +163,22 @@ int MultichainProcessChunkResponse(const CRelayResponsePair *response_pair,map < strError="Chunk data hash mismatch"; goto exitlbl; } - chunk_err=pwalletTxsMain->m_ChunkDB->AddChunk(chunk->m_Hash,&(chunk->m_Entity),(unsigned char*)collect_row->m_TxID,collect_row->m_Vout,ptrOut,NULL,sizeOut,0,0); - if(chunk_err) + if( (collect_row->m_State.m_Status & MC_CCF_DELETED ) == 0 ) { - if(chunk_err != MC_ERR_FOUND) + chunk_err=pwalletTxsMain->m_ChunkDB->AddChunk(chunk->m_Hash,&(chunk->m_Entity),(unsigned char*)collect_row->m_TxID,collect_row->m_Vout,ptrOut,NULL,sizeOut,0,0); + if(chunk_err) { - strError=strprintf("Internal chunk DB error: %d",chunk_err); - goto exitlbl; + if(chunk_err != MC_ERR_FOUND) + { + strError=strprintf("Internal chunk DB error: %d",chunk_err); + goto exitlbl; + } + } + else + { + for(int k=0;k<2;k++)collector->m_StatTotal[k].m_Delivered+=k ? collect_row->m_ChunkDef.m_Size : 1; + LogPrint("chunks","Retrieved chunk %s\n",(*(uint256*)(chunk->m_Hash)).ToString().c_str()); } - } - else - { - for(int k=0;k<2;k++)collector->m_StatTotal[k].m_Delivered+=k ? collect_row->m_ChunkDef.m_Size : 1; - LogPrint("chunks","Retrieved chunk %s\n",(*(uint256*)(chunk->m_Hash)).ToString().c_str()); } collect_row->m_State.m_Status |= MC_CCF_DELETED; } diff --git a/src/rpc/rpcclient.cpp b/src/rpc/rpcclient.cpp index 2efbfc95..192fb11a 100644 --- a/src/rpc/rpcclient.cpp +++ b/src/rpc/rpcclient.cpp @@ -174,9 +174,12 @@ static const std::string vAPINames[] = "publishfrom", "publishmulti", "publishmultifrom", +"purgestreamitems", +"purgepublisheditems", "reconsiderblock", "resendwallettransactions", "resume", +"retrievestreamitems", "revoke", "revokefrom", "runstreamfilter", @@ -210,6 +213,7 @@ static const std::string vAPINames[] = "subscribe", "teststreamfilter", "testtxfilter", +"trimsubscribe", "unsubscribe", "validateaddress", "verifychain", @@ -381,6 +385,10 @@ static const CRPCConvertParam vRPCConvertParams[] = { "create", 3 }, { "subscribe", 0 }, { "subscribe", 1 }, + { "trimsubscribe", 0 }, + { "retrievestreamitems", 1 }, + { "purgestreamitems", 1 }, + { "purgepublisheditems", 0 }, { "unsubscribe", 0 }, { "unsubscribe", 1 }, { "listassettransactions", 1 }, @@ -578,6 +586,10 @@ static const CRPCConvertParamMayBeString vRPCConvertParamsMayBeString[] = { "importaddress", 0 }, { "importprivkey", 0 }, { "subscribe", 0 }, + { "trimsubscribe", 0 }, + { "retrievestreamitems", 1 }, + { "purgestreamitems", 1 }, + { "purgepublisheditems", 0 }, { "unsubscribe", 0 }, { "liststreamkeys", 1 }, { "liststreampublishers", 1 }, diff --git a/src/rpc/rpchelp.cpp b/src/rpc/rpchelp.cpp index fb5a5382..b71c8001 100644 --- a/src/rpc/rpchelp.cpp +++ b/src/rpc/rpchelp.cpp @@ -4757,6 +4757,8 @@ void mc_InitRPCHelpMap() mc_InitRPCHelpMap19(); mc_InitRPCHelpMap20(); + pEF->ENT_InitRPCHelpMap(); + mc_InitRPCLogParamCountMap(); mc_InitRPCAllowedWhenWaitingForUpgradeSet(); mc_InitRPCAllowedWhenOffline(); diff --git a/src/rpc/rpclist.cpp b/src/rpc/rpclist.cpp index fa3f5caf..aa5901e2 100644 --- a/src/rpc/rpclist.cpp +++ b/src/rpc/rpclist.cpp @@ -256,6 +256,10 @@ static const CRPCCommand vRPCCommands[] = { "wallet", "publishmultifrom", &publishmultifrom, false, false, true }, { "wallet", "subscribe", &subscribe, false, false, true }, { "wallet", "unsubscribe", &unsubscribe, false, false, true }, + { "wallet", "trimsubscribe", &trimsubscribe, false, false, true }, + { "wallet", "retrievestreamitems", &retrievestreamitems, false, false, true }, + { "wallet", "purgestreamitems", &purgestreamitems, false, false, true }, + { "wallet", "purgepublisheditems", &purgepublisheditems, false, false, true }, { "wallet", "listassettransactions", &listassettransactions, false, false, true }, { "wallet", "getassettransaction", &getassettransaction, false, false, true }, { "wallet", "getstreamitem", &getstreamitem, false, false, true }, diff --git a/src/rpc/rpcmisc.cpp b/src/rpc/rpcmisc.cpp index df827d76..dd370cdf 100644 --- a/src/rpc/rpcmisc.cpp +++ b/src/rpc/rpcmisc.cpp @@ -510,7 +510,11 @@ Value setruntimeparam(const json_spirit::Array& params, bool fHelp) mode |= MC_WMD_AUTOSUBSCRIBE_STREAMS; mode |= MC_WMD_AUTOSUBSCRIBE_ASSETS; found=true; - } + } + if( autosubscribe=="" ) + { + found=true; + } if(!found) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter value"); diff --git a/src/rpc/rpcserver.cpp b/src/rpc/rpcserver.cpp index 828dd49d..ac9b460f 100644 --- a/src/rpc/rpcserver.cpp +++ b/src/rpc/rpcserver.cpp @@ -251,7 +251,10 @@ string CRPCTable::help(string strCommand) const } catch (std::exception& e) { - strHelp = string(e.what()); + if(strCommand != "") + { + strHelp = string(e.what()); + } } } else diff --git a/src/rpc/rpcserver.h b/src/rpc/rpcserver.h index c44504d2..53e68ae1 100644 --- a/src/rpc/rpcserver.h +++ b/src/rpc/rpcserver.h @@ -268,6 +268,10 @@ extern json_spirit::Value publishmulti(const json_spirit::Array& params, bool fH extern json_spirit::Value publishmultifrom(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value subscribe(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value unsubscribe(const json_spirit::Array& params, bool fHelp); +extern json_spirit::Value trimsubscribe(const json_spirit::Array& params, bool fHelp); +extern json_spirit::Value retrievestreamitems(const json_spirit::Array& params, bool fHelp); +extern json_spirit::Value purgestreamitems(const json_spirit::Array& params, bool fHelp); +extern json_spirit::Value purgepublisheditems(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value listassettransactions(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value getassettransaction(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value getstreamitem(const json_spirit::Array& params, bool fHelp); diff --git a/src/rpc/rpcstreams.cpp b/src/rpc/rpcstreams.cpp index 917fcd1c..d8878ee2 100644 --- a/src/rpc/rpcstreams.cpp +++ b/src/rpc/rpcstreams.cpp @@ -203,7 +203,7 @@ Value liststreams(const Array& params, bool fHelp) { if(paramtobool(params[1])) { - output_level=0x3E; + output_level=0xBE; if(mc_gState->m_Features->StreamFilters()) { output_level |= 0x40; @@ -1053,6 +1053,50 @@ Value publishfrom(const Array& params, bool fHelp) return wtx.GetHash().GetHex(); } +Value trimsubscribe(const Array& params, bool fHelp) +{ + if (fHelp || params.size() != 2) + throw runtime_error("Help message not found\n"); + + pEF->ENT_RPCVerifyEdition(); + + string indexes=params[1].get_str(); + + vector inputEntities; + vector inputStrings; + if(params[0].type() == str_type) + { + inputStrings.push_back(params[0].get_str()); + } + else + { + inputStrings=ParseStringList(params[0]); + } + + for(int is=0;is<(int)inputStrings.size();is++) + { + mc_EntityDetails entity_to_subscribe; + Value param=inputStrings[is]; + ParseEntityIdentifier(param,&entity_to_subscribe, MC_ENT_TYPE_STREAM); + inputEntities.push_back(entity_to_subscribe); + } + + for(int is=0;is<(int)inputStrings.size();is++) + { + mc_EntityDetails* lpEntity; + lpEntity=&inputEntities[is]; + + mc_TxEntity entity; + entity.Zero(); + memcpy(entity.m_EntityID,lpEntity->GetTxID()+MC_AST_SHORT_TXID_OFFSET,MC_AST_SHORT_TXID_SIZE); + entity.m_EntityType=MC_TET_STREAM | MC_TET_CHAINPOS; + pEF->STR_TrimSubscription(&entity,indexes); + } + + return Value::null; +} + + Value subscribe(const Array& params, bool fHelp) { if (fHelp || params.size() < 1 || params.size() > ((pEF->ENT_EditionNumeric() == 0) ? 2 : 3)) @@ -2591,7 +2635,7 @@ int GetAndQueryDirtyList(vector& conditions, mc_EntityDetails return dirty_count; } -void FillContitionsList(vector& conditions, Value param) +void FillConditionsList(vector& conditions, Value param) { bool key_found=false; bool publisher_found=false; @@ -2733,7 +2777,7 @@ Value liststreamqueryitems(const Array& params, bool fHelp) verbose=paramtobool(params[2]); } - FillContitionsList(conditions,params[1]); + FillConditionsList(conditions,params[1]); if(conditions.size() == 0) { @@ -2803,3 +2847,33 @@ Value liststreamqueryitems(const Array& params, bool fHelp) return retArray; } + +Value retrievestreamitems(const Array& params, bool fHelp) +{ + if (fHelp || params.size() != 2) + throw runtime_error("Help message not found\n"); + + pEF->ENT_RPCVerifyEdition(); + + return pEF->STR_RPCRetrieveStreamItems(params); +} + +Value purgestreamitems(const Array& params, bool fHelp) +{ + if (fHelp || params.size() != 2) + throw runtime_error("Help message not found\n"); + + pEF->ENT_RPCVerifyEdition(); + + return pEF->STR_RPCPurgeStreamItems(params); +} + +Value purgepublisheditems(const Array& params, bool fHelp) +{ + if (fHelp || params.size() != 1) + throw runtime_error("Help message not found\n"); + + pEF->ENT_RPCVerifyEdition(); + + return pEF->STR_RPCPurgePublishedItems(params); +} diff --git a/src/rpc/rpcutils.cpp b/src/rpc/rpcutils.cpp index f585308b..da2c979f 100644 --- a/src/rpc/rpcutils.cpp +++ b/src/rpc/rpcutils.cpp @@ -13,6 +13,7 @@ #include "utils/utilmoneystr.h" #include "wallet/wallettxs.h" #include "json/json_spirit_ubjson.h" +#include "community/community.h" #include @@ -797,6 +798,7 @@ Object StreamEntry(const unsigned char *txid,uint32_t output_level,mc_EntityDeta // 0x0010 stats // 0x0020 creators // 0x0040 filters +// 0x0080 subscription // 0x0800 skip name and ref Object entry; @@ -984,14 +986,56 @@ Object StreamEntry(const unsigned char *txid,uint32_t output_level,mc_EntityDeta { if(output_level & 0x0008) { + vector> index_types; + index_types.push_back(pair("items",MC_TET_STREAM | MC_TET_CHAINPOS)); + index_types.push_back(pair("keys",MC_TET_STREAM_KEY | MC_TET_CHAINPOS)); + index_types.push_back(pair("publishers",MC_TET_STREAM_PUBLISHER | MC_TET_CHAINPOS)); + index_types.push_back(pair("items-local",MC_TET_STREAM | MC_TET_TIMERECEIVED)); + index_types.push_back(pair("keys-local",MC_TET_STREAM_KEY | MC_TET_TIMERECEIVED)); + index_types.push_back(pair("publishers-local",MC_TET_STREAM_PUBLISHER | MC_TET_TIMERECEIVED)); entry.push_back(Pair("subscribed",true)); - if(entStat.m_Flags & MC_EFL_NOT_IN_SYNC) - { + if( ((entStat.m_Flags & MC_EFL_NOT_IN_SYNC) != 0) || + (pEF->STR_IsOutOfSync(&(entStat.m_Entity)) != 0) ) + { entry.push_back(Pair("synchronized",false)); } else { - entry.push_back(Pair("synchronized",true)); + bool fSynchronized=true; + if(pEF->ENT_EditionNumeric() == 0) + { + for(unsigned int ind=1;indFindEntity(&entStat)) + { + if( (entStat.m_Flags & MC_EFL_NOT_IN_SYNC) != 0) + { + fSynchronized=false; + } + } + } + } + } + entry.push_back(Pair("synchronized",fSynchronized)); + } + if(output_level & 0x0080) + { + entStat.m_Entity.m_EntityType=MC_TET_STREAM | MC_TET_CHAINPOS; + entry.push_back(Pair("retrieve",(pEF->STR_NoRetrieve(&(entStat.m_Entity))==0))); + + mc_TxImport dummy_import; + dummy_import.m_ImportID=1; + + Object indexes; + for(unsigned int ind=0;indSTR_IsIndexSkipped(&dummy_import,NULL,&(entStat.m_Entity)) == 0))); + } + entry.push_back(Pair("indexes",indexes)); } } if(output_level & 0x0010) diff --git a/src/v8/v8filter.cpp b/src/v8/v8filter.cpp index 710a4462..18b5a990 100644 --- a/src/v8/v8filter.cpp +++ b/src/v8/v8filter.cpp @@ -171,6 +171,10 @@ for (var fn of Object.getOwnPropertyNames(Math)) { delete Date.now; )"; +static std::string jsDeleteDateParse = R"( +delete Date.parse; +)"; + V8Filter::~V8Filter() { if (m_isRunning) @@ -216,6 +220,10 @@ int V8Filter::Initialize(V8Engine *engine, std::string script, std::string funct { jsPreamble += jsLimitMathSet; } + if (mc_gState->m_Features->DisabledJSDateParse()) + { + jsPreamble += jsDeleteDateParse; + } int status = this->CompileAndLoadScript(jsPreamble, "", "preamble", strResult); if (status != MC_ERR_NOERROR || !strResult.empty()) diff --git a/src/v8_win/v8_win.cpp b/src/v8_win/v8_win.cpp index 91596c83..de962eb0 100644 --- a/src/v8_win/v8_win.cpp +++ b/src/v8_win/v8_win.cpp @@ -79,14 +79,14 @@ bool V8Filter_IsRunning(V8Filter_t *filter_) } int V8Filter_Initialize(V8Filter_t *filter_, V8Engine_t *engine_, const char *script_, const char *functionName_, - const char **callbackNames_, size_t nCallbackNames_, bool isFilterLimitedMathSet_, bool isFixedJSDateFunctions_, + const char **callbackNames_, size_t nCallbackNames_, int jsInjectionParams, char *strResult_) { auto filter = reinterpret_cast(filter_); auto engine = reinterpret_cast(engine_); std::vector callbackNames = cstrs2vec(callbackNames_, nCallbackNames_); std::string strResult; - int retval = filter->Initialize(engine, script_, functionName_, callbackNames, isFilterLimitedMathSet_, isFixedJSDateFunctions_, strResult); + int retval = filter->Initialize(engine, script_, functionName_, callbackNames, jsInjectionParams, strResult); strcpy_s(strResult_, RESULT_SIZE, strResult.c_str()); return retval; } @@ -127,13 +127,13 @@ int V8Engine_Initialize(V8Engine_t *engine_, IFilterCallback_t *filterCallback_, } int V8Engine_CreateFilter(V8Engine_t *engine_, const char *script_, const char *mainName_, const char **callbackNames_, - size_t nCallbackNames_, V8Filter_t *filter_, bool isFilterLimitedMathSet_, bool isFixedJSDateFunctions_, char *strResult_) + size_t nCallbackNames_, V8Filter_t *filter_, int jsInjectionParams, char *strResult_) { auto engine = reinterpret_cast(engine_); auto filter = reinterpret_cast(filter_); std::vector callbackNames = cstrs2vec(callbackNames_, nCallbackNames_); std::string strResult; - int retval = engine->CreateFilter(script_, mainName_, callbackNames, filter, isFilterLimitedMathSet_, isFixedJSDateFunctions_, strResult); + int retval = engine->CreateFilter(script_, mainName_, callbackNames, filter, jsInjectionParams, strResult); strcpy_s(strResult_, RESULT_SIZE, strResult.c_str()); return retval; } diff --git a/src/v8_win/v8_win.h b/src/v8_win/v8_win.h index 7c34fc53..8dc8b491 100644 --- a/src/v8_win/v8_win.h +++ b/src/v8_win/v8_win.h @@ -6,6 +6,11 @@ #include "declspec.h" +#define MC_V8W_JS_INJECTION_LIMITED_MATH_SET 0x00000001 +#define MC_V8W_JS_INJECTION_FIXED_DATE_FUNCTIONS 0x00000002 +#define MC_V8W_JS_INJECTION_DISABLED_DATE_PARSE 0x00000004 + + #ifdef __cplusplus extern "C" { #endif @@ -83,7 +88,7 @@ DLLEXPORT bool V8Filter_IsRunning(V8Filter_t* filter_); */ DLLEXPORT int V8Filter_Initialize(V8Filter_t* filter_, V8Engine_t* engine_, const char* script_, const char* functionName_, const char** callbackNames_, size_t nCallbackNames_, - bool isFilterLimitedMathSet_, bool isFixedJSDateFunctions_, char* strResult_); + int jsInjectionParams, char* strResult_); /** * Run the filter function in the JS script. @@ -127,14 +132,13 @@ DLLEXPORT int V8Engine_Initialize(V8Engine_t* engine_, IFilterCallback_t* filter * If empty, register no callback functions. * @param nCallbackNames_ The number of callback names in @p callback_names_ * @param filter_ The filter object to initialize. - * @param isFilterLimitedMathSet_ @c true if JS Math functions have to be suppressed. - * @param isFixedJSDateFunctions_ @c true if limited set of JS Date functions is allowed. + * @param jsInjectionParams @c JC injection params. MC_V8W_JS_INJECTION constants * @param strResult_ Reason for failure if unsuccessful. * @return MC_ERR_INTERNAL_ERROR if the engine failed, MC_ERR_NOERROR otherwise. */ DLLEXPORT int V8Engine_CreateFilter(V8Engine_t* engine_, const char* script_, const char* mainName_, const char** callbackNames_, size_t nCallbackNames_, V8Filter_t* filter_, - bool isFilterLimitedMathSet_, bool FixedJSDateFunctions_, char* strResult_); + int jsInjectionParams, char* strResult_); /** * Run the filter function in the JS script. diff --git a/src/v8_win/v8engine.cpp b/src/v8_win/v8engine.cpp index 418e5917..40fb0039 100644 --- a/src/v8_win/v8engine.cpp +++ b/src/v8_win/v8engine.cpp @@ -49,11 +49,11 @@ int V8Engine::Initialize(IFilterCallback *filterCallback, std::string dataDir_, } int V8Engine::CreateFilter(std::string script, std::string mainName, const std::vector &callbackNames, - V8Filter *filter, bool isFilterLimitedMathSet, bool isFixedJSDateFunctions_, std::string &strResult) + V8Filter *filter, int jsInjectionParams, std::string &strResult) { logger->debug("V8Engine::CreateFilter - enter"); strResult.clear(); - int retval = filter->Initialize(this, script, mainName, callbackNames, isFilterLimitedMathSet, isFixedJSDateFunctions_, strResult); + int retval = filter->Initialize(this, script, mainName, callbackNames, jsInjectionParams, strResult); logger->debug("V8Engine::CreateFilter - leave retval={} strResult='{}'", retval, strResult); return retval; } diff --git a/src/v8_win/v8engine.h b/src/v8_win/v8engine.h index 7fabe121..bd14f7a2 100644 --- a/src/v8_win/v8engine.h +++ b/src/v8_win/v8engine.h @@ -49,7 +49,7 @@ class V8Engine * @return MC_ERR_INTERNAL_ERROR if the engine failed, MC_ERR_NOERROR otherwise. */ int CreateFilter(std::string script, std::string mainName, const std::vector &callbackNames, - V8Filter *filter, bool isFilterLimitedMathSet, bool isFixedJSDateFunctions_, std::string &strResult); + V8Filter *filter, int jsInjectionParams, std::string &strResult); /** * Run the filter function in the JS script. diff --git a/src/v8_win/v8filter.cpp b/src/v8_win/v8filter.cpp index 8d145cd2..89c394ff 100644 --- a/src/v8_win/v8filter.cpp +++ b/src/v8_win/v8filter.cpp @@ -129,6 +129,11 @@ for (var fn of Object.getOwnPropertyNames(Math)) { } delete Date.now; )"; + +static std::string jsDeleteDateParse = R"( +delete Date.parse; +)"; + // clang-format on V8Filter::~V8Filter() @@ -142,7 +147,7 @@ V8Filter::~V8Filter() } int V8Filter::Initialize(V8Engine *engine, std::string script, std::string functionName, - const std::vector &callbackNames, bool isFilterLimitedMathSet, bool isFixedJSDateFunctions, + const std::vector &callbackNames, int jsInjectionParams, std::string &strResult) { logger->debug("V8Filter::Initialize - enter"); @@ -174,14 +179,18 @@ int V8Filter::Initialize(V8Engine *engine, std::string script, std::string funct m_context.Reset(isolate, context); std::string jsPreamble = jsFixture; - if (isFixedJSDateFunctions) + if(jsInjectionParams & MC_V8W_JS_INJECTION_FIXED_DATE_FUNCTIONS) { jsPreamble=jsFixtureDateFunctions; } - if (isFilterLimitedMathSet) + if(jsInjectionParams & MC_V8W_JS_INJECTION_LIMITED_MATH_SET) { jsPreamble += jsLimitMathSet; } + if(jsInjectionParams & MC_V8W_JS_INJECTION_DISABLED_DATE_PARSE) + { + jsPreamble += jsDeleteDateParse; + } logger->debug(" Processing preamble"); int status = this->CompileAndLoadScript(jsPreamble, "", "preamble", strResult); diff --git a/src/v8_win/v8filter.h b/src/v8_win/v8filter.h index 8138bb44..71e1f519 100644 --- a/src/v8_win/v8filter.h +++ b/src/v8_win/v8filter.h @@ -7,6 +7,11 @@ #include "json/json_spirit.h" #include +#define MC_V8W_JS_INJECTION_LIMITED_MATH_SET 0x00000001 +#define MC_V8W_JS_INJECTION_FIXED_DATE_FUNCTIONS 0x00000002 +#define MC_V8W_JS_INJECTION_DISABLED_DATE_PARSE 0x00000004 + + namespace mc_v8 { class V8Engine; @@ -36,7 +41,7 @@ class V8Filter * MC_ERR_NOERROR otherwise. */ int Initialize(V8Engine *engine, std::string script, std::string functionName, - const std::vector &callbackNames, bool isFilterLimitedMathSet, bool isFixedJSDateFunctions, std::string &strResult); + const std::vector &callbackNames, int jsInjectionParams, std::string &strResult); /** * Run the filter function in the JS script. diff --git a/src/version/version.cpp b/src/version/version.cpp index 0363ebff..66a4eaed 100644 --- a/src/version/version.cpp +++ b/src/version/version.cpp @@ -13,8 +13,8 @@ int mc_State::VersionInfo(int version) return custom_version; } - int this_build=20000203; - int this_protocol=20008; + int this_build=20000901; + int this_protocol=20009; if(version < 0) { diff --git a/src/wallet/chunkcollector.cpp b/src/wallet/chunkcollector.cpp index bd550947..2d0629ca 100644 --- a/src/wallet/chunkcollector.cpp +++ b/src/wallet/chunkcollector.cpp @@ -4,6 +4,8 @@ #include "multichain/multichain.h" #include "wallet/chunkcollector.h" +#define MC_IMPOSSIBLE_NEXT_ATTEMPT 0xFFFFFFFF + void mc_ChunkEntityKey::Zero() { memset(this,0, sizeof(mc_ChunkEntityKey)); @@ -169,23 +171,57 @@ void mc_ChunkCollector::GetDBRow(mc_ChunkCollectorRow* collect_row) int mc_ChunkCollector::DeleteDBRow(mc_ChunkCollectorRow *collect_row) { + uint32_t next_attempt; SetDBRow(collect_row); + + next_attempt=m_DBRow.m_QueryNextAttempt; + if(next_attempt) + { + m_DBRow.m_QueryNextAttempt=MC_IMPOSSIBLE_NEXT_ATTEMPT; + m_DB->Delete((char*)&m_DBRow+m_KeyDBOffset,m_KeyDBSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); + m_DBRow.m_QueryNextAttempt=next_attempt; + } return m_DB->Delete((char*)&m_DBRow+m_KeyDBOffset,m_KeyDBSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); } int mc_ChunkCollector::InsertDBRow(mc_ChunkCollectorRow *collect_row) { + uint32_t next_attempt; + uint32_t query_attempts; collect_row->m_State.m_Status &= MC_CCF_ERROR_MASK; SetDBRow(collect_row); collect_row->m_State.m_Status |= MC_CCF_INSERTED; + + next_attempt=m_DBRow.m_QueryNextAttempt; + if(next_attempt) + { + query_attempts=m_DBRow.m_QueryAttempts; + m_DBRow.m_QueryNextAttempt=MC_IMPOSSIBLE_NEXT_ATTEMPT; + m_DBRow.m_QueryAttempts=next_attempt; + m_DB->Write((char*)&m_DBRow+m_KeyDBOffset,m_KeyDBSize,(char*)&m_DBRow+m_ValueDBOffset,m_ValueDBSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); + m_DBRow.m_QueryNextAttempt=next_attempt; + m_DBRow.m_QueryAttempts=query_attempts; + } + return m_DB->Write((char*)&m_DBRow+m_KeyDBOffset,m_KeyDBSize,(char*)&m_DBRow+m_ValueDBOffset,m_ValueDBSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); } int mc_ChunkCollector::UpdateDBRow(mc_ChunkCollectorRow *collect_row) { + uint32_t next_attempt; + uint32_t query_attempts; int err; collect_row->m_State.m_Status &= MC_CCF_ERROR_MASK; SetDBRow(collect_row); + + next_attempt=m_DBRow.m_QueryNextAttempt; + if(next_attempt) + { + m_DBRow.m_QueryNextAttempt=MC_IMPOSSIBLE_NEXT_ATTEMPT; + m_DB->Delete((char*)&m_DBRow+m_KeyDBOffset,m_KeyDBSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); + m_DBRow.m_QueryNextAttempt=next_attempt; + } + err=m_DB->Delete((char*)&m_DBRow+m_KeyDBOffset,m_KeyDBSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); if(err) { @@ -194,6 +230,18 @@ int mc_ChunkCollector::UpdateDBRow(mc_ChunkCollectorRow *collect_row) collect_row->m_DBNextAttempt=collect_row->m_State.m_QueryNextAttempt; m_DBRow.m_QueryNextAttempt=mc_SwapBytes32(collect_row->m_DBNextAttempt); collect_row->m_State.m_Status |= MC_CCF_INSERTED; + + next_attempt=m_DBRow.m_QueryNextAttempt; + if(next_attempt) + { + query_attempts=m_DBRow.m_QueryAttempts; + m_DBRow.m_QueryNextAttempt=MC_IMPOSSIBLE_NEXT_ATTEMPT; + m_DBRow.m_QueryAttempts=next_attempt; + m_DB->Write((char*)&m_DBRow+m_KeyDBOffset,m_KeyDBSize,(char*)&m_DBRow+m_ValueDBOffset,m_ValueDBSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); + m_DBRow.m_QueryNextAttempt=next_attempt; + m_DBRow.m_QueryAttempts=query_attempts; + } + return m_DB->Write((char*)&m_DBRow+m_KeyDBOffset,m_KeyDBSize,(char*)&m_DBRow+m_ValueDBOffset,m_ValueDBSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); } @@ -243,17 +291,21 @@ int mc_ChunkCollector::ReadFromDB(mc_Buffer *mempool,int rows) if(ptr) { memcpy((char*)&m_DBRow,ptr,m_TotalDBSize); - GetDBRow(&collect_row); - collect_row.m_State.m_Status |= MC_CCF_INSERTED; - if(m_ChunkDB->GetChunkDef(&chunk_def,collect_row.m_ChunkDef.m_Hash,&(collect_row.m_ChunkDef.m_Entity),collect_row.m_TxID,collect_row.m_Vout) == MC_ERR_NOERROR) - { - collect_row.m_State.m_Status |= MC_CCF_DELETED; - } - mprow=mempool->Seek(&collect_row); - if(mprow < 0) + if(m_DBRow.m_QueryNextAttempt != MC_IMPOSSIBLE_NEXT_ATTEMPT) { - mempool->Add(&collect_row); - row++; + GetDBRow(&collect_row); + collect_row.m_State.m_Status |= MC_CCF_INSERTED; + mprow=mempool->Seek(&collect_row); + if(mprow < 0) + { + if(m_ChunkDB->GetChunkDefWithLimit(&chunk_def,collect_row.m_ChunkDef.m_Hash,&(collect_row.m_ChunkDef.m_Entity),collect_row.m_TxID,collect_row.m_Vout, + MC_CCW_MAX_ITEMS_PER_CHUNKFOR_CHECK) == MC_ERR_NOERROR) + { + collect_row.m_State.m_Status |= MC_CCF_DELETED; + } + mempool->Add(&collect_row); + row++; + } } } else diff --git a/src/wallet/chunkcollector.h b/src/wallet/chunkcollector.h index 49cbb953..a10dc971 100644 --- a/src/wallet/chunkcollector.h +++ b/src/wallet/chunkcollector.h @@ -29,6 +29,7 @@ #define MC_CCW_MAX_MBS_PER_SECOND 8 #define MC_CCW_MAX_DELAY_BETWEEN_COLLECTS 1000 #define MC_CCW_QUERY_SPLIT 4 +#define MC_CCW_MAX_ITEMS_PER_CHUNKFOR_CHECK 16 typedef struct mc_ChunkEntityKey diff --git a/src/wallet/chunkdb.cpp b/src/wallet/chunkdb.cpp index 6cb6110b..352bc6ab 100644 --- a/src/wallet/chunkdb.cpp +++ b/src/wallet/chunkdb.cpp @@ -3,6 +3,7 @@ #include "multichain/multichain.h" #include "wallet/chunkdb.h" +#include "community/community.h" #define MC_CDB_TMP_FLAG_SHOULD_COMMIT 0x00000001 #define MC_CDB_FILE_PAGE_SIZE 0x00100000 @@ -534,7 +535,7 @@ int mc_ChunkDB::SourceChunksRecovery() return err; } -int mc_ChunkDB::RemoveEntityInternal(mc_TxEntity *entity) +int mc_ChunkDB::RemoveEntityInternal(mc_TxEntity *entity,uint32_t *removed_chunks,uint64_t *removed_size) { int err; mc_SubscriptionDBRow subscription; @@ -552,6 +553,15 @@ int mc_ChunkDB::RemoveEntityInternal(mc_TxEntity *entity) err=MC_ERR_NOERROR; sprintf_hex(enthex,entity->m_EntityID,MC_TDB_ENTITY_ID_SIZE); + if(removed_chunks) + { + *removed_chunks=0; + } + if(removed_size) + { + *removed_size=0; + } + subscription.Zero(); subscription.m_RecordType=MC_CDB_TYPE_SUBSCRIPTION; memcpy(&subscription.m_Entity,entity,sizeof(mc_TxEntity)); @@ -696,6 +706,16 @@ int mc_ChunkDB::RemoveEntityInternal(mc_TxEntity *entity) } } break; + case MC_ENT_SPRM_CHUNK_SIZE: + if(removed_chunks) + { + *removed_chunks+=1; + } + if(removed_size) + { + *removed_size+=(uint32_t)mc_GetLE(buf+param_value_start,bytes); + } + break; case MC_ENT_SPRM_ITEM_COUNT: if(bytes != sizeof(uint32_t)) { @@ -737,6 +757,7 @@ int mc_ChunkDB::RemoveEntityInternal(mc_TxEntity *entity) file_offset=file_size; } } +// pEF->STR_RemoveDataFromFile(FileHan,0,file_size,0); } @@ -789,12 +810,12 @@ int mc_ChunkDB::RemoveEntityInternal(mc_TxEntity *entity) return err; } -int mc_ChunkDB::RemoveEntity(mc_TxEntity *entity) +int mc_ChunkDB::RemoveEntity(mc_TxEntity *entity,uint32_t *removed_chunks,uint64_t *removed_size) { int err; Lock(); - err=RemoveEntityInternal(entity); + err=RemoveEntityInternal(entity,removed_chunks,removed_size); UnLock(); return err; @@ -1105,7 +1126,8 @@ int mc_ChunkDB::GetChunkDefInternal( const void *entity, const unsigned char *txid, const int vout, - int *mempool_row) + int *mempool_row, + int check_limit) { mc_SubscriptionDBRow *subscription; int err,value_len,mprow; @@ -1238,36 +1260,41 @@ int mc_ChunkDB::GetChunkDefInternal( { // ptr=GetChunkInternal(chunk_def,-1,-1,&bytes); total_items=chunk_def->m_ItemCount; - while(chunk_def->m_Pos < on_disk_items) + if( (check_limit == -1) || ((int)on_disk_items <= check_limit) ) { - if( (chunk_def->m_TxIDStart == 0) || (chunk_def->m_TxIDStart == (uint32_t)mc_GetLE((void*)txid,4))) - { - ptr=GetChunkInternal(chunk_def,-1,-1,&bytes); - if(mc_ScriptMatchesTxIDAndVOut(ptr,bytes,txid,vout) == MC_ERR_NOERROR) - { - return MC_ERR_NOERROR; - } - } - chunk_def->m_Pos+=1; - - if(chunk_def->m_Pos < total_items) + while(chunk_def->m_Pos < on_disk_items) { - chunk_def->SwapPosBytes(); - ptr=(unsigned char*)m_DB->Read((char*)chunk_def+m_KeyOffset,m_KeySize,&value_len,0,&err); - chunk_def->SwapPosBytes(); - if(err) + if( (chunk_def->m_TxIDStart == 0) || (chunk_def->m_TxIDStart == (uint32_t)mc_GetLE((void*)txid,4))) { - return err; - } - - if(ptr) - { - memcpy((char*)chunk_def+m_ValueOffset,ptr,m_ValueSize); -// ptr=GetChunkInternal(chunk_def,-1,-1,&bytes); + ptr=GetChunkInternal(chunk_def,-1,-1,&bytes); + if(mc_ScriptMatchesTxIDAndVOut(ptr,bytes,txid,vout) == MC_ERR_NOERROR) + { + return MC_ERR_NOERROR; + } } - else + chunk_def->m_Pos+=1; + + if(chunk_def->m_Pos < total_items) { - chunk_def->m_Pos=on_disk_items; + chunk_def->SwapPosBytes(); + ptr=(unsigned char*)m_DB->Read((char*)chunk_def+m_KeyOffset,m_KeySize,&value_len,0,&err); + chunk_def->SwapPosBytes(); + if(err) + { + return err; + } + + if(ptr) + { + memcpy((char*)chunk_def+m_ValueOffset,ptr,m_ValueSize); + // ptr=GetChunkInternal(chunk_def,-1,-1,&bytes); + } + /* + else + { + chunk_def->m_Pos=on_disk_items; + } + */ } } } @@ -1325,12 +1352,30 @@ int mc_ChunkDB::GetChunkDef( int err; Lock(); - err=GetChunkDefInternal(chunk_def,hash,entity,txid,vout,NULL); + err=GetChunkDefInternal(chunk_def,hash,entity,txid,vout,NULL,-1); UnLock(); return err; } +int mc_ChunkDB::GetChunkDefWithLimit( + mc_ChunkDBRow *chunk_def, + const unsigned char *hash, + const void *entity, + const unsigned char *txid, + const int vout, + int check_limit) +{ + int err; + + Lock(); + err=GetChunkDefInternal(chunk_def,hash,entity,txid,vout,NULL,check_limit); + UnLock(); + + return err; +} + + int mc_ChunkDB::AddChunkInternal( const unsigned char *hash, const mc_TxEntity *entity, @@ -1362,7 +1407,7 @@ int mc_ChunkDB::AddChunkInternal( chunk_def.Zero(); - err=GetChunkDefInternal(&chunk_def,hash,entity,txid,vout,NULL); + err=GetChunkDefInternal(&chunk_def,hash,entity,txid,vout,NULL,-1); if(err == MC_ERR_NOERROR) { return MC_ERR_FOUND; @@ -1398,7 +1443,7 @@ int mc_ChunkDB::AddChunkInternal( if(txid) { - err=GetChunkDefInternal(&entity_chunk_def,hash,entity,NULL,-1,&mempool_entity_row); + err=GetChunkDefInternal(&entity_chunk_def,hash,entity,NULL,-1,&mempool_entity_row,-1); if(err == MC_ERR_NOERROR) { total_items=entity_chunk_def.m_ItemCount; @@ -1425,7 +1470,7 @@ int mc_ChunkDB::AddChunkInternal( if(add_entity_row) { - err=GetChunkDefInternal(&null_chunk_def,hash,NULL,NULL,-1,&mempool_last_null_row); + err=GetChunkDefInternal(&null_chunk_def,hash,NULL,NULL,-1,&mempool_last_null_row,-1); if(err == MC_ERR_NOT_FOUND) { add_null_row=1; @@ -1632,7 +1677,7 @@ unsigned char *mc_ChunkDB::GetChunkInternal(mc_ChunkDBRow *chunk_def, } subscription=(mc_SubscriptionDBRow *)m_Subscriptions->GetRow(subscription_id); - if(GetChunkDefInternal(&chunk_def_zero,chunk_def->m_Hash,&(subscription->m_Entity),NULL,0,NULL) == MC_ERR_NOERROR) + if(GetChunkDefInternal(&chunk_def_zero,chunk_def->m_Hash,&(subscription->m_Entity),NULL,0,NULL,-1) == MC_ERR_NOERROR) { return GetChunkInternal(&chunk_def_zero,offset,len,bytes); } @@ -2125,3 +2170,21 @@ int mc_ChunkDB::Commit(int block,uint32_t flush_mode) return err; } +int mc_ChunkDB::RestoreChunkIfNeeded(mc_ChunkDBRow *chunk_def) +{ + if( (chunk_def->m_StorageFlags & MC_CFL_STORAGE_PURGED) == 0 ) + { + return MC_ERR_NOERROR; + } + + int err=MC_ERR_NOERROR; + chunk_def->m_StorageFlags-=MC_CFL_STORAGE_PURGED; + + chunk_def->SwapPosBytes(); + err=m_DB->Write((char*)chunk_def+m_KeyOffset,m_KeySize, + (char*)chunk_def+m_ValueOffset,m_ValueSize,MC_OPT_DB_DATABASE_DEFAULT); + chunk_def->SwapPosBytes(); + + return err; +} + diff --git a/src/wallet/chunkdb.h b/src/wallet/chunkdb.h index c50849e3..aae14d22 100644 --- a/src/wallet/chunkdb.h +++ b/src/wallet/chunkdb.h @@ -28,6 +28,7 @@ #define MC_CDB_FLUSH_MODE_DATASYNC 0x00000100 #define MC_CFL_STORAGE_FLUSHED 0x01000000 +#define MC_CFL_STORAGE_PURGED 0x02000000 /** File DB Row*/ @@ -159,8 +160,8 @@ typedef struct mc_ChunkDB int AddSubscription(mc_SubscriptionDBRow *subscription); int AddEntity(mc_TxEntity *entity,uint32_t flags); // Adds entity int AddEntityInternal(mc_TxEntity *entity,uint32_t flags); - int RemoveEntity(mc_TxEntity *entity); - int RemoveEntityInternal(mc_TxEntity *entity); + int RemoveEntity(mc_TxEntity *entity,uint32_t *removed_chunks,uint64_t *removed_size); + int RemoveEntityInternal(mc_TxEntity *entity,uint32_t *removed_chunks,uint64_t *removed_size); int SourceChunksRecovery(); mc_SubscriptionDBRow *FindSubscription(const mc_TxEntity *entity); // Finds subscription @@ -194,7 +195,8 @@ typedef struct mc_ChunkDB const void *entity, const unsigned char *txid, const int vout, - int *mempool_entity_row); + int *mempool_entity_row, + int check_limit); int GetChunkDef( @@ -204,6 +206,14 @@ typedef struct mc_ChunkDB const unsigned char *txid, const int vout); + int GetChunkDefWithLimit( + mc_ChunkDBRow *chunk_def, + const unsigned char *hash, // Chunk hash (before chopping) + const void *entity, + const unsigned char *txid, + const int vout, + int check_limit); + unsigned char *GetChunkInternal(mc_ChunkDBRow *chunk_def, int32_t offset, int32_t len, @@ -222,6 +232,7 @@ typedef struct mc_ChunkDB uint32_t fileid, uint32_t flush_mode); + int RestoreChunkIfNeeded(mc_ChunkDBRow *chunk_def); int AddToFile(const void *chunk, uint32_t size, diff --git a/src/wallet/wallettxdb.cpp b/src/wallet/wallettxdb.cpp index c4c35d5a..9148b99f 100644 --- a/src/wallet/wallettxdb.cpp +++ b/src/wallet/wallettxdb.cpp @@ -2611,7 +2611,9 @@ mc_TxImport *mc_TxDB::StartImport(mc_Buffer *lpEntities,int block,int *err) lpChainEntStat=m_Imports->GetEntity(row); if( (lpChainEntStat->m_Flags & MC_EFL_NOT_IN_SYNC) == 0 ) // in-sync rows ordered by timereceived should be copied { - if( (lpChainEntStat->m_Entity.m_EntityType & MC_TET_ORDERMASK) == MC_TET_TIMERECEIVED) + if( ((lpChainEntStat->m_Entity.m_EntityType & MC_TET_ORDERMASK) == MC_TET_TIMERECEIVED) && + (pEF->STR_IsIndexSkipped(NULL,NULL,&(lpChainEntStat->m_Entity)) == 0) && + (pEF->STR_IsIndexSkipped(m_Imports+slot,NULL,&(lpChainEntStat->m_Entity)) == 0) ) { lpEntStat->m_Flags -= MC_EFL_NOT_IN_SYNC; erow.Zero(); @@ -3018,7 +3020,10 @@ int mc_TxDB::CompleteImport(mc_TxImport *import,uint32_t flags) { if( (lpdel->m_Entity.IsSubscription() != 0) || ( (flags & MC_EFL_NOT_IN_SYNC_AFTER_IMPORT) == 0) ) { - lpdel->m_Flags-=MC_EFL_NOT_IN_SYNC; + if(pEF->STR_IsIndexSkipped(import,NULL,&(lpdel->m_Entity)) == 0) + { + lpdel->m_Flags-=MC_EFL_NOT_IN_SYNC; + } } } diff --git a/src/wallet/wallettxs.cpp b/src/wallet/wallettxs.cpp index e2f07a78..370c2876 100644 --- a/src/wallet/wallettxs.cpp +++ b/src/wallet/wallettxs.cpp @@ -5,6 +5,7 @@ #include "wallet/wallettxs.h" #include "utils/core_io.h" +#include "community/community.h" #include "json/json_spirit_utils.h" #include "json/json_spirit_value.h" @@ -1339,7 +1340,7 @@ int mc_WalletTxs::Unsubscribe(mc_Buffer* lpEntities,bool purge) { for(j=0;jGetCount();j++) { - m_ChunkDB->RemoveEntity((mc_TxEntity*)lpEntities->GetRow(j)); + m_ChunkDB->RemoveEntity((mc_TxEntity*)lpEntities->GetRow(j),NULL,NULL); } } } @@ -2373,7 +2374,8 @@ int mc_WalletTxs::AddTx(mc_TxImport *import,const CWalletTx& tx,int block,CDiskT if(imp->FindEntity(&entity) >= 0) { - if(chunk_hashes) + if( (chunk_hashes != NULL) && + (pEF->STR_NoRetrieve(&entity) == 0) ) { mc_ChunkDBRow chunk_def; mc_TxEntity chunk_entity; @@ -2411,6 +2413,11 @@ int mc_WalletTxs::AddTx(mc_TxImport *import,const CWalletTx& tx,int block,CDiskT // Feeding async chunk retriever here } } + else + { + m_ChunkDB->RestoreChunkIfNeeded(&chunk_def); + } + chunk_hashes+=MC_CDB_CHUNK_HASH_SIZE; } }