From b179bd3707ee90931d363e26e4a6498f57e6eed9 Mon Sep 17 00:00:00 2001 From: mike31 Date: Thu, 14 Jun 2018 10:09:11 +0300 Subject: [PATCH 01/26] Fixed stack smasching on asset ref decoding --- src/rpc/rpcutils.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rpc/rpcutils.cpp b/src/rpc/rpcutils.cpp index 3871b525..1afa1525 100644 --- a/src/rpc/rpcutils.cpp +++ b/src/rpc/rpcutils.cpp @@ -75,7 +75,7 @@ bool AssetRefDecode(unsigned char *bin, const char* string, const size_t stringL return false; mc_PutLE(bin+0,&blockNum,4); - mc_PutLE(bin+4,&txOffset,8); + mc_PutLE(bin+4,&txOffset,4); bin[8]=(unsigned char)(txIDPrefixInteger%256); bin[9]=(unsigned char)(txIDPrefixInteger/256); From 60780f3ee58c90df355b2eec04e0d314c81a22b4 Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 18 Jun 2018 17:23:04 +0300 Subject: [PATCH 02/26] Logging RPC request IDs --- src/multichain/multichain-cli.cpp | 7 ++++++- src/rpc/rpcserver.cpp | 34 +++++++++++++++++++------------ src/rpc/rpcserver.h | 2 +- 3 files changed, 28 insertions(+), 15 deletions(-) diff --git a/src/multichain/multichain-cli.cpp b/src/multichain/multichain-cli.cpp index 221f23e2..64c35a6b 100644 --- a/src/multichain/multichain-cli.cpp +++ b/src/multichain/multichain-cli.cpp @@ -241,7 +241,12 @@ Object CallRPC(const string& strMethod, const Array& params) mapRequestHeaders["Authorization"] = string("Basic ") + strUserPass64; // Send request // JSON_NO_DOUBLE_FORMATTING=1; - string strRequest = JSONRPCRequest(strMethod, params, 1); + + int32_t id_nonce; + id_nonce=mc_RandomInRange(10000000,99999999); + Value req_id=strprintf("%08d-%u",id_nonce,mc_TimeNowAsUInt()); + + string strRequest = JSONRPCRequest(strMethod, params, req_id); // JSON_NO_DOUBLE_FORMATTING=0; JSON_DOUBLE_DECIMAL_DIGITS=GetArg("-apidecimaldigits",-1); string strPost = HTTPPost(strRequest, mapRequestHeaders); diff --git a/src/rpc/rpcserver.cpp b/src/rpc/rpcserver.cpp index 5e640147..42870ee5 100644 --- a/src/rpc/rpcserver.cpp +++ b/src/rpc/rpcserver.cpp @@ -73,11 +73,19 @@ string JSONRPCRequestForLog(const string& strMethod, const Array& params, const { request.push_back(Pair("params", params)); } -/* request.push_back(Pair("id", id)); +/* request.push_back(Pair("chain_name", string(mc_gState->m_Params->NetworkName()))); */ - return write_string(Value(request), false) + "\n"; + return write_string(Value(request), false);// + "\n"; +} + +string JSONRPCMethodIDForLog(const string& strMethod, const Value& id) +{ + Object request; + request.push_back(Pair("method", strMethod)); + request.push_back(Pair("id", id)); + return write_string(Value(request), false);// + "\n"; } @@ -979,14 +987,14 @@ static Object JSONRPCExecOne(const Value& req) try { jreq.parse(req); - Value result = tableRPC.execute(jreq.strMethod, jreq.params); + Value result = tableRPC.execute(jreq.strMethod, jreq.params,jreq.id); rpc_result = JSONRPCReplyObj(result, Value::null, jreq.id); } catch (Object& objError) { /* MCHN START */ mc_gState->m_WalletMode=wallet_mode; - if(fDebug)LogPrint("mcapi","mcapi: API request failure A\n"); + if(fDebug)LogPrint("mcapi","mcapi: API request failure A: %s\n",JSONRPCMethodIDForLog(jreq.strMethod,jreq.id).c_str()); /* MCHN END */ rpc_result = JSONRPCReplyObj(Value::null, objError, jreq.id); } @@ -994,7 +1002,7 @@ static Object JSONRPCExecOne(const Value& req) { /* MCHN START */ mc_gState->m_WalletMode=wallet_mode; - if(fDebug)LogPrint("mcapi","mcapi: API request failure B\n"); + if(fDebug)LogPrint("mcapi","mcapi: API request failure B: %s\n",JSONRPCMethodIDForLog(jreq.strMethod,jreq.id).c_str()); /* MCHN END */ rpc_result = JSONRPCReplyObj(Value::null, JSONRPCError(RPC_PARSE_ERROR, e.what()), jreq.id); @@ -1061,7 +1069,7 @@ static bool HTTPReq_JSONRPC(AcceptedConnection *conn, if (valRequest.type() == obj_type) { jreq.parse(valRequest); - Value result = tableRPC.execute(jreq.strMethod, jreq.params); + Value result = tableRPC.execute(jreq.strMethod, jreq.params,jreq.id); // Send reply strReply = JSONRPCReply(result, Value::null, jreq.id); @@ -1078,7 +1086,7 @@ static bool HTTPReq_JSONRPC(AcceptedConnection *conn, { /* MCHN START */ mc_gState->m_WalletMode=wallet_mode; - if(fDebug)LogPrint("mcapi","mcapi: API request failure: %s, code: %d\n",jreq.strMethod.c_str(),find_value(objError, "code").get_int()); + if(fDebug)LogPrint("mcapi","mcapi: API request failure: %s, code: %d\n",JSONRPCMethodIDForLog(jreq.strMethod,jreq.id).c_str(),find_value(objError, "code").get_int()); // if(fDebug)LogPrint("mcapi","mcapi: API request failure C\n"); /* MCHN END */ @@ -1089,7 +1097,7 @@ static bool HTTPReq_JSONRPC(AcceptedConnection *conn, { /* MCHN START */ mc_gState->m_WalletMode=wallet_mode; - if(fDebug)LogPrint("mcapi","mcapi: API request failure D\n"); + if(fDebug)LogPrint("mcapi","mcapi: API request failure D: %s\n",JSONRPCMethodIDForLog(jreq.strMethod,jreq.id).c_str()); /* MCHN END */ ErrorReply(conn->stream(), JSONRPCError(RPC_PARSE_ERROR, e.what()), jreq.id); return false; @@ -1134,7 +1142,7 @@ void ServiceConnection(AcceptedConnection *conn) } } -json_spirit::Value CRPCTable::execute(const std::string &strMethod, const json_spirit::Array ¶ms) const +json_spirit::Value CRPCTable::execute(const std::string &strMethod, const json_spirit::Array ¶ms, const Value& req_id) const { // Find method const CRPCCommand *pcmd = tableRPC[strMethod]; @@ -1190,7 +1198,7 @@ json_spirit::Value CRPCTable::execute(const std::string &strMethod, const json_s // Execute if(fDebug) { - string strRequest = JSONRPCRequestForLog(strMethod, params, 1); + string strRequest = JSONRPCRequestForLog(strMethod, params, req_id); LogPrint("mcapi","mcapi: API request: %s\n",strRequest.c_str()); } @@ -1233,7 +1241,7 @@ json_spirit::Value CRPCTable::execute(const std::string &strMethod, const json_s strResult=JSONRPCReply(result, Value::null, 1); if(strcmp(strResultNone.c_str(),strResult.c_str())) { - string strRequestBad = JSONRPCRequestForLog(strMethod, params, 1); + string strRequestBad = JSONRPCRequestForLog(strMethod, params, req_id); if(fDebug)LogPrint("walletcompare","walletcompare: ERROR: Result mismatch on API request: %s\n",strRequestBad.c_str()); if(fDebug)LogPrint("walletcompare","walletcompare: %s\n",strResultNone.c_str()); if(fDebug)LogPrint("walletcompare","walletcompare: %s\n",strResult.c_str()); @@ -1256,13 +1264,13 @@ json_spirit::Value CRPCTable::execute(const std::string &strMethod, const json_s #endif // !ENABLE_WALLET } /* MCHN START */ - if(fDebug)LogPrint("mcapi","mcapi: API request successful: %s\n",strMethod.c_str()); + if(fDebug)LogPrint("mcapi","mcapi: API request successful: %s\n",JSONRPCMethodIDForLog(strMethod,req_id).c_str()); /* MCHN END */ return result; } catch (std::exception& e) { - if(fDebug)LogPrint("mcapi","mcapi: API request failure: %s\n",strMethod.c_str()); + if(fDebug)LogPrint("mcapi","mcapi: API request failure: %s\n",JSONRPCMethodIDForLog(strMethod,req_id).c_str());//strMethod.c_str()); if(strcmp(e.what(),"Help message not found\n") == 0) { throw JSONRPCError(RPC_MISC_ERROR, mc_RPCHelpString(strMethod).c_str()); diff --git a/src/rpc/rpcserver.h b/src/rpc/rpcserver.h index d53a66cc..ce2010b8 100644 --- a/src/rpc/rpcserver.h +++ b/src/rpc/rpcserver.h @@ -114,7 +114,7 @@ class CRPCTable * @throws an exception (json_spirit::Value) when an error happens. */ void initialize(); - json_spirit::Value execute(const std::string &method, const json_spirit::Array ¶ms) const; + json_spirit::Value execute(const std::string &method, const json_spirit::Array ¶ms, const json_spirit::Value& req_id) const; }; extern CRPCTable tableRPC; From 025784061a25f3234f96ada06bcb8a923191df41 Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 9 Jul 2018 12:09:32 +0300 Subject: [PATCH 03/26] Fixed autosubscribe for offchain items --- src/wallet/wallettxs.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/wallet/wallettxs.cpp b/src/wallet/wallettxs.cpp index af99bcbb..c7c2d036 100644 --- a/src/wallet/wallettxs.cpp +++ b/src/wallet/wallettxs.cpp @@ -2800,7 +2800,13 @@ int mc_WalletTxs::AddTx(mc_TxImport *import,const CWalletTx& tx,int block,CDiskT entity.m_EntityType=MC_TET_STREAM_PUBLISHER | MC_TET_CHAINPOS; m_Database->AddEntity(imp,&entity,0); entity.m_EntityType=MC_TET_STREAM_PUBLISHER | MC_TET_TIMERECEIVED; - m_Database->AddEntity(imp,&entity,0); + m_Database->AddEntity(imp,&entity,0); + + if(mc_gState->m_Features->Chunks()) + { + entity.m_EntityType=MC_TET_STREAM; + m_ChunkDB->AddEntity(&entity,0); + } } } From 47cebfb23b6bf40e2706fdaef98d02f052119caa Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 16 Jul 2018 14:38:28 +0300 Subject: [PATCH 04/26] Fixed txcount in getwalletinfo --- src/rpc/rpcwallet.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rpc/rpcwallet.cpp b/src/rpc/rpcwallet.cpp index ea8eb14a..630639b1 100644 --- a/src/rpc/rpcwallet.cpp +++ b/src/rpc/rpcwallet.cpp @@ -2169,7 +2169,7 @@ Value getwalletinfo(const Array& params, bool fHelp) obj.push_back(Pair("walletdbversion", mc_gState->GetWalletDBVersion())); if(mc_gState->m_WalletMode & MC_WMD_ADDRESS_TXS) { - obj.push_back(Pair("txcount", (int)pwalletTxsMain->m_Database->m_DBStat.m_Count+(int)pwalletTxsMain->m_UnconfirmedSends.size())); + obj.push_back(Pair("txcount",(int)pwalletTxsMain->m_Database->m_DBStat.m_Count+pwalletTxsMain->m_Database->m_RawMemPools[0]->GetCount())); } else { From a4822f27c2f1a45a8c9da98a228de1c25bacee33 Mon Sep 17 00:00:00 2001 From: mike31 Date: Tue, 17 Jul 2018 16:05:12 +0300 Subject: [PATCH 05/26] Fixed Mac compilation error --- src/protocol/relay.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/protocol/relay.cpp b/src/protocol/relay.cpp index 272a3072..5540e225 100644 --- a/src/protocol/relay.cpp +++ b/src/protocol/relay.cpp @@ -1546,7 +1546,7 @@ void mc_RelayManager::CheckTime() } Lock(); - for(map::iterator it = m_RelayRecords.begin(); it != m_RelayRecords.end();) + for(map::iterator it = m_RelayRecords.begin(); it != m_RelayRecords.end();) { if(it->second.m_Timestamp < m_LastTime) { @@ -1623,7 +1623,7 @@ int mc_RelayManager::GetRelayRecord(CNode *pfrom,mc_OffchainMessageID msg_id,uin } // printf("getrr: %d, ts: %u, nc: %u\n",pfrom_id,timestamp,nonce); const mc_RelayRecordKey key=mc_RelayRecordKey(msg_id,pfrom_id); - map::iterator it = m_RelayRecords.find(key); + map::iterator it = m_RelayRecords.find(key); if (it == m_RelayRecords.end()) { return MC_ERR_NOT_FOUND; From caa4d73cafef76910efea5f2911d2c4b316adce1 Mon Sep 17 00:00:00 2001 From: mike31 Date: Tue, 17 Jul 2018 17:10:09 +0300 Subject: [PATCH 06/26] Fixed Mac compilation issue --- src/rpc/rpcdebug.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rpc/rpcdebug.cpp b/src/rpc/rpcdebug.cpp index 20bc773c..64451723 100644 --- a/src/rpc/rpcdebug.cpp +++ b/src/rpc/rpcdebug.cpp @@ -573,7 +573,7 @@ Value debug(const Array& params, bool fHelp) chunk_found=pwalletTxsMain->m_ChunkDB->GetChunk(&chunk_def,0,-1,&chunk_bytes); if(chunk_found) { - chunk_obj.push_back(Pair("size",chunk_bytes)); + chunk_obj.push_back(Pair("size",(int)chunk_bytes)); chunk_obj.push_back(Pair("data",HexStr(chunk_found,chunk_found+chunk_bytes))); } else From 473edb8d2939702a5174ae3bd58437ed906e4275 Mon Sep 17 00:00:00 2001 From: mike31 Date: Wed, 18 Jul 2018 14:36:21 +0300 Subject: [PATCH 07/26] Fixed unicode representation of non-printable characters --- src/json/json_spirit_writer_template.h | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/json/json_spirit_writer_template.h b/src/json/json_spirit_writer_template.h index 4e37ae31..21bc0920 100644 --- a/src/json/json_spirit_writer_template.h +++ b/src/json/json_spirit_writer_template.h @@ -21,6 +21,7 @@ namespace json_spirit inline unsigned int utf8_len_and_mask(unsigned char c,unsigned int *mask) { + if(c<0x20){*mask=0x00;return 0;} if(c<0x80){*mask=0x7F;return 1;} if(c<0xC0){*mask=0x00;return 0;} if(c<0xE0){*mask=0x1F;return 2;} @@ -146,18 +147,24 @@ namespace json_spirit } else { - if( end - i >= charlen) + if(charlen) { - shift=6*(charlen-1); - codepoint |= ( unsigned_c & mask) << shift; - for(j=1;j= charlen) { - shift-=6; - codepoint |= ( *( ++i ) & 0x3F) << shift; + shift=6*(charlen-1); + codepoint |= ( unsigned_c & mask) << shift; + for(j=1;j( codepoint ); } - result += codepoint_to_string< String_type >( codepoint ); } -// result += non_printable_to_string< String_type >( unsigned_c ); + else + { + result += non_printable_to_string< String_type >( unsigned_c ); + } } } From d93d29be8477d2d13fab157a964ded751b8cbe95 Mon Sep 17 00:00:00 2001 From: mike31 Date: Thu, 19 Jul 2018 13:48:41 +0300 Subject: [PATCH 08/26] Boolean rescan for importwallet --- src/rpc/rpcdump.cpp | 26 ++++++++++++++------------ src/rpc/rpchelp.cpp | 3 ++- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/rpc/rpcdump.cpp b/src/rpc/rpcdump.cpp index dedc3881..aec2c379 100644 --- a/src/rpc/rpcdump.cpp +++ b/src/rpc/rpcdump.cpp @@ -408,25 +408,27 @@ Value importwallet(const Array& params, bool fHelp) pwalletMain->nTimeFirstKey = nTimeBegin; /* MCHN START */ - if(mc_gState->m_WalletMode & MC_WMD_ADDRESS_TXS) + if(fRescan) { - if(start_block) + if(mc_gState->m_WalletMode & MC_WMD_ADDRESS_TXS) { - LogPrintf("Rescanning last %i blocks\n", chainActive.Height()-start_block+1); + if(start_block) + { + LogPrintf("Rescanning last %i blocks\n", chainActive.Height()-start_block+1); + } + else + { + LogPrintf("Rescanning all %i blocks\n", chainActive.Height()); + } + pwalletMain->ScanForWalletTransactions(chainActive[start_block],false,true); } else { - LogPrintf("Rescanning all %i blocks\n", chainActive.Height()); + LogPrintf("Rescanning last %i blocks\n", chainActive.Height() - pindex->nHeight + 1); + pwalletMain->ScanForWalletTransactions(pindex,false,true); } - pwalletMain->ScanForWalletTransactions(chainActive[start_block],false,true); - pwalletMain->MarkDirty(); - } - else - { - LogPrintf("Rescanning last %i blocks\n", chainActive.Height() - pindex->nHeight + 1); - pwalletMain->ScanForWalletTransactions(pindex,false,true); - pwalletMain->MarkDirty(); } + pwalletMain->MarkDirty(); if (!fGood) throw JSONRPCError(RPC_WALLET_ERROR, "Error adding some keys to wallet"); diff --git a/src/rpc/rpchelp.cpp b/src/rpc/rpchelp.cpp index 398a7622..fa2d69d3 100644 --- a/src/rpc/rpchelp.cpp +++ b/src/rpc/rpchelp.cpp @@ -2244,7 +2244,8 @@ void mc_InitRPCHelpMap10() "\nImports keys from a wallet dump file (see dumpwallet).\n" "\nArguments:\n" "1. \"filename\" (string, required) The wallet file\n" - "2. rescan (integer, optional, default=0) Rescan from block, if negative - from the end.\n" + "2. rescan (boolean or integer, optional, default=true) Rescan the wallet for transactions. \n" + " If integer rescan from block, if negative - from the end.\n" "\nExamples:\n" "\nDump the wallet\n" + HelpExampleCli("dumpwallet", "\"test\"") + From e9d4a5e9d6edaf89281ded3587171f7111b123d6 Mon Sep 17 00:00:00 2001 From: mike31 Date: Thu, 19 Jul 2018 14:13:13 +0300 Subject: [PATCH 09/26] Fixed fdatasync for Mac compilation --- src/utils/systemdependent.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/utils/systemdependent.cpp b/src/utils/systemdependent.cpp index 27933160..b8921862 100644 --- a/src/utils/systemdependent.cpp +++ b/src/utils/systemdependent.cpp @@ -233,7 +233,11 @@ void __US_FlushFileWithMode(int FileHan,uint32_t use_data_sync) { if(use_data_sync) { +#ifdef MAC_OSX + fcntl(FileHan, F_FULLFSYNC, 0); +#else fdatasync(FileHan); +#endif } else { From 2a1fc7a995b3bab7430b08abc07f50b2ec5bc303 Mon Sep 17 00:00:00 2001 From: mike31 Date: Thu, 19 Jul 2018 15:17:14 +0300 Subject: [PATCH 10/26] Boolean verbose parameter for getrawtransaction --- src/rpc/rpcrawtransaction.cpp | 3 ++- src/rpc/rpcutils.h | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/rpc/rpcrawtransaction.cpp b/src/rpc/rpcrawtransaction.cpp index 3e352d22..69525455 100644 --- a/src/rpc/rpcrawtransaction.cpp +++ b/src/rpc/rpcrawtransaction.cpp @@ -504,7 +504,8 @@ Value getrawtransaction(const Array& params, bool fHelp) bool fVerbose = false; if (params.size() > 1) - fVerbose = (params[1].get_int() != 0); + fVerbose=paramtobool(params[1],false); +// fVerbose = (params[1].get_int() != 0); CTransaction tx; uint256 hashBlock = 0; diff --git a/src/rpc/rpcutils.h b/src/rpc/rpcutils.h index 14f83057..f80c34a1 100644 --- a/src/rpc/rpcutils.h +++ b/src/rpc/rpcutils.h @@ -129,6 +129,7 @@ bool AssetCompareByRef(Value a,Value b); Array AssetArrayFromAmounts(mc_Buffer *asset_amounts,int issue_asset_id,uint256 hash,int show_type); void ParseRawAction(string action,bool& lock_it, bool& sign_it,bool& send_it); bool paramtobool(Value param); +bool paramtobool(Value param,bool strict); int paramtoint(Value param,bool check_for_min,int min_value,string error_message); int64_t paramtoint64(Value param,bool check_for_min,int64_t min_value,string error_message); int ParseBlockIdentifier(Value blockset_identifier); From bef04be3bf3c9d61e1a37d2128075bf335ad85a9 Mon Sep 17 00:00:00 2001 From: mike31 Date: Thu, 26 Jul 2018 18:55:40 +0300 Subject: [PATCH 11/26] Flushing offchain queue on subscribe/unsubscribe --- src/rpc/rpcstreams.cpp | 7 +- src/wallet/chunkcollector.cpp | 144 +++++++++++++++++++++++++++++++--- src/wallet/chunkcollector.h | 4 +- src/wallet/wallettxs.cpp | 5 ++ 4 files changed, 149 insertions(+), 11 deletions(-) diff --git a/src/rpc/rpcstreams.cpp b/src/rpc/rpcstreams.cpp index c07de87f..ea2377a2 100644 --- a/src/rpc/rpcstreams.cpp +++ b/src/rpc/rpcstreams.cpp @@ -889,8 +889,13 @@ Value unsubscribe(const Array& params, bool fHelp) inputEntities.push_back(entity_to_subscribe); } + int32_t buf_mode=MC_BUF_MODE_DEFAULT; + if(inputStrings.size() > 1) + { + buf_mode=MC_BUF_MODE_MAP; + } mc_Buffer *streams=mc_gState->m_TmpBuffers->m_RpcBuffer1; - streams->Initialize(sizeof(mc_TxEntity),sizeof(mc_TxEntity),MC_BUF_MODE_DEFAULT); + streams->Initialize(sizeof(mc_TxEntity),sizeof(mc_TxEntity),buf_mode); bool fNewFound=false; diff --git a/src/wallet/chunkcollector.cpp b/src/wallet/chunkcollector.cpp index d14839e1..5063e4aa 100644 --- a/src/wallet/chunkcollector.cpp +++ b/src/wallet/chunkcollector.cpp @@ -434,7 +434,125 @@ int mc_ChunkCollector::InsertChunk( return err; } + +int mc_ChunkCollector::Unsubscribe(mc_Buffer* lpEntities) +{ + Lock(); + + int i,err; + unsigned char *ptr; + mc_ChunkCollectorRow *row; + mc_ChunkCollectorRow collect_row; + int try_again,commit_required; + mc_TxEntity entity; + + commit_required=0; + for(i=0;iGetCount();i++) + { + row=(mc_ChunkCollectorRow *)m_MemPool->GetRow(i); + memcpy(&entity,&row->m_ChunkDef.m_Entity,sizeof(mc_TxEntity)); + entity.m_EntityType |= MC_TET_CHAINPOS; + if(lpEntities->Seek(&entity) >= 0) + { + row->m_State.m_Status |= MC_CCF_DELETED; + commit_required=1; + } + } + if(commit_required) + { + CommitInternal(0); + } + + if(m_MemPool == m_MemPool1) + { + m_MemPoolNext=m_MemPool2; + } + else + { + m_MemPoolNext=m_MemPool1; + } + + + try_again=1; + m_DBRow.Zero(); + if(m_DB) + { + while(try_again) + { + m_MemPoolNext->Clear(); + try_again=0; + err=SeekDB(&m_DBRow); + + ptr=(unsigned char*)m_DB->MoveNext(&err); + while(ptr) + { + if(err) + { + return MC_ERR_CORRUPTED; + } + memcpy((char*)&m_DBRow,ptr,m_TotalDBSize); + ptr=(unsigned char*)m_DB->MoveNext(&err); + GetDBRow(&collect_row); + memcpy(&entity,&collect_row.m_ChunkDef.m_Entity,sizeof(mc_TxEntity)); + entity.m_EntityType |= MC_TET_CHAINPOS; + if(lpEntities->Seek(&entity) >= 0) + { + m_MemPoolNext->Add(&collect_row); + } + else + { + memcpy(&m_LastDBRow,&m_DBRow,m_TotalDBSize); + } + if( (m_MemPoolNext->GetCount() >= 2*m_MaxMemPoolSize) || (ptr == NULL) ) + { + if(ptr) + { + try_again=1; + ptr=NULL; + memcpy(&m_DBRow,&m_LastDBRow,m_TotalDBSize); + } + if(m_MemPoolNext->GetCount()) + { + commit_required=1; + for(i=0;iGetCount();i++) + { + row=(mc_ChunkCollectorRow *)m_MemPoolNext->GetRow(i); + m_TotalChunkCount--; + m_TotalChunkSize-=row->m_ChunkDef.m_Size; + if(row->m_State.m_Status & MC_CCF_INSERTED) + { + DeleteDBRow(row); + } + } + m_DBRow.Zero(); + m_DBRow.m_TotalChunkSize=m_TotalChunkSize; + m_DBRow.m_TotalChunkCount=m_TotalChunkCount; + m_DB->Write((char*)&m_DBRow+m_KeyDBOffset,m_KeyDBSize,(char*)&m_DBRow+m_ValueDBOffset,m_ValueDBSize,MC_OPT_DB_DATABASE_TRANSACTIONAL); + + err=m_DB->Commit(MC_OPT_DB_DATABASE_TRANSACTIONAL); + if(err) + { + return err; + } + } + } + } + } + } + + m_MemPoolNext->Clear(); + + if(commit_required) + { + CommitInternal(1); + } + + UnLock(); + + return MC_ERR_NOERROR; +} + int mc_ChunkCollector::InsertChunkInternal( const unsigned char *hash, const mc_TxEntity *entity, @@ -459,6 +577,11 @@ int mc_ChunkCollector::InsertChunkInternal( m_MemPool->Add(&collect_row); m_TotalChunkCount++; m_TotalChunkSize+=chunk_size; + + if(m_MemPoolNext->GetCount() >= 2*m_MaxMemPoolSize) + { + CommitInternal(0); + } } return MC_ERR_NOERROR; @@ -573,13 +696,13 @@ int mc_ChunkCollector::Commit() int err; Lock(); - err=CommitInternal(); + err=CommitInternal(1); UnLock(); return err; } -int mc_ChunkCollector::CommitInternal() +int mc_ChunkCollector::CommitInternal(int fill_mempool) { int i; mc_ChunkCollectorRow *row; @@ -650,7 +773,7 @@ int mc_ChunkCollector::CommitInternal() } */ if(!row->m_State.m_Query.IsZero() || - ((row->m_State.m_QueryNextAttempt <= time_now) && (m_MemPoolNext->GetCount() < m_MaxMemPoolSize)) ) + ((fill_mempool != 0) && (row->m_State.m_QueryNextAttempt <= time_now) && (m_MemPoolNext->GetCount() < m_MaxMemPoolSize)) ) { m_MemPoolNext->Add(row); } @@ -671,16 +794,19 @@ int mc_ChunkCollector::CommitInternal() } } - if(m_MemPoolNext->GetCount() < m_MaxMemPoolSize) + if(fill_mempool) { - m_LastDBRow.Zero(); - err=SeekDB(&m_LastDBRow); - if(err == MC_ERR_NOERROR) + if(m_MemPoolNext->GetCount() < m_MaxMemPoolSize) { - ReadFromDB(m_MemPoolNext,m_MaxMemPoolSize); + m_LastDBRow.Zero(); + err=SeekDB(&m_LastDBRow); + if(err == MC_ERR_NOERROR) + { + ReadFromDB(m_MemPoolNext,m_MaxMemPoolSize); + } } } - + m_MemPool->Clear(); m_MemPool=m_MemPoolNext; diff --git a/src/wallet/chunkcollector.h b/src/wallet/chunkcollector.h index ebf31619..ee2c3e2f 100644 --- a/src/wallet/chunkcollector.h +++ b/src/wallet/chunkcollector.h @@ -188,7 +188,9 @@ typedef struct mc_ChunkCollector int FillMarkPoolByFlag(uint32_t flag, uint32_t not_flag); int Commit(); - int CommitInternal(); + int CommitInternal(int fill_mempool); + + int Unsubscribe(mc_Buffer* lpEntities); void Zero(); int Destroy(); diff --git a/src/wallet/wallettxs.cpp b/src/wallet/wallettxs.cpp index c7c2d036..bf819439 100644 --- a/src/wallet/wallettxs.cpp +++ b/src/wallet/wallettxs.cpp @@ -1334,6 +1334,11 @@ int mc_WalletTxs::Unsubscribe(mc_Buffer* lpEntities,bool purge) } if(fDebug)LogPrint("wallet","wtxs: Unsubscribed from %d entities\n",lpEntities->GetCount()); m_Database->UnLock(); + + if(m_ChunkCollector) + { + m_ChunkCollector->Unsubscribe(lpEntities); + } return err; } From 7c9fbd0cc5ea8bb534e62d5340e6172191bcc140 Mon Sep 17 00:00:00 2001 From: mike31 Date: Tue, 31 Jul 2018 15:12:59 +0300 Subject: [PATCH 12/26] Random seed, fixing Mac sempahore issue --- src/multichain/multichaind.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/multichain/multichaind.cpp b/src/multichain/multichaind.cpp index 93d63b49..c9560b10 100644 --- a/src/multichain/multichaind.cpp +++ b/src/multichain/multichaind.cpp @@ -80,6 +80,8 @@ bool AppInit(int argc, char* argv[]) mc_gState->m_Params->Parse(argc, argv, MC_ETP_DAEMON); mc_CheckDataDirInConfFile(); + mc_RandomSeed(mc_TimeNowAsUInt()); + if(mc_gState->m_Params->NetworkName()) { if(strlen(mc_gState->m_Params->NetworkName()) > MC_PRM_NETWORK_NAME_MAX_SIZE) From 4ec9a04c32f1a4a2f7b08ea18e3a1921f89a7780 Mon Sep 17 00:00:00 2001 From: mike31 Date: Tue, 31 Jul 2018 15:38:29 +0300 Subject: [PATCH 13/26] boost downgrading instructions for MacOS compilation --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index c3a7db25..8e1bd8ef 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,12 @@ Install dependencies Install brew (follow instructions on brew.sh) brew install autoconf automake berkeley-db4 libtool boost openssl pkg-config rename +on MacOS High Sierra + + brew uninstall boost + brew install boost@1.57 + brew link boost@1.57 --force + Prepare for static linking -------------------------- Apple does not support statically linked binaries as [documented here](https://developer.apple.com/library/content/qa/qa1118/_index.html), however, it is convenient for end-users to launch a binary without having to first install brew, a third-party system designed for developers. @@ -111,6 +117,8 @@ The default brew cookbook for berkeley-db and boost builds static libraries, but In 'def configure_args' change 'shared' to 'no-shared' brew install openssl --force + + Compile MultiChain for Mac (64-bit) -------------------------- From 0168cea94c56ff57126685531869275f36d6fcc7 Mon Sep 17 00:00:00 2001 From: mike31 Date: Tue, 31 Jul 2018 15:40:28 +0300 Subject: [PATCH 14/26] Fixed help message for verbose parameter in getrawtransaction --- src/rpc/rpchelp.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rpc/rpchelp.cpp b/src/rpc/rpchelp.cpp index fa2d69d3..e803bb4f 100644 --- a/src/rpc/rpchelp.cpp +++ b/src/rpc/rpchelp.cpp @@ -1068,7 +1068,7 @@ void mc_InitRPCHelpMap05() "\nArguments:\n" "1. \"txid\" (string, required) The transaction id\n" - "2. verbose (numeric, optional, default=0) If 0, return a string, other return a json object\n" + "2. verbose (numeric or boolean, optional, default=0(false)) If 0, return a string, other return a json object\n" "\nResult (if verbose is not set or set to 0):\n" "\"data\" (string) The serialized, hex-encoded data for 'txid'\n" From a08285bec48cd576707132861a8ec2780d5d6752 Mon Sep 17 00:00:00 2001 From: mike31 Date: Thu, 2 Aug 2018 09:55:01 +0300 Subject: [PATCH 15/26] Avoiding followon scanning when unnecessary --- src/rpc/rpcutils.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/rpc/rpcutils.cpp b/src/rpc/rpcutils.cpp index 1afa1525..d16262ce 100644 --- a/src/rpc/rpcutils.cpp +++ b/src/rpc/rpcutils.cpp @@ -1562,7 +1562,9 @@ Object AssetEntry(const unsigned char *txid,int64_t quantity,uint32_t output_lev Array issues; int64_t total=0; - if(( (output_level & 0x0020) !=0 )|| mc_gState->m_Assets->HasFollowOns(txid)) + if(( (output_level & 0x0020) !=0 ) || // issuers + // For listassets with followons + ( (mc_gState->m_Assets->HasFollowOns(txid) != 0) && (quantity < 0) && ( (output_level & 0x00C0) == 0) )) { int64_t qty; mc_Buffer *followons; From 61cb4406cf6e29426ee65a154c2812c0d7f14300 Mon Sep 17 00:00:00 2001 From: mike31 Date: Thu, 2 Aug 2018 11:35:06 +0300 Subject: [PATCH 16/26] Fixed watch only flag in getbalance --- src/rpc/rpcwallet.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/rpc/rpcwallet.cpp b/src/rpc/rpcwallet.cpp index 630639b1..277829a9 100644 --- a/src/rpc/rpcwallet.cpp +++ b/src/rpc/rpcwallet.cpp @@ -1002,9 +1002,15 @@ Value getbalance(const Array& params, bool fHelp) { CTxOut txout; out.GetHashAndTxOut(txout); - if(out.IsTrustedNoDepth() || (out.nDepth >= nMinDepth)) + + isminetype fIsMine=pwalletMain->IsMine(txout); + + if(fIsMine & filter) { - nBalance+=txout.nValue; + if(out.IsTrustedNoDepth() || (out.nDepth >= nMinDepth)) + { + nBalance+=txout.nValue; + } } } } From 4635a60feedfec6e99eada98d47eae0c2d091158 Mon Sep 17 00:00:00 2001 From: mike31 Date: Thu, 2 Aug 2018 18:13:21 +0300 Subject: [PATCH 17/26] liststreamqueryitems preparations --- src/chainparams/state.h | 8 ++ src/rpc/rpcclient.cpp | 2 + src/rpc/rpclist.cpp | 1 + src/rpc/rpcserver.h | 1 + src/rpc/rpcstreams.cpp | 242 ++++++++++++++++++++++++++++++++++--- src/rpc/rpcwallet.h | 23 +++- src/rpc/rpcwalletutils.cpp | 233 ++++++++++++++++++++++++++++++++++- 7 files changed, 489 insertions(+), 21 deletions(-) diff --git a/src/chainparams/state.h b/src/chainparams/state.h index 327913c7..7ba26417 100644 --- a/src/chainparams/state.h +++ b/src/chainparams/state.h @@ -176,6 +176,8 @@ typedef struct mc_TmpBuffers mc_Buffer *m_RpcABNoMapBuffer1; mc_Buffer *m_RpcABNoMapBuffer2; mc_Buffer *m_RpcEntityRows; + mc_Buffer *m_RpcEntityRowsToMerge; + mc_Buffer *m_RpcEntityRowsFull; mc_SHA256 *m_RpcHasher1; mc_Script *m_RpcChunkScript1; mc_Script *m_RelayTmpBuffer; @@ -197,6 +199,10 @@ typedef struct mc_TmpBuffers mc_InitABufferDefault(m_RpcABNoMapBuffer2); m_RpcEntityRows=new mc_Buffer; m_RpcEntityRows->Initialize(MC_TDB_ENTITY_KEY_SIZE,MC_TDB_ROW_SIZE,MC_BUF_MODE_DEFAULT); + m_RpcEntityRowsToMerge=new mc_Buffer; + m_RpcEntityRowsToMerge->Initialize(MC_TDB_ENTITY_KEY_SIZE,MC_TDB_ROW_SIZE,MC_BUF_MODE_DEFAULT); + m_RpcEntityRowsFull=new mc_Buffer; + m_RpcEntityRowsFull->Initialize(MC_TDB_ENTITY_KEY_SIZE,MC_TDB_ROW_SIZE,MC_BUF_MODE_DEFAULT); m_RpcHasher1=new mc_SHA256(); m_RpcChunkScript1=new mc_Script(); m_RelayTmpBuffer=new mc_Script(); @@ -214,6 +220,8 @@ typedef struct mc_TmpBuffers delete m_RpcABNoMapBuffer1; delete m_RpcABNoMapBuffer2; delete m_RpcEntityRows; + delete m_RpcEntityRowsToMerge; + delete m_RpcEntityRowsFull; delete m_RpcHasher1; delete m_RpcChunkScript1; delete m_RelayTmpBuffer; diff --git a/src/rpc/rpcclient.cpp b/src/rpc/rpcclient.cpp index 5d49d62a..a5512c9b 100644 --- a/src/rpc/rpcclient.cpp +++ b/src/rpc/rpcclient.cpp @@ -196,6 +196,8 @@ static const CRPCConvertParam vRPCConvertParams[] = { "liststreampublishers", 3 }, { "liststreampublishers", 4 }, { "liststreampublishers", 5 }, + { "liststreamqueryitems", 1 }, + { "liststreamqueryitems", 2 }, { "liststreamkeyitems", 2 }, { "liststreamkeyitems", 3 }, { "liststreamkeyitems", 4 }, diff --git a/src/rpc/rpclist.cpp b/src/rpc/rpclist.cpp index abffe3ff..294e74df 100644 --- a/src/rpc/rpclist.cpp +++ b/src/rpc/rpclist.cpp @@ -240,6 +240,7 @@ static const CRPCCommand vRPCCommands[] = { "wallet", "getstreamitem", &getstreamitem, false, false, true }, { "wallet", "liststreamtxitems", &liststreamtxitems, false, false, true }, { "wallet", "liststreamitems", &liststreamitems, false, false, true }, + { "wallet", "liststreamqueryitems", &liststreamqueryitems, false, false, true }, { "wallet", "liststreamkeyitems", &liststreamkeyitems, false, false, true }, { "wallet", "liststreampublisheritems",&liststreampublisheritems,false, false, true }, { "wallet", "liststreamkeys", &liststreamkeys, false, false, true }, diff --git a/src/rpc/rpcserver.h b/src/rpc/rpcserver.h index ce2010b8..e03f08a1 100644 --- a/src/rpc/rpcserver.h +++ b/src/rpc/rpcserver.h @@ -260,6 +260,7 @@ extern json_spirit::Value getstreamitem(const json_spirit::Array& params, bool f extern json_spirit::Value liststreamtxitems(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value liststreamitems(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value liststreamkeyitems(const json_spirit::Array& params, bool fHelp); +extern json_spirit::Value liststreamqueryitems(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value liststreampublisheritems(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value liststreamkeys(const json_spirit::Array& params, bool fHelp); extern json_spirit::Value liststreampublishers(const json_spirit::Array& params, bool fHelp); diff --git a/src/rpc/rpcstreams.cpp b/src/rpc/rpcstreams.cpp index ea2377a2..3fa0350a 100644 --- a/src/rpc/rpcstreams.cpp +++ b/src/rpc/rpcstreams.cpp @@ -10,6 +10,15 @@ #include "json/json_spirit_reader_template.h" #include "json/json_spirit_writer_template.h" +#define MC_QPR_MAX_UNCHECKED_TX_LIST_SIZE 1048576 +#define MC_QPR_MAX_MERGED_TX_LIST_SIZE 1024 +#define MC_QPR_MAX_DIRTY_TX_LIST_SIZE 256 +#define MC_QPR_MAX_SECONDARY_TX_LIST_SIZE 1048576 +#define MC_QPR_TX_CHECK_COST 100 +#define MC_QPR_MAX_TX_PER_BLOCK 4 + + + Value createupgradefromcmd(const Array& params, bool fHelp); void parseStreamIdentifier(Value stream_identifier,mc_EntityDetails *entity) @@ -988,7 +997,7 @@ Value liststreamtxitems(const Array& params, bool fHelp) int stream_output; while(first_output < (int)wtx.vout.size()) { - Object entry=StreamItemEntry(wtx,first_output,stream_entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET,verbose,NULL,NULL,&stream_output); + Object entry=StreamItemEntry(wtx,first_output,stream_entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET,verbose,NULL,&stream_output); if(stream_output < (int)wtx.vout.size()) { @@ -1116,7 +1125,7 @@ Value liststreamitems(const Array& params, bool fHelp) uint256 hash; int first_output=mc_GetHashAndFirstOutput(lpEntTx,&hash); const CWalletTx& wtx=pwalletTxsMain->GetWalletTx(hash,NULL,NULL); - Object entry=StreamItemEntry(wtx,first_output,stream_entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET,verbose,NULL,NULL,NULL); + Object entry=StreamItemEntry(wtx,first_output,stream_entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET,verbose,NULL,NULL); if(entry.size()) { retArray.push_back(entry); @@ -1236,7 +1245,7 @@ Value liststreamblockitems(const Array& params, bool fHelp) int stream_output; while(first_output < (int)wtx.vout.size()) { - Object entry=StreamItemEntry(wtx,first_output,stream_entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET,verbose,NULL,NULL,&stream_output); + Object entry=StreamItemEntry(wtx,first_output,stream_entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET,verbose,NULL,&stream_output); if(entry.size()) { retArray.push_back(entry); @@ -1327,14 +1336,17 @@ Value getstreamsummary(const Array& params, bool fPublisher) bool fFirstPublisher=false; bool fFirstPublisherAll=false; string key_string=params[1].get_str(); - const char *key_ptr=key_string.c_str(); + vector conditions; + if(fPublisher) { - getSubKeyEntityFromPublisher(params[1].get_str(),entStat,&entity); + getSubKeyEntityFromPublisher(params[1].get_str(),entStat,&entity); + conditions.push_back(mc_QueryCondition(MC_QCT_PUBLISHER,params[1].get_str())); } else { getSubKeyEntityFromKey(params[1].get_str(),entStat,&entity); + conditions.push_back(mc_QueryCondition(MC_QCT_KEY,params[1].get_str())); } set setFirstPublishers; @@ -1441,6 +1453,8 @@ Value getstreamsummary(const Array& params, bool fPublisher) int first_output=mc_GetHashAndFirstOutput(lpEntTx,&hash); const CWalletTx& wtx=pwalletTxsMain->GetWalletTx(hash,NULL,NULL); Object entry; + entry=StreamItemEntry(wtx,first_output,stream_entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET,false,&conditions,NULL); +/* if(fPublisher) { entry=StreamItemEntry(wtx,first_output,stream_entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET,false,NULL,&key_ptr,NULL); @@ -1449,7 +1463,7 @@ Value getstreamsummary(const Array& params, bool fPublisher) { entry=StreamItemEntry(wtx,first_output,stream_entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET,false,&key_ptr,NULL,NULL); } - +*/ if(fFirstPublisher) { pcount=0; @@ -1678,10 +1692,12 @@ Value liststreamkeyitems(const Array& params, bool fHelp) } string key_string=params[1].get_str(); - const char *key_ptr=key_string.c_str(); getSubKeyEntityFromKey(params[1].get_str(),entStat,&entity); - + vector conditions; + + conditions.push_back(mc_QueryCondition(MC_QCT_KEY,params[1].get_str())); + mc_Buffer *entity_rows=mc_gState->m_TmpBuffers->m_RpcEntityRows; entity_rows->Clear(); @@ -1697,7 +1713,7 @@ Value liststreamkeyitems(const Array& params, bool fHelp) uint256 hash; int first_output=mc_GetHashAndFirstOutput(lpEntTx,&hash); const CWalletTx& wtx=pwalletTxsMain->GetWalletTx(hash,NULL,NULL); - Object entry=StreamItemEntry(wtx,first_output,stream_entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET,verbose,&key_ptr,NULL,NULL); + Object entry=StreamItemEntry(wtx,first_output,stream_entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET,verbose,&conditions,NULL); if(entry.size()) { retArray.push_back(entry); @@ -1781,9 +1797,12 @@ Value liststreampublisheritems(const Array& params, bool fHelp) } string key_string=params[1].get_str(); - const char *key_ptr=key_string.c_str(); getSubKeyEntityFromPublisher(params[1].get_str(),entStat,&entity); + vector conditions; + + conditions.push_back(mc_QueryCondition(MC_QCT_PUBLISHER,params[1].get_str())); + mc_Buffer *entity_rows=mc_gState->m_TmpBuffers->m_RpcEntityRows; entity_rows->Clear(); @@ -1799,7 +1818,7 @@ Value liststreampublisheritems(const Array& params, bool fHelp) uint256 hash; int first_output=mc_GetHashAndFirstOutput(lpEntTx,&hash); const CWalletTx& wtx=pwalletTxsMain->GetWalletTx(hash,NULL,NULL); - Object entry=StreamItemEntry(wtx,first_output,stream_entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET,verbose,NULL,&key_ptr,NULL); + Object entry=StreamItemEntry(wtx,first_output,stream_entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET,verbose,&conditions,NULL); if(entry.size()) { retArray.push_back(entry); @@ -1826,8 +1845,6 @@ Value liststreammap_operation(mc_TxEntity *parent_entity,vector& in mc_TxEntityRow erow; uint160 stream_subkey_hash; int row,enitity_count; - const char **given_key; - const char **given_publisher; entity_rows->Clear(); enitity_count=inputEntities.size(); @@ -1879,18 +1896,17 @@ Value liststreammap_operation(mc_TxEntity *parent_entity,vector& in { shift=1; } - const char *key_ptr=key_string.c_str(); - given_key=NULL; - given_publisher=NULL; + vector conditions; + if((parent_entity->m_EntityType & MC_TET_TYPE_MASK) == MC_TET_STREAM_PUBLISHER) { all_entry.push_back(Pair("publisher", key_string)); - given_publisher=&key_ptr; + conditions.push_back(mc_QueryCondition(MC_QCT_PUBLISHER,key_string)); } else { all_entry.push_back(Pair("key", key_string)); - given_key=&key_ptr; + conditions.push_back(mc_QueryCondition(MC_QCT_KEY,key_string)); } all_entry.push_back(Pair("items", total)); all_entry.push_back(Pair("confirmed", confirmed)); @@ -1915,7 +1931,7 @@ Value liststreammap_operation(mc_TxEntity *parent_entity,vector& in Value item_value; - item_value=StreamItemEntry(wtx,first_output,parent_entity->m_EntityID,true,given_key,given_publisher,NULL); + item_value=StreamItemEntry(wtx,first_output,parent_entity->m_EntityID,true,&conditions,NULL); if(row == 1) { all_entry.push_back(Pair("first", item_value)); @@ -2064,3 +2080,191 @@ Value liststreampublishers(const Array& params, bool fHelp) return liststreamkeys_or_publishers(params,true); } +int GetAndQueryDirtyList(vector& conditions, mc_EntityDetails *stream_entity,bool fLocalOrdering,mc_Buffer *entity_rows) +{ + int i,row,out_row; + int conditions_count=(int)conditions.size(); + int conditions_used=0; + int max_size=0; + int clean_count,dirty_count,last_state; + vector vConditionEntities; + vector vConditionListSizes; + vector vConditionMerged; + mc_TxEntityStat entStat; + bool merge_lists=true; + + vConditionEntities.resize(conditions_count+1); + vConditionListSizes.resize(conditions_count+1); + vConditionMerged.resize(conditions_count+1); + + entStat.Zero(); + memcpy(&entStat,stream_entity->GetTxID()+MC_AST_SHORT_TXID_OFFSET,MC_AST_SHORT_TXID_SIZE); + entStat.m_Entity.m_EntityType=MC_TET_STREAM; + if(fLocalOrdering) + { + entStat.m_Entity.m_EntityType |= MC_TET_TIMERECEIVED; + } + else + { + entStat.m_Entity.m_EntityType |= MC_TET_CHAINPOS; + } + if(!pwalletTxsMain->FindEntity(&entStat)) + { + throw JSONRPCError(RPC_NOT_SUBSCRIBED, "Not subscribed to this stream"); + } + + for(i=0;i<=conditions_count;i++) + { + vConditionEntities[i].Zero(); + vConditionListSizes[i]=-1; + vConditionMerged[i]=0; + + entStat.m_Entity.m_EntityType &= MC_TET_ORDERMASK; + if(iGetListSize(&vConditionEntities[i],entStat.m_Generation,NULL); + if(vConditionListSizes[i]>max_size) + { + max_size=vConditionListSizes[i]; + } + } + } + + clean_count=0; + dirty_count=0; + + while(merge_lists) + { + int min_size=max_size+1; + int min_condition=conditions_count; + conditions_count=false; + for(i=0;i<=conditions_count;i++) + { + if(vConditionMerged[i] == 0) + { + if(vConditionListSizes[i]<=min_size) + { + min_size=vConditionListSizes[i]; + min_condition=i; + } + } + } + + if(min_condition MC_QPR_MAX_UNCHECKED_TX_LIST_SIZE) + { + throw JSONRPCError(RPC_NOT_SUBSCRIBED, "Not subscribed to this stream"); + } + pwalletTxsMain->GetList(&vConditionEntities[min_condition],entStat.m_Generation,1,min_size,entity_rows); + conditions_used++; + clean_count=0; + dirty_count=0; + for(row=0;rowGetCount();row++) + { + mc_TxEntityRow *lpEntTx; + lpEntTx=(mc_TxEntityRow*)entity_rows->GetRow(row); + if( (lpEntTx->m_Flags & MC_TFL_IS_EXTENSION) == 0 ) + { + clean_count++; + } + if(!fLocalOrdering) + { + if(lpEntTx->m_Block == -1) + { + lpEntTx->m_Block=chainActive.Height()+1; + } + } + } + } + else + { + merge_lists=false; + } + } + } + + last_state=-2; + clean_count=0; + dirty_count=0; + out_row=0; + for(row=0;rowGetCount();row++) + { + mc_TxEntityRow *lpEntTx; + lpEntTx=(mc_TxEntityRow*)entity_rows->GetRow(row); + if(lpEntTx->m_Flags & MC_TFL_IS_EXTENSION) + { + lpEntTx->m_Generation=last_state; + } + else + { + switch(lpEntTx->m_Generation) + { + case -1: + break; + case -2: + dirty_count++; + break; + default: + if(conditions_used < conditions_count) + { + lpEntTx->m_Generation=-2; + dirty_count++; + } + else + { + clean_count++; + } + break; + } + last_state=lpEntTx->m_Generation; + } + if(lpEntTx->m_Generation != -1) + { + if(out_row < row) + { + memcpy(entity_rows->GetRow(out_row),lpEntTx,entity_rows->m_Size); + out_row++; + } + } + } + + entity_rows->SetCount(out_row); + + return dirty_count; +} + +Value liststreamqueryitems(const Array& params, bool fHelp) +{ + if (fHelp || params.size() < 2 || params.size() > 6) + throw runtime_error("Help message not found\n"); + + if((mc_gState->m_WalletMode & MC_WMD_TXS) == 0) + { + throw JSONRPCError(RPC_NOT_SUPPORTED, "API is not supported with this wallet version. For full streams functionality, run \"multichaind -walletdbversion=2 -rescan\" "); + } + + return Value::null; +} diff --git a/src/rpc/rpcwallet.h b/src/rpc/rpcwallet.h index adf4cf49..b212d645 100644 --- a/src/rpc/rpcwallet.h +++ b/src/rpc/rpcwallet.h @@ -23,12 +23,33 @@ #include "wallet/chunkdb.h" #include "rpc/rpcutils.h" +#define MC_QCT_KEY 1 +#define MC_QCT_PUBLISHER 2 + + +typedef struct mc_QueryCondition +{ + uint32_t m_Type; + string m_Value; + bool m_TmpMatch; + + mc_QueryCondition(int type, string value) + { + m_Type=type; + m_Value=value; + m_TmpMatch=false; + } +} mc_QueryCondition; + + + + void SendMoneyToSeveralAddresses(const std::vector addresses, CAmount nValue, CWalletTx& wtxNew,mc_Script *dropscript,CScript scriptOpReturn,const std::vector& fromaddresses); vector ParseAddresses(string param, bool create_full_list, bool allow_scripthash); void FindAddressesWithPublishPermission(std::vector& fromaddresses,mc_EntityDetails *stream_entity); set ParseAddresses(Value param, isminefilter filter); bool CBitcoinAddressFromTxEntity(CBitcoinAddress &address,mc_TxEntity *lpEntity); -Object StreamItemEntry(const CWalletTx& wtx,int first_output,const unsigned char *stream_id, bool verbose, const char** given_key,const char ** given_publisher,int *output); +Object StreamItemEntry(const CWalletTx& wtx,int first_output,const unsigned char *stream_id, bool verbose, vector *given_conditions,int *output); Object TxOutEntry(const CTxOut& TxOutIn,int vout,const CTxIn& TxIn,uint256 hash,mc_Buffer *amounts,mc_Script *lpScript); void WalletTxToJSON(const CWalletTx& wtx, Object& entry,bool skipWalletConflicts = false, int vout = -1); void MinimalWalletTxToJSON(const CWalletTx& wtx, Object& entry); diff --git a/src/rpc/rpcwalletutils.cpp b/src/rpc/rpcwalletutils.cpp index bb10546a..74804c38 100644 --- a/src/rpc/rpcwalletutils.cpp +++ b/src/rpc/rpcwalletutils.cpp @@ -430,7 +430,7 @@ bool CBitcoinAddressFromTxEntity(CBitcoinAddress &address,mc_TxEntity *lpEntity) return false; } -Object StreamItemEntry(const CWalletTx& wtx,int first_output,const unsigned char *stream_id, bool verbose, const char** given_key,const char ** given_publisher,int *output) +Object StreamItemEntry1(const CWalletTx& wtx,int first_output,const unsigned char *stream_id, bool verbose, const char** given_key,const char ** given_publisher,int *output) { Object entry; Array publishers; @@ -628,6 +628,237 @@ Object StreamItemEntry(const CWalletTx& wtx,int first_output,const unsigned char return entry; } +Object StreamItemEntry(const CWalletTx& wtx,int first_output,const unsigned char *stream_id, bool verbose, vector *given_conditions,int *output) +{ + Object entry; + Array publishers; + set publishers_set; + Array keys; + int stream_output; + const unsigned char *ptr; + unsigned char item_key[MC_ENT_MAX_ITEM_KEY_SIZE+1]; + int item_key_size; +// Value item_value; + uint32_t format; + unsigned char *chunk_hashes; + int chunk_count; + int64_t total_chunk_size,out_size; + uint32_t retrieve_status=0; + Value format_item_value; + string format_text_str; + int start_from=first_output; + + stream_output=-1; + if(output) + { + *output=(int)wtx.vout.size(); + } + while( (stream_output < 0) && (start_from<(int)wtx.vout.size()) ) + { + stream_output=-1; + for (int j = start_from; j < (int)wtx.vout.size(); ++j) + { + if(stream_output < 0) + { + if(given_conditions) + { + for(int c=0;c<(int)(*given_conditions).size();c++) + { + (*given_conditions)[c].m_TmpMatch=false; + } + } + keys.clear(); + const CScript& script1 = wtx.vout[j].scriptPubKey; + CScript::const_iterator pc1 = script1.begin(); + + mc_gState->m_TmpScript->Clear(); + mc_gState->m_TmpScript->SetScript((unsigned char*)(&pc1[0]),(size_t)(script1.end()-pc1),MC_SCR_TYPE_SCRIPTPUBKEY); + + if(mc_gState->m_TmpScript->IsOpReturnScript()) + { + if(mc_gState->m_TmpScript->GetNumElements()) // 2 OP_DROPs + OP_RETURN - item key + { + mc_gState->m_TmpScript->ExtractAndDeleteDataFormat(&format,&chunk_hashes,&chunk_count,&total_chunk_size); +// chunk_hashes=NULL; +// mc_gState->m_TmpScript->ExtractAndDeleteDataFormat(&format); + + unsigned char short_txid[MC_AST_SHORT_TXID_SIZE]; + mc_gState->m_TmpScript->SetElement(0); + + if(mc_gState->m_TmpScript->GetEntity(short_txid) == 0) + { + if(memcmp(short_txid,stream_id,MC_AST_SHORT_TXID_SIZE) == 0) + { + stream_output=j; + for(int e=1;em_TmpScript->GetNumElements()-1;e++) + { + mc_gState->m_TmpScript->SetElement(e); + // Should be spkk + if(mc_gState->m_TmpScript->GetItemKey(item_key,&item_key_size)) // Item key + { + return entry; + } + item_key[item_key_size]=0; + + if(given_conditions) + { + for(int c=0;c<(int)(*given_conditions).size();c++) + { + if((*given_conditions)[c].m_Type == MC_QCT_KEY) + { + if(strcmp((char*)item_key,(*given_conditions)[c].m_Value.c_str()) == 0) + { + (*given_conditions)[c].m_TmpMatch=true; + } + } + } + } + + keys.push_back(string(item_key,item_key+item_key_size)); + } + + if(given_conditions) + { + for(int c=0;c<(int)(*given_conditions).size();c++) + { + if((*given_conditions)[c].m_Type == MC_QCT_KEY) + { + if(!(*given_conditions)[c].m_TmpMatch) + { + stream_output=-1; + } + } + } + } + + const unsigned char *elem; + +// elem = mc_gState->m_TmpScript->GetData(mc_gState->m_TmpScript->GetNumElements()-1,&elem_size); + retrieve_status = GetFormattedData(mc_gState->m_TmpScript,&elem,&out_size,chunk_hashes,chunk_count,total_chunk_size); +// item_value=OpReturnEntry(elem,elem_size,wtx.GetHash(),j); + format_item_value=OpReturnFormatEntry(elem,out_size,wtx.GetHash(),j,format,&format_text_str,retrieve_status); + } + } + } + } + } + } + + if(stream_output < 0) + { + return entry; + } + if(output) + { + *output=stream_output; + } + + publishers.clear(); + publishers_set.clear(); + for (int i = 0; i < (int)wtx.vin.size(); ++i) + { + int op_addr_offset,op_addr_size,is_redeem_script,sighash_type; + + const CScript& script2 = wtx.vin[i].scriptSig; + CScript::const_iterator pc2 = script2.begin(); + + ptr=mc_ExtractAddressFromInputScript((unsigned char*)(&pc2[0]),(int)(script2.end()-pc2),&op_addr_offset,&op_addr_size,&is_redeem_script,&sighash_type,0); + if(ptr) + { + if( (sighash_type == SIGHASH_ALL) || ( (sighash_type == SIGHASH_SINGLE) && (i == stream_output) ) ) + { + uint160 publisher_hash=Hash160(ptr+op_addr_offset,ptr+op_addr_offset+op_addr_size); + if(publishers_set.count(publisher_hash) == 0) + { + publishers_set.insert(publisher_hash); + string publisher_str; + if(is_redeem_script) + { + publisher_str=CBitcoinAddress((CScriptID)publisher_hash).ToString(); + } + else + { + publisher_str=CBitcoinAddress((CKeyID)publisher_hash).ToString(); + } + + if(given_conditions) + { + for(int c=0;c<(int)(*given_conditions).size();c++) + { + if((*given_conditions)[c].m_Type == MC_QCT_PUBLISHER) + { + if(strcmp(publisher_str.c_str(),(*given_conditions)[c].m_Value.c_str()) == 0) + { + (*given_conditions)[c].m_TmpMatch=true; + } + } + } + } + + publishers.push_back(publisher_str); + } + } + } + } + + if(given_conditions) + { + for(int c=0;c<(int)(*given_conditions).size();c++) + { + if((*given_conditions)[c].m_Type == MC_QCT_PUBLISHER) + { + if(!(*given_conditions)[c].m_TmpMatch) + { + stream_output=-1; + } + } + } + } + + if(stream_output < 0) + { + start_from++; + } + } + + if(stream_output < 0) + { + return entry; + } + + + entry.push_back(Pair("publishers", publishers)); + entry.push_back(Pair("keys", keys)); + if(mc_gState->m_Compatibility & MC_VCM_1_0) + { + entry.push_back(Pair("key", keys[0])); + } + entry.push_back(Pair("offchain", (retrieve_status & MC_OST_STORAGE_MASK) == MC_OST_OFF_CHAIN)); + if( ( retrieve_status & MC_OST_CONTROL_NO_DATA ) == 0) + { + entry.push_back(Pair("available", AvailableFromStatus(retrieve_status))); + if(retrieve_status & MC_OST_ERROR_MASK) + { + string error_str; + int errorCode; + error_str=OffChainError(retrieve_status,&errorCode); + entry.push_back(Pair("error", error_str)); + } + } + entry.push_back(Pair("data", format_item_value)); + + if(verbose) + { + WalletTxToJSON(wtx, entry, true, stream_output); + } + else + { + MinimalWalletTxToJSON(wtx, entry); + } + + return entry; +} + Object TxOutEntry(const CTxOut& TxOutIn,int vout,const CTxIn& TxIn,uint256 hash,mc_Buffer *amounts,mc_Script *lpScript) { From 32c6bc0875e5f741705c778406fd3ec9d83b214c Mon Sep 17 00:00:00 2001 From: mike31 Date: Sun, 5 Aug 2018 12:43:14 +0300 Subject: [PATCH 18/26] liststreamqueryitems --- src/rpc/rpcstreams.cpp | 247 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 227 insertions(+), 20 deletions(-) diff --git a/src/rpc/rpcstreams.cpp b/src/rpc/rpcstreams.cpp index 3fa0350a..79448d0e 100644 --- a/src/rpc/rpcstreams.cpp +++ b/src/rpc/rpcstreams.cpp @@ -12,7 +12,8 @@ #define MC_QPR_MAX_UNCHECKED_TX_LIST_SIZE 1048576 #define MC_QPR_MAX_MERGED_TX_LIST_SIZE 1024 -#define MC_QPR_MAX_DIRTY_TX_LIST_SIZE 256 +#define MC_QPR_MAX_DIRTY_TX_LIST_SIZE 5000 +#define MC_QPR_MAX_CLEAN_TX_LIST_SIZE 5000 #define MC_QPR_MAX_SECONDARY_TX_LIST_SIZE 1048576 #define MC_QPR_TX_CHECK_COST 100 #define MC_QPR_MAX_TX_PER_BLOCK 4 @@ -2125,23 +2126,23 @@ int GetAndQueryDirtyList(vector& conditions, mc_EntityDetails switch(conditions[i].m_Type) { case MC_QCT_KEY: - entStat.m_Entity.m_EntityType=MC_TET_STREAM_KEY; + entStat.m_Entity.m_EntityType |= MC_TET_STREAM_KEY; getSubKeyEntityFromKey(conditions[i].m_Value,entStat,&vConditionEntities[i]); break; case MC_QCT_PUBLISHER: - entStat.m_Entity.m_EntityType=MC_TET_STREAM_PUBLISHER; - getSubKeyEntityFromKey(conditions[i].m_Value,entStat,&vConditionEntities[i]); + entStat.m_Entity.m_EntityType |= MC_TET_STREAM_PUBLISHER; + getSubKeyEntityFromPublisher(conditions[i].m_Value,entStat,&vConditionEntities[i]); break; } } else { - entStat.m_Entity.m_EntityType=MC_TET_STREAM; + entStat.m_Entity.m_EntityType |= MC_TET_STREAM; memcpy(&vConditionEntities[i],&entStat.m_Entity,sizeof(mc_TxEntity)); } if(vConditionEntities[i].m_EntityType) { - vConditionListSizes[i]=pwalletTxsMain->GetListSize(&vConditionEntities[i],entStat.m_Generation,NULL); + vConditionListSizes[i]=pwalletTxsMain->GetListSize(&vConditionEntities[i],entStat.m_Generation,NULL); if(vConditionListSizes[i]>max_size) { max_size=vConditionListSizes[i]; @@ -2156,7 +2157,6 @@ int GetAndQueryDirtyList(vector& conditions, mc_EntityDetails { int min_size=max_size+1; int min_condition=conditions_count; - conditions_count=false; for(i=0;i<=conditions_count;i++) { if(vConditionMerged[i] == 0) @@ -2172,11 +2172,11 @@ int GetAndQueryDirtyList(vector& conditions, mc_EntityDetails if(min_condition MC_QPR_MAX_UNCHECKED_TX_LIST_SIZE) { - throw JSONRPCError(RPC_NOT_SUBSCRIBED, "Not subscribed to this stream"); + throw JSONRPCError(RPC_NOT_SUPPORTED, "This query may take too much time"); } pwalletTxsMain->GetList(&vConditionEntities[min_condition],entStat.m_Generation,1,min_size,entity_rows); conditions_used++; @@ -2186,6 +2186,7 @@ int GetAndQueryDirtyList(vector& conditions, mc_EntityDetails { mc_TxEntityRow *lpEntTx; lpEntTx=(mc_TxEntityRow*)entity_rows->GetRow(row); + lpEntTx->m_TempPos=0; if( (lpEntTx->m_Flags & MC_TFL_IS_EXTENSION) == 0 ) { clean_count++; @@ -2204,9 +2205,13 @@ int GetAndQueryDirtyList(vector& conditions, mc_EntityDetails merge_lists=false; } } + else + { + merge_lists=false; + } } - last_state=-2; + last_state=2; clean_count=0; dirty_count=0; out_row=0; @@ -2216,21 +2221,25 @@ int GetAndQueryDirtyList(vector& conditions, mc_EntityDetails lpEntTx=(mc_TxEntityRow*)entity_rows->GetRow(row); if(lpEntTx->m_Flags & MC_TFL_IS_EXTENSION) { - lpEntTx->m_Generation=last_state; + lpEntTx->m_TempPos=last_state; + if(lpEntTx->m_TempPos == 2) + { + dirty_count++; + } } else { - switch(lpEntTx->m_Generation) + switch(lpEntTx->m_TempPos) { - case -1: + case 1: break; - case -2: + case 2: dirty_count++; break; default: if(conditions_used < conditions_count) { - lpEntTx->m_Generation=-2; + lpEntTx->m_TempPos=2; dirty_count++; } else @@ -2239,15 +2248,15 @@ int GetAndQueryDirtyList(vector& conditions, mc_EntityDetails } break; } - last_state=lpEntTx->m_Generation; + last_state=lpEntTx->m_TempPos; } - if(lpEntTx->m_Generation != -1) + if(lpEntTx->m_TempPos != 1) { if(out_row < row) { memcpy(entity_rows->GetRow(out_row),lpEntTx,entity_rows->m_Size); - out_row++; } + out_row++; } } @@ -2256,15 +2265,213 @@ int GetAndQueryDirtyList(vector& conditions, mc_EntityDetails return dirty_count; } +void FillContitionsList(vector& conditions, Value param) +{ + bool key_found=false; + bool publisher_found=false; + bool field_parsed; + + if(param.type() != obj_type) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid query, should be object "); + } + + BOOST_FOREACH(const Pair& d, param.get_obj()) + { + field_parsed=false; + if(d.name_ == "key") + { + if(key_found) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Only one of the key fields can appear in the object"); + } + if(d.value_.type()==str_type) + { + conditions.push_back(mc_QueryCondition(MC_QCT_KEY,d.value_.get_str())); + } + else + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid key, should be string"); + } + field_parsed=true; + key_found=true; + } + + if(d.name_ == "keys") + { + if( mc_gState->m_Features->MultipleStreamKeys() == 0 ) + { + throw JSONRPCError(RPC_NOT_SUPPORTED, "Multiple keys are not supported by this protocol version"); + } + if(key_found) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Only one of the key fields can appear in the object"); + } + if(d.value_.type() == array_type) + { + for(int i=0;i<(int)d.value_.get_array().size();i++) + { + if(d.value_.get_array()[i].type()==str_type) + { + conditions.push_back(mc_QueryCondition(MC_QCT_KEY,d.value_.get_array()[i].get_str())); + } + else + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid key, should be string"); + } + } + } + else + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid keys, should be array"); + } + field_parsed=true; + key_found=true; + } + + if(d.name_ == "publisher") + { + if(publisher_found) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Only one of the publisher fields can appear in the object"); + } + if(d.value_.type()==str_type) + { + conditions.push_back(mc_QueryCondition(MC_QCT_PUBLISHER,d.value_.get_str())); + } + else + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid publisher, should be string"); + } + field_parsed=true; + publisher_found=true; + } + + if(d.name_ == "publishers") + { + if(publisher_found) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Only one of the publisher fields can appear in the object"); + } + if(d.value_.type() == array_type) + { + for(int i=0;i<(int)d.value_.get_array().size();i++) + { + if(d.value_.get_array()[i].type()==str_type) + { + conditions.push_back(mc_QueryCondition(MC_QCT_PUBLISHER,d.value_.get_array()[i].get_str())); + } + else + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid publisher, should be string"); + } + } + } + else + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid publishers, should be array"); + } + field_parsed=true; + publisher_found=true; + } + + if(!field_parsed) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Invalid field: %s",d.name_.c_str())); + } + + } +} + Value liststreamqueryitems(const Array& params, bool fHelp) { - if (fHelp || params.size() < 2 || params.size() > 6) + if (fHelp || params.size() < 2 || params.size() > 3) throw runtime_error("Help message not found\n"); if((mc_gState->m_WalletMode & MC_WMD_TXS) == 0) { throw JSONRPCError(RPC_NOT_SUPPORTED, "API is not supported with this wallet version. For full streams functionality, run \"multichaind -walletdbversion=2 -rescan\" "); } + + vector conditions; + vector * lpConditions; - return Value::null; + mc_EntityDetails stream_entity; + parseStreamIdentifier(params[0],&stream_entity); + + bool verbose=false; + int dirty_count; + + if (params.size() > 2) + { + verbose=paramtobool(params[2]); + } + + FillContitionsList(conditions,params[1]); + + if(conditions.size() == 0) + { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid query, cannot be empty"); + } + + mc_Buffer *entity_rows=mc_gState->m_TmpBuffers->m_RpcEntityRows; + entity_rows->Clear(); + + dirty_count=GetAndQueryDirtyList(conditions,&stream_entity,false,entity_rows); + + if(dirty_count > MC_QPR_MAX_DIRTY_TX_LIST_SIZE) + { + throw JSONRPCError(RPC_NOT_SUPPORTED, "This query may take too much time"); + } + + if(entity_rows->GetCount() > MC_QPR_MAX_CLEAN_TX_LIST_SIZE) + { + throw JSONRPCError(RPC_NOT_SUPPORTED, "Resulting list is too large"); + } + + Array retArray; + int last_output; + uint256 last_hash=0; + for(int i=0;iGetCount();i++) + { + mc_TxEntityRow *lpEntTx; + lpEntTx=(mc_TxEntityRow*)entity_rows->GetRow(i); + lpConditions=NULL; + if(lpEntTx->m_TempPos != 1) + { + if(lpEntTx->m_TempPos == 2) + { + lpConditions=&conditions; + } + uint256 hash; + int first_output=mc_GetHashAndFirstOutput(lpEntTx,&hash); + if(last_hash == hash) + { + if(first_output <= last_output) + { + first_output=-1; + } + } + else + { + last_output=-1; + } + last_hash=hash; + if(first_output >= 0) + { + const CWalletTx& wtx=pwalletTxsMain->GetWalletTx(hash,NULL,NULL); + Object entry=StreamItemEntry(wtx,first_output,stream_entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET,verbose,lpConditions,&last_output); + if(entry.size()) + { + retArray.push_back(entry); + } + else + { + last_output=-1; + } + } + } + } + + return retArray; } From 36c76df99ac43c9319a6e6b3684136b59fdbc1a9 Mon Sep 17 00:00:00 2001 From: mike31 Date: Sun, 5 Aug 2018 13:04:34 +0300 Subject: [PATCH 19/26] liststreamqueryitems help --- src/rpc/rpchelp.cpp | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/rpc/rpchelp.cpp b/src/rpc/rpchelp.cpp index e803bb4f..e9c3f823 100644 --- a/src/rpc/rpchelp.cpp +++ b/src/rpc/rpchelp.cpp @@ -4118,7 +4118,31 @@ void mc_InitRPCHelpMap18() )); - mapHelpStrings.insert(std::make_pair("AAAAAAA", + mapHelpStrings.insert(std::make_pair("liststreamqueryitems", + "liststreamqueryitems \"stream-identifier\" query ( verbose )\n" + "\nReturns stream items for specific query.\n" + "\nArguments:\n" + "1. \"stream-identifier\" (string, required) Stream identifier - one of the following: stream txid, stream reference, stream name.\n" + "2. query (object, required) Query\n" + " {\n" + " \"key\" : \"key\" (string, optional, default: \"\") Item key\n" + " or\n" + " \"keys\" : keys (array, optional) Item keys, array of strings\n" + " or\n" + " \"publisher\" : \"publisher\" (string, optional, default: \"\") Publisher\n" + " or\n" + " \"publishers\" : publishers (array, optional) Publishers, array of strings\n" + " }\n" + "3. verbose (boolean, optional, default=false) If true, returns information about item transaction \n" + "\nResult:\n" + "\"stream-items\" (array) List of stream items for specific query.\n" + "\nExamples:\n" + + HelpExampleCli("liststreamqueryitems", "\"test-stream\" \"{\\\"keys\\\":[\\\"key01\\\",\"key02\"]}\"") + + HelpExampleCli("liststreamqueryitems", "\"test-stream\" \"{\\\"keys\\\":[\\\"key01\\\",\"key02\"],\\\"publisher\\\":\\\"1D1ZrZNe3JUo7ZycKEYQQiQAWd9y54F4XZ\\\"}\" true ") + + HelpExampleRpc("liststreamqueryitems", "\"test-stream\", \"{\\\"keys\\\":[\\\"key01\\\",\"key02\"],\\\"publisher\\\":\\\"1D1ZrZNe3JUo7ZycKEYQQiQAWd9y54F4XZ\\\"}\", false") + )); + + mapHelpStrings.insert(std::make_pair("AAAAAAA", "" )); From 7096905aa65a504de14d5edf532f192c0bf65ef1 Mon Sep 17 00:00:00 2001 From: mike31 Date: Sun, 5 Aug 2018 13:25:45 +0300 Subject: [PATCH 20/26] Multiple txs in liststreamtxitems --- src/rpc/rpcclient.cpp | 2 ++ src/rpc/rpchelp.cpp | 6 ++++-- src/rpc/rpcstreams.cpp | 37 ++++++++++++++++++++++++------------- 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/src/rpc/rpcclient.cpp b/src/rpc/rpcclient.cpp index a5512c9b..b46e6148 100644 --- a/src/rpc/rpcclient.cpp +++ b/src/rpc/rpcclient.cpp @@ -178,6 +178,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "listassettransactions", 4 }, { "getassettransaction", 2 }, { "getstreamitem", 2 }, + { "liststreamtxitems", 1 }, { "liststreamtxitems", 2 }, { "liststreamitems", 1 }, { "liststreamitems", 2 }, @@ -341,6 +342,7 @@ static const CRPCConvertParamMayBeString vRPCConvertParamsMayBeString[] = { "unsubscribe", 0 }, { "liststreamkeys", 1 }, { "liststreampublishers", 1 }, + { "liststreamtxitems", 1 }, { "listassets", 0 }, { "liststreams", 0 }, { "listupgrades", 0 }, diff --git a/src/rpc/rpchelp.cpp b/src/rpc/rpchelp.cpp index e9c3f823..3fd477cd 100644 --- a/src/rpc/rpchelp.cpp +++ b/src/rpc/rpchelp.cpp @@ -3709,11 +3709,13 @@ void mc_InitRPCHelpMap16() )); mapHelpStrings.insert(std::make_pair("liststreamtxitems", - "liststreamtxitems \"stream-identifier\" \"txid\" ( verbose )\n" + "liststreamtxitems \"stream-identifier\" txids ( verbose )\n" "\nReturns stream items.\n" "\nArguments:\n" "1. \"stream-identifier\" (string, required) Stream identifier - one of the following: stream txid, stream reference, stream name.\n" - "2. \"txid\" (string, required) The transaction id\n" + "2. \"txids\" (string, required) Transaction IDs, comma delimited\n" + " or\n" + "2. txids (array, required) Array of transaction IDs\n" "3. verbose (boolean, optional, default=false) If true, returns information about item transaction \n" "\nResult:\n" "\"stream-items\" (array) Array of stream items.\n" diff --git a/src/rpc/rpcstreams.cpp b/src/rpc/rpcstreams.cpp index 79448d0e..30884720 100644 --- a/src/rpc/rpcstreams.cpp +++ b/src/rpc/rpcstreams.cpp @@ -981,9 +981,6 @@ Value liststreamtxitems(const Array& params, bool fHelp) throw JSONRPCError(RPC_NOT_SUBSCRIBED, "Not subscribed to this stream"); } - - uint256 hash = ParseHashV(params[1], "parameter 2"); - bool verbose=false; if (params.size() > 2) @@ -991,21 +988,35 @@ Value liststreamtxitems(const Array& params, bool fHelp) verbose=paramtobool(params[2]); } - const CWalletTx& wtx=pwalletTxsMain->GetWalletTx(hash,NULL,NULL); - Array output_array; - int first_output=0; - int stream_output; - while(first_output < (int)wtx.vout.size()) + + vector inputStrings; + + inputStrings=ParseStringList(params[1]); + + for(int j=0;j<(int)inputStrings.size();j++) { - Object entry=StreamItemEntry(wtx,first_output,stream_entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET,verbose,NULL,&stream_output); + uint256 hash = ParseHashV(inputStrings[j], "txid"); - if(stream_output < (int)wtx.vout.size()) + const CWalletTx& wtx=pwalletTxsMain->GetWalletTx(hash,NULL,NULL); + + int first_output=0; + int stream_output; + while(first_output < (int)wtx.vout.size()) { - output_array.push_back(entry); + Object entry=StreamItemEntry(wtx,first_output,stream_entity.GetTxID()+MC_AST_SHORT_TXID_OFFSET,verbose,NULL,&stream_output); + + if(stream_output < (int)wtx.vout.size()) + { + output_array.push_back(entry); + } + first_output=stream_output+1; } - first_output=stream_output+1; - } + } + +// uint256 hash = ParseHashV(params[1], "parameter 2"); + + return output_array; } From 28f7960f51954aa5145175b6e20706e49929cc11 Mon Sep 17 00:00:00 2001 From: mike31 Date: Sun, 5 Aug 2018 13:34:14 +0300 Subject: [PATCH 21/26] Version 2.0 alpha 4 --- src/version/version.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/version/version.cpp b/src/version/version.cpp index 8f8f2d9c..d1c8e5d2 100644 --- a/src/version/version.cpp +++ b/src/version/version.cpp @@ -13,7 +13,7 @@ int mc_State::VersionInfo(int version) return custom_version; } - int this_build=20000103; + int this_build=20000104; int this_protocol=20003; if(version < 0) From 5579db0b7089d644de70efe1040f7d22e3202c7f Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 6 Aug 2018 10:11:43 +0300 Subject: [PATCH 22/26] maxqueryscanitems runtime parameter --- src/chainparams/globals.h | 1 + src/core/init.cpp | 1 + src/core/main.h | 1 + src/rpc/rpcstreams.cpp | 12 +++++++----- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/chainparams/globals.h b/src/chainparams/globals.h index b0cdc6a9..0e55f380 100644 --- a/src/chainparams/globals.h +++ b/src/chainparams/globals.h @@ -23,6 +23,7 @@ int64_t MAX_MONEY = 21000000 * COIN; unsigned int MAX_SCRIPT_ELEMENT_SIZE=520; // script.h int MIN_BLOCKS_BETWEEN_UPGRADES = 100; int MAX_OP_RETURN_SHOWN=16384; +int MAX_STREAM_QUERY_ITEMS=5000; int MAX_FORMATTED_DATA_DEPTH=100; unsigned int MAX_OP_RETURN_OP_DROP_COUNT=100000000; uint32_t JSON_NO_DOUBLE_FORMATTING=0; diff --git a/src/core/init.cpp b/src/core/init.cpp index baf43e92..9b1e3610 100644 --- a/src/core/init.cpp +++ b/src/core/init.cpp @@ -505,6 +505,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += " -hideknownopdrops= " + strprintf(_("Remove recognized MultiChain OP_DROP metadata from the responses to JSON_RPC calls (default: %u)"), 0) + "\n"; strUsage += " -maxshowndata= " + strprintf(_("The maximum number of bytes to show in the data field of API responses. (default: %u)"), MAX_OP_RETURN_SHOWN) + "\n"; strUsage += " " + _("Pieces of data larger than this will be returned as an object with txid, vout and size fields, for use with the gettxoutdata command.") + "\n"; + strUsage += " -maxqueryscanitems= " + strprintf(_("The maximum number of txs to be decoded in JSON_RPC calls. (default: %u)"), MAX_STREAM_QUERY_ITEMS) + "\n"; strUsage += " -v1apicompatible= " + strprintf(_("JSON_RPC calls responses compatible with MultiChain 1.0 (default: %u)"), 0) + "\n"; // strUsage += " -apidecimaldigits= " + _("maximal number of decimal digits in API output (default: auto)") + "\n"; diff --git a/src/core/main.h b/src/core/main.h index 4095809c..2444f3f7 100644 --- a/src/core/main.h +++ b/src/core/main.h @@ -72,6 +72,7 @@ static const unsigned int DEFAULT_MAX_ORPHAN_TRANSACTIONS = 50000; static const unsigned int DEFAULT_MAX_SUCCESSORS_FROM_ONE_NODE = 10; /* MCHN END */ extern int MAX_OP_RETURN_SHOWN; +extern int MAX_STREAM_QUERY_ITEMS; extern int MAX_FORMATTED_DATA_DEPTH; extern int MIN_BLOCKS_BETWEEN_UPGRADES; extern unsigned int OFFCHAIN_MSG_PADDING; diff --git a/src/rpc/rpcstreams.cpp b/src/rpc/rpcstreams.cpp index 30884720..9d1fbbcd 100644 --- a/src/rpc/rpcstreams.cpp +++ b/src/rpc/rpcstreams.cpp @@ -2411,7 +2411,7 @@ Value liststreamqueryitems(const Array& params, bool fHelp) parseStreamIdentifier(params[0],&stream_entity); bool verbose=false; - int dirty_count; + int dirty_count,max_count; if (params.size() > 2) { @@ -2429,13 +2429,15 @@ Value liststreamqueryitems(const Array& params, bool fHelp) entity_rows->Clear(); dirty_count=GetAndQueryDirtyList(conditions,&stream_entity,false,entity_rows); - - if(dirty_count > MC_QPR_MAX_DIRTY_TX_LIST_SIZE) + max_count=GetArg("-maxqueryscanitems",MAX_STREAM_QUERY_ITEMS); + if(dirty_count > max_count) { - throw JSONRPCError(RPC_NOT_SUPPORTED, "This query may take too much time"); + throw JSONRPCError(RPC_NOT_SUPPORTED, + strprintf("This query requires decoding %d items, which is above the maxqueryscanitems limit of %d.", + dirty_count,max_count)); } - if(entity_rows->GetCount() > MC_QPR_MAX_CLEAN_TX_LIST_SIZE) + if(entity_rows->GetCount() > max_count) { throw JSONRPCError(RPC_NOT_SUPPORTED, "Resulting list is too large"); } From 4c1fe2833cf14424545cebad7a0bc1d7d4f46dd2 Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 6 Aug 2018 11:39:19 +0300 Subject: [PATCH 23/26] Fixed missing creator list in liststreams --- src/rpc/rpcutils.cpp | 49 +++++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/src/rpc/rpcutils.cpp b/src/rpc/rpcutils.cpp index d16262ce..a3b2e5a8 100644 --- a/src/rpc/rpcutils.cpp +++ b/src/rpc/rpcutils.cpp @@ -666,31 +666,42 @@ Object StreamEntry(const unsigned char *txid,uint32_t output_level) string param_value((char*)ptr+value_offset,(char*)ptr+value_offset+value_size); fields.push_back(Pair(param_name, param_value)); } - else - { - if(ptr[offset+1] == MC_ENT_SPRM_ISSUER) - { - if(value_size == 24) - { - unsigned char tptr[4]; - memcpy(tptr,ptr+value_offset+sizeof(uint160),4); - if(mc_GetLE(tptr,4) & MC_PFL_IS_SCRIPTHASH) - { - openers.push_back(CBitcoinAddress(*(CScriptID*)(ptr+value_offset)).ToString()); - } - else - { - openers.push_back(CBitcoinAddress(*(CKeyID*)(ptr+value_offset)).ToString()); - } - } - } - } } offset=new_offset; } vfields=fields; } + offset=0; + while(offset>=0) + { + new_offset=entity.NextParam(offset,&value_offset,&value_size); + if(value_offset > 0) + { + if(ptr[offset] == 0) + { + if(ptr[offset+1] == MC_ENT_SPRM_ISSUER) + { + if(value_size == 24) + { + unsigned char tptr[4]; + memcpy(tptr,ptr+value_offset+sizeof(uint160),4); + if(mc_GetLE(tptr,4) & MC_PFL_IS_SCRIPTHASH) + { + openers.push_back(CBitcoinAddress(*(CScriptID*)(ptr+value_offset)).ToString()); + } + else + { + openers.push_back(CBitcoinAddress(*(CKeyID*)(ptr+value_offset)).ToString()); + } + } + } + } + } + offset=new_offset; + } + + entry.push_back(Pair("details",vfields)); } From d9dbe66ce35cbd87ba5c191dbcbccd2037a407d2 Mon Sep 17 00:00:00 2001 From: mike31 Date: Mon, 6 Aug 2018 14:58:15 +0300 Subject: [PATCH 24/26] maxqueryscanitems in setruntimeparam --- src/core/init.cpp | 6 +++--- src/rpc/rpchelp.cpp | 3 ++- src/rpc/rpcmisc.cpp | 2 ++ 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/core/init.cpp b/src/core/init.cpp index 9b1e3610..07ea282a 100644 --- a/src/core/init.cpp +++ b/src/core/init.cpp @@ -502,11 +502,11 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += " -flushsourcechunks= " + _("Flush offchain items created by this node to disk immediately when created, default 1") + "\n"; strUsage += "\n" + _("MultiChain API response parameters") + "\n"; - strUsage += " -hideknownopdrops= " + strprintf(_("Remove recognized MultiChain OP_DROP metadata from the responses to JSON_RPC calls (default: %u)"), 0) + "\n"; + strUsage += " -hideknownopdrops= " + strprintf(_("Remove recognized MultiChain OP_DROP metadata from the responses to JSON-RPC calls (default: %u)"), 0) + "\n"; strUsage += " -maxshowndata= " + strprintf(_("The maximum number of bytes to show in the data field of API responses. (default: %u)"), MAX_OP_RETURN_SHOWN) + "\n"; strUsage += " " + _("Pieces of data larger than this will be returned as an object with txid, vout and size fields, for use with the gettxoutdata command.") + "\n"; - strUsage += " -maxqueryscanitems= " + strprintf(_("The maximum number of txs to be decoded in JSON_RPC calls. (default: %u)"), MAX_STREAM_QUERY_ITEMS) + "\n"; - strUsage += " -v1apicompatible= " + strprintf(_("JSON_RPC calls responses compatible with MultiChain 1.0 (default: %u)"), 0) + "\n"; + strUsage += " -maxqueryscanitems= " + strprintf(_("The maximum number of txs to be decoded during JSON-RPC querying commands. (default: %u)"), MAX_STREAM_QUERY_ITEMS) + "\n"; + strUsage += " -v1apicompatible= " + strprintf(_("JSON-RPC calls responses compatible with MultiChain 1.0 (default: %u)"), 0) + "\n"; // strUsage += " -apidecimaldigits= " + _("maximal number of decimal digits in API output (default: auto)") + "\n"; strUsage += "\n" + _("Wallet optimization options:") + "\n"; diff --git a/src/rpc/rpchelp.cpp b/src/rpc/rpchelp.cpp index 3fd477cd..de09acda 100644 --- a/src/rpc/rpchelp.cpp +++ b/src/rpc/rpchelp.cpp @@ -3550,6 +3550,7 @@ void mc_InitRPCHelpMap15() " miningturnover,\n" " lockadminminerounds,\n" " maxshowndata, \n" + " maxqueryscanitems, \n" " bantx,\n" " lockblock,\n" " autosubscribe,\n" @@ -4130,7 +4131,7 @@ void mc_InitRPCHelpMap18() " \"key\" : \"key\" (string, optional, default: \"\") Item key\n" " or\n" " \"keys\" : keys (array, optional) Item keys, array of strings\n" - " or\n" + " and/or\n" " \"publisher\" : \"publisher\" (string, optional, default: \"\") Publisher\n" " or\n" " \"publishers\" : publishers (array, optional) Publishers, array of strings\n" diff --git a/src/rpc/rpcmisc.cpp b/src/rpc/rpcmisc.cpp index 8f11ae37..358cd151 100644 --- a/src/rpc/rpcmisc.cpp +++ b/src/rpc/rpcmisc.cpp @@ -225,6 +225,7 @@ Value getruntimeparams(const json_spirit::Array& params, bool fHelp) obj.push_back(Pair("lockblock",GetArg("-lockblock",""))); obj.push_back(Pair("hideknownopdrops",GetBoolArg("-hideknownopdrops",false))); obj.push_back(Pair("maxshowndata",GetArg("-maxshowndata",MAX_OP_RETURN_SHOWN))); + obj.push_back(Pair("maxqueryscanitems",GetArg("-maxqueryscanitems",MAX_STREAM_QUERY_ITEMS))); obj.push_back(Pair("v1apicompatible",GetBoolArg("-v1apicompatible",false))); obj.push_back(Pair("miningrequirespeers",Params().MiningRequiresPeers())); obj.push_back(Pair("mineemptyrounds",Params().MineEmptyRounds())); @@ -360,6 +361,7 @@ Value setruntimeparam(const json_spirit::Array& params, bool fHelp) } if( (param_name == "lockadminminerounds") || (param_name == "maxshowndata") || + (param_name == "maxqueryscanitems") || (param_name == "dropmessagestest") ) { if( (params[1].type() == int_type) || (params[1].type() == str_type) ) From ff46bdb120ebf0c305afad5e473394ca74b8a955 Mon Sep 17 00:00:00 2001 From: mike31 Date: Wed, 8 Aug 2018 10:04:31 +0300 Subject: [PATCH 25/26] Fixed possible crash when inserting new chunk --- src/wallet/chunkcollector.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wallet/chunkcollector.cpp b/src/wallet/chunkcollector.cpp index 5063e4aa..f1b72ed1 100644 --- a/src/wallet/chunkcollector.cpp +++ b/src/wallet/chunkcollector.cpp @@ -578,7 +578,7 @@ int mc_ChunkCollector::InsertChunkInternal( m_TotalChunkCount++; m_TotalChunkSize+=chunk_size; - if(m_MemPoolNext->GetCount() >= 2*m_MaxMemPoolSize) + if(m_MemPool->GetCount() >= 2*m_MaxMemPoolSize) { CommitInternal(0); } From de34537a5e9fec999ce5d10a514c0c8c7d7fb123 Mon Sep 17 00:00:00 2001 From: mike31 Date: Wed, 8 Aug 2018 22:23:48 +0300 Subject: [PATCH 26/26] Fixed memory leak on our tx count --- src/wallet/wallet.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index e63473f4..d344607d 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -3136,7 +3136,7 @@ bool CWallet::CommitTransaction(CWalletTx& wtxNew, CReserveKey& reservekey, stri } // Track how many getdata requests our transaction gets - mapRequestCount[wtxNew.GetHash()] = 0; + // mapRequestCount[wtxNew.GetHash()] = 0; // Broadcast