Memory optimizations for large address queries

integration/regtest-node.js

@@ -28,6 +28,7 @@ var Transaction = index.Transaction;
 var BitcoreNode = index.Node;
 var AddressService = index.services.Address;
 var BitcoinService = index.services.Bitcoin;
+var encoding = require('../lib/services/address/encoding');
 var DBService = index.services.DB;
 var testWIF = 'cSdkPxkAjA4HDr5VHgsebAPDEh9Gyub4HK8UJr2DFGGqKKy4K5sG';
 var testKey;
@@ -43,22 +44,6 @@ describe('Node Functionality', function() {
   before(function(done) {
     this.timeout(30000);
 
-    // Add the regtest network
-    bitcore.Networks.remove(bitcore.Networks.testnet);
-    bitcore.Networks.add({
-      name: 'regtest',
-      alias: 'regtest',
-      pubkeyhash: 0x6f,
-      privatekey: 0xef,
-      scripthash: 0xc4,
-      xpubkey: 0x043587cf,
-      xprivkey: 0x04358394,
-      networkMagic: 0xfabfb5da,
-      port: 18444,
-      dnsSeeds: [ ]
-    });
-    regtest = bitcore.Networks.get('regtest');
-
     var datadir = __dirname + '/data';
 
     testKey = bitcore.PrivateKey(testWIF);
@@ -93,6 +78,9 @@ describe('Node Functionality', function() {
 
       node = new BitcoreNode(configuration);
 
+      regtest = bitcore.Networks.get('regtest');
+      should.exist(regtest);
+
       node.on('error', function(err) {
         log.error(err);
       });
@@ -208,7 +196,7 @@ describe('Node Functionality', function() {
 
       // We need to add a transaction to the mempool so that the next block will
       // have a different hash as the hash has been invalidated.
-      client.sendToAddress(testKey.toAddress().toString(), 10, function(err) {
+      client.sendToAddress(testKey.toAddress(regtest).toString(), 10, function(err) {
         if (err) {
           throw err;
         }
@@ -250,7 +238,7 @@ describe('Node Functionality', function() {
     var address;
     var unspentOutput;
     before(function() {
-      address = testKey.toAddress().toString();
+      address = testKey.toAddress(regtest).toString();
     });
     it('should be able to get the balance of the test address', function(done) {
       node.services.address.getBalance(address, false, function(err, balance) {
@@ -333,19 +321,19 @@ describe('Node Functionality', function() {
         /* jshint maxstatements: 50 */
 
         testKey2 = bitcore.PrivateKey.fromWIF('cNfF4jXiLHQnFRsxaJyr2YSGcmtNYvxQYSakNhuDGxpkSzAwn95x');
-        address2 = testKey2.toAddress().toString();
+        address2 = testKey2.toAddress(regtest).toString();
 
         testKey3 = bitcore.PrivateKey.fromWIF('cVTYQbaFNetiZcvxzXcVMin89uMLC43pEBMy2etgZHbPPxH5obYt');
-        address3 = testKey3.toAddress().toString();
+        address3 = testKey3.toAddress(regtest).toString();
 
         testKey4 = bitcore.PrivateKey.fromWIF('cPNQmfE31H2oCUFqaHpfSqjDibkt7XoT2vydLJLDHNTvcddCesGw');
-        address4 = testKey4.toAddress().toString();
+        address4 = testKey4.toAddress(regtest).toString();
 
         testKey5 = bitcore.PrivateKey.fromWIF('cVrzm9gCmnzwEVMGeCxY6xLVPdG3XWW97kwkFH3H3v722nb99QBF');
-        address5 = testKey5.toAddress().toString();
+        address5 = testKey5.toAddress(regtest).toString();
 
         testKey6 = bitcore.PrivateKey.fromWIF('cPfMesNR2gsQEK69a6xe7qE44CZEZavgMUak5hQ74XDgsRmmGBYF');
-        address6 = testKey6.toAddress().toString();
+        address6 = testKey6.toAddress(regtest).toString();
 
         var tx = new Transaction();
         tx.from(unspentOutput);
@@ -726,7 +714,7 @@ describe('Node Functionality', function() {
         node.services.bitcoind.sendTransaction(tx.serialize());
 
         setImmediate(function() {
-          var addrObj = node.services.address._getAddressInfo(address);
+          var addrObj = encoding.getAddressInfo(address);
           node.services.address._getOutputsMempool(address, addrObj.hashBuffer,
                                                    addrObj.hashTypeBuffer, function(err, outs) {
             if (err) {

lib/services/address/constants.js

@@ -0,0 +1,54 @@
+'use strict';
+
+var exports = {};
+
+exports.PREFIXES = {
+  OUTPUTS: new Buffer('02', 'hex'), // Query outputs by address and/or height
+  SPENTS: new Buffer('03', 'hex'), // Query inputs by address and/or height
+  SPENTSMAP: new Buffer('05', 'hex') // Get the input that spends an output
+};
+
+exports.MEMPREFIXES = {
+  OUTPUTS: new Buffer('01', 'hex'), // Query mempool outputs by address
+  SPENTS: new Buffer('02', 'hex'), // Query mempool inputs by address
+  SPENTSMAP: new Buffer('03', 'hex') // Query mempool for the input that spends an output
+};
+
+// To save space, we're only storing the PubKeyHash or ScriptHash in our index.
+// To avoid intentional unspendable collisions, which have been seen on the blockchain,
+// we must store the hash type (PK or Script) as well.
+exports.HASH_TYPES = {
+  PUBKEY: new Buffer('01', 'hex'),
+  REDEEMSCRIPT: new Buffer('02', 'hex')
+};
+
+// Translates from our enum type back into the hash types returned by
+// bitcore-lib/address.
+exports.HASH_TYPES_READABLE = {
+  '01': 'pubkeyhash',
+  '02': 'scripthash'
+};
+
+exports.HASH_TYPES_MAP = {
+  'pubkeyhash': exports.HASH_TYPES.PUBKEY,
+  'scripthash': exports.HASH_TYPES.REDEEMSCRIPT
+};
+
+exports.SPACER_MIN = new Buffer('00', 'hex');
+exports.SPACER_MAX = new Buffer('ff', 'hex');
+exports.SPACER_HEIGHT_MIN = new Buffer('0000000000', 'hex');
+exports.SPACER_HEIGHT_MAX = new Buffer('ffffffffff', 'hex');
+exports.TIMESTAMP_MIN = new Buffer('0000000000000000', 'hex');
+exports.TIMESTAMP_MAX = new Buffer('ffffffffffffffff', 'hex');
+
+// The maximum number of inputs that can be queried at once
+exports.MAX_INPUTS_QUERY_LENGTH = 50000;
+// The maximum number of outputs that can be queried at once
+exports.MAX_OUTPUTS_QUERY_LENGTH = 50000;
+// The maximum number of transactions that can be queried at once
+exports.MAX_HISTORY_QUERY_LENGTH = 100;
+// The maximum number of addresses that can be queried at once
+exports.MAX_ADDRESSES_QUERY = 10000;
+
+module.exports = exports;
+