lnwire: create new ExtraOpaqueData type for parsing TLV extensions

.gitignore

@@ -36,6 +36,8 @@ lntest/itest/output*.log
 lntest/itest/pprof*.log
 lntest/itest/.backendlogs
 lntest/itest/.minerlogs
+lntest/itest/lnd-itest
+lntest/itest/.logs-*
 
 cmd/cmd
 *.key

.travis.yml

@@ -50,32 +50,30 @@ jobs:
     - stage: Integration Test
       name: Btcd Integration
       script:
-        - make itest
+        - make itest-parallel
 
     - name: Bitcoind Integration (txindex enabled)
       script:
         - bash ./scripts/install_bitcoind.sh
-        - make itest backend=bitcoind
+        - make itest-parallel backend=bitcoind
 
     - name: Bitcoind Integration (txindex disabled)
       script:
         - bash ./scripts/install_bitcoind.sh
-        - make itest backend="bitcoind notxindex"
+        - make itest-parallel backend="bitcoind notxindex"
 
     - name: Neutrino Integration
       script:
-        - make itest backend=neutrino
+        - make itest-parallel backend=neutrino
 
     - name: Btcd Integration ARM
       script:
-        - GOARM=7 GOARCH=arm GOOS=linux CGO_ENABLED=0 make btcd build-itest
-        - file lnd-itest
-        - GOARM=7 GOARCH=arm GOOS=linux CGO_ENABLED=0 make itest-only
+        - GOARM=7 GOARCH=arm GOOS=linux make itest-parallel
       arch: arm64
 
     - name: Btcd Integration Windows
       script:
-        - make itest-windows
+        - make itest-parallel-windows
       os: windows
       before_install:
         - choco upgrade --no-progress -y make netcat curl findutils
@@ -85,7 +83,8 @@ jobs:
           case $TRAVIS_OS_NAME in
             windows)
               echo "Uploading to termbin.com..."
-              for f in ./lntest/itest/*.log; do cat $f | nc termbin.com 9999 | xargs -r0 printf "$f"' uploaded to %s'; done
+              LOG_FILES=$(find ./lntest/itest -name '*.log')
+              for f in $LOG_FILES; do echo -n $f; cat $f | nc termbin.com 9999 | xargs -r0 printf ' uploaded to %s'; done
               ;;
           esac
 
@@ -97,8 +96,8 @@ after_failure:
         ;;
 
       *)
-        LOG_FILES=./lntest/itest/*.log
-        echo "Uploading to termbin.com..." && find $LOG_FILES | xargs -I{} sh -c "cat {} | nc termbin.com 9999 | xargs -r0 printf '{} uploaded to %s'"
+        LOG_FILES=$(find ./lntest/itest -name '*.log')
+        echo "Uploading to termbin.com..." && for f in $LOG_FILES; do echo -n $f; cat $f | nc termbin.com 9999 | xargs -r0 printf ' uploaded to %s'; done
         echo "Uploading to file.io..." && tar -zcvO $LOG_FILES | curl -s -F 'file=@-;filename=logs.tar.gz' https://file.io | xargs -r0 printf 'logs.tar.gz uploaded to %s\n'
         ;;
     esac

Makefile

@@ -175,6 +175,27 @@ itest-only:
 
 itest: btcd build-itest itest-only
 
+itest-parallel: btcd
+	@$(call print, "Building lnd binary")
+	CGO_ENABLED=0 $(GOBUILD) -tags="$(ITEST_TAGS)" -o lntest/itest/lnd-itest $(ITEST_LDFLAGS) $(PKG)/cmd/lnd
+
+	@$(call print, "Building itest binary for $(backend) backend")
+	CGO_ENABLED=0 $(GOTEST) -v ./lntest/itest -tags="$(DEV_TAGS) $(RPC_TAGS) rpctest $(backend)" -logoutput -goroutinedump -c -o lntest/itest/itest.test
+
+	@$(call print, "Running tests")
+	rm -rf lntest/itest/*.log lntest/itest/.logs-*
+	echo -n "$$(seq 0 $$(expr $(NUM_ITEST_TRANCHES) - 1))" | xargs -P $(NUM_ITEST_TRANCHES) -n 1 -I {} scripts/itest_part.sh {} $(NUM_ITEST_TRANCHES) $(TEST_FLAGS)
+
+itest-parallel-windows: btcd
+	@$(call print, "Building lnd binary")
+	CGO_ENABLED=0 $(GOBUILD) -tags="$(ITEST_TAGS)" -o lntest/itest/lnd-itest.exe $(ITEST_LDFLAGS) $(PKG)/cmd/lnd
+
+	@$(call print, "Building itest binary for $(backend) backend")
+	CGO_ENABLED=0 $(GOTEST) -v ./lntest/itest -tags="$(DEV_TAGS) $(RPC_TAGS) rpctest $(backend)" -logoutput -goroutinedump -c -o lntest/itest/itest.test.exe
+
+	@$(call print, "Running tests")
+	EXEC_SUFFIX=".exe" echo -n "$$(seq 0 $$(expr $(NUM_ITEST_TRANCHES) - 1))" | xargs -P $(NUM_ITEST_TRANCHES) -n 1 -I {} scripts/itest_part.sh {} $(NUM_ITEST_TRANCHES) $(TEST_FLAGS)
+
 itest-windows: btcd build-itest-windows itest-only
 
 unit: btcd

chanbackup/multi.go

@@ -63,7 +63,9 @@ func (m Multi) PackToWriter(w io.Writer, keyRing keychain.KeyRing) error {
 	var multiBackupBuffer bytes.Buffer
 
 	// First, we'll write out the version of this multi channel baackup.
-	err := lnwire.WriteElements(&multiBackupBuffer, byte(m.Version))
+	err := lnwire.WriteElements(
+		&multiBackupBuffer, lnwire.ProtocolVersionTLV, byte(m.Version),
+	)
 	if err != nil {
 		return err
 	}
@@ -111,7 +113,9 @@ func (m *Multi) UnpackFromReader(r io.Reader, keyRing keychain.KeyRing) error {
 	// First, we'll need to read the version of this multi-back up so we
 	// can know how to unpack each of the individual SCB's.
 	var multiVersion byte
-	err = lnwire.ReadElements(backupReader, &multiVersion)
+	err = lnwire.ReadElements(
+		backupReader, lnwire.ProtocolVersionTLV, &multiVersion,
+	)
 	if err != nil {
 		return err
 	}
@@ -127,7 +131,9 @@ func (m *Multi) UnpackFromReader(r io.Reader, keyRing keychain.KeyRing) error {
 		// backup is the same size, so we can continue until we've
 		// parsed out everything.
 		var numBackups uint32
-		err = lnwire.ReadElements(backupReader, &numBackups)
+		err = lnwire.ReadElements(
+			backupReader, lnwire.ProtocolVersionTLV, &numBackups,
+		)
 		if err != nil {
 			return err
 		}

chanbackup/single.go

@@ -207,6 +207,7 @@ func (s *Single) Serialize(w io.Writer) error {
 	var singleBytes bytes.Buffer
 	if err := lnwire.WriteElements(
 		&singleBytes,
+		lnwire.ProtocolVersionTLV,
 		s.IsInitiator,
 		s.ChainHash[:],
 		s.FundingOutpoint,
@@ -249,6 +250,7 @@ func (s *Single) Serialize(w io.Writer) error {
 
 	return lnwire.WriteElements(
 		w,
+		lnwire.ProtocolVersionTLV,
 		byte(s.Version),
 		uint16(len(singleBytes.Bytes())),
 		singleBytes.Bytes(),
@@ -290,12 +292,14 @@ func readLocalKeyDesc(r io.Reader) (keychain.KeyDescriptor, error) {
 	var keyDesc keychain.KeyDescriptor
 
 	var keyFam uint32
-	if err := lnwire.ReadElements(r, &keyFam); err != nil {
+	err := lnwire.ReadElements(r, lnwire.ProtocolVersionTLV, &keyFam)
+	if err != nil {
 		return keyDesc, err
 	}
 	keyDesc.Family = keychain.KeyFamily(keyFam)
 
-	if err := lnwire.ReadElements(r, &keyDesc.Index); err != nil {
+	err = lnwire.ReadElements(r, lnwire.ProtocolVersionTLV, &keyDesc.Index)
+	if err != nil {
 		return keyDesc, err
 	}
 
@@ -333,7 +337,7 @@ func (s *Single) Deserialize(r io.Reader) error {
 	// First, we'll need to read the version of this single-back up so we
 	// can know how to unpack each of the SCB.
 	var version byte
-	err := lnwire.ReadElements(r, &version)
+	err := lnwire.ReadElements(r, lnwire.ProtocolVersionTLV, &version)
 	if err != nil {
 		return err
 	}
@@ -350,19 +354,23 @@ func (s *Single) Deserialize(r io.Reader) error {
 	}
 
 	var length uint16
-	if err := lnwire.ReadElements(r, &length); err != nil {
+	err = lnwire.ReadElements(r, lnwire.ProtocolVersionTLV, &length)
+	if err != nil {
 		return err
 	}
 
 	err = lnwire.ReadElements(
-		r, &s.IsInitiator, s.ChainHash[:], &s.FundingOutpoint,
-		&s.ShortChannelID, &s.RemoteNodePub, &s.Addresses, &s.Capacity,
+		r, lnwire.ProtocolVersionTLV, &s.IsInitiator, s.ChainHash[:],
+		&s.FundingOutpoint, &s.ShortChannelID, &s.RemoteNodePub,
+		&s.Addresses, &s.Capacity,
 	)
 	if err != nil {
 		return err
 	}
 
-	err = lnwire.ReadElements(r, &s.LocalChanCfg.CsvDelay)
+	err = lnwire.ReadElements(
+		r, lnwire.ProtocolVersionTLV, &s.LocalChanCfg.CsvDelay,
+	)
 	if err != nil {
 		return err
 	}
@@ -387,7 +395,9 @@ func (s *Single) Deserialize(r io.Reader) error {
 		return err
 	}
 
-	err = lnwire.ReadElements(r, &s.RemoteChanCfg.CsvDelay)
+	err = lnwire.ReadElements(
+		r, lnwire.ProtocolVersionTLV, &s.RemoteChanCfg.CsvDelay,
+	)
 	if err != nil {
 		return err
 	}
@@ -417,7 +427,8 @@ func (s *Single) Deserialize(r io.Reader) error {
 		shaChainPub [33]byte
 		zeroPub     [33]byte
 	)
-	if err := lnwire.ReadElements(r, shaChainPub[:]); err != nil {
+	err = lnwire.ReadElements(r, lnwire.ProtocolVersionTLV, shaChainPub[:])
+	if err != nil {
 		return err
 	}
 
@@ -433,12 +444,17 @@ func (s *Single) Deserialize(r io.Reader) error {
 	}
 
 	var shaKeyFam uint32
-	if err := lnwire.ReadElements(r, &shaKeyFam); err != nil {
+	err = lnwire.ReadElements(
+		r, lnwire.ProtocolVersionTLV, &shaKeyFam,
+	)
+	if err != nil {
 		return err
 	}
 	s.ShaChainRootDesc.KeyLocator.Family = keychain.KeyFamily(shaKeyFam)
 
-	return lnwire.ReadElements(r, &s.ShaChainRootDesc.KeyLocator.Index)
+	return lnwire.ReadElements(
+		r, lnwire.ProtocolVersionTLV, &s.ShaChainRootDesc.KeyLocator.Index,
+	)
 }
 
 // UnpackFromReader is similar to Deserialize method, but it expects the passed

channeldb/channel.go

@@ -1871,7 +1871,8 @@ func serializeCommitDiff(w io.Writer, diff *CommitDiff) error {
 		return err
 	}
 
-	if err := diff.CommitSig.Encode(w, 0); err != nil {
+	err := diff.CommitSig.Encode(w, lnwire.ProtocolVersionTLV)
+	if err != nil {
 		return err
 	}
 
@@ -1918,7 +1919,7 @@ func deserializeCommitDiff(r io.Reader) (*CommitDiff, error) {
 	}
 
 	d.CommitSig = &lnwire.CommitSig{}
-	if err := d.CommitSig.Decode(r, 0); err != nil {
+	if err := d.CommitSig.Decode(r, lnwire.ProtocolVersionTLV); err != nil {
 		return nil, err
 	}
 

channeldb/channel_test.go

@@ -639,7 +639,8 @@ func TestChannelStateTransition(t *testing.T) {
 		{
 			LogIndex: 2,
 			UpdateMsg: &lnwire.UpdateAddHTLC{
-				ChanID: lnwire.ChannelID{1, 2, 3},
+				ChanID:    lnwire.ChannelID{1, 2, 3},
+				ExtraData: make([]byte, 0),
 			},
 		},
 	}
@@ -660,7 +661,9 @@ func TestChannelStateTransition(t *testing.T) {
 	if !reflect.DeepEqual(
 		dbUnsignedAckedUpdates[0], unsignedAckedUpdates[0],
 	) {
-		t.Fatalf("unexpected update")
+		t.Fatalf("unexpected update: expected %v, got %v",
+			spew.Sdump(unsignedAckedUpdates[0]),
+			spew.Sdump(dbUnsignedAckedUpdates))
 	}
 
 	// The balances, new update, the HTLCs and the changes to the fake
@@ -702,22 +705,25 @@ func TestChannelStateTransition(t *testing.T) {
 				wireSig,
 				wireSig,
 			},
+			ExtraData: make([]byte, 0),
 		},
 		LogUpdates: []LogUpdate{
 			{
 				LogIndex: 1,
 				UpdateMsg: &lnwire.UpdateAddHTLC{
-					ID:     1,
-					Amount: lnwire.NewMSatFromSatoshis(100),
-					Expiry: 25,
+					ID:        1,
+					Amount:    lnwire.NewMSatFromSatoshis(100),
+					Expiry:    25,
+					ExtraData: make([]byte, 0),
 				},
 			},
 			{
 				LogIndex: 2,
 				UpdateMsg: &lnwire.UpdateAddHTLC{
-					ID:     2,
-					Amount: lnwire.NewMSatFromSatoshis(200),
-					Expiry: 50,
+					ID:        2,
+					Amount:    lnwire.NewMSatFromSatoshis(200),
+					Expiry:    50,
+					ExtraData: make([]byte, 0),
 				},
 			},
 		},

channeldb/codec.go

@@ -178,7 +178,8 @@ func WriteElement(w io.Writer, element interface{}) error {
 		}
 
 	case lnwire.Message:
-		if _, err := lnwire.WriteMessage(w, e, 0); err != nil {
+		_, err := lnwire.WriteMessage(w, e, lnwire.ProtocolVersionTLV)
+		if err != nil {
 			return err
 		}
 
@@ -394,7 +395,7 @@ func ReadElement(r io.Reader, element interface{}) error {
 		*e = bytes
 
 	case *lnwire.Message:
-		msg, err := lnwire.ReadMessage(r, 0)
+		msg, err := lnwire.ReadMessage(r, lnwire.ProtocolVersionTLV)
 		if err != nil {
 			return err
 		}

channeldb/migration_01_to_11/codec.go

@@ -172,7 +172,8 @@ func WriteElement(w io.Writer, element interface{}) error {
 		}
 
 	case lnwire.Message:
-		if _, err := lnwire.WriteMessage(w, e, 0); err != nil {
+		_, err := lnwire.WriteMessage(w, e, lnwire.ProtocolVersionLegacy)
+		if err != nil {
 			return err
 		}
 
@@ -383,7 +384,7 @@ func ReadElement(r io.Reader, element interface{}) error {
 		*e = bytes
 
 	case *lnwire.Message:
-		msg, err := lnwire.ReadMessage(r, 0)
+		msg, err := lnwire.ReadMessage(r, lnwire.ProtocolVersionLegacy)
 		if err != nil {
 			return err
 		}

channeldb/migration_01_to_11/migrations.go

@@ -724,7 +724,8 @@ func MigrateGossipMessageStoreKeys(tx kvdb.RwTx) error {
 
 		// Serialize the message with its wire encoding.
 		var b bytes.Buffer
-		if _, err := lnwire.WriteMessage(&b, msg, 0); err != nil {
+		_, err := lnwire.WriteMessage(&b, msg, lnwire.ProtocolVersionTLV)
+		if err != nil {
 			return err
 		}
 

channeldb/migration_01_to_11/migrations_test.go

@@ -464,7 +464,10 @@ func TestMigrateGossipMessageStoreKeys(t *testing.T) {
 	// Construct the message which we'll use to test the migration, along
 	// with its old and new key formats.
 	shortChanID := lnwire.ShortChannelID{BlockHeight: 10}
-	msg := &lnwire.AnnounceSignatures{ShortChannelID: shortChanID}
+	msg := &lnwire.AnnounceSignatures{
+		ShortChannelID:  shortChanID,
+		ExtraOpaqueData: make([]byte, 0),
+	}
 
 	var oldMsgKey [33 + 8]byte
 	copy(oldMsgKey[:33], pubKey.SerializeCompressed())
@@ -526,7 +529,9 @@ func TestMigrateGossipMessageStoreKeys(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		gotMsg, err := lnwire.ReadMessage(bytes.NewReader(rawMsg), 0)
+		gotMsg, err := lnwire.ReadMessage(
+			bytes.NewReader(rawMsg), lnwire.ProtocolVersionLegacy,
+		)
 		if err != nil {
 			t.Fatalf("unable to deserialize raw message: %v", err)
 		}