From 20bd3017b91e32bc920708f3e553d8a6ca13d7c1 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <6098974-kittywhiskers@users.noreply.gitlab.com> Date: Tue, 25 May 2021 19:13:13 +0530 Subject: [PATCH 1/2] merge #19845: add support to (un)serialize as ADDRv2 --- src/crypto/common.h | 7 + src/netaddress.cpp | 312 +++++++++++++++++++++++++++++------ src/netaddress.h | 168 ++++++++++++++++++- src/primitives/transaction.h | 1 + src/test/base32_tests.cpp | 3 + src/test/miner_tests.cpp | 11 -- src/test/net_tests.cpp | 298 ++++++++++++++++++++++++++++++++- src/test/test_dash.h | 11 ++ src/utilstrencodings.cpp | 12 +- src/utilstrencodings.h | 16 +- src/utilstring.h | 17 +- src/version.h | 2 + 12 files changed, 779 insertions(+), 79 deletions(-) diff --git a/src/crypto/common.h b/src/crypto/common.h index a4879b4d6c71..f8440aab68ab 100644 --- a/src/crypto/common.h +++ b/src/crypto/common.h @@ -53,6 +53,13 @@ void static inline WriteLE64(unsigned char* ptr, uint64_t x) memcpy(ptr, (char*)&v, 8); } +uint16_t static inline ReadBE16(const unsigned char* ptr) +{ + uint16_t x; + memcpy((char*)&x, ptr, 2); + return be16toh(x); +} + uint32_t static inline ReadBE32(const unsigned char* ptr) { uint32_t x; diff --git a/src/netaddress.cpp b/src/netaddress.cpp index 12e423852cd8..b725fb17247e 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -5,18 +5,112 @@ #include #include +#include +#include #include +#include +#include #include #include -#include +#include #include #include #include +#include #include #include constexpr size_t CNetAddr::V1_SERIALIZATION_SIZE; +constexpr size_t CNetAddr::MAX_ADDRV2_SIZE; + +CNetAddr::BIP155Network CNetAddr::GetBIP155Network() const +{ + switch (m_net) { + case NET_IPV4: + return BIP155Network::IPV4; + case NET_IPV6: + return BIP155Network::IPV6; + case NET_ONION: + switch (m_addr.size()) { + case ADDR_TORV2_SIZE: + return BIP155Network::TORV2; + case ADDR_TORV3_SIZE: + return BIP155Network::TORV3; + default: + assert(false); + } + case NET_I2P: + return BIP155Network::I2P; + case NET_CJDNS: + return BIP155Network::CJDNS; + case NET_INTERNAL: // should have been handled before calling this function + case NET_UNROUTABLE: // m_net is never and should not be set to NET_UNROUTABLE + case NET_MAX: // m_net is never and should not be set to NET_MAX + assert(false); + } // no default case, so the compiler can warn about missing cases + + assert(false); +} + +bool CNetAddr::SetNetFromBIP155Network(uint8_t possible_bip155_net, size_t address_size) +{ + switch (possible_bip155_net) { + case BIP155Network::IPV4: + if (address_size == ADDR_IPV4_SIZE) { + m_net = NET_IPV4; + return true; + } + throw std::ios_base::failure( + strprintf("BIP155 IPv4 address with length %u (should be %u)", address_size, + ADDR_IPV4_SIZE)); + case BIP155Network::IPV6: + if (address_size == ADDR_IPV6_SIZE) { + m_net = NET_IPV6; + return true; + } + throw std::ios_base::failure( + strprintf("BIP155 IPv6 address with length %u (should be %u)", address_size, + ADDR_IPV6_SIZE)); + case BIP155Network::TORV2: + if (address_size == ADDR_TORV2_SIZE) { + m_net = NET_ONION; + return true; + } + throw std::ios_base::failure( + strprintf("BIP155 TORv2 address with length %u (should be %u)", address_size, + ADDR_TORV2_SIZE)); + case BIP155Network::TORV3: + if (address_size == ADDR_TORV3_SIZE) { + m_net = NET_ONION; + return true; + } + throw std::ios_base::failure( + strprintf("BIP155 TORv3 address with length %u (should be %u)", address_size, + ADDR_TORV3_SIZE)); + case BIP155Network::I2P: + if (address_size == ADDR_I2P_SIZE) { + m_net = NET_I2P; + return true; + } + throw std::ios_base::failure( + strprintf("BIP155 I2P address with length %u (should be %u)", address_size, + ADDR_I2P_SIZE)); + case BIP155Network::CJDNS: + if (address_size == ADDR_CJDNS_SIZE) { + m_net = NET_CJDNS; + return true; + } + throw std::ios_base::failure( + strprintf("BIP155 CJDNS address with length %u (should be %u)", address_size, + ADDR_CJDNS_SIZE)); + } + + // Don't throw on addresses with unknown network ids (maybe from the future). + // Instead silently drop them and have the unserialization code consume + // subsequent ones which may be known to us. + return false; +} bool fAllowPrivateNet = DEFAULT_ALLOWPRIVATENET; @@ -38,7 +132,13 @@ void CNetAddr::SetIP(const CNetAddr& ipIn) assert(ipIn.m_addr.size() == ADDR_IPV6_SIZE); break; case NET_ONION: - assert(ipIn.m_addr.size() == ADDR_TORV2_SIZE); + assert(ipIn.m_addr.size() == ADDR_TORV2_SIZE || ipIn.m_addr.size() == ADDR_TORV3_SIZE); + break; + case NET_I2P: + assert(ipIn.m_addr.size() == ADDR_I2P_SIZE); + break; + case NET_CJDNS: + assert(ipIn.m_addr.size() == ADDR_CJDNS_SIZE); break; case NET_INTERNAL: assert(ipIn.m_addr.size() == ADDR_INTERNAL_SIZE); @@ -52,13 +152,6 @@ void CNetAddr::SetIP(const CNetAddr& ipIn) m_addr = ipIn.m_addr; } -template -inline bool HasPrefix(const T1& obj, const std::array& prefix) -{ - return obj.size() >= PREFIX_LEN && - std::equal(std::begin(prefix), std::end(prefix), std::begin(obj)); -} - void CNetAddr::SetLegacyIPv6(Span ipv6) { assert(ipv6.size() == ADDR_IPV6_SIZE); @@ -103,24 +196,80 @@ bool CNetAddr::SetInternal(const std::string &name) return true; } +namespace torv3 { +// https://gitweb.torproject.org/torspec.git/tree/rend-spec-v3.txt#n2135 +static constexpr size_t CHECKSUM_LEN = 2; +static const unsigned char VERSION[] = {3}; +static constexpr size_t TOTAL_LEN = ADDR_TORV3_SIZE + CHECKSUM_LEN + sizeof(VERSION); + +static void Checksum(Span addr_pubkey, uint8_t (&checksum)[CHECKSUM_LEN]) +{ + // TORv3 CHECKSUM = H(".onion checksum" | PUBKEY | VERSION)[:2] + static const unsigned char prefix[] = ".onion checksum"; + static constexpr size_t prefix_len = 15; + + SHA3_256 hasher; + + hasher.Write(MakeSpan(prefix).first(prefix_len)); + hasher.Write(addr_pubkey); + hasher.Write(VERSION); + + uint8_t checksum_full[SHA3_256::OUTPUT_SIZE]; + + hasher.Finalize(checksum_full); + + memcpy(checksum, checksum_full, sizeof(checksum)); +} + +}; // namespace torv3 + /** - * Parse a TORv2 address and set this object to it. + * Parse a TOR address and set this object to it. * * @returns Whether or not the operation was successful. * * @see CNetAddr::IsTor() */ -bool CNetAddr::SetSpecial(const std::string &strName) +bool CNetAddr::SetSpecial(const std::string& str) { - if (strName.size()>6 && strName.substr(strName.size() - 6, 6) == ".onion") { - std::vector vchAddr = DecodeBase32(strName.substr(0, strName.size() - 6).c_str()); - if (vchAddr.size() != ADDR_TORV2_SIZE) { + static const char* suffix{".onion"}; + static constexpr size_t suffix_len{6}; + + if (!ValidAsCString(str) || str.size() <= suffix_len || + str.substr(str.size() - suffix_len) != suffix) { + return false; + } + + bool invalid; + const auto& input = DecodeBase32(str.substr(0, str.size() - suffix_len).c_str(), &invalid); + + if (invalid) { + return false; + } + + switch (input.size()) { + case ADDR_TORV2_SIZE: + m_net = NET_ONION; + m_addr.assign(input.begin(), input.end()); + return true; + case torv3::TOTAL_LEN: { + Span input_pubkey{input.data(), ADDR_TORV3_SIZE}; + Span input_checksum{input.data() + ADDR_TORV3_SIZE, torv3::CHECKSUM_LEN}; + Span input_version{input.data() + ADDR_TORV3_SIZE + torv3::CHECKSUM_LEN, sizeof(torv3::VERSION)}; + + uint8_t calculated_checksum[torv3::CHECKSUM_LEN]; + torv3::Checksum(input_pubkey, calculated_checksum); + + if (input_checksum != calculated_checksum || input_version != torv3::VERSION) { return false; } + m_net = NET_ONION; - m_addr.assign(vchAddr.begin(), vchAddr.end()); + m_addr.assign(input_pubkey.begin(), input_pubkey.end()); return true; } + } + return false; } @@ -231,13 +380,27 @@ bool CNetAddr::IsRFC7343() const (m_addr[3] & 0xF0) == 0x20; } -bool CNetAddr::IsTor() const { return m_net == NET_ONION; } - bool CNetAddr::IsHeNet() const { return IsIPv6() && HasPrefix(m_addr, std::array{0x20, 0x01, 0x04, 0x70}); } +/** + * Check whether this object represents a TOR address. + * @see CNetAddr::SetSpecial(const std::string &) + */ +bool CNetAddr::IsTor() const { return m_net == NET_ONION; } + +/** + * Check whether this object represents an I2P address. + */ +bool CNetAddr::IsI2P() const { return m_net == NET_I2P; } + +/** + * Check whether this object represents a CJDNS address. + */ +bool CNetAddr::IsCJDNS() const { return m_net == NET_CJDNS; } + bool CNetAddr::IsLocal() const { // IPv4 loopback (127.0.0.0/8 or 0.0.0.0/8) @@ -320,32 +483,74 @@ enum Network CNetAddr::GetNetwork() const return m_net; } +static std::string IPv6ToString(Span a) +{ + assert(a.size() == ADDR_IPV6_SIZE); + // clang-format off + return strprintf("%x:%x:%x:%x:%x:%x:%x:%x", + ReadBE16(&a[0]), + ReadBE16(&a[2]), + ReadBE16(&a[4]), + ReadBE16(&a[6]), + ReadBE16(&a[8]), + ReadBE16(&a[10]), + ReadBE16(&a[12]), + ReadBE16(&a[14])); + // clang-format on +} + std::string CNetAddr::ToStringIP(bool fUseGetnameinfo) const { - if (IsTor()) - return EncodeBase32(m_addr) + ".onion"; - if (IsInternal()) - return EncodeBase32(m_addr) + ".internal"; - if (fUseGetnameinfo) - { - CService serv(*this, 0); - struct sockaddr_storage sockaddr; - socklen_t socklen = sizeof(sockaddr); - if (serv.GetSockAddr((struct sockaddr*)&sockaddr, &socklen)) { - char name[1025] = ""; - if (!getnameinfo((const struct sockaddr*)&sockaddr, socklen, name, sizeof(name), nullptr, 0, NI_NUMERICHOST)) { - return std::string(name); + switch (m_net) { + case NET_IPV4: + case NET_IPV6: { + if (fUseGetnameinfo) { + CService serv(*this, 0); + struct sockaddr_storage sockaddr; + socklen_t socklen = sizeof(sockaddr); + if (serv.GetSockAddr((struct sockaddr*)&sockaddr, &socklen)) { + char name[1025] = ""; + if (!getnameinfo((const struct sockaddr*)&sockaddr, socklen, name, + sizeof(name), nullptr, 0, NI_NUMERICHOST)) + return std::string(name); } } + if (m_net == NET_IPV4) { + return strprintf("%u.%u.%u.%u", m_addr[0], m_addr[1], m_addr[2], m_addr[3]); + } + return IPv6ToString(m_addr); } - if (IsIPv4()) - return strprintf("%u.%u.%u.%u", m_addr[0], m_addr[1], m_addr[2], m_addr[3]); - assert(IsIPv6()); - return strprintf("%x:%x:%x:%x:%x:%x:%x:%x", - m_addr[0] << 8 | m_addr[1], m_addr[2] << 8 | m_addr[3], - m_addr[4] << 8 | m_addr[5], m_addr[6] << 8 | m_addr[7], - m_addr[8] << 8 | m_addr[9], m_addr[10] << 8 | m_addr[11], - m_addr[12] << 8 | m_addr[13], m_addr[14] << 8 | m_addr[15]); + case NET_ONION: + switch (m_addr.size()) { + case ADDR_TORV2_SIZE: + return EncodeBase32(m_addr) + ".onion"; + case ADDR_TORV3_SIZE: { + + uint8_t checksum[torv3::CHECKSUM_LEN]; + torv3::Checksum(m_addr, checksum); + + // TORv3 onion_address = base32(PUBKEY | CHECKSUM | VERSION) + ".onion" + prevector address{m_addr.begin(), m_addr.end()}; + address.insert(address.end(), checksum, checksum + torv3::CHECKSUM_LEN); + address.insert(address.end(), torv3::VERSION, torv3::VERSION + sizeof(torv3::VERSION)); + + return EncodeBase32(address) + ".onion"; + } + default: + assert(false); + } + case NET_I2P: + return EncodeBase32(m_addr, false /* don't pad with = */) + ".b32.i2p"; + case NET_CJDNS: + return IPv6ToString(m_addr); + case NET_INTERNAL: + return EncodeBase32(m_addr) + ".internal"; + case NET_UNROUTABLE: // m_net is never and should not be set to NET_UNROUTABLE + case NET_MAX: // m_net is never and should not be set to NET_MAX + assert(false); + } // no default case, so the compiler can warn about missing cases + + assert(false); } std::string CNetAddr::ToString() const @@ -404,21 +609,22 @@ uint32_t CNetAddr::GetLinkedIPv4() const assert(false); } -uint32_t CNetAddr::GetNetClass() const { - uint32_t net_class = NET_IPV6; - if (IsLocal()) { - net_class = 255; - } +uint32_t CNetAddr::GetNetClass() const +{ + // Make sure that if we return NET_IPV6, then IsIPv6() is true. The callers expect that. + + // Check for "internal" first because such addresses are also !IsRoutable() + // and we don't want to return NET_UNROUTABLE in that case. if (IsInternal()) { - net_class = NET_INTERNAL; - } else if (!IsRoutable()) { - net_class = NET_UNROUTABLE; - } else if (HasLinkedIPv4()) { - net_class = NET_IPV4; - } else if (IsTor()) { - net_class = NET_ONION; + return NET_INTERNAL; + } + if (!IsRoutable()) { + return NET_UNROUTABLE; } - return net_class; + if (HasLinkedIPv4()) { + return NET_IPV4; + } + return m_net; } uint32_t CNetAddr::GetMappedAS(const std::vector &asmap) const { @@ -493,7 +699,7 @@ std::vector CNetAddr::GetGroup(const std::vector &asmap) co vchRet.push_back((ipv4 >> 24) & 0xFF); vchRet.push_back((ipv4 >> 16) & 0xFF); return vchRet; - } else if (IsTor()) { + } else if (IsTor() || IsI2P() || IsCJDNS()) { nBits = 4; } else if (IsHeNet()) { // for he.net, use /36 groups @@ -703,7 +909,7 @@ std::string CService::ToStringPort() const std::string CService::ToStringIPPort(bool fUseGetnameinfo) const { - if (IsIPv4() || IsTor() || IsInternal()) { + if (IsIPv4() || IsTor() || IsI2P() || IsInternal()) { return ToStringIP(fUseGetnameinfo) + ":" + ToStringPort(); } else { return "[" + ToStringIP(fUseGetnameinfo) + "]:" + ToStringPort(); diff --git a/src/netaddress.h b/src/netaddress.h index 78f9591338a4..3a9a8cc94c33 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -14,14 +14,25 @@ #include #include #include +#include +#include +#include #include #include +#include #include #include extern bool fAllowPrivateNet; +/** + * A flag that is ORed into the protocol version to designate that addresses + * should be serialized in (unserialized from) v2 format (BIP155). + * Make sure that this does not collide with any of the values in `version.h` + */ +static const int ADDRV2_FORMAT = 0x20000000; + /** * A network type. * @note An address may belong to more than one network, for example `10.0.0.1` @@ -42,9 +53,15 @@ enum Network /// IPv6 NET_IPV6, - /// TORv2 + /// TOR (v2 or v3) NET_ONION, + /// I2P + NET_I2P, + + /// CJDNS + NET_CJDNS, + /// A set of addresses that represent the hash of a string or FQDN. We use /// them in CAddrMan to keep track of which DNS seeds were used. NET_INTERNAL, @@ -85,6 +102,16 @@ static constexpr size_t ADDR_IPV6_SIZE = 16; /// Size of TORv2 address (in bytes). static constexpr size_t ADDR_TORV2_SIZE = 10; +/// Size of TORv3 address (in bytes). This is the length of just the address +/// as used in BIP155, without the checksum and the version byte. +static constexpr size_t ADDR_TORV3_SIZE = 32; + +/// Size of I2P address (in bytes). +static constexpr size_t ADDR_I2P_SIZE = 32; + +/// Size of CJDNS address (in bytes). +static constexpr size_t ADDR_CJDNS_SIZE = 16; + /// Size of "internal" (NET_INTERNAL) address (in bytes). static constexpr size_t ADDR_INTERNAL_SIZE = 10; @@ -154,6 +181,8 @@ class CNetAddr bool IsRFC6145() const; // IPv6 IPv4-translated address (::FFFF:0:0:0/96) (actually defined in RFC2765) bool IsHeNet() const; // IPv6 Hurricane Electric - https://he.net (2001:0470::/36) bool IsTor() const; + bool IsI2P() const; + bool IsCJDNS() const; bool IsLocal() const; bool IsRoutable() const; bool IsInternal() const; @@ -191,7 +220,11 @@ class CNetAddr template void Serialize(Stream& s) const { - SerializeV1Stream(s); + if (s.GetVersion() & ADDRV2_FORMAT) { + SerializeV2Stream(s); + } else { + SerializeV1Stream(s); + } } /** @@ -200,20 +233,58 @@ class CNetAddr template void Unserialize(Stream& s) { - UnserializeV1Stream(s); + if (s.GetVersion() & ADDRV2_FORMAT) { + UnserializeV2Stream(s); + } else { + UnserializeV1Stream(s); + } } friend class CSubNet; private: + /** + * BIP155 network ids recognized by this software. + */ + enum BIP155Network : uint8_t { + IPV4 = 1, + IPV6 = 2, + TORV2 = 3, + TORV3 = 4, + I2P = 5, + CJDNS = 6, + }; + /** * Size of CNetAddr when serialized as ADDRv1 (pre-BIP155) (in bytes). */ static constexpr size_t V1_SERIALIZATION_SIZE = ADDR_IPV6_SIZE; + /** + * Maximum size of an address as defined in BIP155 (in bytes). + * This is only the size of the address, not the entire CNetAddr object + * when serialized. + */ + static constexpr size_t MAX_ADDRV2_SIZE = 512; + + /** + * Get the BIP155 network id of this address. + * Must not be called for IsInternal() objects. + * @returns BIP155 network id + */ + BIP155Network GetBIP155Network() const; + + /** + * Set `m_net` from the provided BIP155 network id and size after validation. + * @retval true the network was recognized, is valid and `m_net` was set + * @retval false not recognised (from future?) and should be silently ignored + * @throws std::ios_base::failure if the network is one of the BIP155 founding + * networks (id 1..6) with wrong address size. + */ + bool SetNetFromBIP155Network(uint8_t possible_bip155_net, size_t address_size); + /** * Serialize in pre-ADDRv2/BIP155 format to an array. - * Some addresses (e.g. TORv3) cannot be serialized in pre-BIP155 format. */ void SerializeV1Array(uint8_t (&arr)[V1_SERIALIZATION_SIZE]) const { @@ -231,6 +302,9 @@ class CNetAddr memcpy(arr + prefix_size, m_addr.data(), m_addr.size()); return; case NET_ONION: + if (m_addr.size() == ADDR_TORV3_SIZE) { + break; + } prefix_size = sizeof(TORV2_IN_IPV6_PREFIX); assert(prefix_size + m_addr.size() == sizeof(arr)); memcpy(arr, TORV2_IN_IPV6_PREFIX.data(), prefix_size); @@ -242,18 +316,22 @@ class CNetAddr memcpy(arr, INTERNAL_IN_IPV6_PREFIX.data(), prefix_size); memcpy(arr + prefix_size, m_addr.data(), m_addr.size()); return; + case NET_I2P: + break; + case NET_CJDNS: + break; case NET_UNROUTABLE: case NET_MAX: assert(false); } // no default case, so the compiler can warn about missing cases - assert(false); + // Serialize TORv3, I2P and CJDNS as all-zeros. + memset(arr, 0x0, V1_SERIALIZATION_SIZE); } public: /** * Serialize in pre-ADDRv2/BIP155 format to a stream. - * Some addresses (e.g. TORv3) cannot be serialized in pre-BIP155 format. */ template void SerializeV1Stream(Stream& s) const @@ -265,6 +343,25 @@ class CNetAddr s << serialized; } + /** + * Serialize as ADDRv2 / BIP155. + */ + template + void SerializeV2Stream(Stream& s) const + { + if (IsInternal()) { + // Serialize NET_INTERNAL as embedded in IPv6. We need to + // serialize such addresses from addrman. + s << static_cast(BIP155Network::IPV6); + s << COMPACTSIZE(ADDR_IPV6_SIZE); + SerializeV1Stream(s); + return; + } + + s << static_cast(GetBIP155Network()); + s << m_addr; + } + /** * Unserialize from a pre-ADDRv2/BIP155 format from an array. */ @@ -287,6 +384,65 @@ class CNetAddr UnserializeV1Array(serialized); } + + /** + * Unserialize from a ADDRv2 / BIP155 format. + */ + template + void UnserializeV2Stream(Stream& s) + { + uint8_t bip155_net; + s >> bip155_net; + + size_t address_size; + s >> COMPACTSIZE(address_size); + + if (address_size > MAX_ADDRV2_SIZE) { + throw std::ios_base::failure(strprintf( + "Address too long: %u > %u", address_size, MAX_ADDRV2_SIZE)); + } + + scopeId = 0; + + if (SetNetFromBIP155Network(bip155_net, address_size)) { + m_addr.resize(address_size); + s >> MakeSpan(m_addr); + + if (m_net != NET_IPV6) { + return; + } + + // Do some special checks on IPv6 addresses. + + // Recognize NET_INTERNAL embedded in IPv6, such addresses are not + // gossiped but could be coming from addrman, when unserializing from + // disk. + if (HasPrefix(m_addr, INTERNAL_IN_IPV6_PREFIX)) { + m_net = NET_INTERNAL; + memmove(m_addr.data(), m_addr.data() + INTERNAL_IN_IPV6_PREFIX.size(), + ADDR_INTERNAL_SIZE); + m_addr.resize(ADDR_INTERNAL_SIZE); + return; + } + + if (!HasPrefix(m_addr, IPV4_IN_IPV6_PREFIX) && + !HasPrefix(m_addr, TORV2_IN_IPV6_PREFIX)) { + return; + } + + // IPv4 and TORv2 are not supposed to be embedded in IPv6 (like in V1 + // encoding). Unserialize as !IsValid(), thus ignoring them. + } else { + // If we receive an unknown BIP155 network id (from the future?) then + // ignore the address - unserialize as !IsValid(). + s.ignore(address_size); + } + + // Mimic a default-constructed CNetAddr object which is !IsValid() and thus + // will not be gossiped, but continue reading next addresses from the stream. + m_net = NET_IPV6; + m_addr.assign(ADDR_IPV6_SIZE, 0x0); + } }; class CSubNet diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index 70c9886358b2..e94f9a6bc057 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -10,6 +10,7 @@ #include