Initial impl of state network bridge 1902 (#1948)

This commit is contained in:
Daniel Sobol 2024-01-09 12:32:29 +03:00 committed by GitHub
parent 79c6bdc214
commit 70013422cd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 365 additions and 141 deletions

View File

@ -68,6 +68,7 @@ TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(TOOLS))
FLUFFY_TOOLS := \
portal_bridge \
beacon_lc_bridge \
state_bridge \
eth_data_exporter \
content_verifier \
blockwalk \
@ -76,6 +77,7 @@ FLUFFY_TOOLS := \
FLUFFY_TOOLS_DIRS := \
fluffy/tools/beacon_lc_bridge \
fluffy/tools/portal_bridge \
fluffy/tools/state_bridge \
fluffy/tools
# comma-separated values for the "clean" target
FLUFFY_TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(FLUFFY_TOOLS))

View File

@ -1,5 +1,5 @@
# Fluffy
# Copyright (c) 2021-2023 Status Research & Development GmbH
# Copyright (c) 2023-2024 Status Research & Development GmbH
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@ -11,12 +11,29 @@
{.push raises: [].}
import
nimcrypto/[hash, sha2, keccak], stew/[objects, results, endians2], stint,
nimcrypto/[hash, sha2, keccak], stew/[objects, results], stint,
ssz_serialization,
../../common/common_types
export ssz_serialization, common_types, hash, results
type JsonAccount* = object
nonce*: int
balance*: string
storage_hash*: string
code_hash*: string
type JsonProof* = object
address*: string
state*: JsonAccount
proof*: seq[string]
type JsonProofVector* = object
`block`*: int
block_hash*: string
state_root*: string
proofs*: seq[JsonProof]
type
NodeHash* = MDigest[32 * 8] # keccak256
CodeHash* = MDigest[32 * 8] # keccak256
@ -61,6 +78,9 @@ type
address*: Address
codeHash*: CodeHash
WitnessNode* = ByteList
AccountTrieProof* = List[WitnessNode, 32]
ContentKey* = object
case contentType*: ContentType
of unused:

View File

@ -1,12 +1,15 @@
# Fluffy
# Copyright (c) 2021-2023 Status Research & Development GmbH
# Copyright (c) 2021-2024 Status Research & Development GmbH
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.
import
std/[sequtils, sugar],
stew/results, chronos, chronicles,
eth/[rlp, common],
eth/trie/hexary_proof_verification,
eth/p2p/discoveryv5/[protocol, enr],
../../database/content_db,
../wire/[portal_protocol, portal_stream, portal_protocol_config],
@ -59,8 +62,59 @@ proc getContent*(n: StateNetwork, key: ContentKey):
# domain types.
return Opt.some(contentResult.content)
proc validateContent(content: openArray[byte], contentKey: ByteList): bool =
true
proc validateContent(
n: StateNetwork,
contentKey: ByteList,
contentValue: seq[byte]): Future[bool] {.async.} =
let key = contentKey.decode().valueOr:
return false
case key.contentType:
of unused:
warn "Received content with unused content type"
false
of accountTrieNode:
true
of contractStorageTrieNode:
true
of accountTrieProof:
let decodedProof = decodeSsz(contentValue, AccountTrieProof).valueOr:
warn "Received invalid account trie proof", error
return false
let
proof = decodedProof.asSeq().map((p: ByteList) => p.toSeq())
trieKey = keccakHash(key.accountTrieProofKey.address).data.toSeq()
value = proof[^1].decode(seq[seq[byte]])[^1]
stateRoot = MDigest[256](data: key.accountTrieProofKey.stateRoot)
verificationResult = verifyMptProof(proof, stateRoot, trieKey, value)
case verificationResult.kind:
of ValidProof:
true
else:
warn "Received invalid account trie proof"
false
of contractStorageTrieProof:
true
of contractBytecode:
true
proc validateContent(
n: StateNetwork,
contentKeys: ContentKeysList,
contentValues: seq[seq[byte]]): Future[bool] {.async.} =
for i, contentValue in contentValues:
let contentKey = contentKeys[i]
if await n.validateContent(contentKey, contentValue):
let contentId = n.portalProtocol.toContentId(contentKey).valueOr:
error "Received offered content with invalid content key", contentKey
return false
n.portalProtocol.storeContent(contentKey, contentId, contentValue)
info "Received offered content validated successfully", contentKey
else:
error "Received offered content failed validation", contentKey
return false
proc new*(
T: type StateNetwork,
@ -91,8 +145,11 @@ proc new*(
proc processContentLoop(n: StateNetwork) {.async.} =
try:
while true:
# Just dropping state date for now
discard await n.contentQueue.popFirst()
let (maybeContentId, contentKeys, contentValues) = await n.contentQueue.popFirst()
if await n.validateContent(contentKeys, contentValues):
asyncSpawn n.portalProtocol.neighborhoodGossipDiscardPeers(
maybeContentId, contentKeys, contentValues
)
except CancelledError:
trace "processContentLoop canceled"

View File

@ -9,8 +9,9 @@
import
./test_portal_wire_protocol,
./test_state_distance,
./test_state_network,
./state_network_tests/test_state_distance,
./state_network_tests/test_state_network,
./state_network_tests/test_state_content,
./test_state_proof_verification,
./test_accumulator,
./test_history_network,

View File

@ -0,0 +1,44 @@
# Fluffy
# Copyright (c) 2023-2024 Status Research & Development GmbH
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.
import
std/[os, json, sequtils],
testutils/unittests,
stew/[byteutils, io2],
eth/keys,
../../network/state/state_content
const testVectorDir =
"./vendor/portal-spec-tests/tests/mainnet/state/"
procSuite "State Content":
let rng = newRng()
test "Encode/decode accountTrieProof":
let file = testVectorDir & "/proofs.full.block.0.json"
let content = readAllFile(file).valueOr:
quit(1)
let decoded =
try:
Json.decode(content, state_content.JsonProofVector)
except SerializationError:
quit(1)
let proof = decoded.proofs[0].proof.map(hexToSeqByte)
var accountTrieProof = AccountTrieProof(@[])
for witness in proof:
let witnessNode = ByteList(witness)
discard accountTrieProof.add(witnessNode)
let
encodedProof = SSZ.encode(accountTrieProof)
decodedProof = decodeSsz(encodedProof, AccountTrieProof).get()
check decodedProof == accountTrieProof

View File

@ -10,7 +10,7 @@
import
std/sequtils,
stint, unittest2,
../network/state/state_distance
../../network/state/state_distance
suite "State network custom distance function":
test "Calculate distance according to spec":

View File

@ -6,16 +6,23 @@
# at your option. This file may not be copied, modified, or distributed except according to those terms.
import
std/os,
std/[os, json, sequtils, strutils, sugar],
stew/[byteutils, io2],
nimcrypto/hash,
testutils/unittests, chronos,
eth/[common/eth_hash, keys],
eth/trie/hexary_proof_verification,
eth/keys,
eth/common/[eth_types, eth_hash],
eth/p2p/discoveryv5/protocol as discv5_protocol, eth/p2p/discoveryv5/routing_table,
../../nimbus/[config, db/core_db, db/state_db],
../../nimbus/common/[chain_config, genesis],
../network/wire/[portal_protocol, portal_stream],
../network/state/[state_content, state_network],
../database/content_db,
./test_helpers
../../../nimbus/[config, db/core_db, db/state_db],
../../../nimbus/common/[chain_config, genesis],
../../network/wire/[portal_protocol, portal_stream],
../../network/state/[state_content, state_network],
../../database/content_db,
.././test_helpers
const testVectorDir =
"./vendor/portal-spec-tests/tests/mainnet/state/"
proc genesisToTrie(filePath: string): CoreDbMptRef =
# TODO: Doing our best here with API that exists, to be improved.
@ -30,8 +37,84 @@ proc genesisToTrie(filePath: string): CoreDbMptRef =
sdb.getTrie
procSuite "State Content Network":
procSuite "State Network":
let rng = newRng()
test "Test account state proof":
let file = testVectorDir & "/proofs.full.block.0.json"
let content = readAllFile(file).valueOr:
quit(1)
let decoded =
try:
Json.decode(content, state_content.JsonProofVector)
except SerializationError:
quit(1)
let
proof = decoded.proofs[0].proof.map(hexToSeqByte)
stateRoot = MDigest[256].fromHex(decoded.state_root)
address = hexToByteArray[20](decoded.proofs[0].address)
key = keccakHash(address).data.toSeq()
value = proof[^1].decode(seq[seq[byte]])[^1]
proofResult = verifyMptProof(proof, stateRoot, key, value)
check proofResult.kind == ValidProof
asyncTest "Decode and use proofs":
let file = testVectorDir & "/proofs.full.block.0.json"
let content = readAllFile(file).valueOr:
quit(1)
let decoded =
try:
Json.decode(content, state_content.JsonProofVector)
except SerializationError:
quit(1)
let
node1 = initDiscoveryNode(
rng, PrivateKey.random(rng[]), localAddress(20302))
sm1 = StreamManager.new(node1)
node2 = initDiscoveryNode(
rng, PrivateKey.random(rng[]), localAddress(20303))
sm2 = StreamManager.new(node2)
proto1 = StateNetwork.new(node1, ContentDB.new("", uint32.high, inMemory = true), sm1)
proto2 = StateNetwork.new(node2, ContentDB.new("", uint32.high, inMemory = true), sm2)
state_root = hexToByteArray[sizeof(state_content.AccountTrieProofKey.stateRoot)](decoded.state_root)
check proto2.portalProtocol.addNode(node1.localNode) == Added
for proof in decoded.proofs:
let
address = hexToByteArray[sizeof(state_content.Address)](proof.address)
key = AccountTrieProofKey(
address: address,
stateRoot: state_root)
contentKey = ContentKey(
contentType: state_content.ContentType.accountTrieProof,
accountTrieProofKey: key)
var accountTrieProof = AccountTrieProof(@[])
for witness in proof.proof:
let witnessNode = ByteList(hexToSeqByte(witness))
discard accountTrieProof.add(witnessNode)
let encodedValue = SSZ.encode(accountTrieProof)
discard proto1.contentDB.put(contentKey.toContentId(), encodedValue, proto1.portalProtocol.localNode.id)
let foundContent = await proto2.getContent(contentKey)
check foundContent.isSome()
check decodeSsz(foundContent.get(), AccountTrieProof).isOk()
await node1.closeWait()
await node2.closeWait()
asyncTest "Test Share Full State":
let
trie = genesisToTrie("fluffy" / "tests" / "custom_genesis" / "chainid7.json")

View File

@ -0,0 +1 @@
-d:"chronicles_sinks=textlines[dynamic],json[dynamic]"

View File

@ -0,0 +1,82 @@
# Nimbus
# Copyright (c) 2023-2024 Status Research & Development GmbH
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.
# This is a fake bridge that reads state from a directory and backfills it to the portal state network.
{.push raises: [].}
import
std/[os, sugar],
confutils, confutils/std/net, chronicles, chronicles/topics_registry,
json_rpc/clients/httpclient,
chronos,
stew/[byteutils, io2],
eth/async_utils,
eth/common/eth_types,
../../network/state/state_content,
../../rpc/portal_rpc_client,
../../logging,
../eth_data_exporter/cl_data_exporter,
./state_bridge_conf
proc run(config: StateBridgeConf) {.raises: [CatchableError].} =
setupLogging(config.logLevel, config.logStdout)
notice "Launching Fluffy fake state bridge",
cmdParams = commandLineParams()
let portalRpcClient = newRpcHttpClient()
proc backfill(rpcAddress: string, rpcPort: Port) {.async raises: [OSError].} =
# info "Backfilling...", config.rpcAddress, ":", config.rpcPort
await portalRpcClient.connect(config.rpcAddress, Port(config.rpcPort), false)
let files = collect(for f in walkDir(config.dataDir): f.path)
for file in files:
let
content = readAllFile(file).valueOr:
warn "Skipping file because of error \n", file=file, error=($error)
continue
decoded =
try:
Json.decode(content, JsonProofVector)
except SerializationError as e:
warn "Skipping file because of error \n", file=file, error = e.msg
continue
state_root = hexToByteArray[sizeof(Bytes32)](decoded.state_root)
for proof in decoded.proofs:
let
address = hexToByteArray[sizeof(state_content.Address)](proof.address)
key = AccountTrieProofKey(
address: address,
stateRoot: state_root)
contentKey = ContentKey(
contentType: ContentType.accountTrieProof,
accountTrieProofKey: key)
encodedKey = encode(contentKey)
var accountTrieProof = AccountTrieProof(@[])
for witness in proof.proof:
let witnessNode = ByteList(hexToSeqByte(witness))
discard accountTrieProof.add(witnessNode)
discard await portalRpcClient.portal_stateGossip(encodedKey.asSeq().toHex(), SSZ.encode(accountTrieProof).toHex())
await portalRpcClient.close()
notice "Backfill done..."
waitFor backfill(config.rpcAddress, Port(config.rpcPort))
while true:
poll()
when isMainModule:
{.pop.}
let config = StateBridgeConf.load()
{.push raises: [].}
case config.cmd
of StateBridgeCmd.noCommand:
run(config)

View File

@ -0,0 +1,54 @@
# Nimbus
# Copyright (c) 2023-2024 Status Research & Development GmbH
# Licensed and distributed under either of
# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
# at your option. This file may not be copied, modified, or distributed except according to those terms.
{.push raises: [].}
import
confutils, confutils/std/net,
nimcrypto/hash,
../../logging
export net
type
StateBridgeCmd* = enum
noCommand
StateBridgeConf* = object
# Logging
logLevel* {.
desc: "Sets the log level"
defaultValue: "INFO"
name: "log-level" .}: string
logStdout* {.
hidden
desc: "Specifies what kind of logs should be written to stdout (auto, colors, nocolors, json)"
defaultValueDesc: "auto"
defaultValue: StdoutLogKind.Auto
name: "log-format" .}: StdoutLogKind
# Portal JSON-RPC API server to connect to
rpcAddress* {.
desc: "Listening address of the Portal JSON-RPC server"
defaultValue: "127.0.0.1"
name: "rpc-address" .}: string
rpcPort* {.
desc: "Listening port of the Portal JSON-RPC server"
defaultValue: 8545
name: "rpc-port" .}: Port
dataDir* {.
desc: "Data directory to lookup state data. Should point to the directory with json files generated by https://github.com/morph-dev/young-ethereum e.g. ./vendor/portal-spec-tests/tests/mainnet/state/"
name: "data-dir".}: string
case cmd* {.
command
defaultValue: noCommand .}: StateBridgeCmd
of noCommand:
discard

View File

@ -1,121 +0,0 @@
# Nimbus
# Copyright (c) 2018-2023 Status Research & Development GmbH
# Licensed under either of
# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or
# http://www.apache.org/licenses/LICENSE-2.0)
# * MIT license ([LICENSE-MIT](LICENSE-MIT) or
# http://opensource.org/licenses/MIT)
# at your option. This file may not be copied, modified, or distributed except
# according to those terms.
mode = ScriptMode.Verbose
packageName = "nimbus"
version = "0.1.0"
author = "Status Research & Development GmbH"
description = "An Ethereum 2.0 Sharding Client for Resource-Restricted Devices"
license = "Apache License 2.0"
skipDirs = @["tests", "examples"]
# we can't have the result of a custom task in the "bin" var - https://github.com/nim-lang/nimble/issues/542
# bin = @["build/nimbus"]
requires "nim >= 1.2.0",
"bncurve",
"chronicles",
"chronos",
"eth",
"json_rpc",
"libbacktrace",
"nimcrypto",
"stew",
"stint",
"rocksdb",
"ethash",
"blscurve",
"evmc",
"web3"
binDir = "build"
when declared(namedBin):
namedBin = {
"nimbus/nimbus": "nimbus",
"fluffy/fluffy": "fluffy",
"lc_proxy/lc_proxy": "lc_proxy",
"fluffy/tools/portalcli": "portalcli",
}.toTable()
proc buildBinary(name: string, srcDir = "./", params = "", lang = "c") =
if not dirExists "build":
mkDir "build"
# allow something like "nim nimbus --verbosity:0 --hints:off nimbus.nims"
var extra_params = params
for i in 2..<paramCount():
extra_params &= " " & paramStr(i)
exec "nim " & lang & " --out:build/" & name & " " & extra_params & " " & srcDir & name & ".nim"
proc test(path: string, name: string, params = "", lang = "c") =
# Verify stack usage is kept low by setting 750k stack limit in tests.
const stackLimitKiB = 750
when not defined(windows):
const (buildOption, runPrefix) = ("", "ulimit -s " & $stackLimitKiB & " && ")
else:
# No `ulimit` in Windows. `ulimit -s` in Bash is accepted but has no effect.
# See https://public-inbox.org/git/alpine.DEB.2.21.1.1709131448390.4132@virtualbox/
# Also, the command passed to NimScript `exec` on Windows is not a shell script.
# Instead, we can set stack size at link time.
const (buildOption, runPrefix) =
(" -d:windowsNoSetStack --passL:-Wl,--stack," & $(stackLimitKiB * 1024), "")
buildBinary name, (path & "/"), params & buildOption
exec runPrefix & "build/" & name
task test, "Run tests":
test "tests", "all_tests", "-d:chronicles_log_level=ERROR -d:unittest2DisableParamFiltering"
task test_rocksdb, "Run rocksdb tests":
test "tests/db", "test_kvstore_rocksdb", "-d:chronicles_log_level=ERROR -d:unittest2DisableParamFiltering"
task test_evm, "Run EVM tests":
test "tests", "evm_tests", "-d:chronicles_log_level=ERROR -d:unittest2DisableParamFiltering"
## Fluffy tasks
task fluffy, "Build fluffy":
buildBinary "fluffy", "fluffy/", "-d:chronicles_log_level=TRACE -d:chronosStrictException -d:PREFER_BLST_SHA256=false"
task lc_bridge, "Build light client bridge":
buildBinary "beacon_light_client_bridge", "fluffy/", "-d:chronicles_log_level=TRACE -d:chronosStrictException -d:PREFER_BLST_SHA256=false -d:libp2p_pki_schemes=secp256k1"
task fluffy_test, "Run fluffy tests":
# Need the nimbus_db_backend in state network tests as we need a Hexary to
# start from, even though it only uses the MemoryDb.
test "fluffy/tests/portal_spec_tests/mainnet", "all_fluffy_portal_spec_tests", "-d:chronicles_log_level=ERROR -d:chronosStrictException -d:nimbus_db_backend=sqlite -d:PREFER_BLST_SHA256=false"
# Running tests with a low `mergeBlockNumber` to make the tests faster.
# Using the real mainnet merge block number is not realistic for these tests.
test "fluffy/tests", "all_fluffy_tests", "-d:chronicles_log_level=ERROR -d:chronosStrictException -d:nimbus_db_backend=sqlite -d:PREFER_BLST_SHA256=false -d:mergeBlockNumber:38130"
# test "fluffy/tests/beacon_light_client_tests", "all_beacon_light_client_tests", "-d:chronicles_log_level=ERROR -d:chronosStrictException -d:nimbus_db_backend=sqlite -d:PREFER_BLST_SHA256=false"
task fluffy_tools, "Build fluffy tools":
buildBinary "beacon_chain_bridge", "fluffy/tools/bridge/", "-d:chronicles_log_level=TRACE -d:chronosStrictException -d:PREFER_BLST_SHA256=false -d:libp2p_pki_schemes=secp256k1"
buildBinary "eth_data_exporter", "fluffy/tools/", "-d:chronicles_log_level=TRACE -d:chronosStrictException -d:PREFER_BLST_SHA256=false"
buildBinary "content_verifier", "fluffy/tools/", "-d:chronicles_log_level=TRACE -d:chronosStrictException -d:PREFER_BLST_SHA256=false"
buildBinary "blockwalk", "fluffy/tools/", "-d:chronicles_log_level=TRACE -d:chronosStrictException"
buildBinary "portalcli", "fluffy/tools/", "-d:chronicles_log_level=TRACE -d:chronosStrictException -d:PREFER_BLST_SHA256=false"
task utp_test_app, "Build uTP test app":
buildBinary "utp_test_app", "fluffy/tools/utp_testing/", "-d:chronicles_log_level=TRACE -d:chronosStrictException"
task utp_test, "Run uTP integration tests":
test "fluffy/tools/utp_testing", "utp_test", "-d:chronicles_log_level=ERROR -d:chronosStrictException"
task fluffy_test_portal_testnet, "Build test_portal_testnet":
buildBinary "test_portal_testnet", "fluffy/scripts/", "-d:chronicles_log_level=DEBUG -d:chronosStrictException -d:unittest2DisableParamFiltering -d:PREFER_BLST_SHA256=false"
## Nimbus Verified Proxy tasks
task nimbus_verified_proxy, "Build Nimbus verified proxy":
buildBinary "nimbus_verified_proxy", "nimbus_verified_proxy/", "-d:chronicles_log_level=TRACE -d:chronosStrictException -d:PREFER_BLST_SHA256=false -d:libp2p_pki_schemes=secp256k1"
task nimbus_verified_proxy_test, "Run Nimbus verified proxy tests":
test "nimbus_verified_proxy/tests", "test_proof_validation", "-d:chronicles_log_level=ERROR -d:chronosStrictException -d:nimbus_db_backend=sqlite -d:PREFER_BLST_SHA256=false"

1
nimbus.nims Symbolic link
View File

@ -0,0 +1 @@
nimbus.nimble

@ -1 +1 @@
Subproject commit eb9edb34eb3ea599a5b14407615cef0e7e48e8b9
Subproject commit c3f11b8713c3e12be7f517d75cbdbafb9753d1ff