feat: Testground tooling

app/app.go

@@ -108,6 +108,9 @@ const (
 	BondDenomAlias = "microtia"
 	// DisplayDenom defines the name, symbol, and display value of the Celestia token.
 	DisplayDenom = "TIA"
+	// SetProtocolVersionOptionKey is the key used to set the protocol version
+	// to a specific value during tests.
+	SetProtocolVersionOptionKey = "set-protocol-version"
 )
 
 // These constants are derived from the above variables.
@@ -259,6 +262,15 @@ func New(
 	cdc := encodingConfig.Amino
 	interfaceRegistry := encodingConfig.InterfaceRegistry
 
+	// set the protocol version to a specific value if specified. This is used
+	// for testing purposes.
+	if pvo := appOpts.Get(SetProtocolVersionOptionKey); pvo != nil {
+		baseAppOptions = append(baseAppOptions, func(ba *baseapp.BaseApp) {
+			appVersion := pvo.(uint64)
+			ba.SetProtocolVersion(appVersion)
+		})
+	}
+
 	bApp := baseapp.NewBaseApp(Name, logger, db, encodingConfig.TxConfig.TxDecoder(), baseAppOptions...)
 	bApp.SetCommitMultiStoreTracer(traceStore)
 	bApp.SetVersion(version.Version)

go.work

@@ -0,0 +1,6 @@
+go 1.21.1
+
+use (
+	.
+	./test/testground
+)

pkg/appconsts/testground/app_consts.go

@@ -0,0 +1,7 @@
+package v1
+
+const (
+	Version              uint64 = 420420420
+	SquareSizeUpperBound int    = 512
+	SubtreeRootThreshold int    = 64
+)

pkg/appconsts/versioned_consts.go

@@ -1,12 +1,12 @@
 package appconsts
 
 import (
+	testground "github.com/celestiaorg/celestia-app/pkg/appconsts/testground"
 	v1 "github.com/celestiaorg/celestia-app/pkg/appconsts/v1"
-	v2 "github.com/celestiaorg/celestia-app/pkg/appconsts/v2"
 )
 
 const (
-	LatestVersion = v2.Version
+	LatestVersion = v1.Version
 )
 
 // SubtreeRootThreshold works as a target upper bound for the number of subtree
@@ -24,8 +24,13 @@ func SubtreeRootThreshold(_ uint64) int {
 // SquareSizeUpperBound is the maximum original square width possible
 // for a version of the state machine. The maximum is decided through
 // governance. See `DefaultGovMaxSquareSize`.
-func SquareSizeUpperBound(_ uint64) int {
-	return v1.SquareSizeUpperBound
+func SquareSizeUpperBound(v uint64) int {
+	switch v {
+	case testground.Version:
+		return testground.SquareSizeUpperBound
+	default:
+		return v1.SquareSizeUpperBound
+	}
 }
 
 var (

test/testground/Dockerfile

@@ -0,0 +1,89 @@
+# BUILD_BASE_IMAGE is the base image to use for the build. It contains a rolling
+# accumulation of Go build/package caches.
+ARG BUILD_BASE_IMAGE=docker.io/golang:1.21.0-alpine3.18
+# This Dockerfile performs a multi-stage build and RUNTIME_IMAGE is the image
+# onto which to copy the resulting binary.
+#
+# Picking a different runtime base image from the build image allows us to
+# slim down the deployable considerably.
+#
+# The user can override the runtime image by passing in the appropriate builder
+# configuration option.
+ARG RUNTIME_IMAGE=alpine:3.18
+
+#:::
+#::: BUILD CONTAINER
+#:::
+FROM ${BUILD_BASE_IMAGE} AS builder
+
+# PLAN_DIR is the location containing the plan source inside the container.
+ENV PLAN_DIR /plan
+
+ENV INFLUXDB_URL=http://influxdb:8086
+
+# SDK_DIR is the location containing the (optional) sdk source inside the container.
+ENV SDK_DIR /sdk
+
+# Delete any prior artifacts, if this is a cached image.
+RUN rm -rf ${PLAN_DIR} ${SDK_DIR} /testground_dep_lists
+
+# TESTPLAN_EXEC_PKG is the executable package of the testplan to build.
+# The image will build that package only.
+ARG TESTPLAN_EXEC_PKG="."
+
+# GO_PROXY is the go proxy that will be used, or direct by default.
+ARG GO_PROXY=https://proxy.golang.org
+
+# BUILD_TAGS is either nothing, or when expanded, it expands to "-tags <comma-separated build tags>"
+ARG BUILD_TAGS
+
+# TESTPLAN_EXEC_PKG is the executable package within this test plan we want to build. 
+ENV TESTPLAN_EXEC_PKG ${TESTPLAN_EXEC_PKG}
+
+# We explicitly set GOCACHE under the /go directory for more tidiness.
+ENV GOCACHE /go/cache
+
+
+# Copy only go.mod files and download deps, in order to leverage Docker caching.
+COPY /plan/go.mod ${PLAN_DIR}/go.mod
+
+RUN apk add gcompat
+
+# Download deps.
+RUN echo "Using go proxy: ${GO_PROXY}" \
+    && cd ${PLAN_DIR} \
+    && go env -w GOPROXY="${GO_PROXY}" \
+    && go mod download
+
+
+# Now copy the rest of the source and run the build.
+COPY . /
+
+
+RUN cd ${PLAN_DIR} \
+    && go env -w GOPROXY="${GO_PROXY}" \
+    && CGO_ENABLED=${CgoEnabled} GOOS=linux GOARCH=amd64 go build -o ${PLAN_DIR}/testplan.bin ${BUILD_TAGS} ${TESTPLAN_EXEC_PKG}
+
+# Store module dependencies
+RUN cd ${PLAN_DIR} \
+  && go list -m all > /testground_dep_list
+
+#:::
+#::: (OPTIONAL) RUNTIME CONTAINER
+#:::
+
+## The 'AS runtime' token is used to parse Docker stdout to extract the build image ID to cache.
+FROM ${RUNTIME_IMAGE} AS runtime
+RUN apk add --no-cache bash gcompat curl
+# PLAN_DIR is the location containing the plan source inside the build container.
+ENV PLAN_DIR /plan
+
+
+# HOME ENV is crucial for app/sdk -> remove at your OWN RISK!
+ENV HOME /
+
+COPY --from=builder /testground_dep_list /
+COPY --from=builder ${PLAN_DIR}/testplan.bin /testplan
+
+EXPOSE 9090 26657 26656 1317 26658 26660 26659 30000
+ENTRYPOINT [ "/testplan"]

test/testground/README.md

@@ -0,0 +1,132 @@
+# Testground Experiement Tooling
+
+## Test Instance Communication and Experiment Flow
+
+```go
+// Role is the interface between a testground test entrypoint and the actual
+// test logic. Testground creates many instances and passes each instance a
+// configuration from the plan and manifest toml files. From those
+// configurations a Role is created for each node, and the three methods below
+// are ran in order.
+type Role interface {
+	// Plan is the first function called in a test by each node. It is
+	// responsible for creating the genesis block, configuring nodes, and
+	// starting the network.
+	Plan(ctx context.Context, runenv *runtime.RunEnv, initCtx *run.InitContext) error
+	// Execute is the second function called in a test by each node. It is
+	// responsible for running any experiments. This is phase where commands are
+	// sent and received.
+	Execute(ctx context.Context, runenv *runtime.RunEnv, initCtx *run.InitContext) error
+	// Retro is the last function called in a test by each node. It is
+	// responsible for collecting any data from the node and/or running any
+	// retrospective tests or benchmarks.
+	Retro(ctx context.Context, runenv *runtime.RunEnv, initCtx *run.InitContext) error
+}
+
+var _ Role = (*Leader)(nil)
+
+var _ Role = (*Follower)(nil)
+```
+
+```mermaid
+sequenceDiagram
+    participant I as Initializer Node
+    participant L as Leader Node
+    participant F1 as Follower Node 1
+    participant F2 as Follower Node 2
+    participant Fn as Follower Node N
+
+    Note over I, Fn: Testground Initialization
+    I->>L: Create Leader Node Instance
+    I->>F1: Create Follower Node 1 Instance
+    I->>F2: Create Follower Node 2 Instance
+    I->>Fn: Create Follower Node N Instance
+
+    Note over L, Fn: EntryPoint(runenv *runtime.RunEnv, initCtx *run.InitContext)
+
+    Note over L, Fn: Plan(ctx context.Context, runenv *runtime.RunEnv, initCtx *run.InitContext)
+    F1->>L: Send PeerPacket
+    F2->>L: Send PeerPacket
+    Fn->>L: Send PeerPacket
+
+    Note over L: Genesis Creation
+    L->>L: Collect GenTx
+
+    L->>F1: Send Genesis File
+    L->>F2: Send Genesis File
+    L->>Fn: Send Genesis File
+
+    Note over L: Configuration
+    L->>L: Configurators
+
+    L->>F1: Send Config Files
+    L->>F2: Send Config Files
+    L->>Fn: Send Config Files
+
+    Note over L, Fn: Start Network
+
+    Note over L, Fn: Execute(ctx context.Context, runenv *runtime.RunEnv, initCtx *run.InitContext)
+
+    L->>F1: Send Arbitrary Commands
+    L->>F2: Send Arbitrary Commands
+    L->>Fn: Send Arbitrary Commands
+
+    L->>F1: Send EndTest Command
+    L->>F2: Send EndTest Command
+    L->>Fn: Send EndTest Command
+
+    Note over L, Fn: Retro(ctx context.Context, runenv *runtime.RunEnv, initCtx *run.InitContext)
+
+    Note over L: Process log local data
+```
+
+## Configuring an Experiment
+
+### Defining Topologies and Configs
+
+Per the diagram above, the leader node initializes and modifies the configs used
+by each node. This allows for arbitrary network topologies to be created.
+
+## Implemented Experiments
+
+### Standard
+
+The `standard` test runs an experiment that is as close to mainnet as possible.
+This is used as a base for other experiements.
+
+## Running the Experiment
+
+```sh
+cd ./test/testground
+testground plan import --from . --name celestia
+
+# This command should be executed in the 1st terminal
+testground daemon
+
+# This command should be executed in the 2nd terminal
+testground run composition -f compositions/standard/plan.toml --wait
+
+# After the test has been completed, run this command to cleanup remaining instance resources
+testground terminate --runner cluster:k8s
+```
+
+## Collecting Data
+
+### Grafana
+
+All metrics data is logged to a separate testground specific grafana/influx
+node. To access that node, forward the ports use kubectl.
+
+```sh
+export POD_NAME=$(kubectl get pods --namespace default -l "app.kubernetes.io/name=grafana,app.kubernetes.io/instance=tg-monitoring" -o jsonpath="{.items[0].metadata.name}")
+
+kubectl --namespace default port-forward $POD_NAME 3000
+
+contact members of the devops team or testground admins to get the creds for accessing this node.
+```
+
+### Tracing
+
+The tracing infrastructure in celestia-core can be used by using `tracing_nodes`
+plan parameter greater than 0, along with specifying the tracing URL and tracing
+token as plan parameters in the `plan.toml`.

test/testground/compositions/standard/plan.toml

@@ -0,0 +1,54 @@
+[metadata]
+name = "standard"
+author = "core-app"
+
+[global]
+plan = "core-app"
+case = "entrypoint"
+total_instances = 40
+builder = "docker:generic"
+runner = "cluster:k8s"
+disable_metrics = false
+
+[global.run.test_params]
+chain_id = "standard-x"
+timeout = "25m"
+halt_height = "50"
+latency = "0"
+bandwidth = "1Gib"
+validators = "40"
+topology = "seed"
+pex = "true"
+timeout_propose = "10s"
+timeout_commit = "11s"
+per_peer_bandwidth = "5Mib"
+blob_sequences = "2"
+blob_sizes = "130000"
+blobs_per_sequence = "1"
+inbound_peer_count = "20"
+outbound_peer_count = "10"
+gov_max_square_size = "128"
+max_block_bytes = "2000000"
+mempool = "v1"
+broadcast_txs = "true"
+tracing_nodes = "0"
+tracing_token = ""
+tracing_url = ""
+
+[[groups]]
+id = "validators"
+builder = "docker:generic"
+[groups.resources]
+memory = "8Gi"
+cpu = "6"
+[groups.instances]
+count = 40
+percentage = 0.0
+[groups.build_config]
+build_base_image = "golang:1.21.3"
+enable_go_build_cache = true
+enabled = true
+go_version = "1.21"
+[groups.build]
+[groups.run]
+artifact = ""