[QT-504] Add scenario sampling

Signed-off-by: Ryan Cragun <[email protected]>
hashicorp · Aug 23, 2023 · 804bf90 · 804bf90
1 parent 63fbb38
commit 804bf90
Show file tree

Hide file tree

Showing 90 changed files with 13,285 additions and 3,327 deletions.
diff --git a/Makefile b/Makefile
@@ -8,6 +8,7 @@ GO_BUILD_TAGS=-tags osusergo,netgo
 GO_LD_FLAGS=-ldflags="-extldflags=-static -X ${REPO}/internal/version.Version=${VERSION} -X ${REPO}/internal/version.GitSHA=${GIT_SHA}"
 GO_GC_FLAGS=
 LINT_OUT_FORMAT?=colored-line-number
+BUF_LINT_OUT_FORMAT?=github-actions
 GORACE=GORACE=log_path=/tmp/enos-gorace.log
 TEST_ACC=ENOS_ACC=1
 TEST_ACC_EXT=ENOS_ACC=1 ENOS_EXT=1
@@ -57,13 +58,13 @@ lint-fix-golang:
 
 .PHONY: lint-proto
 lint-proto:
-	pushd proto && buf lint --error-format=$(LINT_OUT_FORMAT)
+	pushd proto && buf lint --error-format=$(BUF_LINT_OUT_FORMAT)
 
 .PHONY: fmt
-fmt: fmt-golang fmt-proto
+fmt: fmt-golang fmt-proto fmt-terraform fmt-enos
 
 .PHONY: fmt-check
-fmt-check: fmt-check-golang fmt-check-proto
+fmt-check: fmt-check-golang fmt-check-proto fmt-check-terraform fmt-check-enos
 
 .PHONY: fmt-golang
 fmt-golang:
@@ -81,6 +82,22 @@ fmt-proto:
 fmt-check-proto:
 	buf format proto -d --exit-code
 
+.PHONY: fmt-terraform
+fmt-terraform:
+	terraform fmt -recursive acceptance/scenarios
+
+.PHONY: fmt-check-terraform
+fmt-check-terraform:
+	terraform fmt -check -diff -recursive acceptance/scenarios
+
+.PHONY: fmt-enos
+fmt-enos:
+	dist/enos fmt --recursive acceptance/scenarios
+
+.PHONY: fmt-check-enos
+fmt-check-enos:
+	dist/enos fmt --check --diff --recursive acceptance/scenarios
+
 .PHONY: clean
 clean:
 	rm -rf dist enos

diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go
@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/davecgh/go-spew/spew"
 	"github.com/stretchr/testify/require"
 	"google.golang.org/protobuf/encoding/protojson"
 
@@ -239,7 +240,8 @@ func requireEqualOperationResponses(t *testing.T, expected *pb.OperationResponse
 
 	got := &pb.OperationResponses{}
 	require.NoErrorf(t, protojson.Unmarshal(out, got), string(out))
-	require.Len(t, got.GetResponses(), len(expected.GetResponses()))
+	require.Lenf(t, expected.GetResponses(), len(got.GetResponses()),
+		spew.Sdump(expected.GetResponses(), got.GetResponses()))
 	expectedResponses := expected.GetResponses()
 	gotResponses := got.GetResponses()
 	sortResponses(expectedResponses)

diff --git a/acceptance/fmt_test.go b/acceptance/fmt_test.go
@@ -19,7 +19,7 @@ func TestAcc_Cmd_Fmt(t *testing.T) {
 
 	enos := newAcceptanceRunner(t)
 
-	path, err := filepath.Abs("./scenarios/scenario_generate_pass_0")
+	path, err := filepath.Abs("./invalid_scenarios/scenario_not_formatted")
 	require.NoError(t, err)
 
 	cmd := fmt.Sprintf("fmt %s -d -c --format json", path)

diff --git a/...ios/scenario_list_fail_malformed/enos.hcl → ...ios/scenario_list_fail_malformed/enos.hcl b/...ios/scenario_list_fail_malformed/enos.hcl → ...ios/scenario_list_fail_malformed/enos.hcl
diff --git a/...ios/scenario_list_fail_malformed/enos.txt → ...ios/scenario_list_fail_malformed/enos.txt b/...ios/scenario_list_fail_malformed/enos.txt → ...ios/scenario_list_fail_malformed/enos.txt
diff --git a/acceptance/invalid_scenarios/scenario_not_formatted/enos.hcl b/acceptance/invalid_scenarios/scenario_not_formatted/enos.hcl
@@ -0,0 +1,94 @@
+variable "defaultinput" {
+  type = object({
+    value = string
+  })
+  default = {
+    value = "defaultval"
+  }
+  sensitive = false
+}
+
+variable "setinput" {
+  type = object({
+    value = string
+  })
+  default = {
+    value = "defaultval"
+  }
+  sensitive = false
+}
+
+terraform_cli "debug" {
+  env = {
+    TF_LOG_CORE     = "off"
+    TF_LOG_PROVIDER = "debug"
+  }
+}
+
+module "foo" {
+  source = "./modules/foo"
+
+  input        = var.defaultinput.value
+  anotherinput = ["anotherfoo"]
+}
+
+module "bar" {
+  source = "./modules/bar"
+
+  input        = "bar"
+  anotherinput = "anotherbar"
+}
+
+module "fooref" {
+  source = "./modules/foo"
+}
+
+module "barref" {
+  source = "./modules/bar"
+}
+
+scenario "test" {
+  terraform_cli = terraform_cli.debug
+
+  matrix {
+    foo = ["matrixfoo", "matrixbar"]
+  }
+
+  step "foo" {
+    module = module.foo
+  }
+
+  step "bar" {
+    module = module.bar
+  }
+
+  step "fooover" {
+    module = module.foo
+
+    variables {
+      input        = matrix.foo
+      anotherinput = ["fooover"]
+    }
+  }
+
+  step "barover" {
+    module = module.bar
+
+   variables {
+      anotherinput = var.setinput.value
+    }
+  }
+
+  step "fooref" {
+    module = module.fooref
+  }
+
+  step "barref" {
+    module = module.barref
+
+   variables {
+      input        = step.fooref.input
+      anotherinput = step.fooref.anotherinput
+    }
+  }
+}
diff --git a/acceptance/invalid_scenarios/scenario_not_formatted/enos.vars.hcl b/acceptance/invalid_scenarios/scenario_not_formatted/enos.vars.hcl
@@ -0,0 +1,3 @@
+setinput = {
+  value = "itisset"
+}
diff --git a/acceptance/invalid_scenarios/scenario_not_formatted/modules/bar/main.tf b/acceptance/invalid_scenarios/scenario_not_formatted/modules/bar/main.tf
@@ -0,0 +1,17 @@
+output "input" {
+  value = var.input
+}
+
+output "anotherinput" {
+  value = var.input
+}
+
+variable "input" {
+  type = string
+  default = "notset"
+}
+
+variable "anotherinput" {
+  type = string
+  default = "notset"
+}
diff --git a/acceptance/invalid_scenarios/scenario_not_formatted/modules/foo/main.tf b/acceptance/invalid_scenarios/scenario_not_formatted/modules/foo/main.tf
@@ -0,0 +1,17 @@
+output "input" {
+  value = var.input
+}
+
+output "anotherinput" {
+  value = var.input
+}
+
+variable "input" {
+  type = string
+  default = "notset"
+}
+
+variable "anotherinput" {
+  type = list(string)
+  default = ["one"]
+}
diff --git a/acceptance/scenario_check_test.go b/acceptance/scenario_check_test.go
@@ -60,20 +60,23 @@ func TestAcc_Cmd_Scenario_Check(t *testing.T) {
 
 			for _, variant := range test.variants {
 				name := test.name
-				elements := []*pb.Scenario_Filter_Element{}
+				filter := test.name
+				elements := []*pb.Matrix_Element{}
 				if len(variant) == 2 {
 					name = fmt.Sprintf("%s [%s:%s]", name, variant[0], variant[1])
-					elements = append(elements, &pb.Scenario_Filter_Element{
+					filter = fmt.Sprintf("%s %s:%s", test.name, variant[0], variant[1])
+					elements = append(elements, &pb.Matrix_Element{
 						Key:   variant[0],
 						Value: variant[1],
 					})
 				}
 				uid := fmt.Sprintf("%x", sha256.Sum256([]byte(name)))
 				scenarioRef := &pb.Ref_Scenario{
 					Id: &pb.Scenario_ID{
-						Name: test.name,
-						Uid:  uid,
-						Variants: &pb.Scenario_Filter_Vector{
+						Name:   test.name,
+						Filter: filter,
+						Uid:    uid,
+						Variants: &pb.Matrix_Vector{
 							Elements: elements,
 						},
 					},
@@ -152,8 +155,8 @@ func TestAcc_Cmd_Scenario_Check_WithWarnings(t *testing.T) {
 
 			for _, variant := range []string{"has_warning", "valid"} {
 				name := fmt.Sprintf("warning [mod:%s]", variant)
-				elements := []*pb.Scenario_Filter_Element{}
-				elements = append(elements, &pb.Scenario_Filter_Element{
+				elements := []*pb.Matrix_Element{}
+				elements = append(elements, &pb.Matrix_Element{
 					Key:   "mod",
 					Value: variant,
 				})
@@ -162,7 +165,7 @@ func TestAcc_Cmd_Scenario_Check_WithWarnings(t *testing.T) {
 					Id: &pb.Scenario_ID{
 						Name: "warning",
 						Uid:  uid,
-						Variants: &pb.Scenario_Filter_Vector{
+						Variants: &pb.Matrix_Vector{
 							Elements: elements,
 						},
 					},

diff --git a/acceptance/scenario_destroy_test.go b/acceptance/scenario_destroy_test.go
@@ -57,10 +57,10 @@ func TestAcc_Cmd_Scenario_Destroy(t *testing.T) {
 			require.NoError(t, err)
 
 			filter := test.name
-			elements := []*pb.Scenario_Filter_Element{}
+			elements := []*pb.Matrix_Element{}
 			for _, variant := range test.variants {
 				filter = fmt.Sprintf("%s %s:%s", filter, variant[0], variant[1])
-				elements = append(elements, &pb.Scenario_Filter_Element{
+				elements = append(elements, &pb.Matrix_Element{
 					Key:   variant[0],
 					Value: variant[1],
 				})
@@ -79,9 +79,10 @@ func TestAcc_Cmd_Scenario_Destroy(t *testing.T) {
 
 			scenarioRef := &pb.Ref_Scenario{
 				Id: &pb.Scenario_ID{
-					Name: test.name,
-					Uid:  test.uid,
-					Variants: &pb.Scenario_Filter_Vector{
+					Name:   test.name,
+					Filter: filter,
+					Uid:    test.uid,
+					Variants: &pb.Matrix_Vector{
 						Elements: elements,
 					},
 				},

diff --git a/acceptance/scenario_e2e_aws_test.go b/acceptance/scenario_e2e_aws_test.go
@@ -101,9 +101,9 @@ func TestAcc_Cmd_Scenario_E2E_AWS(t *testing.T) {
 				Responses: []*pb.Operation_Response{},
 			}
 			for _, variant := range test.variants {
-				elements := []*pb.Scenario_Filter_Element{}
+				elements := []*pb.Matrix_Element{}
 				for _, v := range variant.variants {
-					elements = append(elements, &pb.Scenario_Filter_Element{
+					elements = append(elements, &pb.Matrix_Element{
 						Key:   v[0],
 						Value: v[1],
 					})
@@ -113,7 +113,7 @@ func TestAcc_Cmd_Scenario_E2E_AWS(t *testing.T) {
 					Id: &pb.Scenario_ID{
 						Name: test.name,
 						Uid:  variant.uid,
-						Variants: &pb.Scenario_Filter_Vector{
+						Variants: &pb.Matrix_Vector{
 							Elements: elements,
 						},
 					},

diff --git a/acceptance/scenario_exec_test.go b/acceptance/scenario_exec_test.go
@@ -57,20 +57,21 @@ func TestAcc_Cmd_Scenario_Exec(t *testing.T) {
 			require.NoError(t, err)
 
 			filter := test.name
-			elements := []*pb.Scenario_Filter_Element{}
+			elements := []*pb.Matrix_Element{}
 			for _, variant := range test.variants {
 				filter = fmt.Sprintf("%s %s:%s", filter, variant[0], variant[1])
-				elements = append(elements, &pb.Scenario_Filter_Element{
+				elements = append(elements, &pb.Matrix_Element{
 					Key:   variant[0],
 					Value: variant[1],
 				})
 			}
 
 			scenarioRef := &pb.Ref_Scenario{
 				Id: &pb.Scenario_ID{
-					Name: test.name,
-					Uid:  test.uid,
-					Variants: &pb.Scenario_Filter_Vector{
+					Name:   test.name,
+					Filter: filter,
+					Uid:    test.uid,
+					Variants: &pb.Matrix_Vector{
 						Elements: elements,
 					},
 				},
@@ -103,7 +104,7 @@ func TestAcc_Cmd_Scenario_Exec(t *testing.T) {
 										Id: &pb.Scenario_ID{
 											Name: test.name,
 											Uid:  test.uid,
-											Variants: &pb.Scenario_Filter_Vector{
+											Variants: &pb.Matrix_Vector{
 												Elements: elements,
 											},
 										},

diff --git a/acceptance/scenario_generate_test.go b/acceptance/scenario_generate_test.go
@@ -86,20 +86,21 @@ func TestAcc_Cmd_Scenario_Generate(t *testing.T) {
 			require.NoError(t, err)
 
 			filter := test.name
-			elements := []*pb.Scenario_Filter_Element{}
+			elements := []*pb.Matrix_Element{}
 			for _, variant := range test.variants {
 				filter = fmt.Sprintf("%s %s:%s", filter, variant[0], variant[1])
-				elements = append(elements, &pb.Scenario_Filter_Element{
+				elements = append(elements, &pb.Matrix_Element{
 					Key:   variant[0],
 					Value: variant[1],
 				})
 			}
 
 			scenarioRef := &pb.Ref_Scenario{
 				Id: &pb.Scenario_ID{
-					Name: test.name,
-					Uid:  test.uid,
-					Variants: &pb.Scenario_Filter_Vector{
+					Name:   test.name,
+					Filter: filter,
+					Uid:    test.uid,
+					Variants: &pb.Matrix_Vector{
 						Elements: elements,
 					},
 				},

diff --git a/acceptance/scenario_launch_test.go b/acceptance/scenario_launch_test.go
@@ -56,20 +56,21 @@ func TestAcc_Cmd_Scenario_Launch(t *testing.T) {
 			require.NoError(t, err)
 
 			filter := test.name
-			elements := []*pb.Scenario_Filter_Element{}
+			elements := []*pb.Matrix_Element{}
 			for _, variant := range test.variants {
 				filter = fmt.Sprintf("%s %s:%s", filter, variant[0], variant[1])
-				elements = append(elements, &pb.Scenario_Filter_Element{
+				elements = append(elements, &pb.Matrix_Element{
 					Key:   variant[0],
 					Value: variant[1],
 				})
 			}
 
 			scenarioRef := &pb.Ref_Scenario{
 				Id: &pb.Scenario_ID{
-					Name: test.name,
-					Uid:  test.uid,
-					Variants: &pb.Scenario_Filter_Vector{
+					Name:   test.name,
+					Filter: filter,
+					Uid:    test.uid,
+					Variants: &pb.Matrix_Vector{
 						Elements: elements,
 					},
 				},