Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[QT-709] Validate scenario samples #134

Merged
merged 1 commit into from
May 2, 2024
Merged

[QT-709] Validate scenario samples #134

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,15 +40,15 @@ build-race:

.PHONY: test
test:
${GORACE} go test -race ./... -v $(TESTARGS) -timeout=5m -parallel=4
${GORACE} go test -race ./... $(TESTARGS) -timeout=5m -parallel=4

.PHONY: test-acc
test-acc: build-race
${TEST_ACC} ${GORACE} ENOS_BINARY_PATH=${BUILD_BINARY_PATH} go test -race ./... -v $(TESTARGS) -timeout 120m
${TEST_ACC} ${GORACE} ENOS_BINARY_PATH=${BUILD_BINARY_PATH} go test -race ./... $(TESTARGS) -timeout 120m

.PHONY: test-acc-ext
test-acc-ext: build-race
${TEST_ACC_EXT} ${GORACE} ENOS_BINARY_PATH=${BUILD_BINARY_PATH} go test -race ./... -v $(TESTARGS) -timeout 120m
${TEST_ACC_EXT} ${GORACE} ENOS_BINARY_PATH=${BUILD_BINARY_PATH} go test -race ./... $(TESTARGS) -timeout 120m

.PHONY: lint
lint: lint-golang lint-proto
Expand Down
14 changes: 11 additions & 3 deletions acceptance/acceptance_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ package acceptance

import (
"context"
"errors"
"fmt"
"io"
"os"
Expand Down Expand Up @@ -196,10 +197,10 @@ type acceptanceRunner struct {
}

// run runs an Enos sub-command.
func (r *acceptanceRunner) run(ctx context.Context, subCommand string) ([]byte, error) {
func (r *acceptanceRunner) run(ctx context.Context, subCommand string) ([]byte, []byte, error) {
path, err := filepath.Abs(r.enosBinPath)
if err != nil {
return nil, err
return nil, nil, err
}

cmdParts := strings.Split(subCommand, " ")
Expand All @@ -209,7 +210,14 @@ func (r *acceptanceRunner) run(ctx context.Context, subCommand string) ([]byte,
cmd := exec.CommandContext(ctx, path, cmdParts...)
cmd.Env = os.Environ()

return cmd.CombinedOutput()
stdout, err := cmd.Output()
var stderr []byte
var exitErr *exec.ExitError
if err != nil && errors.As(err, &exitErr) {
stderr = exitErr.Stderr
}

return stdout, stderr, err
}

func (r *acceptanceRunner) validate(t *testing.T) {
Expand Down
2 changes: 1 addition & 1 deletion acceptance/fmt_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ func TestAcc_Cmd_Fmt(t *testing.T) {
require.NoError(t, err)

cmd := fmt.Sprintf("fmt %s -d -c --format json", path)
out, err := enos.run(context.Background(), cmd)
out, _, err := enos.run(context.Background(), cmd)
target := &exec.ExitError{}
require.Error(t, err)
if errors.As(err, &target) {
Expand Down
37 changes: 37 additions & 0 deletions acceptance/invalid_scenarios/sample_empty_frame/enos.hcl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
# Copyright (c) HashiCorp, Inc.
# SPDX-License-Identifier: MPL-2.0

module "foo" {
source = "../scenario_generate_pass_0/modules/foo"
}

module "bar" {
source = "../scenario_generate_pass_0/modules/bar"
}

scenario "smoke" {
matrix {
arch = ["amd64", "arm64"]
distro = ["ubuntu", "rhel"]
}

step "one" {
module = module.foo

variables {
input = matrix.arch
anotherinput = matrix.distro
}
}
}

sample "smoke_empty_frame" {
subset "smoke" {
matrix {
// Since we're filtering on a variant that does not exist our "smoke" frame will be empty.
// That will cause validate to fail since there's no reason to include empty frames in a
// sample.
arch = ["not_a_variant"]
}
}
}
4 changes: 2 additions & 2 deletions acceptance/scenario_check_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ func TestAcc_Cmd_Scenario_Check(t *testing.T) {
require.NoError(t, err)

cmd := fmt.Sprintf("scenario check --chdir %s --out %s --format json", path, outDir)
out, err := enos.run(context.Background(), cmd)
out, _, err := enos.run(context.Background(), cmd)
require.NoError(t, err, string(out))

expected := &pb.OperationResponses{
Expand Down Expand Up @@ -142,7 +142,7 @@ func TestAcc_Cmd_Scenario_Check_WithWarnings(t *testing.T) {
if failOnWarnings {
cmd = cmd + " --fail-on-warnings"
}
out, err := enos.run(context.Background(), cmd)
out, _, err := enos.run(context.Background(), cmd)
if failOnWarnings {
require.Error(t, err, string(out))

Expand Down
4 changes: 2 additions & 2 deletions acceptance/scenario_destroy_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,12 +71,12 @@ func TestAcc_Cmd_Scenario_Destroy(t *testing.T) {
// Test destroying a scenario with it launched or not
if test.launch {
cmd := fmt.Sprintf("scenario launch --chdir %s --out %s %s", path, outDir, filter)
out, err := enos.run(context.Background(), cmd)
out, _, err := enos.run(context.Background(), cmd)
require.NoError(t, err, string(out))
}

cmd := fmt.Sprintf("scenario destroy --chdir %s --out %s --format json %s", path, outDir, filter)
out, err := enos.run(context.Background(), cmd)
out, _, err := enos.run(context.Background(), cmd)
require.NoError(t, err, string(out))

scenarioRef := &pb.Ref_Scenario{
Expand Down
4 changes: 2 additions & 2 deletions acceptance/scenario_e2e_aws_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ func TestAcc_Cmd_Scenario_E2E_AWS(t *testing.T) {

// Lets try one more time to destroy resources that might have been
// created
out, err := enos.run(context.Background(), fmt.Sprintf("scenario destroy --chdir %s --out %s", path, outDir))
out, _, err := enos.run(context.Background(), fmt.Sprintf("scenario destroy --chdir %s --out %s", path, outDir))
require.NoErrorf(t, err, string(out))
})

Expand Down Expand Up @@ -154,7 +154,7 @@ func TestAcc_Cmd_Scenario_E2E_AWS(t *testing.T) {
}

cmd := fmt.Sprintf("scenario run --chdir %s --out %s --format json", path, outDir)
out, err := enos.run(context.Background(), cmd)
out, _, err := enos.run(context.Background(), cmd)
if err != nil {
failed = true
}
Expand Down
4 changes: 2 additions & 2 deletions acceptance/scenario_exec_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,11 +80,11 @@ func TestAcc_Cmd_Scenario_Exec(t *testing.T) {
}

cmd := fmt.Sprintf("scenario launch --chdir %s --out %s %s", path, outDir, filter)
out, err := enos.run(context.Background(), cmd)
out, _, err := enos.run(context.Background(), cmd)
require.NoError(t, err, string(out))

cmd = fmt.Sprintf(`scenario exec --cmd version --chdir %s --out %s --format json %s`, path, outDir, filter)
out, err = enos.run(context.Background(), cmd)
out, _, err = enos.run(context.Background(), cmd)
require.NoError(t, err, string(out))

expected := &pb.OperationResponses{
Expand Down
2 changes: 1 addition & 1 deletion acceptance/scenario_generate_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ func TestAcc_Cmd_Scenario_Generate(t *testing.T) {
}

cmd := fmt.Sprintf("scenario generate --chdir %s --out %s %s --format json", path, outDir, filter)
out, err := enos.run(context.Background(), cmd)
out, _, err := enos.run(context.Background(), cmd)
require.NoErrorf(t, err, string(out))

expected := &pb.OperationResponses{
Expand Down
2 changes: 1 addition & 1 deletion acceptance/scenario_launch_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ func TestAcc_Cmd_Scenario_Launch(t *testing.T) {
}

cmd := fmt.Sprintf("scenario launch --chdir %s --out %s --format json %s", path, outDir, filter)
out, err := enos.run(context.Background(), cmd)
out, _, err := enos.run(context.Background(), cmd)
require.NoError(t, err, string(out))

expected := &pb.OperationResponses{
Expand Down
2 changes: 1 addition & 1 deletion acceptance/scenario_list_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ func TestAcc_Cmd_Scenario_List(t *testing.T) {
require.NoError(t, err)
cmd := fmt.Sprintf("scenario list --chdir %s --format json", path)
fmt.Println(path)
out, err := enos.run(context.Background(), cmd)
out, _, err := enos.run(context.Background(), cmd)
if test.fail {
require.Error(t, err)

Expand Down
2 changes: 1 addition & 1 deletion acceptance/scenario_outline_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ func TestAcc_Cmd_Scenario_Outline(t *testing.T) {
path, err := filepath.Abs(filepath.Join("./", test.dir))
require.NoError(t, err)
cmd := fmt.Sprintf("scenario outline --chdir %s --format json", path)
out, err := enos.run(context.Background(), cmd)
out, _, err := enos.run(context.Background(), cmd)
if test.fail {
require.Error(t, err)

Expand Down
6 changes: 3 additions & 3 deletions acceptance/scenario_output_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,16 +74,16 @@ func TestAcc_Cmd_Scenario_Output(t *testing.T) {

t.Cleanup(func() {
cmd := fmt.Sprintf("scenario destroy --chdir %s --out %s %s", path, outDir, filter)
out, err := enos.run(context.Background(), cmd)
out, _, err := enos.run(context.Background(), cmd)
require.NoError(t, err, string(out))
})

cmd := fmt.Sprintf("scenario launch --chdir %s --out %s %s", path, outDir, filter)
out, err := enos.run(context.Background(), cmd)
out, _, err := enos.run(context.Background(), cmd)
require.NoError(t, err, string(out))

cmd = fmt.Sprintf(`scenario output --name step_reference_unknown --chdir %s --out %s --format json %s`, path, outDir, filter)
out, err = enos.run(context.Background(), cmd)
out, _, err = enos.run(context.Background(), cmd)
require.NoError(t, err, string(out))

expected := &pb.OperationResponses{
Expand Down
4 changes: 2 additions & 2 deletions acceptance/scenario_run_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ func TestAcc_Cmd_Scenario_Run(t *testing.T) {
}

cmd := fmt.Sprintf("scenario run --chdir %s --out %s --format json %s", path, outDir, filter)
out, err := enos.run(context.Background(), cmd)
out, _, err := enos.run(context.Background(), cmd)
require.NoError(t, err, string(out))

expected := &pb.OperationResponses{
Expand Down Expand Up @@ -150,7 +150,7 @@ func TestAcc_Cmd_Scenario_Run_Timeout(t *testing.T) {
require.NoError(t, err)

cmd := fmt.Sprintf("scenario run --chdir %s --out %s --format json --timeout 1s %s", path, outDir, test.name)
out, err := enos.run(context.Background(), cmd)
out, _, err := enos.run(context.Background(), cmd)
require.Error(t, err, string(out))
})
}
Expand Down
2 changes: 1 addition & 1 deletion acceptance/scenario_sample_list_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ func TestAcc_Cmd_Scenario_Sample_List(t *testing.T) {
require.NoError(t, err)
cmd := fmt.Sprintf("scenario sample list --chdir %s --format json", path)
fmt.Println(path)
out, err := enos.run(context.Background(), cmd)
out, _, err := enos.run(context.Background(), cmd)
if test.fail {
require.Error(t, err)

Expand Down
48 changes: 44 additions & 4 deletions acceptance/scenario_sample_observe_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ package acceptance

import (
"context"
"encoding/json"
"fmt"
"path/filepath"
"testing"
Expand All @@ -13,6 +14,7 @@ import (
"google.golang.org/protobuf/encoding/protojson"
"google.golang.org/protobuf/types/known/structpb"

"github.com/hashicorp/enos/internal/ui/machine"
"github.com/hashicorp/enos/proto/hashicorp/enos/v1/pb"
)

Expand All @@ -26,7 +28,31 @@ func TestAcc_Cmd_Scenario_Sample_Observe(t *testing.T) {
fail bool
}{
{
dir: "sample_observe",
dir: "invalid_scenarios/sample_empty_frame",
filter: &pb.Sample_Filter{
Sample: &pb.Ref_Sample{
Id: &pb.Sample_ID{
Name: "smoke_empty_frame",
},
},
Seed: 1234,
MaxElements: 1,
MinElements: 1,
},
out: &pb.ObserveSampleResponse{
Diagnostics: []*pb.Diagnostic{},
Decode: &pb.DecodeResponse{
Diagnostics: []*pb.Diagnostic{
{
Summary: "the sampling frame for smoke_empty_frame/smoke is invalid: perhaps the matrix variants specified in the subset matrix exclude all possible combinations:\n[arch:not_a_variant]",
},
},
},
},
fail: true,
},
{
dir: "scenarios/sample_observe",
filter: &pb.Sample_Filter{
Sample: &pb.Ref_Sample{
Id: &pb.Sample_ID{
Expand Down Expand Up @@ -144,7 +170,7 @@ func TestAcc_Cmd_Scenario_Sample_Observe(t *testing.T) {
t.Parallel()
enos := newAcceptanceRunner(t)

path, err := filepath.Abs(filepath.Join("./scenarios", test.dir))
path, err := filepath.Abs(filepath.Join(".", test.dir))
require.NoError(t, err)
cmd := fmt.Sprintf("scenario sample observe %s --chdir %s --format json --min %d --max %d --seed %d",
test.filter.GetSample().GetId().GetName(),
Expand All @@ -154,16 +180,30 @@ func TestAcc_Cmd_Scenario_Sample_Observe(t *testing.T) {
test.filter.GetSeed(),
)
fmt.Println(path)
out, err := enos.run(context.Background(), cmd)
stdout, stderr, err := enos.run(context.Background(), cmd)
if test.fail {
require.Error(t, err)

got := &pb.ObserveSampleResponse{}
require.NoErrorf(t, protojson.Unmarshal(stdout, got), string(stdout))
require.Len(t, got.GetDiagnostics(), len(test.out.GetDiagnostics()))
require.Len(t, got.GetDecode().GetDiagnostics(), len(test.out.GetDecode().GetDiagnostics()))
for i, d := range test.out.GetDiagnostics() {
require.Equal(t, got.GetDiagnostics()[i].GetSummary(), d.GetSummary())
}
for i, d := range test.out.GetDecode().GetDiagnostics() {
require.Equal(t, got.GetDecode().GetDiagnostics()[i].GetSummary(), d.GetSummary())
}
errMsg := &machine.ErrJSON{}
require.NoError(t, json.Unmarshal(stderr, errMsg))
require.Len(t, errMsg.Errors, 1)

return
}

require.NoError(t, err)
got := &pb.ObserveSampleResponse{}
require.NoError(t, protojson.Unmarshal(out, got))
require.NoError(t, protojson.Unmarshal(stdout, got))
require.Len(t, got.GetObservation().GetElements(), len(test.out.GetObservation().GetElements()))
for i := range test.out.GetObservation().GetElements() {
require.Equal(t, test.out.GetObservation().GetElements()[i].String(), got.GetObservation().GetElements()[i].String())
Expand Down
8 changes: 6 additions & 2 deletions acceptance/scenario_validate_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,10 @@ func TestAcc_Cmd_Scenario_Validate(t *testing.T) {
dir: "invalid_scenarios/scenario_list_fail_malformed",
fail: true,
},
{
dir: "invalid_scenarios/sample_empty_frame",
fail: true,
},
} {
t.Run(test.dir, func(t *testing.T) {
t.Parallel()
Expand All @@ -39,7 +43,7 @@ func TestAcc_Cmd_Scenario_Validate(t *testing.T) {
require.NoError(t, err)
cmd := fmt.Sprintf("scenario validate --chdir %s --format json", path)
fmt.Println(path)
out, err := enos.run(context.Background(), cmd)
out, _, err := enos.run(context.Background(), cmd)
if test.fail {
require.Error(t, err)

Expand Down Expand Up @@ -77,7 +81,7 @@ func TestAcc_Cmd_Scenario_Validate_filtered(t *testing.T) {
require.NoError(t, err)
cmd := fmt.Sprintf("scenario validate %s --chdir %s --format json", filter, path)
fmt.Println(path)
out, err := enos.run(context.Background(), cmd)
out, _, err := enos.run(context.Background(), cmd)
if test.fail {
require.Error(t, err)

Expand Down
2 changes: 1 addition & 1 deletion acceptance/version_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ func TestAcc_Cmd_Version(t *testing.T) {
t.Parallel()

enos := newAcceptanceRunner(t)
out, err := enos.run(context.Background(), test.cmd)
out, _, err := enos.run(context.Background(), test.cmd)
require.NoError(t, err)
require.True(t, test.out.Match(out))
})
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ require (
golang.org/x/term v0.19.0
golang.org/x/text v0.14.0
google.golang.org/grpc v1.63.2
google.golang.org/protobuf v1.33.0
google.golang.org/protobuf v1.34.0
)

require (
Expand Down
2 changes: 2 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -271,6 +271,8 @@ google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM=
google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA=
google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
google.golang.org/protobuf v1.34.0 h1:Qo/qEd2RZPCf2nKuorzksSknv0d3ERwp1vFG38gSmH4=
google.golang.org/protobuf v1.34.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
Expand Down
Loading
Loading