start the sorting rewrite of matrix for performance

Signed-off-by: Ryan Cragun <[email protected]>

internal/flightplan/matrix.go

@@ -4,7 +4,6 @@ import (
 	"cmp"
 	"fmt"
 	"slices"
-	"sort"
 	"strings"
 
 	"github.com/zclconf/go-cty/cty"
@@ -103,8 +102,10 @@ func (e Element) Equal(other Element) bool {
 	return true
 }
 
-func NewVector() *Vector {
-	return &Vector{}
+func NewVector(elms ...Element) *Vector {
+	return &Vector{
+		elements: elms,
+	}
 }
 
 // String returns the vector as a string.
@@ -315,6 +316,7 @@ func (v *Vector) SortedElements() []Element {
 	return v.sortedElements
 }
 
+// sort() sorts a vectors shadow sortedElements.
 func (v *Vector) sort() {
 	if v.elements == nil {
 		return
@@ -330,17 +332,24 @@ func (v *Vector) sort() {
 		return
 	}
 
-	slices.SortStableFunc(v.sortedElements, func(a, b Element) int {
-		if i := cmp.Compare(a.Key, b.Key); i != 0 {
-			return i
-		}
-
-		return cmp.Compare(a.Val, b.Val)
-	})
+	slices.SortStableFunc(v.sortedElements, compareElement)
 
 	v.dirty = false
 }
 
+// SortElements sorts a vectors elements.
+func (v *Vector) SortElements() {
+	if v.elements == nil {
+		return
+	}
+
+	if len(v.elements) < 2 {
+		return
+	}
+
+	slices.SortStableFunc(v.elements, compareElement)
+}
+
 // NewVectorFromProto takes a proto filter vector and returns a new Vector.
 func NewVectorFromProto(pbv *pb.Matrix_Vector) *Vector {
 	v := NewVector()
@@ -370,17 +379,33 @@ func (m *Matrix) String() string {
 
 // AddVector adds a vector the matrix.
 func (m *Matrix) AddVector(vec *Vector) {
-	if vec == nil || len(vec.elements) == 0 {
+	if m == nil || vec == nil || len(vec.elements) == 0 {
 		return
 	}
 
 	if m.Vectors == nil {
-		m.Vectors = []*Vector{}
+		m.Vectors = []*Vector{vec}
+		return
 	}
 
 	m.Vectors = append(m.Vectors, vec)
 }
 
+// AddVectorSorted adds a sorted vector to a sorted matrix.
+func (m *Matrix) AddVectorSorted(vec *Vector) {
+	if m == nil || vec == nil || len(vec.elements) == 0 {
+		return
+	}
+
+	if m.Vectors == nil {
+		m.Vectors = []*Vector{vec}
+		return
+	}
+
+	i, _ := slices.BinarySearchFunc(m.Vectors, vec, compareVector)
+	m.Vectors = slices.Insert(m.Vectors, i, vec)
+}
+
 // Copy creates a new Copy of the Matrix.
 func (m *Matrix) Copy() *Matrix {
 	nm := NewMatrix()
@@ -485,6 +510,18 @@ func (m *Matrix) HasVector(other *Vector) bool {
 	return false
 }
 
+// HasVectorSorted returns whether or not a sorted matrix has a sorted vector. It assumes
+// the matrix and vector have both already been sorted.
+func (m *Matrix) HasVectorSorted(other *Vector) bool {
+	if other == nil {
+		return false
+	}
+
+	_, has := slices.BinarySearchFunc(m.Vectors, other, compareVector)
+
+	return has
+}
+
 // HasVectorUnordered returns whether or not a matrix has a vector whose unordered
 // values match exactly with another that is given.
 func (m *Matrix) HasVectorUnordered(other *Vector) bool {
@@ -531,14 +568,37 @@ func (m *Matrix) Unique() *Matrix {
 
 // UniqueValues returns a new Matrix with all Vectors that have unique values.
 func (m *Matrix) UniqueValues() *Matrix {
-	nm := NewMatrix()
-	for _, v := range m.Vectors {
-		if !nm.HasVectorUnordered(v) {
-			nm.AddVector(v)
+	if m == nil {
+		return nil
+	}
+
+	if len(m.Vectors) < 2 {
+		return m.Copy()
+	}
+
+	nmUnsorted := NewMatrix()
+	nmSorted := NewMatrix()
+	for i := range m.Vectors {
+		v := m.Vectors[i].Copy()
+		v.SortElements()
+		if !nmSorted.HasVectorSorted(v) {
+			nmSorted.AddVectorSorted(v)
+			nmUnsorted.AddVector(m.Vectors[i])
 		}
 	}
 
-	return nm
+	return nmUnsorted
+}
+
+// Compact removes duplicate vectors from a matrix
+func (m *Matrix) Compact() {
+	if m == nil || len(m.Vectors) < 2 {
+		return
+	}
+
+	slices.CompactFunc(m.Vectors, func(a, b *Vector) bool {
+		return a.Equal(b)
+	})
 }
 
 // Filter takes a scenario filter returns a new filtered matrix.
@@ -683,23 +743,11 @@ func (m *Matrix) EqualUnordered(other *Matrix) bool {
 	}
 
 	mSorted := m.Copy()
+	mSorted.Sort()
 	otherSorted := other.Copy()
+	otherSorted.Sort()
 
-	sort.Slice(mSorted.Vectors, func(i, j int) bool {
-		return mSorted.Vectors[i].StringSorted() < mSorted.Vectors[j].StringSorted()
-	})
-
-	sort.Slice(otherSorted.Vectors, func(i, j int) bool {
-		return otherSorted.Vectors[i].StringSorted() < otherSorted.Vectors[j].StringSorted()
-	})
-
-	for i := range mSorted.Vectors {
-		if !mSorted.Vectors[i].EqualUnordered(otherSorted.Vectors[i]) {
-			return false
-		}
-	}
-
-	return true
+	return mSorted.Equal(otherSorted)
 }
 
 // Proto returns the matrix as a proto message. If a matrix is created with a scenario filter
@@ -743,6 +791,25 @@ func (m *Matrix) FromProto(in *pb.Matrix) {
 	}
 }
 
+func (m *Matrix) SortVectorElements() {
+	if m == nil {
+		return
+	}
+
+	for i := range m.Vectors {
+		m.Vectors[i].SortElements()
+	}
+}
+
+func (m *Matrix) Sort() {
+	if m == nil || len(m.Vectors) < 1 {
+		return
+	}
+
+	m.SortVectorElements()
+	slices.SortStableFunc(m.Vectors, compareVector)
+}
+
 // Match determines if Exclude directive matches the vector.
 func (ex *Exclude) Match(vec *Vector) bool {
 	if vec == nil {
@@ -788,3 +855,50 @@ func (ex *Exclude) FromProto(pfe *pb.Matrix_Exclude) {
 	ex.Vector = NewVectorFromProto(pfe.GetVector())
 	ex.Mode = pfe.GetMode()
 }
+
+// compareElement takes two elements and does a sort commparison
+func compareElement(a, b Element) int {
+	if iv := cmp.Compare(a.Key, b.Key); iv != 0 {
+		return iv
+	}
+
+	return cmp.Compare(a.Val, b.Val)
+}
+
+// compareVector takes two vectors and does a sort comparison.
+func compareVector(a, b *Vector) int {
+	// Compare by existence
+	if a == nil && b == nil {
+		return 0
+	}
+
+	if a != nil && b == nil {
+		return 1
+	}
+
+	if a == nil && b != nil {
+		return -1
+	}
+
+	// Compare by number of variants
+	var aElms, bElms []Element
+	if len(a.elements) > 0 {
+		aElms = a.elements
+	}
+	if len(b.elements) > 0 {
+		bElms = b.elements
+	}
+	if i := cmp.Compare(len(aElms), len(bElms)); i != 0 {
+		return i
+	}
+
+	// Compare by element value
+	for i := range a.elements {
+		if iv := compareElement(a.elements[i], b.elements[i]); iv != 0 {
+			return iv
+		}
+	}
+
+	// We have equal vectors
+	return 0
+}

internal/flightplan/matrix_benchmark_test.go

@@ -0,0 +1,25 @@
+package flightplan
+
+import (
+	"testing"
+)
+
+var benchmarkMatrix = &Matrix{Vectors: []*Vector{
+	NewVector(NewElement("arch", "amd64"), NewElement("arch", "arm64")),
+	NewVector(NewElement("backend", "consul"), NewElement("backend", "raft")),
+	NewVector(NewElement("source", "local"), NewElement("source", "crt"), NewElement("source", "art")),
+	NewVector(NewElement("type", "bundle"), NewElement("source", "package")),
+	NewVector(NewElement("backend_ver", "1.14.2"), NewElement("backend_ver", "1.13.2"), NewElement("backend_ver", "1.12.2"), NewElement("backend_ver", "1.11.2")),
+	NewVector(NewElement("distro", "rhel"), NewElement("distro", "ubuntu")),
+	NewVector(NewElement("edition", "ce"), NewElement("edition", "ent"), NewElement("edition", "ent.hms"), NewElement("edition", "ent.fips1402"), NewElement("edition", "ent.hsm.fips1402")),
+	NewVector(NewElement("seal", "shamir"), NewElement("awskms", "awkms")),
+}}
+
+func Benchmark_Matrix_UniqueValues(b *testing.B) {
+	b.Run("UniqueValuesNew", func(b *testing.B) {
+		benchmarkMatrix.Copy().CartesianProduct().UniqueValues()
+	})
+	b.Run("UniqueValuesOld", func(b *testing.B) {
+		benchmarkMatrix.Copy().CartesianProduct().UniqueValuesOld()
+	})
+}