Skip to content

Commit

Permalink
Merge pull request #666 from planetscale/dump-support-invisible
Browse files Browse the repository at this point in the history 
Dump: support invisible columns
  • Loading branch information
@mscoutermarsh
mscoutermarsh committed May 1, 2023
2 parents 1e30660 + ba4423c commit 37460ef
Show file tree
Hide file tree
Showing 2 changed files with 70 additions and 78 deletions.
47 changes: 17 additions & 30 deletions internal/dumper/dumper.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -223,43 +223,28 @@ func (d *Dumper) dumpTable(conn *Connection, database string, table string) erro
var where string
var selfields []string

isGenerated, err := d.generatedFields(conn, table)
if err != nil {
return err
}

fields := make([]string, 0)
{
cursor, err := conn.StreamFetch(fmt.Sprintf("SELECT * FROM `%s`.`%s` LIMIT 1", database, table))
flds, err := d.dumpableFieldNames(conn, table)
if err != nil {
return err
}

flds := cursor.Fields()
for _, fld := range flds {
d.log.Debug("dump", zap.Any("filters", d.cfg.Filters), zap.String("table", table), zap.String("field_name", fld.Name))

if _, ok := d.cfg.Filters[table][fld.Name]; ok {
continue
}
for _, name := range flds {
d.log.Debug("dump", zap.Any("filters", d.cfg.Filters), zap.String("table", table), zap.String("field_name", name))

if isGenerated[fld.Name] {
if _, ok := d.cfg.Filters[table][name]; ok {
continue
}

fields = append(fields, fmt.Sprintf("`%s`", fld.Name))
replacement, ok := d.cfg.Selects[table][fld.Name]
fields = append(fields, fmt.Sprintf("`%s`", name))
replacement, ok := d.cfg.Selects[table][name]
if ok {
selfields = append(selfields, fmt.Sprintf("%s AS `%s`", replacement, fld.Name))
selfields = append(selfields, fmt.Sprintf("%s AS `%s`", replacement, name))
} else {
selfields = append(selfields, fmt.Sprintf("`%s`", fld.Name))
selfields = append(selfields, fmt.Sprintf("`%s`", name))
}
}

err = cursor.Close()
if err != nil {
return err
}
}

if v, ok := d.cfg.Wheres[table]; ok {
Expand Down Expand Up @@ -406,16 +391,15 @@ func (d *Dumper) filterDatabases(conn *Connection, filter *regexp.Regexp, invert
return databases, nil
}

// generatedFields returns a map that contains fields that are virtually
// generated.
func (d *Dumper) generatedFields(conn *Connection, table string) (map[string]bool, error) {
// dumpableFieldNames returns a slice that contains valid field names for the dump.
func (d *Dumper) dumpableFieldNames(conn *Connection, table string) ([]string, error) {
qr, err := conn.Fetch(fmt.Sprintf("SHOW FIELDS FROM `%s`", table))
if err != nil {
return nil, err

}

fields := map[string]bool{}
fields := make([]string, 0, len(qr.Rows))
for _, t := range qr.Rows {
if len(t) != 6 {
return nil, fmt.Errorf("error fetching fields, expecting to have 6 columns, have: %d", len(t))
Expand All @@ -424,10 +408,13 @@ func (d *Dumper) generatedFields(conn *Connection, table string) (map[string]boo
name := t[0].String()
extra := t[5].String()

// Can be either "VIRTUAL GENERATED" or "VIRTUAL STORED"
// Can be either "VIRTUAL GENERATED" or "STORED GENERATED"
// https://dev.mysql.com/doc/refman/8.0/en/show-columns.html
if strings.Contains(extra, "VIRTUAL") {
fields[name] = true
if strings.Contains(extra, "GENERATED") {
// Skip generated columns
continue
} else {
fields = append(fields, name)
}
}

Expand Down
101 changes: 53 additions & 48 deletions internal/dumper/dumper_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,17 @@ import (
qt "github.com/frankban/quicktest"
)

func testRow(name, extra string) []sqltypes.Value {
return []sqltypes.Value{
sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte(name)),
sqltypes.MakeTrusted(querypb.Type_BLOB, []byte("varchar(255)")),
sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte("YES")),
sqltypes.MakeTrusted(querypb.Type_BINARY, []byte("")),
sqltypes.MakeTrusted(querypb.Type_NULL_TYPE, []byte("NULL")),
sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte(extra)),
}
}

func TestDumper(t *testing.T) {
c := qt.New(t)

Expand Down Expand Up @@ -114,14 +125,13 @@ func TestDumper(t *testing.T) {
{Name: "Extra", Type: querypb.Type_VARCHAR},
},
Rows: [][]sqltypes.Value{
{
sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte("not_deleted")),
sqltypes.MakeTrusted(querypb.Type_BLOB, []byte("int")),
sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte("YES")),
sqltypes.MakeTrusted(querypb.Type_BINARY, []byte("")),
sqltypes.MakeTrusted(querypb.Type_NULL_TYPE, []byte("NULL")),
sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte("VIRTUAL GENERATED")),
},
testRow("id", ""),
testRow("name", ""),
testRow("namei1", ""),
testRow("null", ""),
testRow("decimal", ""),
testRow("datetime", ""),
testRow("not_deleted", "virtual generated"),
},
}

Expand Down Expand Up @@ -261,14 +271,13 @@ func TestDumperGeneratedFields(t *testing.T) {
{Name: "Extra", Type: querypb.Type_VARCHAR},
},
Rows: [][]sqltypes.Value{
{
sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte("name")),
sqltypes.MakeTrusted(querypb.Type_BLOB, []byte("varchar(255)")),
sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte("YES")),
sqltypes.MakeTrusted(querypb.Type_BINARY, []byte("")),
sqltypes.MakeTrusted(querypb.Type_NULL_TYPE, []byte("NULL")),
sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte("VIRTUAL GENERATED")),
},
testRow("id", ""),
testRow("name", "VIRTUAL GENERATED"),
testRow("namei1", ""),
testRow("null", ""),
testRow("decimal", ""),
testRow("datetime", ""),
testRow("not_deleted", "VIRTUAL GENERATED"),
},
}

Expand Down Expand Up @@ -442,14 +451,13 @@ func TestDumperAll(t *testing.T) {
{Name: "Extra", Type: querypb.Type_VARCHAR},
},
Rows: [][]sqltypes.Value{
{
sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte("not_deleted")),
sqltypes.MakeTrusted(querypb.Type_BLOB, []byte("int")),
sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte("YES")),
sqltypes.MakeTrusted(querypb.Type_BINARY, []byte("")),
sqltypes.MakeTrusted(querypb.Type_NULL_TYPE, []byte("NULL")),
sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte("VIRTUAL GENERATED")),
},
testRow("id", ""),
testRow("name", ""),
testRow("namei1", ""),
testRow("null", ""),
testRow("decimal", ""),
testRow("datetime", ""),
testRow("not_deleted", "virtual generated"),
},
}

Expand Down Expand Up @@ -630,14 +638,13 @@ func TestDumperMultiple(t *testing.T) {
{Name: "Extra", Type: querypb.Type_VARCHAR},
},
Rows: [][]sqltypes.Value{
{
sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte("not_deleted")),
sqltypes.MakeTrusted(querypb.Type_BLOB, []byte("int")),
sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte("YES")),
sqltypes.MakeTrusted(querypb.Type_BINARY, []byte("")),
sqltypes.MakeTrusted(querypb.Type_NULL_TYPE, []byte("NULL")),
sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte("VIRTUAL GENERATED")),
},
testRow("id", ""),
testRow("name", ""),
testRow("namei1", ""),
testRow("null", ""),
testRow("decimal", ""),
testRow("datetime", ""),
testRow("not_deleted", "virtual generated"),
},
}

Expand Down Expand Up @@ -828,14 +835,13 @@ func TestDumperSimpleRegexp(t *testing.T) {
{Name: "Extra", Type: querypb.Type_VARCHAR},
},
Rows: [][]sqltypes.Value{
{
sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte("not_deleted")),
sqltypes.MakeTrusted(querypb.Type_BLOB, []byte("int")),
sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte("YES")),
sqltypes.MakeTrusted(querypb.Type_BINARY, []byte("")),
sqltypes.MakeTrusted(querypb.Type_NULL_TYPE, []byte("NULL")),
sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte("VIRTUAL GENERATED")),
},
testRow("id", ""),
testRow("name", ""),
testRow("namei1", ""),
testRow("null", ""),
testRow("decimal", ""),
testRow("datetime", ""),
testRow("not_deleted", "virtual generated"),
},
}

Expand Down Expand Up @@ -1026,14 +1032,13 @@ func TestDumperComplexRegexp(t *testing.T) {
{Name: "Extra", Type: querypb.Type_VARCHAR},
},
Rows: [][]sqltypes.Value{
{
sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte("not_deleted")),
sqltypes.MakeTrusted(querypb.Type_BLOB, []byte("int")),
sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte("YES")),
sqltypes.MakeTrusted(querypb.Type_BINARY, []byte("")),
sqltypes.MakeTrusted(querypb.Type_NULL_TYPE, []byte("NULL")),
sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte("VIRTUAL GENERATED")),
},
testRow("id", ""),
testRow("name", ""),
testRow("namei1", ""),
testRow("null", ""),
testRow("decimal", ""),
testRow("datetime", ""),
testRow("not_deleted", "virtual generated"),
},
}

Expand Down

0 comments on commit 37460ef

Please sign in to comment.