diff --git a/internal/dumper/dumper.go b/internal/dumper/dumper.go index a333189a..cfdabee0 100644 --- a/internal/dumper/dumper.go +++ b/internal/dumper/dumper.go @@ -223,43 +223,28 @@ func (d *Dumper) dumpTable(conn *Connection, database string, table string) erro var where string var selfields []string - isGenerated, err := d.generatedFields(conn, table) - if err != nil { - return err - } - fields := make([]string, 0) { - cursor, err := conn.StreamFetch(fmt.Sprintf("SELECT * FROM `%s`.`%s` LIMIT 1", database, table)) + flds, err := d.dumpableFieldNames(conn, table) if err != nil { return err } - flds := cursor.Fields() - for _, fld := range flds { - d.log.Debug("dump", zap.Any("filters", d.cfg.Filters), zap.String("table", table), zap.String("field_name", fld.Name)) - - if _, ok := d.cfg.Filters[table][fld.Name]; ok { - continue - } + for _, name := range flds { + d.log.Debug("dump", zap.Any("filters", d.cfg.Filters), zap.String("table", table), zap.String("field_name", name)) - if isGenerated[fld.Name] { + if _, ok := d.cfg.Filters[table][name]; ok { continue } - fields = append(fields, fmt.Sprintf("`%s`", fld.Name)) - replacement, ok := d.cfg.Selects[table][fld.Name] + fields = append(fields, fmt.Sprintf("`%s`", name)) + replacement, ok := d.cfg.Selects[table][name] if ok { - selfields = append(selfields, fmt.Sprintf("%s AS `%s`", replacement, fld.Name)) + selfields = append(selfields, fmt.Sprintf("%s AS `%s`", replacement, name)) } else { - selfields = append(selfields, fmt.Sprintf("`%s`", fld.Name)) + selfields = append(selfields, fmt.Sprintf("`%s`", name)) } } - - err = cursor.Close() - if err != nil { - return err - } } if v, ok := d.cfg.Wheres[table]; ok { @@ -406,16 +391,15 @@ func (d *Dumper) filterDatabases(conn *Connection, filter *regexp.Regexp, invert return databases, nil } -// generatedFields returns a map that contains fields that are virtually -// generated. -func (d *Dumper) generatedFields(conn *Connection, table string) (map[string]bool, error) { +// dumpableFieldNames returns a slice that contains valid field names for the dump. +func (d *Dumper) dumpableFieldNames(conn *Connection, table string) ([]string, error) { qr, err := conn.Fetch(fmt.Sprintf("SHOW FIELDS FROM `%s`", table)) if err != nil { return nil, err } - fields := map[string]bool{} + fields := make([]string, 0, len(qr.Rows)) for _, t := range qr.Rows { if len(t) != 6 { return nil, fmt.Errorf("error fetching fields, expecting to have 6 columns, have: %d", len(t)) @@ -424,10 +408,13 @@ func (d *Dumper) generatedFields(conn *Connection, table string) (map[string]boo name := t[0].String() extra := t[5].String() - // Can be either "VIRTUAL GENERATED" or "VIRTUAL STORED" + // Can be either "VIRTUAL GENERATED" or "STORED GENERATED" // https://dev.mysql.com/doc/refman/8.0/en/show-columns.html - if strings.Contains(extra, "VIRTUAL") { - fields[name] = true + if strings.Contains(extra, "GENERATED") { + // Skip generated columns + continue + } else { + fields = append(fields, name) } } diff --git a/internal/dumper/dumper_test.go b/internal/dumper/dumper_test.go index 22a364b4..15bd53aa 100644 --- a/internal/dumper/dumper_test.go +++ b/internal/dumper/dumper_test.go @@ -14,6 +14,17 @@ import ( qt "github.com/frankban/quicktest" ) +func testRow(name, extra string) []sqltypes.Value { + return []sqltypes.Value{ + sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte(name)), + sqltypes.MakeTrusted(querypb.Type_BLOB, []byte("varchar(255)")), + sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte("YES")), + sqltypes.MakeTrusted(querypb.Type_BINARY, []byte("")), + sqltypes.MakeTrusted(querypb.Type_NULL_TYPE, []byte("NULL")), + sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte(extra)), + } +} + func TestDumper(t *testing.T) { c := qt.New(t) @@ -114,14 +125,13 @@ func TestDumper(t *testing.T) { {Name: "Extra", Type: querypb.Type_VARCHAR}, }, Rows: [][]sqltypes.Value{ - { - sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte("not_deleted")), - sqltypes.MakeTrusted(querypb.Type_BLOB, []byte("int")), - sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte("YES")), - sqltypes.MakeTrusted(querypb.Type_BINARY, []byte("")), - sqltypes.MakeTrusted(querypb.Type_NULL_TYPE, []byte("NULL")), - sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte("VIRTUAL GENERATED")), - }, + testRow("id", ""), + testRow("name", ""), + testRow("namei1", ""), + testRow("null", ""), + testRow("decimal", ""), + testRow("datetime", ""), + testRow("not_deleted", "virtual generated"), }, } @@ -261,14 +271,13 @@ func TestDumperGeneratedFields(t *testing.T) { {Name: "Extra", Type: querypb.Type_VARCHAR}, }, Rows: [][]sqltypes.Value{ - { - sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte("name")), - sqltypes.MakeTrusted(querypb.Type_BLOB, []byte("varchar(255)")), - sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte("YES")), - sqltypes.MakeTrusted(querypb.Type_BINARY, []byte("")), - sqltypes.MakeTrusted(querypb.Type_NULL_TYPE, []byte("NULL")), - sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte("VIRTUAL GENERATED")), - }, + testRow("id", ""), + testRow("name", "VIRTUAL GENERATED"), + testRow("namei1", ""), + testRow("null", ""), + testRow("decimal", ""), + testRow("datetime", ""), + testRow("not_deleted", "VIRTUAL GENERATED"), }, } @@ -442,14 +451,13 @@ func TestDumperAll(t *testing.T) { {Name: "Extra", Type: querypb.Type_VARCHAR}, }, Rows: [][]sqltypes.Value{ - { - sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte("not_deleted")), - sqltypes.MakeTrusted(querypb.Type_BLOB, []byte("int")), - sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte("YES")), - sqltypes.MakeTrusted(querypb.Type_BINARY, []byte("")), - sqltypes.MakeTrusted(querypb.Type_NULL_TYPE, []byte("NULL")), - sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte("VIRTUAL GENERATED")), - }, + testRow("id", ""), + testRow("name", ""), + testRow("namei1", ""), + testRow("null", ""), + testRow("decimal", ""), + testRow("datetime", ""), + testRow("not_deleted", "virtual generated"), }, } @@ -630,14 +638,13 @@ func TestDumperMultiple(t *testing.T) { {Name: "Extra", Type: querypb.Type_VARCHAR}, }, Rows: [][]sqltypes.Value{ - { - sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte("not_deleted")), - sqltypes.MakeTrusted(querypb.Type_BLOB, []byte("int")), - sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte("YES")), - sqltypes.MakeTrusted(querypb.Type_BINARY, []byte("")), - sqltypes.MakeTrusted(querypb.Type_NULL_TYPE, []byte("NULL")), - sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte("VIRTUAL GENERATED")), - }, + testRow("id", ""), + testRow("name", ""), + testRow("namei1", ""), + testRow("null", ""), + testRow("decimal", ""), + testRow("datetime", ""), + testRow("not_deleted", "virtual generated"), }, } @@ -828,14 +835,13 @@ func TestDumperSimpleRegexp(t *testing.T) { {Name: "Extra", Type: querypb.Type_VARCHAR}, }, Rows: [][]sqltypes.Value{ - { - sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte("not_deleted")), - sqltypes.MakeTrusted(querypb.Type_BLOB, []byte("int")), - sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte("YES")), - sqltypes.MakeTrusted(querypb.Type_BINARY, []byte("")), - sqltypes.MakeTrusted(querypb.Type_NULL_TYPE, []byte("NULL")), - sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte("VIRTUAL GENERATED")), - }, + testRow("id", ""), + testRow("name", ""), + testRow("namei1", ""), + testRow("null", ""), + testRow("decimal", ""), + testRow("datetime", ""), + testRow("not_deleted", "virtual generated"), }, } @@ -1026,14 +1032,13 @@ func TestDumperComplexRegexp(t *testing.T) { {Name: "Extra", Type: querypb.Type_VARCHAR}, }, Rows: [][]sqltypes.Value{ - { - sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte("not_deleted")), - sqltypes.MakeTrusted(querypb.Type_BLOB, []byte("int")), - sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte("YES")), - sqltypes.MakeTrusted(querypb.Type_BINARY, []byte("")), - sqltypes.MakeTrusted(querypb.Type_NULL_TYPE, []byte("NULL")), - sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte("VIRTUAL GENERATED")), - }, + testRow("id", ""), + testRow("name", ""), + testRow("namei1", ""), + testRow("null", ""), + testRow("decimal", ""), + testRow("datetime", ""), + testRow("not_deleted", "virtual generated"), }, }