Merge pull request #16 from ironbeer/dev-v0.0.8

Changes for v0.0.8

cmd/start.go

@@ -69,7 +69,10 @@ func runStartCmd(cmd *cobra.Command, args []string) {
 	}
 
 	// setup database
-	db, err := database.NewDatabase(conf.DatabasePath())
+	if conf.Database.Path == "" {
+		conf.Database.Path = conf.DatabasePath()
+	}
+	db, err := database.NewDatabase(&conf.Database)
 	if err != nil {
 		log.Crit("Failed to open database", "err", err)
 	}

config/config.go

@@ -37,6 +37,8 @@ var (
 		"beacon.enable":                          true,
 		"beacon.endpoint":                        "https://script.google.com/macros/s/AKfycbzJpDKyn271jbm5otk_BxGkrS2b1YdMQerVq2-XxLdTOdhUPKCZICqvagvGgByxx_nq0Q/exec",
 		"beacon.interval":                        15 * time.Minute,
+		"database.long_query_time":               200 * time.Millisecond,
+		"database.min_examined_row_limit":        10000,
 	}
 )
 
@@ -117,6 +119,9 @@ type Config struct {
 
 	// Beacon worker configuration.
 	Beacon Beacon `json:"beacon" mapstructure:"beacon"`
+
+	// Database configuration.
+	Database Database `json:"database" mapstructure:"database"`
 }
 
 func (c *Config) DatabasePath() string {
@@ -264,3 +269,12 @@ type Beacon struct {
 	// Interval for send beacon.
 	Interval time.Duration `json:"interval"`
 }
+
+type Database struct {
+	// File path of the SQLite database.
+	Path string `json:"path" mapstructure:"path"`
+
+	// Slow query log configurations.
+	LongQueryTime       time.Duration `json:"long_query_time"        mapstructure:"long_query_time"`
+	MinExaminedRowLimit int           `json:"min_examined_row_limit" mapstructure:"min_examined_row_limit"`
+}

config/config_test.go

@@ -84,6 +84,10 @@ func (s *ConfigTestSuite) TestParseConfig() {
 		enable: true
 		endpoint: http://127.0.0.1/beacon
 		interval: 1s
+
+	database:
+		long_query_time: 1s
+		min_examined_row_limit: 100
 	`)
 
 	got, _ := NewConfig([]byte(strings.ReplaceAll(input, "\t", "  ")))
@@ -172,6 +176,11 @@ func (s *ConfigTestSuite) TestParseConfig() {
 		Endpoint: "http://127.0.0.1/beacon",
 		Interval: time.Second,
 	}, got.Beacon)
+
+	s.Equal(Database{
+		LongQueryTime:       time.Second,
+		MinExaminedRowLimit: 100,
+	}, got.Database)
 }
 
 func (s *ConfigTestSuite) TestValidate() {
@@ -278,4 +287,7 @@ func (s *ConfigTestSuite) TestDefaultValues() {
 		got.Beacon.Endpoint,
 	)
 	s.Equal(15*time.Minute, got.Beacon.Interval)
+
+	s.Equal(200*time.Millisecond, got.Database.LongQueryTime)
+	s.Equal(10000, got.Database.MinExaminedRowLimit)
 }

database/block_test.go

@@ -5,6 +5,7 @@ import (
 
 	"gorm.io/gorm"
 
+	"github.com/oasysgames/oasys-optimism-verifier/config"
 	"github.com/oasysgames/oasys-optimism-verifier/testhelper"
 	"github.com/stretchr/testify/suite"
 )
@@ -21,7 +22,7 @@ type BlockDatabaseTestSuite struct {
 }
 
 func (s *BlockDatabaseTestSuite) SetupTest() {
-	db, err := NewDatabase(":memory:")
+	db, err := NewDatabase(&config.Database{Path: ":memory:"})
 	if err != nil {
 		panic(err)
 	}

database/database.go

@@ -3,8 +3,10 @@ package database
 import (
 	"errors"
 
+	"github.com/oasysgames/oasys-optimism-verifier/config"
 	"gorm.io/driver/sqlite"
 	"gorm.io/gorm"
+	gormlog "gorm.io/gorm/logger"
 )
 
 var (
@@ -26,9 +28,13 @@ type Database struct {
 	Optimism *OptimismDatabase
 }
 
-func NewDatabase(filepath string) (*Database, error) {
-	config := &gorm.Config{Logger: &mylogger{}}
-	db, err := gorm.Open(sqlite.Open(filepath), config)
+func NewDatabase(cfg *config.Database) (*Database, error) {
+	config := &gorm.Config{Logger: &mylogger{
+		LogLevel:            gormlog.Info,
+		LongQueryTime:       cfg.LongQueryTime,
+		MinExaminedRowLimit: cfg.MinExaminedRowLimit,
+	}}
+	db, err := gorm.Open(sqlite.Open(cfg.Path), config)
 	if err != nil {
 		return nil, err
 	}

database/database_test.go

@@ -3,6 +3,7 @@ package database
 import (
 	"math/rand"
 
+	"github.com/oasysgames/oasys-optimism-verifier/config"
 	"github.com/oasysgames/oasys-optimism-verifier/testhelper"
 	"github.com/oasysgames/oasys-optimism-verifier/util"
 	"gorm.io/gorm"
@@ -17,7 +18,7 @@ type DatabaseTestSuite struct {
 
 func (s *DatabaseTestSuite) SetupTest() {
 	// Setup database
-	db, err := NewDatabase(":memory:")
+	db, err := NewDatabase(&config.Database{Path: ":memory:"})
 	if err != nil {
 		panic(err)
 	}

database/logger.go

@@ -10,45 +10,71 @@ import (
 	gormlog "gorm.io/gorm/logger"
 )
 
-type mylogger struct{}
+type mylogger struct {
+	LogLevel            gormlog.LogLevel
+	LongQueryTime       time.Duration
+	MinExaminedRowLimit int
+}
 
-func (m *mylogger) LogMode(gormlog.LogLevel) gormlog.Interface {
-	return &mylogger{}
+func (l *mylogger) LogMode(level gormlog.LogLevel) gormlog.Interface {
+	newlogger := *l
+	newlogger.LogLevel = level
+	return &newlogger
 }
 
-func (m *mylogger) Info(ctx context.Context, msg string, args ...interface{}) {
-	log.Info(msg, args)
+func (l *mylogger) Info(ctx context.Context, msg string, args ...interface{}) {
+	if l.LogLevel >= gormlog.Info {
+		log.Info(msg, args)
+	}
 }
 
-func (m *mylogger) Warn(ctx context.Context, msg string, args ...interface{}) {
-	log.Warn(msg, args)
+func (l *mylogger) Warn(ctx context.Context, msg string, args ...interface{}) {
+	if l.LogLevel >= gormlog.Warn {
+		log.Warn(msg, args)
+	}
 }
 
-func (m *mylogger) Error(ctx context.Context, msg string, args ...interface{}) {
-	log.Error(msg, args)
+func (l *mylogger) Error(ctx context.Context, msg string, args ...interface{}) {
+	if l.LogLevel >= gormlog.Error {
+		log.Error(msg, args)
+	}
 }
 
-func (m *mylogger) Trace(
+func (l *mylogger) Trace(
 	ctx context.Context,
 	begin time.Time,
 	fc func() (sql string, rowsAffected int64),
 	err error,
 ) {
+	if l.LogLevel <= gormlog.Silent {
+		return
+	}
+
 	elapsed := time.Since(begin)
 	sql, rows := fc()
+	if elapsed >= l.LongQueryTime || rows > int64(l.MinExaminedRowLimit) {
+		log.Warn(
+			"Slow query",
+			"elapsed", elapsed,
+			"affected", rows,
+			"sql", sql,
+		)
+	}
+
 	if err != nil && !errors.Is(err, gorm.ErrRecordNotFound) {
 		log.Error(
 			"gorm",
-			"sql",
-			sql,
-			"affected",
-			rows,
-			"time",
-			float64(elapsed.Nanoseconds())/1e6,
-			"err",
-			err,
+			"elapsed", elapsed,
+			"affected", rows,
+			"sql", sql,
+			"err", err,
 		)
 	} else {
-		log.Debug("gorm", "sql", sql, "affected", rows, "time", float64(elapsed.Nanoseconds())/1e6)
+		log.Debug(
+			"gorm",
+			"elapsed", elapsed,
+			"affected", rows,
+			"sql", sql,
+		)
 	}
 }

database/models.go

@@ -32,24 +32,24 @@ type OptimismState struct {
 	OptimismScc   OptimismScc
 
 	// Value of `StateBatchAppended` event.
-	BatchIndex        uint64 `gorm:"uniqueIndex:optimism_state_idx0,priority:2"`
+	BatchIndex        uint64 `gorm:"uniqueIndex:optimism_state_idx0,priority:2;index:optimism_state_idx1"`
 	BatchRoot         common.Hash
 	BatchSize         uint64
 	PrevTotalElements uint64
 	ExtraData         []byte
 }
 
 type OptimismSignature struct {
-	ID         string `gorm:"primarykey"`
+	ID         string `gorm:"primarykey;index:optimism_signature_idx3,priority:2"`
 	PreviousID string
 
-	SignerID uint64 `gorm:"uniqueIndex:optimism_signature_idx1,priority:1"`
+	SignerID uint64 `gorm:"uniqueIndex:optimism_signature_idx1,priority:1;index:optimism_signature_idx3,priority:1"`
 	Signer   Signer
 
 	OptimismSccID uint64 `gorm:"uniqueIndex:optimism_signature_idx1,priority:2"`
 	OptimismScc   OptimismScc
 
-	BatchIndex        uint64 `gorm:"uniqueIndex:optimism_signature_idx1,priority:3"`
+	BatchIndex        uint64 `gorm:"uniqueIndex:optimism_signature_idx1,priority:3;index:optimism_signature_idx2"`
 	BatchRoot         common.Hash
 	BatchSize         uint64
 	PrevTotalElements uint64