From db7de4b31b1836abed822077840665639f0d66da Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Fri, 19 Jan 2024 23:27:27 -0500 Subject: Add ability to use sequential or timestamp-based versions --- internal/services/migration.go | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) (limited to 'internal/services/migration.go') diff --git a/internal/services/migration.go b/internal/services/migration.go index 8b5898e..91921bc 100644 --- a/internal/services/migration.go +++ b/internal/services/migration.go @@ -3,7 +3,10 @@ package services import ( "fmt" "sort" + "strconv" + "time" + "github.com/eugenetriguba/bolt/internal/configloader" "github.com/eugenetriguba/bolt/internal/models" "github.com/eugenetriguba/bolt/internal/output" "github.com/eugenetriguba/bolt/internal/repositories" @@ -195,7 +198,26 @@ func (ms *MigrationService) RevertMigration(migration *models.Migration) error { return nil } -func (ms *MigrationService) CreateMigration(migration *models.Migration) error { +func (ms *MigrationService) CreateMigration(versionStyle configloader.VersionStyle, message string) error { + var migration *models.Migration + + if versionStyle == configloader.VersionStyleTimestamp { + migration = models.NewTimestampMigration(time.Now(), message) + } else { + var currentVerison uint64 = 0 + latestMigration, err := ms.fsRepo.Latest() + if err != nil { + return err + } + if latestMigration != nil { + currentVerison, err = strconv.ParseUint(latestMigration.Version, 10, 64) + if err != nil { + return err + } + } + migration = models.NewSequentialMigration(currentVerison+1, message) + } + err := ms.fsRepo.Create(migration) if err != nil { return err -- cgit v1.2.3 From b8b23e393dcda33cc98b22f4083977b543861bf2 Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Sun, 28 Jan 2024 17:29:19 -0500 Subject: Add test for invalid version style in config --- internal/services/migration.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'internal/services/migration.go') diff --git a/internal/services/migration.go b/internal/services/migration.go index 91921bc..6836980 100644 --- a/internal/services/migration.go +++ b/internal/services/migration.go @@ -198,7 +198,10 @@ func (ms *MigrationService) RevertMigration(migration *models.Migration) error { return nil } -func (ms *MigrationService) CreateMigration(versionStyle configloader.VersionStyle, message string) error { +func (ms *MigrationService) CreateMigration( + versionStyle configloader.VersionStyle, + message string, +) error { var migration *models.Migration if versionStyle == configloader.VersionStyleTimestamp { -- cgit v1.2.3 From 052bfc3c1f25ff0f4a9506472c50857dc07a8664 Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Sun, 25 Feb 2024 15:56:37 -0500 Subject: Add error handling to unhandled errors during migration apply/revert --- internal/services/migration.go | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) (limited to 'internal/services/migration.go') diff --git a/internal/services/migration.go b/internal/services/migration.go index 6836980..b6b00bb 100644 --- a/internal/services/migration.go +++ b/internal/services/migration.go @@ -41,7 +41,14 @@ func (ms *MigrationService) ApplyAllMigrations() error { for _, migration := range migrations { if !migration.Applied { - ms.ApplyMigration(migration) + err := ms.ApplyMigration(migration) + if err != nil { + return fmt.Errorf( + "unable to apply migration %s: %w", + migration.Name(), + err, + ) + } } } @@ -75,7 +82,11 @@ func (ms *MigrationService) ApplyUpToVersion(version string) error { if !migration.Applied { err := ms.ApplyMigration(migration) if err != nil { - return err + return fmt.Errorf( + "unable to apply migration %s: %w", + migration.Name(), + err, + ) } } @@ -124,7 +135,14 @@ func (ms *MigrationService) RevertAllMigrations() error { for _, migration := range migrations { if migration.Applied { - ms.RevertMigration(migration) + err := ms.RevertMigration(migration) + if err != nil { + return fmt.Errorf( + "unable to revert migration %s: %w", + migration.Name(), + err, + ) + } } } @@ -158,7 +176,11 @@ func (ms *MigrationService) RevertDownToVersion(version string) error { if migration.Applied { err := ms.RevertMigration(migration) if err != nil { - return err + return fmt.Errorf( + "unable to revert migration %s: %w", + migration.Name(), + err, + ) } } -- cgit v1.2.3 From bf1d3b3e1fc42132678b244e479839416a03a6ce Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Thu, 29 Feb 2024 19:53:20 -0500 Subject: Add service tests --- internal/services/migration.go | 233 ++++++++++++++++++++++++++++++++--------- 1 file changed, 186 insertions(+), 47 deletions(-) (limited to 'internal/services/migration.go') diff --git a/internal/services/migration.go b/internal/services/migration.go index b6b00bb..48859d6 100644 --- a/internal/services/migration.go +++ b/internal/services/migration.go @@ -1,29 +1,62 @@ package services import ( + "database/sql" "fmt" "sort" "strconv" + "strings" "time" "github.com/eugenetriguba/bolt/internal/configloader" "github.com/eugenetriguba/bolt/internal/models" "github.com/eugenetriguba/bolt/internal/output" "github.com/eugenetriguba/bolt/internal/repositories" + "github.com/eugenetriguba/bolt/internal/sqlparse" ) type MigrationService struct { - dbRepo *repositories.MigrationDBRepo - fsRepo *repositories.MigrationFsRepo + dbRepo repositories.MigrationDBRepo + fsRepo repositories.MigrationFsRepo + cfg configloader.Config outputter output.Outputter } func NewMigrationService( - dbRepo *repositories.MigrationDBRepo, - fsRepo *repositories.MigrationFsRepo, + dbRepo repositories.MigrationDBRepo, + fsRepo repositories.MigrationFsRepo, + cfg configloader.Config, outputter output.Outputter, -) *MigrationService { - return &MigrationService{dbRepo: dbRepo, fsRepo: fsRepo, outputter: outputter} +) MigrationService { + return MigrationService{ + dbRepo: dbRepo, + fsRepo: fsRepo, + cfg: cfg, + outputter: outputter, + } +} + +func NewMigrationServiceFromConfig( + db *sql.DB, + outputter output.Outputter, + cfg configloader.Config, +) (MigrationService, error) { + migrationDBRepo, err := repositories.NewMigrationDBRepo(db) + if err != nil { + return MigrationService{}, err + } + + migrationFsRepo, err := repositories.NewMigrationFsRepo(&cfg.Migrations) + if err != nil { + return MigrationService{}, err + } + + return NewMigrationService( + migrationDBRepo, + migrationFsRepo, + cfg, + outputter, + ), nil } type sortOrder int @@ -33,7 +66,7 @@ const ( SortOrderAsc ) -func (ms *MigrationService) ApplyAllMigrations() error { +func (ms MigrationService) ApplyAllMigrations() error { migrations, err := ms.ListMigrations(SortOrderAsc) if err != nil { return err @@ -55,7 +88,7 @@ func (ms *MigrationService) ApplyAllMigrations() error { return nil } -func (ms *MigrationService) ApplyUpToVersion(version string) error { +func (ms MigrationService) ApplyUpToVersion(version string) error { migrations, err := ms.ListMigrations(SortOrderAsc) if err != nil { return err @@ -98,7 +131,7 @@ func (ms *MigrationService) ApplyUpToVersion(version string) error { return nil } -func (ms *MigrationService) ApplyMigration(migration *models.Migration) error { +func (ms MigrationService) ApplyMigration(migration *models.Migration) error { ms.outputter.Output( fmt.Sprintf( "Applying migration %s_%s..", @@ -112,10 +145,22 @@ func (ms *MigrationService) ApplyMigration(migration *models.Migration) error { return err } - err = ms.dbRepo.Apply(scriptContents, migration) + sqlParser := sqlparse.NewSqlParser() + execOptions, err := sqlParser.Parse(strings.NewReader(scriptContents)) if err != nil { - return err + return fmt.Errorf("unable to parse sql file for upgrade script: %w", err) } + + if execOptions.UseTransaction { + err = ms.dbRepo.ApplyWithTx(scriptContents, migration) + } else { + err = ms.dbRepo.Apply(scriptContents, migration) + } + + if err != nil { + return fmt.Errorf("unable to apply migration %s: %w", migration.Name(), err) + } + ms.outputter.Output( fmt.Sprintf( "Successfully applied migration %s_%s!", @@ -127,7 +172,7 @@ func (ms *MigrationService) ApplyMigration(migration *models.Migration) error { return nil } -func (ms *MigrationService) RevertAllMigrations() error { +func (ms MigrationService) RevertAllMigrations() error { migrations, err := ms.ListMigrations(SortOrderDesc) if err != nil { return err @@ -149,7 +194,7 @@ func (ms *MigrationService) RevertAllMigrations() error { return nil } -func (ms *MigrationService) RevertDownToVersion(version string) error { +func (ms MigrationService) RevertDownToVersion(version string) error { migrations, err := ms.ListMigrations(SortOrderDesc) if err != nil { return err @@ -192,7 +237,7 @@ func (ms *MigrationService) RevertDownToVersion(version string) error { return nil } -func (ms *MigrationService) RevertMigration(migration *models.Migration) error { +func (ms MigrationService) RevertMigration(migration *models.Migration) error { ms.outputter.Output( fmt.Sprintf( "Reverting migration %s_%s..", @@ -205,10 +250,22 @@ func (ms *MigrationService) RevertMigration(migration *models.Migration) error { return err } - err = ms.dbRepo.Revert(scriptContents, migration) + sqlParser := sqlparse.NewSqlParser() + execOptions, err := sqlParser.Parse(strings.NewReader(scriptContents)) + if err != nil { + return fmt.Errorf("unable to parse sql file for downgrade script: %w", err) + } + + if execOptions.UseTransaction { + err = ms.dbRepo.ApplyWithTx(scriptContents, migration) + } else { + err = ms.dbRepo.Revert(scriptContents, migration) + } + if err != nil { return err } + ms.outputter.Output( fmt.Sprintf( "Successfully reverted migration %s_%s!", @@ -220,70 +277,152 @@ func (ms *MigrationService) RevertMigration(migration *models.Migration) error { return nil } -func (ms *MigrationService) CreateMigration( - versionStyle configloader.VersionStyle, - message string, -) error { +func (ms MigrationService) CreateMigration(message string) (*models.Migration, error) { var migration *models.Migration - if versionStyle == configloader.VersionStyleTimestamp { + if ms.cfg.Migrations.VersionStyle == configloader.VersionStyleTimestamp { migration = models.NewTimestampMigration(time.Now(), message) } else { - var currentVerison uint64 = 0 - latestMigration, err := ms.fsRepo.Latest() + currentVersion, err := ms.getCurrentSequentialMigrationVersion() if err != nil { - return err + return nil, err } - if latestMigration != nil { - currentVerison, err = strconv.ParseUint(latestMigration.Version, 10, 64) - if err != nil { - return err - } - } - migration = models.NewSequentialMigration(currentVerison+1, message) + migration = models.NewSequentialMigration(currentVersion+1, message) } err := ms.fsRepo.Create(migration) if err != nil { - return err + return nil, err } ms.outputter.Output( fmt.Sprintf("Created migration %s - %s.", migration.Version, migration.Message), ) - return nil + return migration, nil +} + +func (ms MigrationService) getCurrentSequentialMigrationVersion() (uint64, error) { + var currentVersion uint64 = 0 + + latestMigration, err := ms.fsRepo.Latest() + if err != nil { + return 0, err + } + + if latestMigration != nil { + currentVersion, err = strconv.ParseUint(latestMigration.Version, 10, 64) + if err != nil { + return 0, err + } + } + + return currentVersion, nil } -func (ms *MigrationService) ListMigrations(order sortOrder) ([]*models.Migration, error) { +func (ms MigrationService) ListMigrations(order sortOrder) ([]*models.Migration, error) { // Assumption: All migration versions that have been applied // to the database exist locally in the list of filesystem migrations. // This would hold true unless someone decided to apply a migration and, - // for whatever reason, delete it later from their filesystem. - migrations, err := ms.fsRepo.List() + // for whatever reason, delete it later from their filesystem. If there is + // an applied migration that doesn't exist locally, it would not be shown + // here. + localMigrations, err := ms.fsRepo.List() if err != nil { - return nil, err + return nil, fmt.Errorf("unable to list out local filesystem migrations: %w", err) } appliedMigrations, err := ms.dbRepo.List() if err != nil { - return nil, err + return nil, fmt.Errorf( + "unable to list out applied migrations from remote db: %w", + err, + ) } - var flatMigrations []*models.Migration - for _, migration := range migrations { - _, ok := appliedMigrations[migration.Version] + migrations := make([]*models.Migration, 0) + for _, localMigration := range localMigrations { + _, ok := appliedMigrations[localMigration.Version] if ok { - migration.Applied = true + localMigration.Applied = true } - flatMigrations = append(flatMigrations, migration) + migrations = append(migrations, localMigration) } - sort.Slice(flatMigrations, func(i, j int) bool { - if order == SortOrderAsc { - return flatMigrations[i].Version < flatMigrations[j].Version + err = ms.sortMigrations(migrations, order) + if err != nil { + return nil, err + } + + return migrations, err +} + +func (ms MigrationService) sortMigrations(migrations []*models.Migration, order sortOrder) error { + sortErrs := make([]error, 0) + sort.Slice(migrations, func(i, j int) bool { + var comparison bool + var err error + + if ms.cfg.Migrations.VersionStyle == configloader.VersionStyleSequential { + comparison, err = ms.compareSequentialMigrations( + migrations[i], + migrations[j], + order, + ) } else { - return flatMigrations[i].Version > flatMigrations[j].Version + comparison, err = ms.compareTimestampMigrations(migrations[i], migrations[j], order) } + + if err != nil { + sortErrs = append(sortErrs, err) + } + + return comparison }) + if len(sortErrs) != 0 { + return fmt.Errorf("unable to sort migrations: %v", sortErrs) + } + return nil +} + +func (ms MigrationService) compareSequentialMigrations( + m1 *models.Migration, + m2 *models.Migration, + order sortOrder, +) (bool, error) { + m1Version, err := strconv.ParseInt(m1.Version, 10, 64) + if err != nil { + return false, err + } + + m2Version, err := strconv.ParseInt(m2.Version, 10, 64) + if err != nil { + return false, err + } - return flatMigrations, nil + if order == SortOrderAsc { + return m1Version < m2Version, nil + } else { + return m1Version > m2Version, nil + } +} + +func (ms MigrationService) compareTimestampMigrations( + m1 *models.Migration, + m2 *models.Migration, + order sortOrder, +) (bool, error) { + m1Version, err := time.Parse("20060102150405", m1.Version) + if err != nil { + return false, err + } + + m2Version, err := time.Parse("20060102150405", m2.Version) + if err != nil { + return false, err + } + + if order == SortOrderAsc { + return m1Version.Before(m2Version), nil + } else { + return m1Version.After(m2Version), nil + } } -- cgit v1.2.3 From 55d3999d9efdeb0a87959e93d473cb64c8ba8c48 Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Sat, 2 Mar 2024 07:35:21 -0500 Subject: Add proper tests for ApplyAllMigrations and ApplyUpToVersion --- internal/services/migration.go | 2 ++ 1 file changed, 2 insertions(+) (limited to 'internal/services/migration.go') diff --git a/internal/services/migration.go b/internal/services/migration.go index 48859d6..c0d043b 100644 --- a/internal/services/migration.go +++ b/internal/services/migration.go @@ -105,6 +105,8 @@ func (ms MigrationService) ApplyUpToVersion(version string) error { return fmt.Errorf("migration with version %s does not exist", version) } if targetMigration.Applied { + // Assumption: If the target migration is applied, all migrations before + // it must also be applied. return fmt.Errorf( "migration with version %s is already applied, nothing to apply", version, -- cgit v1.2.3 From 75a1bfa521ef1e10514330b984f46b84e1cef926 Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Sat, 2 Mar 2024 08:31:26 -0500 Subject: Remove NewMigrationServiceFromConfig --- internal/services/migration.go | 26 +------------------------- 1 file changed, 1 insertion(+), 25 deletions(-) (limited to 'internal/services/migration.go') diff --git a/internal/services/migration.go b/internal/services/migration.go index c0d043b..bc4b9d1 100644 --- a/internal/services/migration.go +++ b/internal/services/migration.go @@ -1,7 +1,6 @@ package services import ( - "database/sql" "fmt" "sort" "strconv" @@ -36,29 +35,6 @@ func NewMigrationService( } } -func NewMigrationServiceFromConfig( - db *sql.DB, - outputter output.Outputter, - cfg configloader.Config, -) (MigrationService, error) { - migrationDBRepo, err := repositories.NewMigrationDBRepo(db) - if err != nil { - return MigrationService{}, err - } - - migrationFsRepo, err := repositories.NewMigrationFsRepo(&cfg.Migrations) - if err != nil { - return MigrationService{}, err - } - - return NewMigrationService( - migrationDBRepo, - migrationFsRepo, - cfg, - outputter, - ), nil -} - type sortOrder int const ( @@ -259,7 +235,7 @@ func (ms MigrationService) RevertMigration(migration *models.Migration) error { } if execOptions.UseTransaction { - err = ms.dbRepo.ApplyWithTx(scriptContents, migration) + err = ms.dbRepo.RevertWithTx(scriptContents, migration) } else { err = ms.dbRepo.Revert(scriptContents, migration) } -- cgit v1.2.3 From 4daf960b457fe1ab27e405c1c1d56e9cfc7c904e Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Sat, 2 Mar 2024 08:50:27 -0500 Subject: Add proper tests to revert service methods --- internal/services/migration.go | 2 ++ 1 file changed, 2 insertions(+) (limited to 'internal/services/migration.go') diff --git a/internal/services/migration.go b/internal/services/migration.go index bc4b9d1..446f267 100644 --- a/internal/services/migration.go +++ b/internal/services/migration.go @@ -189,6 +189,8 @@ func (ms MigrationService) RevertDownToVersion(version string) error { return fmt.Errorf("migration with version %s does not exist", version) } if !targetMigration.Applied { + // Assumption: Every migration from the latest down to the target + // migration hasn't been applied. return fmt.Errorf( "migration with version %s isn't applied, nothing to revert", version, -- cgit v1.2.3 From 018083b9259c2265ad3ea2432f315641e5b996ce Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Sat, 2 Mar 2024 10:18:41 -0500 Subject: Clean up fs repo and fs repo tests --- internal/services/migration.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'internal/services/migration.go') diff --git a/internal/services/migration.go b/internal/services/migration.go index 446f267..a91ae2a 100644 --- a/internal/services/migration.go +++ b/internal/services/migration.go @@ -335,7 +335,10 @@ func (ms MigrationService) ListMigrations(order sortOrder) ([]*models.Migration, return migrations, err } -func (ms MigrationService) sortMigrations(migrations []*models.Migration, order sortOrder) error { +func (ms MigrationService) sortMigrations( + migrations []*models.Migration, + order sortOrder, +) error { sortErrs := make([]error, 0) sort.Slice(migrations, func(i, j int) bool { var comparison bool -- cgit v1.2.3 From a37df93853def914dbda165cdcf09ae8e0305ec3 Mon Sep 17 00:00:00 2001 From: Eugene Triguba Date: Sat, 2 Mar 2024 11:04:00 -0500 Subject: Remove Latest() from migration fs repo in favor of using List on service --- internal/services/migration.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'internal/services/migration.go') diff --git a/internal/services/migration.go b/internal/services/migration.go index a91ae2a..4be10cc 100644 --- a/internal/services/migration.go +++ b/internal/services/migration.go @@ -283,12 +283,13 @@ func (ms MigrationService) CreateMigration(message string) (*models.Migration, e func (ms MigrationService) getCurrentSequentialMigrationVersion() (uint64, error) { var currentVersion uint64 = 0 - latestMigration, err := ms.fsRepo.Latest() + migrations, err := ms.ListMigrations(SortOrderDesc) if err != nil { return 0, err } - if latestMigration != nil { + if len(migrations) > 0 { + latestMigration := migrations[0] currentVersion, err = strconv.ParseUint(latestMigration.Version, 10, 64) if err != nil { return 0, err -- cgit v1.2.3