Add db controlled mutex to prevent database init occuring during another test as this breaks the tests on MariaDB

internal/api/api_test.go

@@ -16,6 +16,7 @@ import (
 	"github.com/photoprism/photoprism/internal/form"
 	"github.com/photoprism/photoprism/internal/photoprism/get"
 	"github.com/photoprism/photoprism/internal/server/limiter"
+	"github.com/photoprism/photoprism/internal/testextras"
 	"github.com/photoprism/photoprism/pkg/header"
 )
 
@@ -38,6 +39,14 @@ func TestMain(m *testing.M) {
 	log.SetLevel(logrus.TraceLevel)
 	event.AuditLog = log
 
+	caller := "internal/api/api_test.go/TestMain"
+	dbc, err := testextras.AcquireDBMutex(log, caller)
+	if err != nil {
+		log.Error("FAIL")
+		os.Exit(1)
+	}
+	defer testextras.UnlockDBMutex(dbc.Db())
+
 	// Init test config.
 	c := config.TestConfig()
 	get.SetConfig(c)
@@ -49,6 +58,7 @@ func TestMain(m *testing.M) {
 	// Run unit tests.
 	code := m.Run()
 
+	testextras.ReleaseDBMutex(dbc.Db(), log, caller, code)
 	os.Exit(code)
 }
 

internal/auth/session/session_test.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/photoprism/photoprism/internal/config"
 	"github.com/photoprism/photoprism/internal/event"
+	"github.com/photoprism/photoprism/internal/testextras"
 )
 
 func TestMain(m *testing.M) {
@@ -15,10 +16,20 @@ func TestMain(m *testing.M) {
 	log.SetLevel(logrus.TraceLevel)
 	event.AuditLog = log
 
+	caller := "internal/auth/session/session_test.go/TestMain"
+	dbc, err := testextras.AcquireDBMutex(log, caller)
+	if err != nil {
+		log.Error("FAIL")
+		os.Exit(1)
+	}
+	defer testextras.UnlockDBMutex(dbc.Db())
+
 	c := config.TestConfig()
 	defer c.CloseDb()
 
 	code := m.Run()
 
+	testextras.ReleaseDBMutex(dbc.Db(), log, caller, code)
+
 	os.Exit(code)
 }

internal/commands/commands_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/photoprism/photoprism/internal/config"
 	"github.com/photoprism/photoprism/internal/event"
 	"github.com/photoprism/photoprism/internal/photoprism/get"
+	"github.com/photoprism/photoprism/internal/testextras"
 )
 
 func TestMain(m *testing.M) {
@@ -18,6 +19,14 @@ func TestMain(m *testing.M) {
 	log.SetLevel(logrus.TraceLevel)
 	event.AuditLog = log
 
+	caller := "internal/commands/commands_test.go/TestMain"
+	dbc, err := testextras.AcquireDBMutex(log, caller)
+	if err != nil {
+		log.Error("FAIL")
+		os.Exit(1)
+	}
+	defer testextras.UnlockDBMutex(dbc.Db())
+
 	c := config.NewTestConfig("commands")
 	get.SetConfig(c)
 
@@ -29,6 +38,8 @@ func TestMain(m *testing.M) {
 	// Run unit tests.
 	code := m.Run()
 
+	testextras.ReleaseDBMutex(dbc.Db(), log, caller, code)
+
 	// Close database connection.
 	c.CloseDb()
 

internal/config/config_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 
+	"github.com/photoprism/photoprism/internal/testextras"
 	"github.com/photoprism/photoprism/pkg/fs"
 )
 
@@ -18,11 +19,21 @@ func TestMain(m *testing.M) {
 	log = logrus.StandardLogger()
 	log.SetLevel(logrus.TraceLevel)
 
+	caller := "internal/config/config_test.go/TestMain"
+	dbc, err := testextras.AcquireDBMutex(log, caller)
+	if err != nil {
+		log.Error("FAIL")
+		os.Exit(1)
+	}
+	defer testextras.UnlockDBMutex(dbc.Db())
+
 	c := TestConfig()
 	defer c.CloseDb()
 
 	code := m.Run()
 
+	testextras.ReleaseDBMutex(dbc.Db(), log, caller, code)
+
 	os.Exit(code)
 }
 

internal/entity/dbtest/dbtest_test.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/photoprism/photoprism/internal/entity"
 	"github.com/photoprism/photoprism/internal/event"
+	"github.com/photoprism/photoprism/internal/testextras"
 	"github.com/photoprism/photoprism/pkg/clean"
 )
 
@@ -25,6 +26,14 @@ func TestMain(m *testing.M) {
 	log.SetLevel(logrus.TraceLevel)
 	event.AuditLog = log
 
+	caller := "internal/entity/dbtest/dbtest_test.go/TestMain"
+	dbc, err := testextras.AcquireDBMutex(log, caller)
+	if err != nil {
+		log.Error("FAIL")
+		os.Exit(1)
+	}
+	defer testextras.UnlockDBMutex(dbc.Db())
+
 	driver := os.Getenv("PHOTOPRISM_TEST_DRIVER")
 	dsn := os.Getenv("PHOTOPRISM_TEST_DSN")
 
@@ -52,5 +61,7 @@ func TestMain(m *testing.M) {
 
 	code := m.Run()
 
+	testextras.ReleaseDBMutex(dbc.Db(), log, caller, code)
+
 	os.Exit(code)
 }

internal/entity/entity_test.go

@@ -8,6 +8,7 @@ import (
 	"github.com/stretchr/testify/assert"
 
 	"github.com/photoprism/photoprism/internal/event"
+	"github.com/photoprism/photoprism/internal/testextras"
 )
 
 func TestMain(m *testing.M) {
@@ -15,6 +16,14 @@ func TestMain(m *testing.M) {
 	log.SetLevel(logrus.TraceLevel)
 	event.AuditLog = log
 
+	caller := "internal/entity/entity_test.go/TestMain"
+	dbc, err := testextras.AcquireDBMutex(log, caller)
+	if err != nil {
+		log.Error("FAIL")
+		os.Exit(1)
+	}
+	defer testextras.UnlockDBMutex(dbc.Db())
+
 	db := InitTestDb(
 		os.Getenv("PHOTOPRISM_TEST_DRIVER"),
 		os.Getenv("PHOTOPRISM_TEST_DSN"))
@@ -23,6 +32,8 @@ func TestMain(m *testing.M) {
 
 	code := m.Run()
 
+	testextras.ReleaseDBMutex(dbc.Db(), log, caller, code)
+
 	os.Exit(code)
 }
 

internal/entity/query/query_test.go

@@ -8,12 +8,21 @@ import (
 	"github.com/stretchr/testify/assert"
 
 	"github.com/photoprism/photoprism/internal/entity"
+	"github.com/photoprism/photoprism/internal/testextras"
 )
 
 func TestMain(m *testing.M) {
 	log = logrus.StandardLogger()
 	log.SetLevel(logrus.TraceLevel)
 
+	caller := "internal/entity/query/query_test.go/TestMain"
+	dbc, err := testextras.AcquireDBMutex(log, caller)
+	if err != nil {
+		log.Error("FAIL")
+		os.Exit(1)
+	}
+	defer testextras.UnlockDBMutex(dbc.Db())
+
 	db := entity.InitTestDb(
 		os.Getenv("PHOTOPRISM_TEST_DRIVER"),
 		os.Getenv("PHOTOPRISM_TEST_DSN"))
@@ -22,6 +31,8 @@ func TestMain(m *testing.M) {
 
 	code := m.Run()
 
+	testextras.ReleaseDBMutex(dbc.Db(), log, caller, code)
+
 	os.Exit(code)
 }
 

internal/entity/search/search_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 
 	"github.com/photoprism/photoprism/internal/entity"
+	"github.com/photoprism/photoprism/internal/testextras"
 	"github.com/sirupsen/logrus"
 )
 
@@ -12,6 +13,14 @@ func TestMain(m *testing.M) {
 	log = logrus.StandardLogger()
 	log.SetLevel(logrus.TraceLevel)
 
+	caller := "internal/entity/search/search_test.go/TestMain"
+	dbc, err := testextras.AcquireDBMutex(log, caller)
+	if err != nil {
+		log.Error("FAIL")
+		os.Exit(1)
+	}
+	defer testextras.UnlockDBMutex(dbc.Db())
+
 	db := entity.InitTestDb(
 		os.Getenv("PHOTOPRISM_TEST_DRIVER"),
 		os.Getenv("PHOTOPRISM_TEST_DSN"))
@@ -20,5 +29,7 @@ func TestMain(m *testing.M) {
 
 	code := m.Run()
 
+	testextras.ReleaseDBMutex(dbc.Db(), log, caller, code)
+
 	os.Exit(code)
 }

internal/photoprism/backup/backup_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/photoprism/photoprism/internal/event"
 	"github.com/photoprism/photoprism/internal/photoprism"
 	"github.com/photoprism/photoprism/internal/photoprism/get"
+	"github.com/photoprism/photoprism/internal/testextras"
 )
 
 func TestMain(m *testing.M) {
@@ -17,6 +18,14 @@ func TestMain(m *testing.M) {
 	log.SetLevel(logrus.TraceLevel)
 	event.AuditLog = log
 
+	caller := "internal/photoprism/backup/backup_test.go/TestMain"
+	dbc, err := testextras.AcquireDBMutex(log, caller)
+	if err != nil {
+		log.Error("FAIL")
+		os.Exit(1)
+	}
+	defer testextras.UnlockDBMutex(dbc.Db())
+
 	c := config.TestConfig()
 	defer c.CloseDb()
 
@@ -25,5 +34,7 @@ func TestMain(m *testing.M) {
 
 	code := m.Run()
 
+	testextras.ReleaseDBMutex(dbc.Db(), log, caller, code)
+
 	os.Exit(code)
 }

internal/photoprism/get/get_test.go

@@ -5,14 +5,26 @@ import (
 	"testing"
 
 	"github.com/photoprism/photoprism/internal/config"
+	"github.com/photoprism/photoprism/internal/testextras"
 )
 
 func TestMain(m *testing.M) {
+
+	caller := "internal/photoprism/get/get_test.go/TestMain"
+	dbc, err := testextras.AcquireDBMutex(log, caller)
+	if err != nil {
+		log.Error("FAIL")
+		os.Exit(1)
+	}
+	defer testextras.UnlockDBMutex(dbc.Db())
+
 	c := config.NewTestConfig("service")
 	SetConfig(c)
 	defer c.CloseDb()
 
 	code := m.Run()
 
+	testextras.ReleaseDBMutex(dbc.Db(), log, caller, code)
+
 	os.Exit(code)
 }

internal/photoprism/photoprism_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 
 	"github.com/photoprism/photoprism/internal/config"
+	"github.com/photoprism/photoprism/internal/testextras"
 	"github.com/sirupsen/logrus"
 )
 
@@ -12,11 +13,21 @@ func TestMain(m *testing.M) {
 	log = logrus.StandardLogger()
 	log.SetLevel(logrus.TraceLevel)
 
+	caller := "internal/photoprism/photoprism_test.go/TestMain"
+	dbc, err := testextras.AcquireDBMutex(log, caller)
+	if err != nil {
+		log.Error("FAIL")
+		os.Exit(1)
+	}
+	defer testextras.UnlockDBMutex(dbc.Db())
+
 	c := config.NewTestConfig("photoprism")
 	SetConfig(c)
 	defer c.CloseDb()
 
 	code := m.Run()
 
+	testextras.ReleaseDBMutex(dbc.Db(), log, caller, code)
+
 	os.Exit(code)
 }

internal/server/server_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/photoprism/photoprism/internal/event"
 	"github.com/photoprism/photoprism/internal/photoprism/get"
 	"github.com/photoprism/photoprism/internal/server/limiter"
+	"github.com/photoprism/photoprism/internal/testextras"
 )
 
 func TestMain(m *testing.M) {
@@ -18,6 +19,14 @@ func TestMain(m *testing.M) {
 	log.SetLevel(logrus.TraceLevel)
 	event.AuditLog = log
 
+	caller := "internal/server/server_test.go/TestMain"
+	dbc, err := testextras.AcquireDBMutex(log, caller)
+	if err != nil {
+		log.Error("FAIL")
+		os.Exit(1)
+	}
+	defer testextras.UnlockDBMutex(dbc.Db())
+
 	// Init test config.
 	c := config.TestConfig()
 	get.SetConfig(c)
@@ -29,5 +38,7 @@ func TestMain(m *testing.M) {
 	// Run unit tests.
 	code := m.Run()
 
+	testextras.ReleaseDBMutex(dbc.Db(), log, caller, code)
+
 	os.Exit(code)
 }

internal/testextras/createextras.go

@@ -0,0 +1,14 @@
+package testextras
+
+import (
+	"gorm.io/gorm"
+)
+
+func MigrateTestExtras(db *gorm.DB) {
+	if err := db.AutoMigrate(&TestLog{}); err != nil {
+		panic(err)
+	}
+	if err := db.AutoMigrate(&TestDBMutex{}); err != nil {
+		panic(err)
+	}
+}

internal/testextras/db_conn.go

@@ -0,0 +1,129 @@
+package testextras
+
+import (
+	"fmt"
+	"sync"
+	"time"
+
+	"gorm.io/driver/mysql"
+	"gorm.io/driver/postgres"
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+	"gorm.io/gorm/logger"
+)
+
+// Supported test databases.
+const (
+	MySQL           = "mysql"
+	Postgres        = "postgres"
+	SQLite3         = "sqlite"
+	SQLiteTestDB    = ".test.db"
+	SQLiteMemoryDSN = ":memory:?cache=shared&_foreign_keys=on"
+)
+
+var drivers = map[string]func(string) gorm.Dialector{
+	MySQL:    mysql.Open,
+	Postgres: postgres.Open,
+	SQLite3:  sqlite.Open,
+}
+
+// dbConn is the global gorm.DB connection provider.
+var dbConn Gorm
+
+// Gorm is a gorm.DB connection provider interface.
+type Gorm interface {
+	Db() *gorm.DB
+}
+
+// DbConn is a gorm.DB connection provider.
+type DbConn struct {
+	Driver string
+	Dsn    string
+
+	once sync.Once
+	db   *gorm.DB
+}
+
+// Db returns the gorm db connection.
+func (g *DbConn) Db() *gorm.DB {
+	g.once.Do(g.Open)
+
+	if g.db == nil {
+		log.Fatal("migrate: database not connected")
+	}
+
+	return g.db
+}
+
+// Open creates a new gorm db connection.
+func (g *DbConn) Open() {
+	log.Infof("Opening DB connection with driver %s", g.Driver)
+	db, err := gorm.Open(drivers[g.Driver](g.Dsn), gormConfig())
+
+	if err != nil || db == nil {
+		for i := 1; i <= 12; i++ {
+			fmt.Printf("gorm.Open(%s, %s) %d\n", g.Driver, g.Dsn, i)
+			db, err = gorm.Open(drivers[g.Driver](g.Dsn), gormConfig())
+
+			if db != nil && err == nil {
+				break
+			} else {
+				time.Sleep(5 * time.Second)
+			}
+		}
+
+		if err != nil || db == nil {
+			fmt.Println(err)
+			log.Fatal(err)
+		}
+	}
+	log.Info("DB connection established successfully")
+
+	sqlDB, _ := db.DB()
+
+	sqlDB.SetMaxIdleConns(4)   // in config_db it uses c.DatabaseConnsIdle(), but we don't have the c here.
+	sqlDB.SetMaxOpenConns(256) // in config_db it uses c.DatabaseConns(), but we don't have the c here.
+
+	g.db = db
+}
+
+// Close closes the gorm db connection.
+func (g *DbConn) Close() {
+	if g.db != nil {
+		sqlDB, _ := g.db.DB()
+		if err := sqlDB.Close(); err != nil {
+			log.Fatal(err)
+		}
+
+		g.db = nil
+	}
+}
+
+// SetDbProvider sets the Gorm database connection provider.
+func SetDbProvider(conn Gorm) {
+	dbConn = conn
+}
+
+// HasDbProvider returns true if a db provider exists.
+func HasDbProvider() bool {
+	return dbConn != nil
+}
+
+func gormConfig() *gorm.Config {
+	return &gorm.Config{
+		Logger: logger.New(
+			log,
+			logger.Config{
+				SlowThreshold:             time.Second,   // Slow SQL threshold
+				LogLevel:                  logger.Silent, // Log level
+				IgnoreRecordNotFoundError: true,          // Ignore ErrRecordNotFound error for logger
+				ParameterizedQueries:      true,          // Don't include params in the SQL log
+				Colorful:                  false,         // Disable color
+			},
+		),
+		// Set UTC as the default for created and updated timestamps.
+		NowFunc: func() time.Time {
+			return time.Now().UTC()
+		},
+	}
+}

internal/testextras/testdbmutex.go

@@ -0,0 +1,103 @@
+package testextras
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/photoprism/photoprism/internal/event"
+	"github.com/photoprism/photoprism/pkg/clean"
+	"gorm.io/gorm"
+)
+
+// Test DB Mutex structure to store the currently active mutex
+type TestDBMutex struct {
+	ID        uint      `gorm:"primaryKey;"`
+	CreateAt  time.Time `sql:"index:idx_testdbmutex_create_at"`
+	ProcessId int
+}
+
+// Attempts to acquire a database controlled mutex.  Using the table primary key to prevent more than 1 insert succeeding.
+// Will retry 60 times with 10s interval, before returning false on failure to get mutex.
+// The mutex uses the process id to ensure uniqueness between processes.
+func LockDBMutex(db *gorm.DB, log event.Logger, caller string) bool {
+	pid := os.Getpid()
+	err := errors.New("so i am not nil")
+	counter := 0
+	for err != nil {
+		record := TestDBMutex{ID: 1, CreateAt: time.Now().UTC(), ProcessId: pid}
+		if err = db.Create(&record).Error; err != nil {
+			counter += 1
+			LogMessage(db, fmt.Sprintf("%v LockDBMutex Failed Attempt %v", caller, counter))
+			if counter > 60 { // There is 10 minutes of wait time here.
+				return false
+			}
+			time.Sleep(10 * time.Second)
+		}
+	}
+	return true
+}
+
+// delete the mutex using the processes id.  This should be called with a defer to try and ensure that it always get cleared.
+// But, if it's a really nasty internal error (eg. SIGFAULT) then go wont free the mutex and this will require manual intervention.
+// The photoprism makefile tests drop the database, which will clear the mutex at the start of the testing.
+func UnlockDBMutex(db *gorm.DB) {
+	pid := os.Getpid()
+	record := TestDBMutex{ProcessId: pid}
+	db.Where("process_id = ?", pid).Delete(&record)
+}
+
+// Clears out a mutex lock and logs messages about it
+func ReleaseDBMutex(db *gorm.DB, log event.Logger, caller string, code int) {
+	LogMessage(db, fmt.Sprintf("%v UnlockDBMutex", caller))
+	UnlockDBMutex(db)
+	log.Info("database mutex released")
+	LogMessage(db, fmt.Sprintf("%v ending with %v", caller, code))
+}
+
+// Opens a database connection, and then attempts to acquire a mutex for this process.
+func AcquireDBMutex(log event.Logger, caller string) (dbc *DbConn, err error) {
+
+	err = nil
+
+	driver := os.Getenv("PHOTOPRISM_TEST_DRIVER")
+	dsn := os.Getenv("PHOTOPRISM_TEST_DSN")
+
+	// Set default test database driver.
+	if driver == "test" || driver == "sqlite" || driver == "" || dsn == "" {
+		driver = SQLite3
+	}
+
+	// Set default database DSN.
+	if driver == SQLite3 {
+		if dsn == "" {
+			dsn = SQLiteMemoryDSN
+		} else if dsn != SQLiteTestDB {
+			// Continue.
+		} else if err := os.Remove(dsn); err == nil {
+			log.Debugf("sqlite: test file %s removed", clean.Log(dsn))
+		}
+	}
+
+	// Create gorm.DB connection provider.
+	dbc = &DbConn{
+		Driver: driver,
+		Dsn:    dsn,
+	}
+
+	SetDbProvider(dbc)
+	log.Info("migrating test extras")
+	MigrateTestExtras(dbc.Db())
+	LogMessage(dbc.Db(), fmt.Sprintf("%v starting", caller))
+	if LockDBMutex(dbc.Db(), log, caller) {
+		LogMessage(dbc.Db(), fmt.Sprintf("%v LockDBMutex", caller))
+		log.Info("database mutex acquired")
+
+	} else {
+		log.Error("Unable to get DBMutex")
+		err = errors.New("unable to acquire DBMutex")
+	}
+
+	return dbc, err
+}

internal/testextras/testextras.go

@@ -0,0 +1,14 @@
+package testextras
+
+import (
+	"github.com/photoprism/photoprism/internal/event"
+)
+
+var log = event.Log
+
+// Log logs the error if any and keeps quiet otherwise.
+func Log(model, action string, err error) {
+	if err != nil {
+		log.Errorf("%s: %s (%s)", model, err, action)
+	}
+}

internal/testextras/testlog.go

@@ -0,0 +1,22 @@
+package testextras
+
+import (
+	"os"
+	"time"
+
+	"gorm.io/gorm"
+)
+
+// Test Logging structure
+type TestLog struct {
+	ID        uint      `gorm:"primaryKey;"`
+	LogTime   time.Time `sql:"index:idx_testlog_log_time"`
+	ProcessId int
+	Message   string `gorm:"size:200;default:''"`
+}
+
+func LogMessage(db *gorm.DB, message string) {
+	pid := os.Getpid()
+	record := TestLog{LogTime: time.Now().UTC(), ProcessId: pid, Message: message}
+	db.Create(&record)
+}

internal/thumb/avatar/avatar_test.go

@@ -9,12 +9,21 @@ import (
 	"github.com/photoprism/photoprism/internal/config"
 	"github.com/photoprism/photoprism/internal/photoprism"
 	"github.com/photoprism/photoprism/internal/photoprism/get"
+	"github.com/photoprism/photoprism/internal/testextras"
 )
 
 func TestMain(m *testing.M) {
 	log = logrus.StandardLogger()
 	log.SetLevel(logrus.TraceLevel)
 
+	caller := "internal/thumb/avatar/avatar_test.go/TestMain"
+	dbc, err := testextras.AcquireDBMutex(log, caller)
+	if err != nil {
+		log.Error("FAIL")
+		os.Exit(1)
+	}
+	defer testextras.UnlockDBMutex(dbc.Db())
+
 	c := config.NewTestConfig("avatar")
 	get.SetConfig(c)
 	photoprism.SetConfig(c)
@@ -22,5 +31,7 @@ func TestMain(m *testing.M) {
 
 	code := m.Run()
 
+	testextras.ReleaseDBMutex(dbc.Db(), log, caller, code)
+
 	os.Exit(code)
 }

internal/workers/auto/auto_test.go

@@ -7,18 +7,27 @@ import (
 	"github.com/sirupsen/logrus"
 
 	"github.com/photoprism/photoprism/internal/config"
+	"github.com/photoprism/photoprism/internal/testextras"
 )
 
 func TestMain(m *testing.M) {
 	log = logrus.StandardLogger()
 	log.SetLevel(logrus.TraceLevel)
 
+	caller := "internal/workers/auto/auto_test.go/TestMain"
+	dbc, err := testextras.AcquireDBMutex(log, caller)
+	if err != nil {
+		log.Error("FAIL")
+		os.Exit(1)
+	}
+	defer testextras.UnlockDBMutex(dbc.Db())
+
 	c := config.TestConfig()
 	defer c.CloseDb()
 
 	// Run unit tests.
 	code := m.Run()
-
+	testextras.ReleaseDBMutex(dbc.Db(), log, caller, code)
 	// Close database connection.
 	_ = c.CloseDb()
 

internal/workers/workers_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/photoprism/photoprism/internal/event"
 	"github.com/photoprism/photoprism/internal/photoprism"
 	"github.com/photoprism/photoprism/internal/photoprism/get"
+	"github.com/photoprism/photoprism/internal/testextras"
 )
 
 func TestMain(m *testing.M) {
@@ -17,6 +18,14 @@ func TestMain(m *testing.M) {
 	log.SetLevel(logrus.TraceLevel)
 	event.AuditLog = log
 
+	caller := "internal/workers/workers_test.go/TestMain"
+	dbc, err := testextras.AcquireDBMutex(log, caller)
+	if err != nil {
+		log.Error("FAIL")
+		os.Exit(1)
+	}
+	defer testextras.UnlockDBMutex(dbc.Db())
+
 	c := config.TestConfig()
 	defer c.CloseDb()
 
@@ -25,5 +34,7 @@ func TestMain(m *testing.M) {
 
 	code := m.Run()
 
+	testextras.ReleaseDBMutex(dbc.Db(), log, caller, code)
+
 	os.Exit(code)
 }

teststatus.md

@@ -1,4 +1,90 @@
-The following is the status of unit testing.
+The following is the status of unit testing against sqlite.  
+
+Removing test database files...  
+find ./internal -type f -name ".test.*" -delete  
+Running all Go tests...  
+richgo test -parallel 1 -count 1 -cpu 1 -tags slow -timeout 20m ./pkg/... ./internal/...  
+
+| Status | Path/Test |
+| ------ | --------------------------------------------------------------------- |
+| PASS | github.com/photoprism/photoprism/pkg/authn |
+| PASS | github.com/photoprism/photoprism/pkg/capture |
+| PASS | github.com/photoprism/photoprism/pkg/checksum |
+| PASS | github.com/photoprism/photoprism/pkg/clean |
+| PASS | github.com/photoprism/photoprism/pkg/clusters |
+| PASS | github.com/photoprism/photoprism/pkg/env |
+| PASS | github.com/photoprism/photoprism/pkg/fs |
+| PASS | github.com/photoprism/photoprism/pkg/fs/fastwalk |
+| PASS | github.com/photoprism/photoprism/pkg/geo |
+| PASS | github.com/photoprism/photoprism/pkg/geo/pluscode |
+| PASS | github.com/photoprism/photoprism/pkg/geo/s2 |
+| PASS | github.com/photoprism/photoprism/pkg/header |
+| PASS | github.com/photoprism/photoprism/pkg/i18n |
+| PASS | github.com/photoprism/photoprism/pkg/list |
+| PASS | github.com/photoprism/photoprism/pkg/log/dummy |
+| PASS | github.com/photoprism/photoprism/pkg/log/level |
+| PASS | github.com/photoprism/photoprism/pkg/media |
+| PASS | github.com/photoprism/photoprism/pkg/media/colors |
+| PASS | github.com/photoprism/photoprism/pkg/media/projection |
+| PASS | github.com/photoprism/photoprism/pkg/media/video |
+| PASS | github.com/photoprism/photoprism/pkg/react |
+| PASS | github.com/photoprism/photoprism/pkg/rnd |
+| PASS | github.com/photoprism/photoprism/pkg/time/unix |
+| PASS | github.com/photoprism/photoprism/pkg/txt |
+| PASS | github.com/photoprism/photoprism/pkg/txt/clip |
+| PASS | github.com/photoprism/photoprism/pkg/txt/report |
+| PASS | github.com/photoprism/photoprism/internal/ai/classify |
+| PASS | github.com/photoprism/photoprism/internal/ai/face |
+| PASS | github.com/photoprism/photoprism/internal/ai/nsfw |
+| SKIP | github.com/photoprism/photoprism/internal/entity/legacy	[no test files] |
+| PASS | github.com/photoprism/photoprism/internal/api |
+| PASS | github.com/photoprism/photoprism/internal/auth/acl |
+| PASS | github.com/photoprism/photoprism/internal/auth/oidc |
+| PASS | github.com/photoprism/photoprism/internal/auth/session |
+| PASS | github.com/photoprism/photoprism/internal/commands |
+| PASS | github.com/photoprism/photoprism/internal/config |
+| PASS | github.com/photoprism/photoprism/internal/config/customize |
+| PASS | github.com/photoprism/photoprism/internal/config/pwa |
+| PASS | github.com/photoprism/photoprism/internal/config/ttl |
+| PASS | github.com/photoprism/photoprism/internal/entity |
+| PASS | github.com/photoprism/photoprism/internal/entity/dbtest |
+| PASS | github.com/photoprism/photoprism/internal/entity/migrate |
+| PASS | github.com/photoprism/photoprism/internal/entity/query |
+| PASS | github.com/photoprism/photoprism/internal/entity/search |
+| PASS | github.com/photoprism/photoprism/internal/entity/search/viewer |
+| PASS | github.com/photoprism/photoprism/internal/entity/sortby |
+| PASS | github.com/photoprism/photoprism/internal/event |
+| PASS | github.com/photoprism/photoprism/internal/ffmpeg |
+| PASS | github.com/photoprism/photoprism/internal/form |
+| PASS | github.com/photoprism/photoprism/internal/functions |
+| PASS | github.com/photoprism/photoprism/internal/meta |
+| PASS | github.com/photoprism/photoprism/internal/mutex |
+| SKIP | github.com/photoprism/photoprism/internal/testextras	[no test files] |
+| PASS | github.com/photoprism/photoprism/internal/photoprism |
+| PASS | github.com/photoprism/photoprism/internal/photoprism/backup |
+| PASS | github.com/photoprism/photoprism/internal/photoprism/get |
+| PASS | github.com/photoprism/photoprism/internal/server |
+| PASS | github.com/photoprism/photoprism/internal/server/limiter |
+| PASS | github.com/photoprism/photoprism/internal/server/wellknown |
+| PASS | github.com/photoprism/photoprism/internal/service |
+| PASS | github.com/photoprism/photoprism/internal/service/hub |
+| PASS | github.com/photoprism/photoprism/internal/service/hub/places |
+| PASS | github.com/photoprism/photoprism/internal/service/maps |
+| PASS | github.com/photoprism/photoprism/internal/service/webdav |
+| PASS | github.com/photoprism/photoprism/internal/thumb |
+| PASS | github.com/photoprism/photoprism/internal/thumb/avatar |
+| PASS | github.com/photoprism/photoprism/internal/thumb/crop |
+| PASS | github.com/photoprism/photoprism/internal/thumb/frame |
+| PASS | github.com/photoprism/photoprism/internal/workers |
+| PASS | github.com/photoprism/photoprism/internal/workers/auto |
+
+
+
+The following is the status of unit testing against mariadb, which drops the database as part of the init.  
+Resetting acceptance database...  
+mysql < scripts/sql/reset-acceptance.sql  
+Running all Go tests on MariaDB...
+PHOTOPRISM_TEST_DRIVER="mysql" PHOTOPRISM_TEST_DSN="root:photoprism@tcp(mariadb:4001)/acceptance?charset=utf8mb4,utf8&collation=utf8mb4_unicode_ci&parseTime=true" richgo test -parallel 1 -count 1 -cpu 1 -tags slow -timeout 20m ./pkg/... ./internal/...  
 | Status | Path/Test |
 | ------ | --------------------------------------------------------------------- |
 | PASS | github.com/photoprism/photoprism/pkg/authn |
@@ -34,13 +120,16 @@ The following is the status of unit testing.
 | PASS | github.com/photoprism/photoprism/internal/auth/acl |
 | PASS | github.com/photoprism/photoprism/internal/auth/oidc |
 | PASS | github.com/photoprism/photoprism/internal/auth/session |
+| SKIP | github.com/photoprism/photoprism/internal/entity/legacy	[no test files] |
 | PASS | github.com/photoprism/photoprism/internal/commands |
 | PASS | github.com/photoprism/photoprism/internal/config |
 | PASS | github.com/photoprism/photoprism/internal/config/customize |
 | PASS | github.com/photoprism/photoprism/internal/config/pwa |
 | PASS | github.com/photoprism/photoprism/internal/config/ttl |
 | PASS | github.com/photoprism/photoprism/internal/entity |
+| PASS | github.com/photoprism/photoprism/internal/entity/dbtest |
 | PASS | github.com/photoprism/photoprism/internal/entity/migrate |
+| SKIP | github.com/photoprism/photoprism/internal/testextras	[no test files] |
 | PASS | github.com/photoprism/photoprism/internal/entity/query |
 | PASS | github.com/photoprism/photoprism/internal/entity/search |
 | PASS | github.com/photoprism/photoprism/internal/entity/search/viewer |
@@ -69,11 +158,3 @@ The following is the status of unit testing.
 | PASS | github.com/photoprism/photoprism/internal/workers |
 | PASS | github.com/photoprism/photoprism/internal/workers/auto |
 
-| Status | Path/Test |
-| ------ | --------------------------------------------------------------------- |
-| FAIL | Init |
-| FAIL |   Init/PhotoLabelCounts |
-| FAIL |   Init/PhotoKeywordCounts |
-| FAIL | 	github.com/photoprism/photoprism/internal/entity/dbtest |
-
-Issues found in tests that don't cause a failure: