support SQLite's multi-thread model

doc.go

@@ -7,10 +7,10 @@
 //
 // Changelog
 //
-// 2019-12-18 v1.1.0 Pre-alpha release using the new cc/v3, gocc, qbe
+// 2019-12-18 v1.1.0 First alpha release using the new cc/v3, gocc, qbe
 // toolchain. Some primitive tests pass on linux_{amd64,386}. Not yet safe for
-// concurrent access by multiple goroutines. Alpha release is planed to arrive
-// before the end of this year.
+// concurrent access by multiple goroutines. Next alpha release is planed to
+// arrive before the end of this year.
 //
 // 2017-06-10 Windows/Intel no more uses the VM (thanks Steffen Butzer).
 //

generator.go

@@ -29,11 +29,13 @@ var (
 		"-DSQLITE_LIKE_DOESNT_MATCH_BLOBS",
 		"-DSQLITE_MAX_EXPR_DEPTH=0",
 		"-DSQLITE_MEMDEBUG", //TODO-
+		"-DSQLITE_MUTEX_APPDEF=1",
+		"-DSQLITE_MUTEX_NOOP",
 		"-DSQLITE_OMIT_DECLTYPE",
 		"-DSQLITE_OMIT_DEPRECATED",
 		"-DSQLITE_OMIT_PROGRESS_CALLBACK",
 		"-DSQLITE_OMIT_SHARED_CACHE",
-		"-DSQLITE_THREADSAFE=0",
+		"-DSQLITE_THREADSAFE=2", // Multi-thread
 		"-DSQLITE_USE_ALLOCA",
 	}
 

go.mod

@@ -2,4 +2,4 @@ module modernc.org/sqlite
 
 go 1.13
 
-require modernc.org/crt/v2 v2.0.0-alpha.1
+require modernc.org/crt/v2 v2.0.0-20191219143825-5728f219e36a

go.sum

@@ -5,9 +5,8 @@ github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uY
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191218084908-4a24b4065292 h1:Y8q0zsdcgAd+JU8VUA8p8Qv2YhuY9zevDG2ORt5qBUI=
 golang.org/x/sys v0.0.0-20191218084908-4a24b4065292/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-modernc.org/crt v1.0.0 h1:avY0RResvfWYzwlZTAO31E5B1mUr0JF6n6gaLJHeO+I=
-modernc.org/crt/v2 v2.0.0-alpha.1 h1:zWuZqfDO54bDrWHYOPXHSthbdCAuk8uXIQF/uVzhkqc=
-modernc.org/crt/v2 v2.0.0-alpha.1/go.mod h1:5b/3yxJX576AKnw3EWIUhZ0MqklKuZKO0udiE+EQgPE=
+modernc.org/crt/v2 v2.0.0-20191219143825-5728f219e36a h1:SrbTBDKoF2Q62x9YAsaVpt2e650s8groKXRjQ2Y2iZs=
+modernc.org/crt/v2 v2.0.0-20191219143825-5728f219e36a/go.mod h1:5b/3yxJX576AKnw3EWIUhZ0MqklKuZKO0udiE+EQgPE=
 modernc.org/internal v1.0.0/go.mod h1:VUD/+JAkhCpvkUitlEOnhpVxCgsBI90oTzSCRcqQVSM=
 modernc.org/mathutil v1.0.0 h1:93vKjrJopTPrtTNpZ8XIovER7iCIH1QU7wNbOQXC60I=
 modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k=

internal/bin/h_linux_amd64.go

@@ -1,4 +1,4 @@
-// Code generated by `gocc /tmp/go-generate-780942695/x.c -o internal/bin/h_linux_amd64.go -Itestdata/sqlite-amalgamation-3300100 -qbec-defines -qbec-enumconsts -qbec-import <none> -qbec-pkgname bin -qbec-structs -DLONGDOUBLE_TYPE=double -DSQLITE_DEBUG -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_DEFAULT_WAL_SYNCHRONOUS=1 -DSQLITE_DQS=0 -DSQLITE_LIKE_DOESNT_MATCH_BLOBS -DSQLITE_MAX_EXPR_DEPTH=0 -DSQLITE_MEMDEBUG -DSQLITE_OMIT_DECLTYPE -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_THREADSAFE=0 -DSQLITE_USE_ALLOCA`, DO NOT EDIT.
+// Code generated by `gocc /tmp/go-generate-795667064/x.c -o internal/bin/h_linux_amd64.go -Itestdata/sqlite-amalgamation-3300100 -qbec-defines -qbec-enumconsts -qbec-import <none> -qbec-pkgname bin -qbec-structs -DLONGDOUBLE_TYPE=double -DSQLITE_DEBUG -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_DEFAULT_WAL_SYNCHRONOUS=1 -DSQLITE_DQS=0 -DSQLITE_LIKE_DOESNT_MATCH_BLOBS -DSQLITE_MAX_EXPR_DEPTH=0 -DSQLITE_MEMDEBUG -DSQLITE_MUTEX_APPDEF=1 -DSQLITE_MUTEX_NOOP -DSQLITE_OMIT_DECLTYPE -DSQLITE_OMIT_DEPRECATED -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_THREADSAFE=2 -DSQLITE_USE_ALLOCA`, DO NOT EDIT.
 
 package bin
 
@@ -289,7 +289,9 @@ const (
 	DSQLITE_MEMDEBUG                       = 1
 	DSQLITE_MISMATCH                       = 20
 	DSQLITE_MISUSE                         = 21
+	DSQLITE_MUTEX_APPDEF                   = 1
 	DSQLITE_MUTEX_FAST                     = 0
+	DSQLITE_MUTEX_NOOP                     = 1
 	DSQLITE_MUTEX_RECURSIVE                = 1
 	DSQLITE_MUTEX_STATIC_APP1              = 8
 	DSQLITE_MUTEX_STATIC_APP2              = 9
@@ -428,7 +430,7 @@ const (
 	DSQLITE_TESTCTRL_SORTER_MMAP           = 24
 	DSQLITE_TESTCTRL_VDBE_COVERAGE         = 21
 	DSQLITE_TEXT                           = 3
-	DSQLITE_THREADSAFE                     = 0
+	DSQLITE_THREADSAFE                     = 2
 	DSQLITE_TOOBIG                         = 18
 	DSQLITE_TRACE_CLOSE                    = 8
 	DSQLITE_TRACE_PROFILE                  = 2

mutex.go

@@ -0,0 +1,270 @@
+// Copyright 2019 The Sqlite Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package sqlite // import "modernc.org/sqlite"
+
+import (
+	"sync"
+	"sync/atomic"
+	"unsafe"
+
+	"modernc.org/crt/v2"
+	"modernc.org/sqlite/internal/bin"
+)
+
+var (
+	mutexMethods = bin.Ssqlite3_mutex_methods{
+		FxMutexInit: *(*uintptr)(unsafe.Pointer(&struct{ f func(*crt.TLS) int32 }{mutexInit})),
+		FxMutexEnd:  *(*uintptr)(unsafe.Pointer(&struct{ f func(*crt.TLS) int32 }{mutexEnd})),
+		FxMutexAlloc: *(*uintptr)(unsafe.Pointer(&struct {
+			f func(*crt.TLS, int32) crt.Intptr
+		}{mutexAlloc})),
+		FxMutexFree:  *(*uintptr)(unsafe.Pointer(&struct{ f func(*crt.TLS, crt.Intptr) }{mutexFree})),
+		FxMutexEnter: *(*uintptr)(unsafe.Pointer(&struct{ f func(*crt.TLS, crt.Intptr) }{mutexEnter})),
+		FxMutexTry: *(*uintptr)(unsafe.Pointer(&struct {
+			f func(*crt.TLS, crt.Intptr) int32
+		}{mutexTry})),
+		FxMutexLeave: *(*uintptr)(unsafe.Pointer(&struct{ f func(*crt.TLS, crt.Intptr) }{mutexLeave})),
+		FxMutexHeld: *(*uintptr)(unsafe.Pointer(&struct {
+			f func(*crt.TLS, crt.Intptr) int32
+		}{mutexHeld})),
+		FxMutexNotheld: *(*uintptr)(unsafe.Pointer(&struct {
+			f func(*crt.TLS, crt.Intptr) int32
+		}{mutexNotheld})),
+	}
+
+	mutexApp1   mutex
+	mutexApp2   mutex
+	mutexApp3   mutex
+	mutexLRU    mutex
+	mutexMaster mutex
+	mutexMem    mutex
+	mutexOpen   mutex
+	mutexPMem   mutex
+	mutexPRNG   mutex
+	mutexVFS1   mutex
+	mutexVFS2   mutex
+	mutexVFS3   mutex
+)
+
+type mutex struct {
+	cnt int32
+	id  int32
+	sync.Mutex
+	wait sync.Mutex
+}
+
+func (m *mutex) enter(id int32) {
+	for {
+		m.Lock()
+		switch m.id {
+		case 0:
+			m.cnt = 1
+			m.id = id
+			m.wait.Lock()
+			m.Unlock()
+			return
+		case id:
+			m.cnt++
+			m.Unlock()
+			return
+		}
+
+		m.Unlock()
+		m.wait.Lock()
+		m.wait.Unlock()
+	}
+}
+
+func (m *mutex) leave() {
+	m.Lock()
+	m.cnt--
+	if m.cnt == 0 {
+		m.id = 0
+		m.wait.Unlock()
+	}
+	m.Unlock()
+}
+
+// int (*xMutexInit)(void);
+//
+// The xMutexInit method defined by this structure is invoked as part of system
+// initialization by the sqlite3_initialize() function. The xMutexInit routine
+// is called by SQLite exactly once for each effective call to
+// sqlite3_initialize().
+//
+// The xMutexInit() method must be threadsafe. It must be harmless to invoke
+// xMutexInit() multiple times within the same process and without intervening
+// calls to xMutexEnd(). Second and subsequent calls to xMutexInit() must be
+// no-ops. xMutexInit() must not use SQLite memory allocation (sqlite3_malloc()
+// and its associates).
+//
+// If xMutexInit fails in any way, it is expected to clean up after itself
+// prior to returning.
+func mutexInit(tls *crt.TLS) int32 { return bin.DSQLITE_OK }
+
+// int (*xMutexEnd)(void);
+func mutexEnd(tls *crt.TLS) int32 { return bin.DSQLITE_OK }
+
+// sqlite3_mutex *(*xMutexAlloc)(int);
+//
+// The sqlite3_mutex_alloc() routine allocates a new mutex and returns a
+// pointer to it. The sqlite3_mutex_alloc() routine returns NULL if it is
+// unable to allocate the requested mutex. The argument to
+// sqlite3_mutex_alloc() must one of these integer constants:
+//
+//	SQLITE_MUTEX_FAST
+//	SQLITE_MUTEX_RECURSIVE
+//	SQLITE_MUTEX_STATIC_MASTER
+//	SQLITE_MUTEX_STATIC_MEM
+//	SQLITE_MUTEX_STATIC_OPEN
+//	SQLITE_MUTEX_STATIC_PRNG
+//	SQLITE_MUTEX_STATIC_LRU
+//	SQLITE_MUTEX_STATIC_PMEM
+//	SQLITE_MUTEX_STATIC_APP1
+//	SQLITE_MUTEX_STATIC_APP2
+//	SQLITE_MUTEX_STATIC_APP3
+//	SQLITE_MUTEX_STATIC_VFS1
+//	SQLITE_MUTEX_STATIC_VFS2
+//	SQLITE_MUTEX_STATIC_VFS3
+//
+// The first two constants (SQLITE_MUTEX_FAST and SQLITE_MUTEX_RECURSIVE) cause
+// sqlite3_mutex_alloc() to create a new mutex. The new mutex is recursive when
+// SQLITE_MUTEX_RECURSIVE is used but not necessarily so when SQLITE_MUTEX_FAST
+// is used. The mutex implementation does not need to make a distinction
+// between SQLITE_MUTEX_RECURSIVE and SQLITE_MUTEX_FAST if it does not want to.
+// SQLite will only request a recursive mutex in cases where it really needs
+// one. If a faster non-recursive mutex implementation is available on the host
+// platform, the mutex subsystem might return such a mutex in response to
+// SQLITE_MUTEX_FAST.
+//
+// The other allowed parameters to sqlite3_mutex_alloc() (anything other than
+// SQLITE_MUTEX_FAST and SQLITE_MUTEX_RECURSIVE) each return a pointer to a
+// static preexisting mutex. Nine static mutexes are used by the current
+// version of SQLite. Future versions of SQLite may add additional static
+// mutexes. Static mutexes are for internal use by SQLite only. Applications
+// that use SQLite mutexes should use only the dynamic mutexes returned by
+// SQLITE_MUTEX_FAST or SQLITE_MUTEX_RECURSIVE.
+//
+// Note that if one of the dynamic mutex parameters (SQLITE_MUTEX_FAST or
+// SQLITE_MUTEX_RECURSIVE) is used then sqlite3_mutex_alloc() returns a
+// different mutex on every call. For the static mutex types, the same mutex is
+// returned on every call that has the same type number.
+func mutexAlloc(tls *crt.TLS, typ int32) (r crt.Intptr) {
+	defer func() {
+	}()
+	switch typ {
+	case
+		bin.DSQLITE_MUTEX_FAST,
+		bin.DSQLITE_MUTEX_RECURSIVE:
+
+		return crt.Xcalloc(tls, 1, crt.Intptr(unsafe.Sizeof(mutex{})))
+	case bin.DSQLITE_MUTEX_STATIC_MASTER:
+		return crt.Intptr(uintptr(unsafe.Pointer(&mutexMaster)))
+	case bin.DSQLITE_MUTEX_STATIC_MEM:
+		return crt.Intptr(uintptr(unsafe.Pointer(&mutexMem)))
+	case bin.DSQLITE_MUTEX_STATIC_OPEN:
+		return crt.Intptr(uintptr(unsafe.Pointer(&mutexOpen)))
+	case bin.DSQLITE_MUTEX_STATIC_PRNG:
+		return crt.Intptr(uintptr(unsafe.Pointer(&mutexPRNG)))
+	case bin.DSQLITE_MUTEX_STATIC_LRU:
+		return crt.Intptr(uintptr(unsafe.Pointer(&mutexLRU)))
+	case bin.DSQLITE_MUTEX_STATIC_PMEM:
+		return crt.Intptr(uintptr(unsafe.Pointer(&mutexPMem)))
+	case bin.DSQLITE_MUTEX_STATIC_APP1:
+		return crt.Intptr(uintptr(unsafe.Pointer(&mutexApp1)))
+	case bin.DSQLITE_MUTEX_STATIC_APP2:
+		return crt.Intptr(uintptr(unsafe.Pointer(&mutexApp2)))
+	case bin.DSQLITE_MUTEX_STATIC_APP3:
+		return crt.Intptr(uintptr(unsafe.Pointer(&mutexApp3)))
+	case bin.DSQLITE_MUTEX_STATIC_VFS1:
+		return crt.Intptr(uintptr(unsafe.Pointer(&mutexVFS1)))
+	case bin.DSQLITE_MUTEX_STATIC_VFS2:
+		return crt.Intptr(uintptr(unsafe.Pointer(&mutexVFS2)))
+	case bin.DSQLITE_MUTEX_STATIC_VFS3:
+		return crt.Intptr(uintptr(unsafe.Pointer(&mutexVFS3)))
+	default:
+		return 0
+	}
+}
+
+// void (*xMutexFree)(sqlite3_mutex *);
+func mutexFree(tls *crt.TLS, m crt.Intptr) { crt.Xfree(tls, m) }
+
+// The sqlite3_mutex_enter() and sqlite3_mutex_try() routines attempt to enter
+// a mutex. If another thread is already within the mutex,
+// sqlite3_mutex_enter() will block and sqlite3_mutex_try() will return
+// SQLITE_BUSY. The sqlite3_mutex_try() interface returns SQLITE_OK upon
+// successful entry. Mutexes created using SQLITE_MUTEX_RECURSIVE can be
+// entered multiple times by the same thread. In such cases, the mutex must be
+// exited an equal number of times before another thread can enter. If the same
+// thread tries to enter any mutex other than an SQLITE_MUTEX_RECURSIVE more
+// than once, the behavior is undefined.
+//
+// If the argument to sqlite3_mutex_enter(), sqlite3_mutex_try(), or
+// sqlite3_mutex_leave() is a NULL pointer, then all three routines behave as
+// no-ops.
+
+// void (*xMutexEnter)(sqlite3_mutex *);
+func mutexEnter(tls *crt.TLS, m crt.Intptr) {
+	if m == 0 {
+		return
+	}
+
+	(*mutex)(unsafe.Pointer(uintptr(m))).enter(tls.ID)
+}
+
+// int (*xMutexTry)(sqlite3_mutex *);
+func mutexTry(tls *crt.TLS, m crt.Intptr) int32 { return bin.DSQLITE_BUSY }
+
+// void (*xMutexLeave)(sqlite3_mutex *);
+func mutexLeave(tls *crt.TLS, m crt.Intptr) {
+	if m == 0 {
+		return
+	}
+
+	(*mutex)(unsafe.Pointer(uintptr(m))).leave()
+}
+
+// The sqlite3_mutex_held() and sqlite3_mutex_notheld() routines are intended
+// for use inside assert() statements. The SQLite core never uses these
+// routines except inside an assert() and applications are advised to follow
+// the lead of the core. The SQLite core only provides implementations for
+// these routines when it is compiled with the SQLITE_DEBUG flag. External
+// mutex implementations are only required to provide these routines if
+// SQLITE_DEBUG is defined and if NDEBUG is not defined.
+//
+// These routines should return true if the mutex in their argument is held or
+// not held, respectively, by the calling thread.
+//
+// The implementation is not required to provide versions of these routines
+// that actually work. If the implementation does not provide working versions
+// of these routines, it should at least provide stubs that always return true
+// so that one does not get spurious assertion failures.
+//
+// If the argument to sqlite3_mutex_held() is a NULL pointer then the routine
+// should return 1. This seems counter-intuitive since clearly the mutex cannot
+// be held if it does not exist. But the reason the mutex does not exist is
+// because the build is not using mutexes. And we do not want the assert()
+// containing the call to sqlite3_mutex_held() to fail, so a non-zero return is
+// the appropriate thing to do. The sqlite3_mutex_notheld() interface should
+// also return 1 when given a NULL pointer.
+
+// int (*xMutexHeld)(sqlite3_mutex *);
+func mutexHeld(tls *crt.TLS, m crt.Intptr) int32 {
+	if m == 0 {
+		return 1
+	}
+
+	return crt.Bool32(atomic.LoadInt32(&(*mutex)(unsafe.Pointer(uintptr(m))).id) == tls.ID)
+}
+
+// int (*xMutexNotheld)(sqlite3_mutex *);
+func mutexNotheld(tls *crt.TLS, m crt.Intptr) int32 {
+	if m == 0 {
+		return 1
+	}
+
+	return crt.Bool32(atomic.LoadInt32(&(*mutex)(unsafe.Pointer(uintptr(m))).id) != tls.ID)
+}

sqlite.go

@@ -40,10 +40,24 @@ const (
 
 func init() {
 	tls := crt.NewTLS()
-	if bin.Xsqlite3_threadsafe(tls) != 0 { //TODO implement mutexes
+	if bin.Xsqlite3_threadsafe(tls) == 0 {
 		panic(fmt.Errorf("sqlite: thread safety configuration error"))
 	}
 
+	varArgs := crt.Xmalloc(tls, crt.Intptr(ptrSize))
+	if varArgs == 0 {
+		panic(fmt.Errorf("cannot allocate memory"))
+	}
+
+	*(*uintptr)(unsafe.Pointer(uintptr(varArgs))) = uintptr(unsafe.Pointer(&mutexMethods))
+	// int sqlite3_config(int, ...);
+	if rc := bin.Xsqlite3_config(tls, bin.DSQLITE_CONFIG_MUTEX, uintptr(varArgs)); rc != bin.DSQLITE_OK {
+		p := bin.Xsqlite3_errstr(tls, rc)
+		str := crt.GoString(p)
+		panic(fmt.Errorf("sqlite: failed to configure mutex methods: %v", str))
+	}
+
+	crt.Xfree(tls, varArgs)
 	sql.Register(driverName, newDrv())
 }
 
@@ -726,10 +740,6 @@ func newConn(s *Driver, name string) (_ *conn, err error) {
 		}
 	}()
 
-	c.Lock()
-
-	defer c.Unlock()
-
 	c.tls = crt.NewTLS()
 	if err = c.openV2(
 		name,