Merge all upstream changes

This commit is contained in:
glebarez
2023-01-28 20:44:52 +07:00
parent 4143dc8465
commit b8c64c30ab
15 changed files with 791 additions and 67 deletions

View File

@@ -10,11 +10,14 @@
Artyom Pervukhin <github@artyom.dev>
Dan Peterson <danp@danp.net>
David Walton <david@davidwalton.com>
Davsk Ltd Co <skinner.david@gmail.com>
Jaap Aarts <jaap.aarts1@gmail.com>
Jan Mercl <0xjnml@gmail.com>
Josh Bleecher Snyder <josharian@gmail.com>
Logan Snow <logansnow@protonmail.com>
Michael Hoffmann <mhoffm@posteo.de>
Ross Light <ross@zombiezen.com>
Steffen Butzer <steffen(dot)butzer@outlook.com>
Saed SayedAhmed <saadmtsa@gmail.com>
Steffen Butzer <steffen(dot)butzer@outlook.com>
Michael Rykov <mrykov@gmail.com>

View File

@@ -10,13 +10,19 @@ Alexander Menzhinsky <amenzhinsky@gmail.com>
Artyom Pervukhin <github@artyom.dev>
Dan Peterson <danp@danp.net>
David Skinner <skinner.david@gmail.com>
David Walton <david@davidwalton.com>
Elle Mouton <elle.mouton@gmail.com>
FlyingOnion <731677080@qq.com>
Gleb Sakhnov <gleb.sakhnov@gmail.com>
Jaap Aarts <jaap.aarts1@gmail.com>
Jan Mercl <0xjnml@gmail.com>
Josh Bleecher Snyder <josharian@gmail.com>
Logan Snow <logansnow@protonmail.com>
Matthew Gabeler-Lee <fastcat@gmail.com>
Michael Hoffmann <mhoffm@posteo.de>
Ross Light <ross@zombiezen.com>
Saed SayedAhmed <saadmtsa@gmail.com>
Steffen Butzer <steffen(dot)butzer@outlook.com>
Yaacov Akiba Slama <ya@slamail.org>
Saed SayedAhmed <saadmtsa@gmail.com>
Michael Rykov <mrykov@gmail.com>

64
addport.go Normal file
View File

@@ -0,0 +1,64 @@
// Copyright 2022 The Sqlite Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build ignore
// +build ignore
package main
import (
"fmt"
"os"
"path/filepath"
"strings"
)
func fail(rc int, msg string, args ...interface{}) {
fmt.Fprintf(os.Stderr, msg, args...)
os.Exit(rc)
}
func main() {
if len(os.Args) != 3 {
fail(1, "expected 2 args: pattern and replacement\n")
}
pattern := os.Args[1]
replacement := os.Args[2]
if err := filepath.Walk(".", func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() {
return nil
}
dir, file := filepath.Split(path)
if x := strings.Index(file, pattern); x >= 0 {
// pattern freebsd
// replacement netbsd
// file libc_freebsd_amd64.go
// replaced libc_netbsd_amd64.go
// 01234567890123456789
// 1
// x 5
file = file[:x] + replacement + file[x+len(pattern):]
dst := filepath.Join(dir, file)
b, err := os.ReadFile(path)
if err != nil {
return fmt.Errorf("reading %s: %v", path, err)
}
if err := os.WriteFile(dst, b, 0640); err != nil {
return fmt.Errorf("writing %s: %v", dst, err)
}
fmt.Printf("%s -> %s\n", path, dst)
}
return nil
}); err != nil {
fail(1, "%s", err)
}
}

View File

@@ -9,10 +9,11 @@ import (
"context"
"database/sql"
"database/sql/driver"
"embed"
"errors"
"flag"
"fmt"
"io/ioutil"
"io"
"math/rand"
"net/url"
"os"
@@ -23,16 +24,20 @@ import (
"regexp"
"runtime"
"runtime/debug"
"runtime/pprof"
"strconv"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"unsafe"
"github.com/google/pprof/profile"
"modernc.org/libc"
"modernc.org/mathutil"
sqlite3 "modernc.org/sqlite/lib"
"modernc.org/sqlite/vfs"
)
func caller(s string, va ...interface{}) {
@@ -124,7 +129,7 @@ func TestMain(m *testing.M) {
func testMain(m *testing.M) int {
var err error
tempDir, err = ioutil.TempDir("", "sqlite-test-")
tempDir, err = os.MkdirTemp("", "sqlite-test-")
if err != nil {
panic(err) //TODOOK
}
@@ -135,7 +140,7 @@ func testMain(m *testing.M) int {
}
func tempDB(t testing.TB) (string, *sql.DB) {
dir, err := ioutil.TempDir("", "sqlite-test-")
dir, err := os.MkdirTemp("", "sqlite-test-")
if err != nil {
t.Fatal(err)
}
@@ -149,6 +154,75 @@ func tempDB(t testing.TB) (string, *sql.DB) {
return dir, db
}
// https://gitlab.com/cznic/sqlite/issues/118
func TestIssue118(t *testing.T) {
// Many iterations generate enough objects to ensure pprof
// profile captures the samples that we are seeking below
for i := 0; i < 10000; i++ {
func() {
db, err := sql.Open("sqlite", ":memory:")
if err != nil {
t.Fatal(err)
}
defer db.Close()
if _, err := db.Exec(`CREATE TABLE t1(v TEXT)`); err != nil {
t.Fatal(err)
}
var val []byte
if _, err := db.Exec(`INSERT INTO t1(v) VALUES(?)`, val); err != nil {
t.Fatal(err)
}
var count int
err = db.QueryRow("SELECT MAX(_ROWID_) FROM t1").Scan(&count)
if err != nil || count <= 0 {
t.Fatalf("Query failure: %d, %s", count, err)
}
}()
}
// Dump & read heap sample
var buf bytes.Buffer
if err := pprof.Lookup("heap").WriteTo(&buf, 0); err != nil {
t.Fatalf("Error dumping heap profile: %s", err)
}
heapProfile, err := profile.Parse(&buf)
if err != nil {
t.Fatalf("Error parsing heap profile: %s", err)
}
// Profile.SampleType indexes map into Sample.Values below. We are
// looking for "inuse_*" values, and skip the "alloc_*" ones
inUseIndexes := make([]int, 0, 2)
for i, t := range heapProfile.SampleType {
if strings.HasPrefix(t.Type, "inuse_") {
inUseIndexes = append(inUseIndexes, i)
}
}
// Look for samples from "libc.NewTLS" and insure that they have nothing in-use
for _, sample := range heapProfile.Sample {
isInUse := false
for _, idx := range inUseIndexes {
isInUse = isInUse || sample.Value[idx] > 0
}
if !isInUse {
continue
}
isNewTLS := false
sampleStack := []string{}
for _, location := range sample.Location {
for _, line := range location.Line {
sampleStack = append(sampleStack, fmt.Sprintf("%s (%s:%d)", line.Function.Name, line.Function.Filename, line.Line))
isNewTLS = isNewTLS || strings.Contains(line.Function.Name, "libc.NewTLS")
}
}
if isNewTLS {
t.Errorf("Memory leak via libc.NewTLS:\n%s\n", strings.Join(sampleStack, "\n"))
}
}
}
// https://gitlab.com/cznic/sqlite/issues/100
func TestIssue100(t *testing.T) {
db, err := sql.Open("sqlite", ":memory:")
@@ -776,7 +850,7 @@ func TestConcurrentGoroutines(t *testing.T) {
nrows = 5000
)
dir, err := ioutil.TempDir("", "sqlite-test-")
dir, err := os.MkdirTemp("", "sqlite-test-")
if err != nil {
t.Fatal(err)
}
@@ -883,12 +957,7 @@ func TestConcurrentProcesses(t *testing.T) {
t.Skip("skipping test in short mode")
}
//TODO The current riscv64 board seems too slow for the hardcoded timeouts.
if runtime.GOARCH == "riscv64" {
t.Skip("skipping test")
}
dir, err := ioutil.TempDir("", "sqlite-test-")
dir, err := os.MkdirTemp("", "sqlite-test-")
if err != nil {
t.Fatal(err)
}
@@ -907,7 +976,7 @@ func TestConcurrentProcesses(t *testing.T) {
continue
}
b, err := ioutil.ReadFile(v)
b, err := os.ReadFile(v)
if err != nil {
t.Fatal(err)
}
@@ -920,7 +989,7 @@ func TestConcurrentProcesses(t *testing.T) {
b = bytes.ReplaceAll(b, []byte("\r\n"), []byte("\n"))
}
if err := ioutil.WriteFile(filepath.Join(dir, filepath.Base(v)), b, 0666); err != nil {
if err := os.WriteFile(filepath.Join(dir, filepath.Base(v)), b, 0666); err != nil {
t.Fatal(err)
}
}
@@ -958,7 +1027,7 @@ outer:
}
fmt.Printf("exec: %s db %s\n", filepath.FromSlash(bin), script)
out, err := exec.Command(filepath.FromSlash(bin), "db", "--timeout", "60000", script).CombinedOutput()
out, err := exec.Command(filepath.FromSlash(bin), "db", "--timeout", "6000000", script).CombinedOutput()
if err != nil {
t.Fatalf("%s\n%v", out, err)
}
@@ -1024,7 +1093,7 @@ INSERT INTO "products" ("id", "user_id", "name", "description", "created_at", "c
`
)
dir, err := ioutil.TempDir("", "sqlite-test-")
dir, err := os.MkdirTemp("", "sqlite-test-")
if err != nil {
t.Fatal(err)
}
@@ -1122,7 +1191,7 @@ func mustExec(t *testing.T, db *sql.DB, sql string, args ...interface{}) sql.Res
func TestIssue20(t *testing.T) {
const TablePrefix = "gosqltest_"
tempDir, err := ioutil.TempDir("", "")
tempDir, err := os.MkdirTemp("", "")
if err != nil {
t.Fatal(err)
}
@@ -1131,7 +1200,13 @@ func TestIssue20(t *testing.T) {
os.RemoveAll(tempDir)
}()
db, err := sql.Open("sqlite", filepath.Join(tempDir, "foo.db")+"?_pragma=busy_timeout(30000)")
// go1.20rc1, linux/ppc64le VM
// 10000 FAIL
// 20000 FAIL
// 40000 PASS
// 30000 PASS
// 25000 PASS
db, err := sql.Open("sqlite", filepath.Join(tempDir, "foo.db")+"?_pragma=busy_timeout%3d50000")
if err != nil {
t.Fatalf("foo.db open fail: %v", err)
}
@@ -1182,7 +1257,7 @@ func TestIssue20(t *testing.T) {
}
func TestNoRows(t *testing.T) {
tempDir, err := ioutil.TempDir("", "")
tempDir, err := os.MkdirTemp("", "")
if err != nil {
t.Fatal(err)
}
@@ -1275,7 +1350,7 @@ func TestColumnsNoRows(t *testing.T) {
// https://gitlab.com/cznic/sqlite/-/issues/28
func TestIssue28(t *testing.T) {
tempDir, err := ioutil.TempDir("", "")
tempDir, err := os.MkdirTemp("", "")
if err != nil {
t.Fatal(err)
}
@@ -1304,7 +1379,7 @@ func TestIssue28(t *testing.T) {
// https://gitlab.com/cznic/sqlite/-/issues/30
func TestColumnTypes(t *testing.T) {
tempDir, err := ioutil.TempDir("", "")
tempDir, err := os.MkdirTemp("", "")
if err != nil {
t.Fatal(err)
}
@@ -1374,7 +1449,7 @@ Col 3: DatabaseTypeName "DATE", DecimalSize 0 0 false, Length 922337203685477580
// https://gitlab.com/cznic/sqlite/-/issues/32
func TestColumnTypesNoRows(t *testing.T) {
tempDir, err := ioutil.TempDir("", "")
tempDir, err := os.MkdirTemp("", "")
if err != nil {
t.Fatal(err)
}
@@ -1802,7 +1877,7 @@ func TestIssue51(t *testing.T) {
t.Skip("skipping test in short mode")
}
tempDir, err := ioutil.TempDir("", "")
tempDir, err := os.MkdirTemp("", "")
if err != nil {
t.Fatal(err)
}
@@ -1941,7 +2016,7 @@ const charset = "abcdefghijklmnopqrstuvwxyz" +
// https://gitlab.com/cznic/sqlite/-/issues/53
func TestIssue53(t *testing.T) {
tempDir, err := ioutil.TempDir("", "")
tempDir, err := os.MkdirTemp("", "")
if err != nil {
t.Fatal(err)
}
@@ -1998,7 +2073,7 @@ CREATE TABLE IF NOT EXISTS loginst (
// https://gitlab.com/cznic/sqlite/-/issues/37
func TestPersistPragma(t *testing.T) {
tempDir, err := ioutil.TempDir("", "")
tempDir, err := os.MkdirTemp("", "")
if err != nil {
t.Fatal(err)
}
@@ -2099,7 +2174,7 @@ func checkPragmas(db *sql.DB, pragmas []pragmaCfg) error {
}
func TestInMemory(t *testing.T) {
tempDir, err := ioutil.TempDir("", "")
tempDir, err := os.MkdirTemp("", "")
if err != nil {
t.Fatal(err)
}
@@ -2153,13 +2228,13 @@ func testInMemory(db *sql.DB) error {
return err
}
files, err := ioutil.ReadDir("./")
dirEntries, err := os.ReadDir("./")
if err != nil {
return err
}
for _, file := range files {
if strings.Contains(file.Name(), "memory") {
for _, dirEntry := range dirEntries {
if strings.Contains(dirEntry.Name(), "memory") {
return fmt.Errorf("file was created for in memory database")
}
}
@@ -2224,7 +2299,7 @@ func TestIssue70(t *testing.T) {
// https://gitlab.com/cznic/sqlite/-/issues/66
func TestIssue66(t *testing.T) {
tempDir, err := ioutil.TempDir("", "")
tempDir, err := os.MkdirTemp("", "")
if err != nil {
t.Fatal(err)
}
@@ -2288,7 +2363,7 @@ func TestIssue66(t *testing.T) {
// https://gitlab.com/cznic/sqlite/-/issues/65
func TestIssue65(t *testing.T) {
tempDir, err := ioutil.TempDir("", "")
tempDir, err := os.MkdirTemp("", "")
if err != nil {
t.Fatal(err)
}
@@ -2304,7 +2379,12 @@ func TestIssue65(t *testing.T) {
testIssue65(t, db, true)
if db, err = sql.Open("sqlite", filepath.Join(tempDir, "testissue65b.sqlite")+"?_pragma=busy_timeout%3d10000"); err != nil {
// go1.20rc1, linux/ppc64le VM
// 10000 FAIL
// 20000 PASS, FAIL
// 40000 FAIL
// 80000 PASS, PASS
if db, err = sql.Open("sqlite", filepath.Join(tempDir, "testissue65b.sqlite")+"?_pragma=busy_timeout%3d80000"); err != nil {
t.Fatalf("Failed to open database: %v", err)
}
@@ -2468,7 +2548,7 @@ func TestConstraintUniqueError(t *testing.T) {
// https://gitlab.com/cznic/sqlite/-/issues/92
func TestBeginMode(t *testing.T) {
tempDir, err := ioutil.TempDir("", "")
tempDir, err := os.MkdirTemp("", "")
if err != nil {
t.Fatal(err)
}
@@ -2534,7 +2614,7 @@ func TestBeginMode(t *testing.T) {
// https://gitlab.com/cznic/sqlite/-/issues/94
func TestCancelRace(t *testing.T) {
tempDir, err := ioutil.TempDir("", "")
tempDir, err := os.MkdirTemp("", "")
if err != nil {
t.Fatal(err)
}
@@ -2614,3 +2694,510 @@ func TestCancelRace(t *testing.T) {
})
}
}
//go:embed embed.db
var fs embed.FS
//go:embed embed2.db
var fs2 embed.FS
func TestVFS(t *testing.T) {
fn, f, err := vfs.New(fs)
if err != nil {
t.Fatal(err)
}
defer func() {
if err := f.Close(); err != nil {
t.Error(err)
}
}()
f2n, f2, err := vfs.New(fs2)
if err != nil {
t.Fatal(err)
}
defer func() {
if err := f2.Close(); err != nil {
t.Error(err)
}
}()
db, err := sql.Open("sqlite", "file:embed.db?vfs="+fn)
if err != nil {
t.Fatal(err)
}
defer db.Close()
db2, err := sql.Open("sqlite", "file:embed2.db?vfs="+f2n)
if err != nil {
t.Fatal(err)
}
defer db2.Close()
rows, err := db.Query("select * from t order by i;")
if err != nil {
t.Fatal(err)
}
var a []int
for rows.Next() {
var i, j, k int
if err := rows.Scan(&i, &j, &k); err != nil {
t.Fatal(err)
}
a = append(a, i, j, k)
}
if err := rows.Err(); err != nil {
t.Fatal(err)
}
t.Log(a)
if g, e := fmt.Sprint(a), "[1 2 3 40 50 60]"; g != e {
t.Fatalf("got %q, expected %q", g, e)
}
if rows, err = db2.Query("select * from u order by s;"); err != nil {
t.Fatal(err)
}
var b []string
for rows.Next() {
var x, y string
if err := rows.Scan(&x, &y); err != nil {
t.Fatal(err)
}
b = append(b, x, y)
}
if err := rows.Err(); err != nil {
t.Fatal(err)
}
t.Log(b)
if g, e := fmt.Sprint(b), "[123 xyz abc def]"; g != e {
t.Fatalf("got %q, expected %q", g, e)
}
}
// y = 2^n, except for n < 0 y = 0.
func exp(n int) int {
if n < 0 {
return 0
}
return 1 << n
}
func BenchmarkConcurrent(b *testing.B) {
benchmarkConcurrent(b, "sqlite", []string{"sql", "drv"})
}
func benchmarkConcurrent(b *testing.B, drv string, modes []string) {
for _, mode := range modes {
for _, measurement := range []string{"reads", "writes"} {
for _, writers := range []int{0, 1, 10, 100, 100} {
for _, readers := range []int{0, 1, 10, 100, 100} {
if measurement == "reads" && readers == 0 || measurement == "writes" && writers == 0 {
continue
}
tag := fmt.Sprintf("%s %s readers %d writers %d %s", mode, measurement, readers, writers, drv)
b.Run(tag, func(b *testing.B) { c := &concurrentBenchmark{}; c.run(b, readers, writers, drv, measurement, mode) })
}
}
}
}
}
// The code for concurrentBenchmark is derived from/heavily inspired by
// original code available at
//
// https://github.com/kalafut/go-sqlite-bench
//
// # MIT License
//
// # Copyright (c) 2022 Jim Kalafut
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
type concurrentBenchmark struct {
b *testing.B
drv string
fn string
start chan struct{}
stop chan struct{}
wg sync.WaitGroup
reads int32
records int32
writes int32
}
func (c *concurrentBenchmark) run(b *testing.B, readers, writers int, drv, measurement, mode string) {
c.b = b
c.drv = drv
b.ReportAllocs()
dir := b.TempDir()
fn := filepath.Join(dir, "test.db")
sqlite3.MutexCounters.Disable()
sqlite3.MutexEnterCallers.Disable()
c.makeDB(fn)
b.ResetTimer()
for i := 0; i < b.N; i++ {
b.StopTimer()
c.start = make(chan struct{})
c.stop = make(chan struct{})
sqlite3.MutexCounters.Disable()
sqlite3.MutexEnterCallers.Disable()
c.makeReaders(readers, mode)
c.makeWriters(writers, mode)
sqlite3.MutexCounters.Clear()
sqlite3.MutexCounters.Enable()
sqlite3.MutexEnterCallers.Clear()
//sqlite3.MutexEnterCallers.Enable()
time.AfterFunc(time.Second, func() { close(c.stop) })
b.StartTimer()
close(c.start)
c.wg.Wait()
}
switch measurement {
case "reads":
b.ReportMetric(float64(c.reads), "reads/s")
case "writes":
b.ReportMetric(float64(c.writes), "writes/s")
}
// b.Log(sqlite3.MutexCounters)
// b.Log(sqlite3.MutexEnterCallers)
}
func (c *concurrentBenchmark) randString(n int) string {
b := make([]byte, n)
for i := range b {
b[i] = byte(65 + rand.Intn(26))
}
return string(b)
}
func (c *concurrentBenchmark) mustExecSQL(db *sql.DB, sql string) {
var err error
for i := 0; i < 100; i++ {
if _, err = db.Exec(sql); err != nil {
if c.retry(err) {
continue
}
c.b.Fatalf("%s: %v", sql, err)
}
return
}
c.b.Fatalf("%s: %v", sql, err)
}
func (c *concurrentBenchmark) mustExecDrv(db driver.Conn, sql string) {
var err error
for i := 0; i < 100; i++ {
if _, err = db.(driver.Execer).Exec(sql, nil); err != nil {
if c.retry(err) {
continue
}
c.b.Fatalf("%s: %v", sql, err)
}
return
}
c.b.Fatalf("%s: %v", sql, err)
}
func (c *concurrentBenchmark) makeDB(fn string) {
const quota = 1e6
c.fn = fn
db := c.makeSQLConn()
defer db.Close()
c.mustExecSQL(db, "CREATE TABLE foo (id INTEGER NOT NULL PRIMARY KEY, name TEXT)")
tx, err := db.Begin()
if err != nil {
c.b.Fatal(err)
}
stmt, err := tx.Prepare("INSERT INTO FOO(name) VALUES($1)")
if err != nil {
c.b.Fatal(err)
}
for i := int32(0); i < quota; i++ {
if _, err = stmt.Exec(c.randString(30)); err != nil {
c.b.Fatal(err)
}
}
if err := tx.Commit(); err != nil {
c.b.Fatal(err)
}
c.records = quota
// Warm the cache.
rows, err := db.Query("SELECT * FROM foo")
if err != nil {
c.b.Fatal(err)
}
for rows.Next() {
var id int
var name string
err = rows.Scan(&id, &name)
if err != nil {
c.b.Fatal(err)
}
}
}
func (c *concurrentBenchmark) makeSQLConn() *sql.DB {
db, err := sql.Open(c.drv, c.fn)
if err != nil {
c.b.Fatal(err)
}
db.SetMaxOpenConns(0)
c.mustExecSQL(db, "PRAGMA busy_timeout=10000")
c.mustExecSQL(db, "PRAGMA synchronous=NORMAL")
c.mustExecSQL(db, "PRAGMA journal_mode=WAL")
return db
}
func (c *concurrentBenchmark) makeDrvConn() driver.Conn {
db, err := sql.Open(c.drv, c.fn)
if err != nil {
c.b.Fatal(err)
}
drv := db.Driver()
if err := db.Close(); err != nil {
c.b.Fatal(err)
}
conn, err := drv.Open(c.fn)
if err != nil {
c.b.Fatal(err)
}
c.mustExecDrv(conn, "PRAGMA busy_timeout=10000")
c.mustExecDrv(conn, "PRAGMA synchronous=NORMAL")
c.mustExecDrv(conn, "PRAGMA journal_mode=WAL")
return conn
}
func (c *concurrentBenchmark) retry(err error) bool {
s := strings.ToLower(err.Error())
return strings.Contains(s, "lock") || strings.Contains(s, "busy")
}
func (c *concurrentBenchmark) makeReaders(n int, mode string) {
var wait sync.WaitGroup
wait.Add(n)
c.wg.Add(n)
for i := 0; i < n; i++ {
switch mode {
case "sql":
go func() {
db := c.makeSQLConn()
defer func() {
db.Close()
c.wg.Done()
}()
wait.Done()
<-c.start
for i := 1; ; i++ {
select {
case <-c.stop:
return
default:
}
recs := atomic.LoadInt32(&c.records)
id := recs * int32(i) % recs
rows, err := db.Query("SELECT * FROM foo WHERE id=$1", id)
if err != nil {
if c.retry(err) {
continue
}
c.b.Fatal(err)
}
for rows.Next() {
var id int
var name string
err = rows.Scan(&id, &name)
if err != nil {
c.b.Fatal(err)
}
}
if err := rows.Close(); err != nil {
c.b.Fatal(err)
}
atomic.AddInt32(&c.reads, 1)
}
}()
case "drv":
go func() {
conn := c.makeDrvConn()
defer func() {
conn.Close()
c.wg.Done()
}()
q := conn.(driver.Queryer)
wait.Done()
<-c.start
for i := 1; ; i++ {
select {
case <-c.stop:
return
default:
}
recs := atomic.LoadInt32(&c.records)
id := recs * int32(i) % recs
rows, err := q.Query("SELECT * FROM foo WHERE id=$1", []driver.Value{int64(id)})
if err != nil {
if c.retry(err) {
continue
}
c.b.Fatal(err)
}
var dest [2]driver.Value
for {
if err := rows.Next(dest[:]); err != nil {
if err != io.EOF {
c.b.Fatal(err)
}
break
}
}
if err := rows.Close(); err != nil {
c.b.Fatal(err)
}
atomic.AddInt32(&c.reads, 1)
}
}()
default:
panic(todo(""))
}
}
wait.Wait()
}
func (c *concurrentBenchmark) makeWriters(n int, mode string) {
var wait sync.WaitGroup
wait.Add(n)
c.wg.Add(n)
for i := 0; i < n; i++ {
switch mode {
case "sql":
go func() {
db := c.makeSQLConn()
defer func() {
db.Close()
c.wg.Done()
}()
wait.Done()
<-c.start
for {
select {
case <-c.stop:
return
default:
}
if _, err := db.Exec("INSERT INTO FOO(name) VALUES($1)", c.randString(30)); err != nil {
if c.retry(err) {
continue
}
c.b.Fatal(err)
}
atomic.AddInt32(&c.records, 1)
atomic.AddInt32(&c.writes, 1)
}
}()
case "drv":
go func() {
conn := c.makeDrvConn()
defer func() {
conn.Close()
c.wg.Done()
}()
e := conn.(driver.Execer)
wait.Done()
<-c.start
for {
select {
case <-c.stop:
return
default:
}
if _, err := e.Exec("INSERT INTO FOO(name) VALUES($1)", []driver.Value{c.randString(30)}); err != nil {
if c.retry(err) {
continue
}
c.b.Fatal(err)
}
atomic.AddInt32(&c.records, 1)
atomic.AddInt32(&c.writes, 1)
}
}()
default:
panic(todo(""))
}
}
wait.Wait()
}

View File

@@ -28,9 +28,6 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201126233918-771906719818/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210902050250-f475640dd07b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac h1:oN6lz7iLW/YC7un8pq+9bOLyXrprv2+DKfkJY+2LJJw=
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -142,8 +139,6 @@ modernc.org/libc v1.13.2 h1:GCFjY9bmwDZ/TJC4OZOUWaNgxIxwb104C/QZrqpcVEA=
modernc.org/libc v1.13.2/go.mod h1:npFeGWjmZTjFeWALQLrvklVmAxv4m80jnG3+xI8FdJk=
modernc.org/mathutil v1.1.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
modernc.org/mathutil v1.4.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
modernc.org/mathutil v1.4.1 h1:ij3fYGe8zBF4Vu+g0oT7mB06r8sqGWKuJu1yXeR4by8=
modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
modernc.org/memory v1.0.4/go.mod h1:nV2OApxradM3/OVbs2/0OsP6nPfakXpi50C7dcoHXlc=
modernc.org/memory v1.0.5 h1:XRch8trV7GgvTec2i7jc33YlUI0RKVDBvZ5eZ5m8y14=

View File

@@ -188,7 +188,7 @@ func pronounceNum(n uint32) string {
num uint32
name string
}{
{num: 1000000000, name: `billion`},
{num: 1000000000, name: `1e9`},
{num: 1000000, name: `million`},
{num: 1000, name: `thousand`},
{num: 100, name: `hundred`},

BIN
embed.db Normal file

Binary file not shown.

BIN
embed2.db Normal file

Binary file not shown.

View File

@@ -3,7 +3,6 @@ package main
import (
"database/sql"
"fmt"
"io/ioutil"
"os"
"path/filepath"
@@ -18,7 +17,7 @@ func main() {
}
func main1() error {
dir, err := ioutil.TempDir("", "test-")
dir, err := os.MkdirTemp("", "test-")
if err != nil {
return err
}

15
go.mod
View File

@@ -1,10 +1,19 @@
module github.com/glebarez/go-sqlite
go 1.16
go 1.17
require (
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab
github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26
golang.org/x/sys v0.4.0
modernc.org/libc v1.22.2
modernc.org/mathutil v1.5.0
modernc.org/sqlite v1.20.2
modernc.org/sqlite v1.20.3
)
require (
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/mattn/go-isatty v0.0.17 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230126093431-47fa9a501578 // indirect
modernc.org/memory v1.5.0 // indirect
)

20
go.sum
View File

@@ -1,21 +1,25 @@
github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4MedaY=
github.com/chzyer/readline v1.5.0/go.mod h1:x22KAscuvRqlLoK9CsoYsmxoXZMMFVyOl86cAH8qUic=
github.com/chzyer/test v0.0.0-20210722231415-061457976a23/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ=
github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk=
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/remyoudompheng/bigfft v0.0.0-20230126093431-47fa9a501578 h1:VstopitMQi3hZP0fzvnsLmzXZdQGc4bEcgu24cp+d4M=
github.com/remyoudompheng/bigfft v0.0.0-20230126093431-47fa9a501578/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -30,8 +34,9 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab h1:2QkjZIsXupsJbJIdSjjUOgWK3aEtzyuh2mPt3l/CkeU=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18=
golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -62,12 +67,13 @@ modernc.org/libc v1.22.2/go.mod h1:uvQavJ1pZ0hIoC/jfqNoMLURIMhKzINIWypNM17puug=
modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ=
modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
modernc.org/memory v1.3.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU=
modernc.org/memory v1.4.0 h1:crykUfNSnMAXaOJnnxcSzbUGMqkLWjklJKkBK2nwZwk=
modernc.org/memory v1.4.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU=
modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds=
modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU=
modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
modernc.org/sqlite v1.20.2 h1:9AaVzJH1Yf0u9iOZRjjuvqxLoGqybqVFbAUC5rvi9u8=
modernc.org/sqlite v1.20.2/go.mod h1:zKcGyrICaxNTMEHSr1HQ2GUraP0j+845GYw37+EyT6A=
modernc.org/sqlite v1.20.3 h1:SqGJMMxjj1PHusLxdYxeQSodg7Jxn9WWkaAQjKrntZs=
modernc.org/sqlite v1.20.3/go.mod h1:zKcGyrICaxNTMEHSr1HQ2GUraP0j+845GYw37+EyT6A=
modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw=
modernc.org/tcl v1.15.0/go.mod h1:xRoGotBZ6dU+Zo2tca+2EqVEeMmOUBzHnhIwq4YrVnE=
modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=

10
norlimit.go Normal file
View File

@@ -0,0 +1,10 @@
// Copyright 2021 The Sqlite Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build windows
// +build windows
package sqlite // import "modernc.org/sqlite"
func setMaxOpenFiles(n int) error { return nil }

View File

@@ -2,9 +2,18 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !freebsd
// +build !freebsd
//go:build freebsd
// +build freebsd
package sqlite // import "modernc.org/sqlite"
func setMaxOpenFiles(n int) error { return nil }
import (
"golang.org/x/sys/unix"
)
func setMaxOpenFiles(n int64) error {
var rLimit unix.Rlimit
rLimit.Max = n
rLimit.Cur = n
return unix.Setrlimit(unix.RLIMIT_NOFILE, &rLimit)
}

View File

@@ -2,6 +2,9 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build linux || darwin || netbsd || openbsd
// +build linux darwin netbsd openbsd
package sqlite // import "modernc.org/sqlite"
import (
@@ -10,7 +13,7 @@ import (
func setMaxOpenFiles(n int64) error {
var rLimit unix.Rlimit
rLimit.Max = n
rLimit.Cur = n
rLimit.Max = uint64(n)
rLimit.Cur = uint64(n)
return unix.Setrlimit(unix.RLIMIT_NOFILE, &rLimit)
}

View File

@@ -2,7 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:generate go run generator.go
//go:generate go run generator.go -full-path-comments
package sqlite // import "modernc.org/sqlite"
@@ -695,14 +695,14 @@ type tx struct {
c *conn
}
func newTx(c *conn) (*tx, error) {
func newTx(c *conn, opts driver.TxOptions) (*tx, error) {
r := &tx{c: c}
var sql string
if c.beginMode != "" {
sql := "begin"
if !opts.ReadOnly && c.beginMode != "" {
sql = "begin " + c.beginMode
} else {
sql = "begin"
}
if err := r.exec(context.Background(), sql); err != nil {
return nil, err
}
@@ -790,7 +790,7 @@ type conn struct {
}
func newConn(dsn string) (*conn, error) {
var query string
var query, vfsName string
// Parse the query parameters from the dsn and them from the dsn if not prefixed by file:
// https://github.com/mattn/go-sqlite3/blob/3392062c729d77820afc1f5cae3427f0de39e954/sqlite3.go#L1046
@@ -798,6 +798,12 @@ func newConn(dsn string) (*conn, error) {
pos := strings.IndexRune(dsn, '?')
if pos >= 1 {
query = dsn[pos+1:]
var err error
vfsName, err = getVFSName(query)
if err != nil {
return nil, err
}
if !strings.HasPrefix(dsn, "file:") {
dsn = dsn[:pos]
}
@@ -806,6 +812,7 @@ func newConn(dsn string) (*conn, error) {
c := &conn{tls: libc.NewTLS()}
db, err := c.openV2(
dsn,
vfsName,
sqlite3.SQLITE_OPEN_READWRITE|sqlite3.SQLITE_OPEN_CREATE|
sqlite3.SQLITE_OPEN_FULLMUTEX|
sqlite3.SQLITE_OPEN_URI,
@@ -846,6 +853,23 @@ func stmtLog(tls *libc.TLS, type1 uint32, cd uintptr, pd uintptr, xd uintptr) in
return sqlite3.SQLITE_OK
}
func getVFSName(query string) (r string, err error) {
q, err := url.ParseQuery(query)
if err != nil {
return "", err
}
for _, v := range q["vfs"] {
if r != "" && r != v {
return "", fmt.Errorf("conflicting vfs query parameters: %v", q["vfs"])
}
r = v
}
return r, nil
}
func applyQueryParams(c *conn, query string) error {
q, err := url.ParseQuery(query)
if err != nil {
@@ -1285,8 +1309,8 @@ func (c *conn) extendedResultCodes(on bool) error {
// const char *zVfs /* Name of VFS module to use */
//
// );
func (c *conn) openV2(name string, flags int32) (uintptr, error) {
var p, s uintptr
func (c *conn) openV2(name, vfsName string, flags int32) (uintptr, error) {
var p, s, vfs uintptr
defer func() {
if p != 0 {
@@ -1295,6 +1319,9 @@ func (c *conn) openV2(name string, flags int32) (uintptr, error) {
if s != 0 {
c.free(s)
}
if vfs != 0 {
c.free(vfs)
}
}()
p, err := c.malloc(int(ptrSize))
@@ -1306,7 +1333,13 @@ func (c *conn) openV2(name string, flags int32) (uintptr, error) {
return 0, err
}
if rc := sqlite3.Xsqlite3_open_v2(c.tls, s, p, flags, 0); rc != sqlite3.SQLITE_OK {
if vfsName != "" {
if vfs, err = libc.CString(vfsName); err != nil {
return 0, err
}
}
if rc := sqlite3.Xsqlite3_open_v2(c.tls, s, p, flags, vfs); rc != sqlite3.SQLITE_OK {
return 0, c.errstr(rc)
}
@@ -1352,7 +1385,7 @@ func (c *conn) Begin() (driver.Tx, error) {
}
func (c *conn) begin(ctx context.Context, opts driver.TxOptions) (t driver.Tx, err error) {
return newTx(c)
return newTx(c, opts)
}
// Close invalidates and potentially stops any current prepared statements and