add tpch. it builds/runs but it's broken. updates #27.

tpch/driver/driver.go

@@ -0,0 +1,54 @@
+// Copyright 2021 The Sqlite Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package driver
+
+import (
+	"database/sql"
+	"fmt"
+)
+
+// System Under Test.
+type SUT interface {
+	CreateTables() error
+	InsertCustomer() string
+	InsertLineItem() string
+	InsertNation() string
+	InsertOrders() string
+	InsertPart() string
+	InsertPartSupp() string
+	InsertProperty() string
+	InsertRegion() string
+	InsertSupplier() string
+	Name() string
+	OpenDB() (*sql.DB, error)
+	OpenMem() (SUT, *sql.DB, error)
+	Q1() string
+	Q2() string
+	QProperty() string
+	SetWD(path string) error
+}
+
+var registered = map[string]SUT{}
+
+func Open(name string) SUT {
+	return registered[name]
+}
+
+func Register(sut SUT) {
+	nm := sut.Name()
+	if _, ok := registered[nm]; ok {
+		panic(fmt.Errorf("already registered: %s", nm))
+	}
+
+	registered[nm] = sut
+}
+
+func List() []string {
+	r := []string{}
+	for k := range registered {
+		r = append(r, k)
+	}
+	return r
+}

tpch/driver/drivers/sqlite.go

@@ -0,0 +1,49 @@
+// Copyright 2021 The Sqlite Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package drivers
+
+import (
+	"database/sql"
+	"path/filepath"
+
+	_ "modernc.org/sqlite"
+	"modernc.org/sqlite/tpch/driver"
+)
+
+func init() {
+	driver.Register(newSQLite())
+}
+
+var _ driver.SUT = (*sqlite)(nil)
+
+type sqlite struct {
+	*sqlite3
+}
+
+func newSQLite() *sqlite {
+	return &sqlite{newSQLite3()}
+}
+
+func (b *sqlite) Name() string { return "sqlite" }
+
+func (b *sqlite) OpenDB() (*sql.DB, error) {
+	pth := filepath.Join(b.wd, "sqlite.db")
+	db, err := sql.Open(b.Name(), pth)
+	if err != nil {
+		return nil, err
+	}
+
+	b.db = db
+	return db, nil
+}
+
+func (b *sqlite) OpenMem() (driver.SUT, *sql.DB, error) {
+	db, err := sql.Open(b.Name(), "file::memory:")
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return &sqlite{&sqlite3{db: db}}, db, nil
+}

tpch/driver/drivers/sqlite3.go

@@ -0,0 +1,272 @@
+// Copyright 2032 The Sqlite Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package drivers
+
+import (
+	"database/sql"
+	"path/filepath"
+
+	_ "github.com/mattn/go-sqlite3"
+	"modernc.org/sqlite/tpch/driver"
+)
+
+const (
+	aQ1 = `select
+			l_returnflag,
+			l_linestatus,
+			sum(l_quantity)/100. as sum_qty,
+			sum(l_extendedprice)/100. as sum_base_price,
+			sum(l_extendedprice*(100-l_discount))/10000. as sum_disc_price,
+			sum(l_extendedprice*(100-l_discount)*(100+l_tax))/1000000. as sum_charge,
+			avg(l_quantity)/100 as avg_qty,
+			avg(l_extendedprice)/100 as avg_price,
+			avg(l_discount)/100 as avg_disc,
+			count(*) as count_order
+		from
+			lineitem
+		where
+			l_shipdate <= date('1998-12-01', printf('-%d day', ?1))
+		group by
+			l_returnflag,
+			l_linestatus
+		order by
+			l_returnflag,
+			l_linestatus;
+`
+	aQ2 = `select
+			s_acctbal,
+			s_name,
+			n_name,
+			p_partkey,
+			p_mfgr,
+			s_address,
+			s_phone,
+			s_comment
+		from
+			part,
+			supplier,
+			partsupp,
+			nation,
+			region
+		where
+			p_partkey == ps_partkey
+			and s_suppkey == ps_suppkey
+			and p_size == ?1
+			and p_type like '%' || ?2
+			and s_nationkey == n_nationkey
+			and n_regionkey == r_regionkey
+			and r_name == ?3
+			and ps_supplycost == (
+				select
+					min(ps_supplycost)
+				from
+					partsupp, supplier,
+					nation, region
+				where
+					p_partkey == ps_partkey
+					and s_suppkey == ps_suppkey
+					and s_nationkey == n_nationkey
+					and n_regionkey == r_regionkey
+					and r_name == ?3
+			)
+		order by
+			s_acctbal desc,
+			n_name,
+			s_name,
+			p_partkey
+		limit 100;
+`
+)
+
+func init() {
+	driver.Register(newSQLite3())
+}
+
+var _ driver.SUT = (*sqlite3)(nil)
+
+type sqlite3 struct {
+	db *sql.DB
+	wd string
+}
+
+func newSQLite3() *sqlite3 {
+	return &sqlite3{}
+}
+
+func (b *sqlite3) Name() string            { return "sqlite3" }
+func (b *sqlite3) SetWD(path string) error { b.wd = path; return nil }
+
+func (b *sqlite3) OpenDB() (*sql.DB, error) {
+	pth := filepath.Join(b.wd, "sqlite3.db")
+	db, err := sql.Open(b.Name(), pth)
+	if err != nil {
+		return nil, err
+	}
+
+	b.db = db
+	return db, nil
+}
+
+func (b *sqlite3) OpenMem() (driver.SUT, *sql.DB, error) {
+	db, err := sql.Open(b.Name(), "file::memory:")
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return &sqlite3{db: db}, db, nil
+}
+
+func (b *sqlite3) CreateTables() error {
+	tx, err := b.db.Begin()
+	if err != nil {
+		return err
+	}
+
+	if _, err = tx.Exec(`
+		create table part (
+			p_partkey      int, -- SF*200,000 are populated
+			p_name         string,
+			p_mfgr         string,
+			p_brand        string,
+			p_type         string,
+			p_size         int,
+			p_container    string,
+			p_retail_price int,
+			p_comment      string
+		);
+
+		create table supplier (
+			s_suppkey   int, -- SF*10,000 are populated
+			s_name      string,
+			s_address   string,
+			s_nationkey int, -- Foreign Key to N_NATIONKEY
+			s_phone     string,
+			s_acctbal    int,
+			s_comment   string
+		);
+
+		create table partsupp (
+			ps_partkey    int, -- Foreign Key to P_PARTKEY
+			ps_suppkey    int, -- Foreign Key to S_SUPPKEY
+			ps_availqty   int,
+			ps_supplycost int,
+			ps_comment    string
+		);
+
+		create table customer (
+			c_custkey    int, -- SF*150,000 are populated
+			c_name       string,
+			c_address    string,
+			c_nationkey  int, -- Foreign Key to N_NATIONKEY
+			c_phone      string,
+			c_acctbal    int,
+			c_mktsegment string,
+			c_commnet    string
+		);
+
+		create table orders (
+			o_orderkey     int, -- SF*1,500,000 are sparsely populated
+			o_custkey       int, -- Foreign Key to C_CUSTKEY
+			o_orderstatus   string,
+			o_totalprice    int,
+			o_orderdate     time,
+			o_orderpriority string,
+			o_clerk         string,
+			o_shippriority  int,
+			o_comment       string
+		);
+
+		create table lineitem (
+			l_orderkey      int, -- Foreign Key to O_ORDERKEY
+			l_partkey       int, -- Foreign key to P_PARTKEY, first part of the compound Foreign Key to (PS_PARTKEY, PS_SUPPKEY) with L_SUPPKEY
+			l_suppkey       int, -- Foreign key to S_SUPPKEY, second part of the compound Foreign Key to (PS_PARTKEY, PS_SUPPKEY) with L_PARTKEY
+			l_linenumber    int,
+			l_quantity      int,
+			l_extendedprice int,
+			l_discount      int,
+			l_tax           int,
+			l_returnflag    string,
+			l_linestatus    string,
+			l_shipdate      time,
+			l_commitdate    time,
+			l_receiptdate   time,
+			l_shipinstruct  string,
+			l_shipmode      string,
+			l_comment       string
+		);
+
+		create table nation (
+			n_nationkey int, -- 25 nations are populated
+			n_name      string,
+			n_regionkey int, -- Foreign Key to R_REGIONKEY
+			n_comment   string
+		);
+
+		create table region (
+			r_regionkey int, -- 5 regions are populated
+			r_name      string,
+			r_comment   string
+		);
+
+		create table _property (
+			ctime   time,
+			sf      int,
+			recs    int
+		);
+
+		`); err != nil {
+		return err
+	}
+
+	return tx.Commit()
+}
+
+func (b *sqlite3) InsertSupplier() string {
+	return "insert into supplier values (?1, ?2, ?3, ?4, ?5, ?6, ?7)"
+}
+
+func (b *sqlite3) InsertPart() string {
+	return "insert into part values (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)"
+}
+
+func (b *sqlite3) InsertPartSupp() string {
+	return "insert into partsupp values (?1, ?2, ?3, ?4, ?5)"
+}
+
+func (b *sqlite3) InsertCustomer() string {
+	return "insert into customer values (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)"
+}
+
+func (b *sqlite3) InsertOrders() string {
+	return "insert into orders values (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)"
+}
+
+func (b *sqlite3) InsertLineItem() string {
+	return "insert into lineitem values (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16)"
+}
+
+func (b *sqlite3) InsertNation() string {
+	return "insert into nation values (?1, ?2, ?3, ?4)"
+}
+
+func (b *sqlite3) InsertRegion() string {
+	return "insert into region values (?1, ?2, ?3)"
+}
+
+func (b *sqlite3) InsertProperty() string {
+	return "insert into _property values (?1, ?2, ?3)"
+}
+
+func (b *sqlite3) QProperty() string {
+	return "select * from _property"
+}
+
+func (b *sqlite3) Q1() string {
+	return aQ1
+}
+
+func (b *sqlite3) Q2() string {
+	return aQ2
+}

tpch/main.go

@@ -0,0 +1,93 @@
+// Copyright 2021 The Sqlite Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"strings"
+
+	"modernc.org/sqlite/tpch/driver"
+	_ "modernc.org/sqlite/tpch/driver/drivers"
+)
+
+// 4.1.3.1 Scale factors used for the test database must be chosen from the set
+// of fixed scale factors defined as follows:
+//
+// 	1, 10, 30, 100, 300, 1000, 3000, 10000, 30000, 100000
+//
+// The database size is defined with reference to scale factor 1 (i.e., SF = 1;
+// approximately 1GB as per Clause 4.2.5), the minimum required size for a test
+// database. Therefore, the following series of database sizes corresponds to
+// the series of scale factors and must be used in the metric names QphH@Size
+// and Price-per-QphH@Size (see Clause 5.4), as well as in the executive
+// summary statement (see Appendix E):
+//
+//	1GB, 10GB, 30GB, 100GB, 300GB, 1000GB, 3000GB, 10000GB, 30000GB, 100000GB
+//
+//	Where GB stands for gigabyte, defined to be 2^30 bytes.
+//
+// Comment 1: Although the minimum size of the test database for a valid
+// performance test is 1GB (i.e., SF = 1), a test database of 3GB (i.e., SF =
+// 3) is not permitted. This requirement is intended to encourage comparability
+// of results at the low end and to ensure a substantial actual difference in
+// test database sizes.
+//
+// Comment 2: The maximum size of the test database for a valid performance
+// test is currently set at 100000 (i.e., SF = 100,000). The TPC recognizes
+// that additional benchmark development work is necessary to allow TPC-H to
+// scale beyond that limit.
+
+func main() {
+	log.SetFlags(0)
+
+	dbgen := flag.Bool("dbgen", false, "Generate test DB. (Several GB)")
+	list := flag.Bool("list", false, "List registered drivers")
+	maxrecs := flag.Int("recs", -1, "Limit table recs. Use specs if < 0.")
+	mem := flag.Bool("mem", false, "Run test with DB in mem, if SUT supports that.")
+	pseudotext := flag.Bool("pseudotext", false, "generate testdata/pseudotext (300MB).")
+	q := flag.Int("q", 0, "Query to run, if > 0. Valid values in [1, 2].")
+	sf := flag.Int("sf", 1, "Scale factor.")
+	sutName := flag.String("sut", "", "System Under Test name.")
+	verbose := flag.Bool("v", false, "Verbose.")
+
+	flag.Parse()
+	maxRecs = *maxrecs
+	switch *sf {
+	case 1, 10, 30, 100, 300, 1000, 3000, 10000, 30000, 100000:
+		// nop
+	default:
+		log.Fatalf("Invalid -sf value: %v", *sf)
+	}
+
+	var sut driver.SUT
+	nm := strings.TrimSpace(*sutName)
+	if nm == "" && !*pseudotext && !*list {
+		log.Fatal("Missing SUT name")
+	}
+
+	if nm != "" {
+		if sut = driver.Open(nm); sut == nil {
+			log.Fatalf("SUT not registered: %s", nm)
+		}
+	}
+
+	var err error
+	switch {
+	case *list:
+		fmt.Println(driver.List())
+	case *pseudotext:
+		err = genPseudotext()
+	case *dbgen:
+		err = dbGen(sut, *sf)
+	case *q > 0:
+		err = run(sut, *mem, *q, *sf, *verbose)
+	}
+
+	if err != nil {
+		log.Fatal(err)
+	}
+}

tpch/q.go

@@ -0,0 +1,188 @@
+// Copyright 2032 The Sqlite Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"database/sql"
+	"fmt"
+	"math"
+	"time"
+
+	"modernc.org/sqlite/tpch/driver"
+)
+
+func cpDB(sut driver.SUT, src, dest *sql.DB) error {
+	for _, v := range []struct {
+		t, i string
+		int
+	}{
+		{"_property", sut.InsertProperty(), 3},
+		{"customer", sut.InsertCustomer(), 8},
+		{"lineitem", sut.InsertLineItem(), 16},
+		{"nation", sut.InsertNation(), 4},
+		{"orders", sut.InsertOrders(), 9},
+		{"part", sut.InsertPart(), 9},
+		{"partsupp", sut.InsertPartSupp(), 5},
+		{"region", sut.InsertRegion(), 3},
+		{"supplier", sut.InsertSupplier(), 7},
+	} {
+		if err := cpTable(src, dest, v.t, v.i, v.int); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func cpTable(src, dest *sql.DB, tn, ins string, ncols int) (err error) {
+	rows, err := src.Query(fmt.Sprintf("select * from %s", tn))
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		if e := rows.Close(); e != nil && err == nil {
+			err = e
+		}
+	}()
+
+	var tx *sql.Tx
+	var stmt *sql.Stmt
+	row := make([]interface{}, ncols)
+	data := make([]interface{}, ncols)
+	for i := range data {
+		data[i] = &row[i]
+	}
+	for i := 0; rows.Next(); i++ {
+		if i%1000 == 0 {
+			if i != 0 {
+				if err = tx.Commit(); err != nil {
+					return err
+				}
+			}
+
+			if tx, err = dest.Begin(); err != nil {
+				return err
+			}
+
+			if stmt, err = tx.Prepare(ins); err != nil {
+				return err
+			}
+		}
+
+		if err = rows.Scan(data...); err != nil {
+			return err
+		}
+
+		for i, v := range row {
+			switch x := v.(type) {
+			case time.Time:
+			case int64:
+			case []byte:
+				row[i] = string(x)
+			default:
+				panic("TODO")
+			}
+		}
+		if _, err = stmt.Exec(row...); err != nil {
+			return err
+		}
+
+	}
+	if err = rows.Err(); err != nil {
+		return err
+	}
+
+	return tx.Commit()
+}
+
+func run(sut driver.SUT, mem bool, n, sf int, verbose bool) (err error) {
+	pth := pthForSUT(sut, sf)
+	if err := sut.SetWD(pth); err != nil {
+		return err
+	}
+
+	db, err := sut.OpenDB()
+	if err != nil {
+		return err
+	}
+
+	defer func(db *sql.DB) {
+		if cerr := db.Close(); cerr != nil && err == nil {
+			err = cerr
+		}
+	}(db)
+
+	if mem {
+		msut, mdb, err := sut.OpenMem()
+		if err != nil {
+			return err
+		}
+
+		if err = msut.CreateTables(); err != nil {
+			return err
+		}
+
+		if err = cpDB(sut, db, mdb); err != nil {
+			return err
+		}
+
+		sut, db = msut, mdb
+	}
+
+	rng := newRng(0, math.MaxInt64)
+	rng.r.Seed(time.Now().UnixNano())
+	t0 := time.Now()
+
+	defer func() {
+		fmt.Println(time.Since(t0)) //TODO -> result.db
+	}()
+
+	switch n {
+	case 1:
+		return exec(db, 10, sut.Q1(), verbose, rng.randomValue(60, 120))
+	case 2:
+		return exec(db, 8, sut.Q2(), verbose, rng.randomValue(1, 50), rng.types(), rng.regions())
+	default:
+		return fmt.Errorf("No query/test #%d", n)
+	}
+}
+
+func exec(db *sql.DB, ncols int, q string, verbose bool, arg ...interface{}) error {
+	rec := make([]interface{}, ncols)
+	data := make([]interface{}, ncols)
+	for i := range data {
+		data[i] = &rec[i]
+	}
+
+	rows, err := db.Query(q, arg...)
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		if e := rows.Close(); e != nil && err == nil {
+			err = e
+		}
+	}()
+
+	for rows.Next() {
+		if !verbose {
+			continue
+		}
+
+		if err = rows.Scan(data...); err != nil {
+			return err
+		}
+
+		for i, v := range rec {
+			if b, ok := v.([]byte); ok {
+				rec[i] = string(b)
+			}
+		}
+
+		fmt.Println(rec)
+	}
+	return rows.Err()
+}