release v1.4.0-beta1v1.4.0-beta1

1 files changed, 224 insertions, 52 deletions

diff --git a/generator.go b/generator.go
index 030d0e5..2f7c4f0 100644
--- a/generator.go
+++ b/generator.go
@@ -8,6 +8,7 @@ package main
 
 import (
 	"archive/zip"
+	"bufio"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -23,23 +24,30 @@ var (
 	config = []string{
 		"-DHAVE_USLEEP",
 		"-DLONGDOUBLE_TYPE=double",
+		//TODO- "-DSQLITE_DEBUG", //TODO-
 		"-DSQLITE_DEFAULT_MEMSTATUS=0",
 		"-DSQLITE_DEFAULT_WAL_SYNCHRONOUS=1",
 		"-DSQLITE_DQS=0",
+		//TODO- "-DSQLITE_ENABLE_API_ARMOR",     //TODO-
 		"-DSQLITE_ENABLE_UNLOCK_NOTIFY", // Adds sqlite3_unlock_notify().
 		"-DSQLITE_LIKE_DOESNT_MATCH_BLOBS",
 		"-DSQLITE_MAX_EXPR_DEPTH=0",
-		"-DSQLITE_MAX_MMAP_SIZE=0", // mmap somehow fails on linux/386
+		//TODO 386 "-DSQLITE_MAX_MMAP_SIZE=0", // mmap somehow fails on linux/386
+		//TODO- "-DSQLITE_MEMDEBUG", //TODO-
 		"-DSQLITE_MUTEX_APPDEF=1",
 		"-DSQLITE_MUTEX_NOOP",
 		"-DSQLITE_OMIT_DECLTYPE",
 		"-DSQLITE_OMIT_PROGRESS_CALLBACK",
-		"-DSQLITE_OMIT_SHARED_CACHE",
+		"-DSQLITE_OMIT_UTF16",
+		"-DSQLITE_TEST",
 		"-DSQLITE_THREADSAFE=2", // Multi-thread
-		"-DSQLITE_USE_ALLOCA",
+		"-ccgo-long-double-is-double",
+		//TODO- "-ccgo-verify-structs", //TODO-
 		// "-DSQLITE_OMIT_DEPRECATED", // mptest needs deprecated sqlite3_trace.
-		//TODO- "-DSQLITE_DEBUG",    //TODO-
-		//TODO- "-DSQLITE_MEMDEBUG", //TODO-
+		// "-DSQLITE_OMIT_LOAD_EXTENSION", // mptest needs this
+		// "-DSQLITE_OMIT_SHARED_CACHE",
+		// "-DSQLITE_USE_ALLOCA",
+		//TODO "-DHAVE_MALLOC_USABLE_SIZE"
 	}
 
 	downloads = []struct {
@@ -47,12 +55,12 @@ var (
 		sz       int
 		dev      bool
 	}{
-		{sqliteDir, "https://www.sqlite.org/2019/sqlite-amalgamation-3300100.zip", 2240, false},
-		{sqliteSrcDir, "https://www.sqlite.org/2019/sqlite-src-3300100.zip", 12060, false},
+		{sqliteDir, "https://www.sqlite.org/2020/sqlite-amalgamation-3320300.zip", 2240, false},
+		{sqliteSrcDir, "https://www.sqlite.org/2020/sqlite-src-3320300.zip", 12060, false},
 	}
 
-	sqliteDir    = filepath.FromSlash("testdata/sqlite-amalgamation-3300100")
-	sqliteSrcDir = filepath.FromSlash("testdata/sqlite-src-3300100")
+	sqliteDir    = filepath.FromSlash("testdata/sqlite-amalgamation-3320300")
+	sqliteSrcDir = filepath.FromSlash("testdata/sqlite-src-3320300")
 )
 
 func download() {
@@ -133,16 +141,21 @@ func download() {
 
 						defer rc.Close()
 
-						dname := filepath.Join(root, f.Name)
-						g, err := os.Create(dname)
+						file, err := os.OpenFile(filepath.Join(root, f.Name), os.O_CREATE|os.O_WRONLY, fi.Mode())
 						if err != nil {
 							return err
 						}
 
-						defer g.Close()
+						w := bufio.NewWriter(file)
+						if _, err = io.Copy(w, rc); err != nil {
+							return err
+						}
 
-						n, err = io.Copy(g, rc)
-						return err
+						if err := w.Flush(); err != nil {
+							return err
+						}
+
+						return file.Close()
 					}(); err != nil {
 						return err
 					}
@@ -163,65 +176,224 @@ func fail(s string, args ...interface{}) {
 
 func main() {
 	download()
-	out, err := exec.Command(
-		"gocc",
-		append(
-			[]string{
-				filepath.Join(sqliteDir, "sqlite3.c"),
-				"-o", filepath.FromSlash(fmt.Sprintf("internal/bin/sqlite_%s_%s.go", runtime.GOOS, runtime.GOARCH)),
-				"-qbec-pkgname", "bin",
-			},
-			config...)...,
-	).CombinedOutput()
+	makeSqlite()
+	makeMpTest()
+	makeSpeedTest()
+	makeTestfixture()
+
+	dst := filepath.FromSlash("testdata/tcl")
+	if err := os.MkdirAll(dst, 0770); err != nil {
+		fail("cannot create %q: %v", dst, err)
+	}
+
+	m, err := filepath.Glob(filepath.Join(sqliteSrcDir, "test/*.test"))
 	if err != nil {
-		fail("%s\n%s\n", out, err)
+		fail("cannot glob *.test: %v", err)
 	}
 
-	dir, err := ioutil.TempDir("", "go-generate-")
+	m2, err := filepath.Glob(filepath.Join(sqliteSrcDir, "test/*.tcl"))
 	if err != nil {
-		fail("s\n", err)
+		fail("cannot glob *.tcl: %v", err)
+	}
+
+	m = append(m, m2...)
+	for _, v := range m {
+		f, err := ioutil.ReadFile(v)
+		if err != nil {
+			fail("cannot read %v: %v", v, err)
+		}
+
+		fn := filepath.Join(dst, filepath.Base(v))
+		if err := ioutil.WriteFile(fn, f, 0660); err != nil {
+			fail("cannot write %v: %v", fn, err)
+		}
+	}
+}
+
+func configure() {
+	wd, err := os.Getwd()
+	if err != nil {
+		fail("%s", err)
+	}
+
+	defer os.Chdir(wd)
+
+	if err := os.Chdir(sqliteSrcDir); err != nil {
+		fail("%s", err)
 	}
 
-	defer os.RemoveAll(dir)
+	cmd := newCmd("./configure")
+	if err = cmd.Run(); err != nil {
+		fail("%s\n", err)
+	}
 
-	src := "#include \"sqlite3.h\"\nstatic char _;\n"
-	fn := filepath.Join(dir, "x.c")
-	if err := ioutil.WriteFile(fn, []byte(src), 0660); err != nil {
-		fail("s\n", err)
+	cmd = newCmd("make", "parse.h", "opcodes.h")
+	if err = cmd.Run(); err != nil {
+		fail("%s\n", err)
 	}
+}
 
-	out, err = exec.Command(
-		"gocc",
+func newCmd(bin string, args ...string) *exec.Cmd {
+	fmt.Printf("==== newCmd %s\n", bin)
+	for _, v := range args {
+		fmt.Printf("\t%v\n", v)
+	}
+	r := exec.Command(bin, args...)
+	r.Stdout = os.Stdout
+	r.Stderr = os.Stderr
+	return r
+}
+
+func makeTestfixture() {
+	dir := filepath.FromSlash(fmt.Sprintf("internal/testfixture"))
+	configure()
+	cmd := newCmd(
+		"ccgo",
 		append(
 			[]string{
-				fn,
-				"-o", filepath.FromSlash(fmt.Sprintf("internal/bin/h_%s_%s.go", runtime.GOOS, runtime.GOARCH)),
+				"-DSQLITE_OMIT_LOAD_EXTENSION",
+				"-DTCLSH",
+				"-DTCLSH_INIT_PROC=sqlite3TestInit",
+				"-I/usr/include/tcl8.6",
+				"-ccgo-export-defines", "",
+				"-ccgo-export-fields", "F",
+				"-ccgo-pkgname", "testfixture",
+				//TODO- "-ccgo-watch-instrumentation", //TODO-
+				"-o", filepath.Join(dir, fmt.Sprintf("testfixture_%s_%s.go", runtime.GOOS, runtime.GOARCH)),
+				filepath.Join(sqliteSrcDir, "ext", "expert", "sqlite3expert.c"),
+				filepath.Join(sqliteSrcDir, "ext", "expert", "test_expert.c"),
+				filepath.Join(sqliteSrcDir, "ext", "fts5", "fts5_tcl.c"),
+				filepath.Join(sqliteSrcDir, "ext", "misc", "amatch.c"),
+				filepath.Join(sqliteSrcDir, "ext", "misc", "carray.c"),
+				filepath.Join(sqliteSrcDir, "ext", "misc", "closure.c"),
+				filepath.Join(sqliteSrcDir, "ext", "misc", "csv.c"),
+				filepath.Join(sqliteSrcDir, "ext", "misc", "eval.c"),
+				filepath.Join(sqliteSrcDir, "ext", "misc", "explain.c"),
+				filepath.Join(sqliteSrcDir, "ext", "misc", "fileio.c"),
+				filepath.Join(sqliteSrcDir, "ext", "misc", "fuzzer.c"),
+				filepath.Join(sqliteSrcDir, "ext", "misc", "ieee754.c"),
+				filepath.Join(sqliteSrcDir, "ext", "misc", "mmapwarm.c"),
+				filepath.Join(sqliteSrcDir, "ext", "misc", "nextchar.c"),
+				filepath.Join(sqliteSrcDir, "ext", "misc", "normalize.c"),
+				filepath.Join(sqliteSrcDir, "ext", "misc", "percentile.c"),
+				filepath.Join(sqliteSrcDir, "ext", "misc", "prefixes.c"),
+				filepath.Join(sqliteSrcDir, "ext", "misc", "regexp.c"),
+				filepath.Join(sqliteSrcDir, "ext", "misc", "remember.c"),
+				filepath.Join(sqliteSrcDir, "ext", "misc", "series.c"),
+				filepath.Join(sqliteSrcDir, "ext", "misc", "spellfix.c"),
+				filepath.Join(sqliteSrcDir, "ext", "misc", "totype.c"),
+				filepath.Join(sqliteSrcDir, "ext", "misc", "unionvtab.c"),
+				filepath.Join(sqliteSrcDir, "ext", "misc", "wholenumber.c"),
+				filepath.Join(sqliteSrcDir, "ext", "rbu", "sqlite3rbu.c"),
+				filepath.Join(sqliteSrcDir, "ext", "rbu", "test_rbu.c"),
+				filepath.Join(sqliteSrcDir, "src", "tclsqlite.c"),
+				filepath.Join(sqliteSrcDir, "src", "test1.c"),
+				filepath.Join(sqliteSrcDir, "src", "test2.c"),
+				filepath.Join(sqliteSrcDir, "src", "test3.c"),
+				filepath.Join(sqliteSrcDir, "src", "test4.c"),
+				filepath.Join(sqliteSrcDir, "src", "test5.c"),
+				filepath.Join(sqliteSrcDir, "src", "test6.c"),
+				filepath.Join(sqliteSrcDir, "src", "test7.c"),
+				filepath.Join(sqliteSrcDir, "src", "test8.c"),
+				filepath.Join(sqliteSrcDir, "src", "test9.c"),
+				filepath.Join(sqliteSrcDir, "src", "test_async.c"),
+				filepath.Join(sqliteSrcDir, "src", "test_autoext.c"),
+				filepath.Join(sqliteSrcDir, "src", "test_backup.c"),
+				filepath.Join(sqliteSrcDir, "src", "test_bestindex.c"),
+				filepath.Join(sqliteSrcDir, "src", "test_blob.c"),
+				filepath.Join(sqliteSrcDir, "src", "test_btree.c"),
+				filepath.Join(sqliteSrcDir, "src", "test_config.c"),
+				filepath.Join(sqliteSrcDir, "src", "test_delete.c"),
+				filepath.Join(sqliteSrcDir, "src", "test_demovfs.c"),
+				filepath.Join(sqliteSrcDir, "src", "test_devsym.c"),
+				filepath.Join(sqliteSrcDir, "src", "test_fs.c"),
+				filepath.Join(sqliteSrcDir, "src", "test_func.c"),
+				filepath.Join(sqliteSrcDir, "src", "test_hexio.c"),
+				filepath.Join(sqliteSrcDir, "src", "test_init.c"),
+				filepath.Join(sqliteSrcDir, "src", "test_intarray.c"),
+				filepath.Join(sqliteSrcDir, "src", "test_journal.c"),
+				filepath.Join(sqliteSrcDir, "src", "test_malloc.c"),
+				filepath.Join(sqliteSrcDir, "src", "test_md5.c"),
+				filepath.Join(sqliteSrcDir, "src", "test_multiplex.c"),
+				filepath.Join(sqliteSrcDir, "src", "test_mutex.c"),
+				filepath.Join(sqliteSrcDir, "src", "test_onefile.c"),
+				filepath.Join(sqliteSrcDir, "src", "test_osinst.c"),
+				filepath.Join(sqliteSrcDir, "src", "test_pcache.c"),
+				filepath.Join(sqliteSrcDir, "src", "test_quota.c"),
+				filepath.Join(sqliteSrcDir, "src", "test_rtree.c"),
+				filepath.Join(sqliteSrcDir, "src", "test_schema.c"),
+				filepath.Join(sqliteSrcDir, "src", "test_superlock.c"),
+				filepath.Join(sqliteSrcDir, "src", "test_syscall.c"),
+				filepath.Join(sqliteSrcDir, "src", "test_tclsh.c"),
+				filepath.Join(sqliteSrcDir, "src", "test_tclvar.c"),
+				filepath.Join(sqliteSrcDir, "src", "test_thread.c"),
+				filepath.Join(sqliteSrcDir, "src", "test_vdbecov.c"),
+				filepath.Join(sqliteSrcDir, "src", "test_vfs.c"),
+				filepath.Join(sqliteSrcDir, "src", "test_window.c"),
 				fmt.Sprintf("-I%s", sqliteDir),
-				"-qbec-defines",
-				"-qbec-enumconsts",
-				"-qbec-import", "<none>",
-				"-qbec-pkgname", "bin",
-				"-qbec-structs",
+				fmt.Sprintf("-I%s", sqliteSrcDir),
+				"-l", "modernc.org/tcl/lib,modernc.org/sqlite/internal/crt2,modernc.org/sqlite/lib",
 			},
 			config...)...,
-	).CombinedOutput()
-	if err != nil {
-		fail("%s\n%s\n", out, err)
+	)
+	if err := cmd.Run(); err != nil {
+		fail("%s\n", err)
 	}
+	os.Remove(filepath.Join(dir, fmt.Sprintf("capi_%s_%s.go", runtime.GOOS, runtime.GOARCH)))
+}
 
-	out, err = exec.Command(
-		"gocc",
+func makeSpeedTest() {
+	cmd := newCmd(
+		"ccgo",
+		append(
+			[]string{
+				"-o", filepath.FromSlash(fmt.Sprintf("speedtest1/main_%s_%s.go", runtime.GOOS, runtime.GOARCH)),
+				filepath.Join(sqliteSrcDir, "test", "speedtest1.c"),
+				fmt.Sprintf("-I%s", sqliteDir),
+				"-l", "modernc.org/sqlite/lib",
+			},
+			config...)...,
+	)
+	if err := cmd.Run(); err != nil {
+		fail("%s\n", err)
+	}
+}
+
+func makeMpTest() {
+	cmd := newCmd(
+		"ccgo",
 		append(
 			[]string{
-				filepath.Join(sqliteSrcDir, "mptest", "mptest.c"),
 				"-o", filepath.FromSlash(fmt.Sprintf("internal/mptest/main_%s_%s.go", runtime.GOOS, runtime.GOARCH)),
+				filepath.Join(sqliteSrcDir, "mptest", "mptest.c"),
 				fmt.Sprintf("-I%s", sqliteDir),
-				"-qbec-dot-import", "modernc.org/sqlite/internal/bin",
-				"-qbec-static-prefix", "s_",
+				"-l", "modernc.org/sqlite/lib",
 			},
 			config...)...,
-	).CombinedOutput()
-	if err != nil {
-		fail("%s\n%s\n", out, err)
+	)
+	if err := cmd.Run(); err != nil {
+		fail("%s\n", err)
+	}
+}
+
+func makeSqlite() {
+	cmd := newCmd(
+		"ccgo",
+		append(
+			[]string{
+				"-DSQLITE_PRIVATE=",
+				"-ccgo-export-defines", "",
+				"-ccgo-export-externs", "X",
+				"-ccgo-export-fields", "F",
+				"-ccgo-export-typedefs", "",
+				"-ccgo-pkgname", "sqlite3",
+				"-o", filepath.FromSlash(fmt.Sprintf("lib/sqlite_%s_%s.go", runtime.GOOS, runtime.GOARCH)),
+				//TODO "-ccgo-volatile", "sqlite3_io_error_pending,sqlite3_open_file_count,sqlite3_pager_readdb_count,sqlite3_search_count,sqlite3_sort_count",
+				filepath.Join(sqliteDir, "sqlite3.c"),
+			},
+			config...)...,
+	)
+	if err := cmd.Run(); err != nil {
+		fail("%s\n", err)
 	}
 }