diff options
Diffstat (limited to 'generator.go')
-rw-r--r-- | generator.go | 793 |
1 files changed, 150 insertions, 643 deletions
diff --git a/generator.go b/generator.go index 001593a..ce25033 100644 --- a/generator.go +++ b/generator.go @@ -7,691 +7,198 @@ package main import ( - "bytes" - "flag" + "archive/zip" "fmt" - "go/format" - "go/scanner" - "go/token" "io" "io/ioutil" + "net/http" "os" "os/exec" "path/filepath" "runtime" - "sort" "strings" - - "log" - - "modernc.org/cc" - "modernc.org/ccgo" - "modernc.org/ccir" - "modernc.org/internal/buffer" - "modernc.org/strutil" - "modernc.org/xc" ) var ( - cpp = flag.Bool("cpp", false, "") - dict = xc.Dict - errLimit = flag.Int("errlimit", 10, "") - filter = flag.String("re", "", "") - ndebug = flag.Bool("ndebug", false, "") - noexec = flag.Bool("noexec", false, "") - oLog = flag.Bool("log", false, "") - trace = flag.Bool("trc", false, "") - unconvertBin string - yydebug = flag.Int("yydebug", 0, "") + config = []string{ + "-DLONGDOUBLE_TYPE=double", + "-DSQLITE_DEBUG", //TODO- + "-DSQLITE_DEFAULT_MEMSTATUS=0", + "-DSQLITE_DEFAULT_WAL_SYNCHRONOUS=1", + "-DSQLITE_DQS=0", + "-DSQLITE_LIKE_DOESNT_MATCH_BLOBS", + "-DSQLITE_MAX_EXPR_DEPTH=0", + "-DSQLITE_MEMDEBUG", //TODO- + "-DSQLITE_OMIT_DECLTYPE", + "-DSQLITE_OMIT_DEPRECATED", + "-DSQLITE_OMIT_PROGRESS_CALLBACK", + "-DSQLITE_OMIT_SHARED_CACHE", + "-DSQLITE_THREADSAFE=0", + "-DSQLITE_USE_ALLOCA", + } + + downloads = []struct { + dir, url string + sz int + dev bool + }{ + {sqliteDir, "https://www.sqlite.org/2019/sqlite-amalgamation-3300100.zip", 2400, false}, + } + + sqliteDir = filepath.FromSlash("testdata/sqlite-amalgamation-3300100") ) -const ( - sqliteRepo = "sqlite.org" - version = "3190300" - - prologueSqlite = `// Code generated by ccgo. DO NOT EDIT. - -/* - -%s -*/ - -// Package sqlite is an in-process implementation of a self-contained, -// serverless, zero-configuration, transactional SQL database engine. (Work In Progress) -%s -package bin - -import ( - "fmt" - "math" - "os" - "path" - "runtime" - "unsafe" - - "modernc.org/ccgo/crt" -) - -func ftrace(s string, args ...interface{}) { - _, fn, fl, _ := runtime.Caller(1) - fmt.Fprintf(os.Stderr, "# %%s:%%d: %%v\n", path.Base(fn), fl, fmt.Sprintf(s, args...)) - os.Stderr.Sync() -} -` - - prologueTest = `// Code generated by ccgo. DO NOT EDIT. - -// %s -%s -package main - -import ( - "math" - "os" - "unsafe" - - "modernc.org/ccgo/crt" - "modernc.org/sqlite/internal/bin" -) - -var argv []*int8 - -func main() { - for _, v := range os.Args { - argv = append(argv, (*int8)(crt.CString(v))) +func download() { + tmp, err := ioutil.TempDir("", "") + if err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err) + return } - argv = append(argv, nil) - X_start(crt.NewTLS(), int32(len(os.Args)), &argv[0]) -} - -` - - defines = ` - #define HAVE_MALLOC_H 1 - #define HAVE_MALLOC_USABLE_SIZE 1 - #define HAVE_USLEEP 1 - #define SQLITE_DEBUG 1 - #define SQLITE_ENABLE_API_ARMOR 1 - #define SQLITE_USE_URI 1 - #define SQLITE_WITHOUT_MSIZE 1 - int sqlite3PendingByte; - ` -) + defer os.RemoveAll(tmp) -func findRepo(s string) string { - s = filepath.FromSlash(s) - for _, v := range strings.Split(strutil.Gopath(), string(os.PathListSeparator)) { - p := filepath.Join(v, "src", s) - fi, err := os.Lstat(p) - if err != nil { + for _, v := range downloads { + dir := filepath.FromSlash(v.dir) + root := filepath.Dir(v.dir) + fi, err := os.Stat(dir) + switch { + case err == nil: + if !fi.IsDir() { + fmt.Fprintf(os.Stderr, "expected %s to be a directory\n", dir) + } continue + default: + if !os.IsNotExist(err) { + fmt.Fprintf(os.Stderr, "%s", err) + continue + } } - if fi.IsDir() { - wd, err := os.Getwd() + if err := func() error { + fmt.Printf("Downloading %v MB from %s\n", float64(v.sz)/1000, v.url) + resp, err := http.Get(v.url) if err != nil { - log.Fatal(err) - } - - if p, err = filepath.Rel(wd, p); err != nil { - log.Fatal(err) + return err } - return p - } - } - return "" -} + defer resp.Body.Close() -func errStr(err error) string { - switch x := err.(type) { - case scanner.ErrorList: - if len(x) != 1 { - x.RemoveMultiples() - } - var b bytes.Buffer - for i, v := range x { - if i != 0 { - b.WriteByte('\n') - } - b.WriteString(v.Error()) - if i == 9 { - fmt.Fprintf(&b, "\n\t... and %v more errors", len(x)-10) - break + base := filepath.Base(v.url) + name := filepath.Join(tmp, base) + f, err := os.Create(name) + if err != nil { + return err } - } - return b.String() - default: - return err.Error() - } -} -func build(predef string, tus [][]string, ccgoOpts []ccgo.Option, opts ...cc.Opt) ([]*cc.TranslationUnit, []byte) { - ndbg := "" - if *ndebug { - ndbg = "#define NDEBUG 1" - } + defer os.Remove(name) - var lpos token.Position - if *cpp { - opts = append(opts, cc.Cpp(func(toks []xc.Token) { - if len(toks) != 0 { - p := toks[0].Position() - if p.Filename != lpos.Filename { - fmt.Fprintf(os.Stderr, "# %d %q\n", p.Line, p.Filename) - } - lpos = p - } - for _, v := range toks { - os.Stderr.WriteString(cc.TokSrc(v)) + n, err := io.Copy(f, resp.Body) + if err != nil { + return err } - os.Stderr.WriteString("\n") - })) - } - - var build []*cc.TranslationUnit - tus = append(tus, []string{ccir.CRT0Path}) - for _, src := range tus { - model, err := ccir.NewModel() - if err != nil { - log.Fatal(err) - } - - ast, err := cc.Parse( - fmt.Sprintf(` -%s -#define _CCGO 1 -#define __arch__ %s -#define __os__ %s -#include <builtin.h> -%s -`, ndbg, runtime.GOARCH, runtime.GOOS, predef), - src, - model, - append([]cc.Opt{ - cc.AllowCompatibleTypedefRedefinitions(), - cc.EnableEmptyStructs(), - cc.EnableImplicitFuncDef(), - cc.EnableNonConstStaticInitExpressions(), - cc.EnableWideBitFieldTypes(), - cc.ErrLimit(*errLimit), - cc.KeepComments(), - cc.SysIncludePaths([]string{ccir.LibcIncludePath}), - }, opts...)..., - ) - if err != nil { - log.Fatal(errStr(err)) - } - - build = append(build, ast) - } - - var out buffer.Bytes - if err := ccgo.New(build, &out, ccgoOpts...); err != nil { - log.Fatal(err) - } - - return build, out.Bytes() -} -func macros(buf io.Writer, ast *cc.TranslationUnit) { - fmt.Fprintf(buf, `const ( -`) - var a []string - for k, v := range ast.Macros { - if v.Value != nil && v.Type.Kind() != cc.Bool { - switch fn := v.DefTok.Position().Filename; { - case - fn == "builtin.h", - fn == "<predefine>", - strings.HasPrefix(fn, "predefined_"): - // ignore - default: - a = append(a, string(dict.S(k))) + if _, err := f.Seek(0, io.SeekStart); err != nil { + return err } - } - } - sort.Strings(a) - for _, v := range a { - m := ast.Macros[dict.SID(v)] - if m.Value == nil { - log.Fatal("TODO") - } - switch t := m.Type; t.Kind() { - case - cc.Int, cc.UInt, cc.Long, cc.ULong, cc.LongLong, cc.ULongLong, - cc.Float, cc.Double, cc.LongDouble, cc.Bool: - fmt.Fprintf(buf, "X%s = %v\n", v, m.Value) - case cc.Ptr: - switch t := t.Element(); t.Kind() { - case cc.Char: - fmt.Fprintf(buf, "X%s = %q\n", v, dict.S(int(m.Value.(cc.StringLitID)))) - default: - log.Fatalf("%v", t.Kind()) - } - default: - log.Fatalf("%v", t.Kind()) - } - } + switch { + case strings.HasSuffix(base, ".zip"): + r, err := zip.NewReader(f, n) + if err != nil { + return err + } - a = a[:0] - for _, v := range ast.Declarations.Identifiers { - switch x := v.Node.(type) { - case *cc.DirectDeclarator: - d := x.TopDeclarator() - id, _ := d.Identifier() - if x.EnumVal == nil { - break + for _, f := range r.File { + fi := f.FileInfo() + if fi.IsDir() { + if err := os.MkdirAll(filepath.Join(root, f.Name), 0770); err != nil { + return err + } + + continue + } + + if err := func() error { + rc, err := f.Open() + if err != nil { + return err + } + + defer rc.Close() + + dname := filepath.Join(root, f.Name) + g, err := os.Create(dname) + if err != nil { + return err + } + + defer g.Close() + + n, err = io.Copy(g, rc) + return err + }(); err != nil { + return err + } + } + return nil } - - a = append(a, string(dict.S(id))) - default: - log.Fatalf("%T", x) - } - } - sort.Strings(a) - for _, v := range a { - dd := ast.Declarations.Identifiers[dict.SID(v)].Node.(*cc.DirectDeclarator) - fmt.Fprintf(buf, "X%s = %v\n", v, dd.EnumVal) - } - fmt.Fprintf(buf, ")\n") -} - -func unconvert(pth string) { - wd, err := os.Getwd() - if err != nil { - log.Fatal(err) - } - - defer func() { - if err := os.Chdir(wd); err != nil { - log.Fatal(err) + panic("internal error") //TODOOK + }(); err != nil { + fmt.Fprintln(os.Stderr, err) } - }() - - if err := os.Chdir(filepath.Dir(pth)); err != nil { - log.Fatal(err) - } - - if out, err := exec.Command(unconvertBin, "-apply").CombinedOutput(); err != nil { - log.Fatalf("unconvert: %s\n%s", err, out) } } -func cp(dst, src, glob string) { - pat := filepath.Join(filepath.FromSlash(src), glob) - m, err := filepath.Glob(pat) - if err != nil { - log.Fatal(err) - } - - if len(m) == 0 { - log.Fatalf("cp(%q, %q, %q): no files for %q", dst, src, glob, pat) - } - - dst = filepath.FromSlash(dst) - for _, v := range m { - f, err := ioutil.ReadFile(v) - if err != nil { - log.Fatal(err) - } - - _, nm := filepath.Split(v) - if err := ioutil.WriteFile(filepath.Join(dst, nm), f, 0664); err != nil { - log.Fatal(err) - } - } +func fail(s string, args ...interface{}) { + fmt.Fprintf(os.Stderr, s, args...) + os.Exit(1) } -func header(f string) []byte { - b, err := ioutil.ReadFile(f) - if err != nil { - log.Fatal(err) - } - - var s scanner.Scanner - s.Init(token.NewFileSet().AddFile(f, -1, len(b)), b, nil, scanner.ScanComments) - var buf buffer.Bytes - for { - _, tok, lit := s.Scan() - switch tok { - case token.COMMENT: - buf.WriteString(lit) - buf.WriteByte('\n') - default: - return buf.Bytes() - } - } -} - -func tidyComment(s string) string { - switch { - case strings.HasPrefix(s, "/*"): - a := strings.Split("/"+s[1:len(s)-1], "\n") - for i, v := range a { - a[i] = "// " + v - } - return strings.Join(a, "\n") + "/\n" - case strings.HasPrefix(s, "//"): - return "// " + s[2:] + "\n" - default: - panic("internal error") - } -} - -func tidyComments(b []byte) string { - var s scanner.Scanner - s.Init(token.NewFileSet().AddFile("", -1, len(b)), b, nil, scanner.ScanComments) - var a []string - for { - _, tok, lit := s.Scan() - if tok == token.EOF { - return strings.Join(a, "\n") - } - - a = append(a, tidyComment(lit)) - } -} - -func sqlite() { - repo := findRepo(sqliteRepo) - if repo == "" { - log.Fatalf("repository not found: %v", sqliteRepo) - return - } - - pth := filepath.Join(repo, "sqlite-amalgamation-"+version) - ast, _ := build( - defines, - [][]string{ - {filepath.Join(pth, "sqlite3.h")}, - {"main.c"}, - }, - []ccgo.Option{ccgo.Library(), ccgo.LibcTypes()}, - cc.EnableAnonymousStructFields(), - cc.IncludePaths([]string{pth}), - ) - sqlite3 := filepath.Join(pth, "sqlite3.c") - _, src := build( - defines, - [][]string{ - {sqlite3}, - {"main.c"}, - }, - []ccgo.Option{ccgo.Library(), ccgo.LibcTypes()}, - cc.EnableAnonymousStructFields(), - cc.IncludePaths([]string{pth}), - ) - - var b bytes.Buffer - lic, err := ioutil.ReadFile("SQLITE-LICENSE") - if err != nil { - log.Fatal(err) - } - - fmt.Fprintf(&b, prologueSqlite, lic, strings.TrimSpace(tidyComments(header(sqlite3)))) - macros(&b, ast[0]) - b.Write(src) - b2, err := format.Source(b.Bytes()) - if err != nil { - b2 = b.Bytes() - } - if err := os.MkdirAll(filepath.Join("internal", "bin"), 0775); err != nil { - log.Fatal(err) - } - - dst := fmt.Sprintf(filepath.Join("internal", "bin", "bin_%s_%s.go"), runtime.GOOS, runtime.GOARCH) - b2 = bytes.Replace(b2, []byte("var Xsqlite3PendingByte int32"), []byte("func Xsqlite3PendingByte() int32 { return _sqlite3PendingByte }"), 1) - if err := ioutil.WriteFile(dst, b2, 0664); err != nil { - log.Fatal(err) - } - - unconvert(dst) -} - -func mpTest() { - repo := findRepo(sqliteRepo) - if repo == "" { - log.Fatalf("repository not found: %v", sqliteRepo) - return - } - - sqlitePth := filepath.Join(repo, "sqlite-amalgamation-"+version) - pth := filepath.Join(repo, "sqlite-src-"+version, "mptest") - tag := "mptest" - test := filepath.Join(pth, tag+".c") - _, src := build( - defines, - [][]string{ - {filepath.Join(sqlitePth, "sqlite3.c")}, - {test}, - }, - []ccgo.Option{ccgo.Packages([]string{"bin"}), ccgo.LibcTypes()}, - cc.EnableAnonymousStructFields(), - cc.IncludePaths([]string{sqlitePth}), - ) - - var b bytes.Buffer - fmt.Fprintf(&b, prologueTest, tag, strings.TrimSpace(tidyComments(header(test)))) - b.Write(src) - b2, err := format.Source(b.Bytes()) - if err != nil { - b2 = b.Bytes() - } - if err := os.MkdirAll(filepath.Join("internal", tag), 0775); err != nil { - log.Fatal(err) - } - - if err := os.MkdirAll(filepath.Join("testdata", tag), 0775); err != nil { - log.Fatal(err) - } - - dst := fmt.Sprintf(filepath.Join("internal", tag, tag+"_%s_%s.go"), runtime.GOOS, runtime.GOARCH) - if err := ioutil.WriteFile(dst, b2, 0664); err != nil { - log.Fatal(err) - } - - unconvert(dst) - cp(filepath.Join("testdata", tag), pth, "*.test") - cp(filepath.Join("testdata", tag), pth, "*.subtest") -} - -func threadTest1() { - repo := findRepo(sqliteRepo) - if repo == "" { - log.Fatalf("repository not found: %v", sqliteRepo) - return - } - - sqlitePth := filepath.Join(repo, "sqlite-amalgamation-"+version) - tag := "threadtest1" - test := filepath.Join("internal", "sqlite.org", "sqlite-src-3190300", "test", "threadtest1.c") - _, src := build( - defines, - [][]string{ - {filepath.Join(sqlitePth, "sqlite3.c")}, - {test}, - }, - []ccgo.Option{ccgo.Packages([]string{"bin"}), ccgo.LibcTypes()}, - cc.EnableAnonymousStructFields(), - cc.IncludePaths([]string{".", sqlitePth, filepath.Join(repo, "sqlite-src-"+version, "src")}), - ) - - var b bytes.Buffer - fmt.Fprintf(&b, prologueTest, tag, strings.TrimSpace(tidyComments(header(test)))) - b.Write(src) - b2, err := format.Source(b.Bytes()) - if err != nil { - b2 = b.Bytes() - } - if err := os.MkdirAll(filepath.Join("internal", tag), 0775); err != nil { - log.Fatal(err) - } - - if err := os.MkdirAll(filepath.Join("testdata", tag), 0775); err != nil { - log.Fatal(err) - } - - dst := fmt.Sprintf(filepath.Join("internal", tag, tag+"_%s_%s.go"), runtime.GOOS, runtime.GOARCH) - if err := ioutil.WriteFile(dst, b2, 0664); err != nil { - log.Fatal(err) - } - - unconvert(dst) -} - -func threadTest2() { - repo := findRepo(sqliteRepo) - if repo == "" { - log.Fatalf("repository not found: %v", sqliteRepo) - return - } - - sqlitePth := filepath.Join(repo, "sqlite-amalgamation-"+version) - pth := filepath.Join(repo, "sqlite-src-"+version, "test") - tag := "threadtest2" - test := filepath.Join(pth, tag+".c") - _, src := build( - defines, - [][]string{ - {filepath.Join(sqlitePth, "sqlite3.c")}, - {test}, - }, - []ccgo.Option{ccgo.Packages([]string{"bin"}), ccgo.LibcTypes()}, - cc.EnableAnonymousStructFields(), - cc.IncludePaths([]string{".", sqlitePth, filepath.Join(repo, "sqlite-src-"+version, "src")}), - ) - - var b bytes.Buffer - fmt.Fprintf(&b, prologueTest, tag, strings.TrimSpace(tidyComments(header(test)))) - b.Write(src) - b2, err := format.Source(b.Bytes()) +func main() { + download() + out, err := exec.Command( + "gocc", + append( + []string{ + filepath.Join(sqliteDir, "sqlite3.c"), + "-o", filepath.FromSlash(fmt.Sprintf("internal/bin/sqlite_%s_%s.go", runtime.GOOS, runtime.GOARCH)), + "-qbec-pkgname", "bin", + }, + config...)..., + ).CombinedOutput() if err != nil { - b2 = b.Bytes() + fail("%s\n%s\n", out, err) } - if err := os.MkdirAll(filepath.Join("internal", tag), 0775); err != nil { - log.Fatal(err) - } - - if err := os.MkdirAll(filepath.Join("testdata", tag), 0775); err != nil { - log.Fatal(err) - } - - dst := fmt.Sprintf(filepath.Join("internal", tag, tag+"_%s_%s.go"), runtime.GOOS, runtime.GOARCH) - if err := ioutil.WriteFile(dst, b2, 0664); err != nil { - log.Fatal(err) - } - - unconvert(dst) -} - -func threadTest3() { - n := 3 - repo := findRepo(sqliteRepo) - if repo == "" { - log.Fatalf("repository not found: %v", sqliteRepo) - return - } - - sqlitePth := filepath.Join(repo, "sqlite-amalgamation-"+version) - pth := filepath.Join(repo, "sqlite-src-"+version, "test") - tag := fmt.Sprintf("threadtest%v", n) - test := filepath.Join(pth, tag+".c") - _, src := build( - defines, - [][]string{ - {filepath.Join(sqlitePth, "sqlite3.c")}, - {filepath.Join(repo, "sqlite-src-"+version, "src", "test_multiplex.c")}, - {test}, - }, - []ccgo.Option{ccgo.Packages([]string{"bin"}), ccgo.LibcTypes()}, - cc.EnableAnonymousStructFields(), - cc.IncludePaths([]string{".", sqlitePth, filepath.Join(repo, "sqlite-src-"+version, "src")}), - ) - var b bytes.Buffer - fmt.Fprintf(&b, prologueTest, tag, strings.TrimSpace(tidyComments(header(test)))) - b.Write(src) - b2, err := format.Source(b.Bytes()) + dir, err := ioutil.TempDir("", "go-generate-") if err != nil { - b2 = b.Bytes() - } - if err := os.MkdirAll(filepath.Join("internal", tag), 0775); err != nil { - log.Fatal(err) - } - - if err := os.MkdirAll(filepath.Join("testdata", tag), 0775); err != nil { - log.Fatal(err) - } - - dst := fmt.Sprintf(filepath.Join("internal", tag, tag+"_%s_%s.go"), runtime.GOOS, runtime.GOARCH) - b2 = bytes.Replace(b2, []byte("Xsqlite3PendingByte"), []byte("bin.Xsqlite3PendingByte()"), -1) - if err := ioutil.WriteFile(dst, b2, 0664); err != nil { - log.Fatal(err) - } - - unconvert(dst) -} - -func threadTest4() { - repo := findRepo(sqliteRepo) - if repo == "" { - log.Fatalf("repository not found: %v", sqliteRepo) - return - } - - sqlitePth := filepath.Join(repo, "sqlite-amalgamation-"+version) - tag := "threadtest4" - test := filepath.Join("internal", "sqlite.org", "sqlite-src-3190300", "test", "threadtest4.c") - _, src := build( - defines, - [][]string{ - {filepath.Join(sqlitePth, "sqlite3.c")}, - {test}, - }, - []ccgo.Option{ccgo.Packages([]string{"bin"}), ccgo.LibcTypes()}, - cc.EnableAnonymousStructFields(), - cc.IncludePaths([]string{".", sqlitePth, filepath.Join(repo, "sqlite-src-"+version, "src")}), - ) - - var b bytes.Buffer - fmt.Fprintf(&b, prologueTest, tag, strings.TrimSpace(tidyComments(header(test)))) - b.Write(src) - b2, err := format.Source(b.Bytes()) + fail("s\n", err) + } + + defer os.RemoveAll(dir) + + src := "#include \"sqlite3.h\"\nstatic char _;\n" + fn := filepath.Join(dir, "x.c") + if err := ioutil.WriteFile(fn, []byte(src), 0660); err != nil { + fail("s\n", err) + } + + out, err = exec.Command( + "gocc", + append( + []string{ + fn, + "-o", filepath.FromSlash(fmt.Sprintf("internal/bin/h_%s_%s.go", runtime.GOOS, runtime.GOARCH)), + fmt.Sprintf("-I%s", sqliteDir), + "-qbec-defines", + "-qbec-enumconsts", + "-qbec-import", "<none>", + "-qbec-pkgname", "bin", + "-qbec-structs", + }, + config...)..., + ).CombinedOutput() if err != nil { - b2 = b.Bytes() + fail("%s\n%s\n", out, err) } - if err := os.MkdirAll(filepath.Join("internal", tag), 0775); err != nil { - log.Fatal(err) - } - - if err := os.MkdirAll(filepath.Join("testdata", tag), 0775); err != nil { - log.Fatal(err) - } - - dst := fmt.Sprintf(filepath.Join("internal", tag, tag+"_%s_%s.go"), runtime.GOOS, runtime.GOARCH) - if err := ioutil.WriteFile(dst, b2, 0664); err != nil { - log.Fatal(err) - } - - unconvert(dst) -} - -func main() { - log.SetFlags(log.Lshortfile | log.Lmicroseconds) - var err error - if unconvertBin, err = exec.LookPath("unconvert"); err != nil { - log.Fatal("Please install the unconvert tool (go get -u github.com/mdempsky/unconvert)") - } - - flag.Parse() - - sqlite() - mpTest() - threadTest1() - threadTest2() - threadTest3() - threadTest4() } |