generator.go (28884B)
1 // Copyright 2017 The Sqlite Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 //go:build generator 6 // +build generator 7 8 //TODO 2023-02-23, netbsd/amd64 fails generating SQLite 3.41: 9 // 10 // C front end 36/85: testdata/sqlite-src-3410000/ext/recover/sqlite3recover.c ... testdata/sqlite-src-3410000/ext/recover/sqlite3recover.c:2023:41: front-end: undefined: SQLITE_FCNTL_RESET_CACHE 11 12 package main 13 14 import ( 15 "archive/zip" 16 "bufio" 17 "bytes" 18 "flag" 19 "fmt" 20 "io" 21 "net/http" 22 "os" 23 "os/exec" 24 "path/filepath" 25 "runtime" 26 "sort" 27 "strings" 28 29 "modernc.org/ccgo/v3/lib" 30 ) 31 32 // gcc 33 // -g 34 // -O2 35 // -DSQLITE_OS_UNIX=1 36 // -I. 37 // -I/home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src 38 // -I/home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/ext/rtree 39 // -I/home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/ext/icu 40 // -I/home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/ext/fts3 41 // -I/home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/ext/async 42 // -I/home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/ext/session 43 // -I/home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/ext/userauth 44 // -D_HAVE_SQLITE_CONFIG_H 45 // -DBUILD_sqlite 46 // -DNDEBUG 47 // -I/usr/include/tcl8.6 48 // -DSQLITE_THREADSAFE=1 49 // -DSQLITE_HAVE_ZLIB=1 50 // -DSQLITE_NO_SYNC=1 51 // -DSQLITE_TEMP_STORE=1 52 // -DSQLITE_TEST=1 53 // -DSQLITE_CRASH_TEST=1 54 // -DTCLSH_INIT_PROC=sqlite3TestInit 55 // -DSQLITE_SERVER=1 56 // -DSQLITE_PRIVATE= 57 // -DSQLITE_CORE 58 // -DBUILD_sqlite 59 // -DSQLITE_SERIES_CONSTRAINT_VERIFY=1 60 // -DSQLITE_DEFAULT_PAGE_SIZE=1024 61 // -DSQLITE_ENABLE_STMTVTAB 62 // -DSQLITE_ENABLE_DBPAGE_VTAB 63 // -DSQLITE_ENABLE_BYTECODE_VTAB 64 // -DSQLITE_ENABLE_DESERIALIZE 65 // -o testfixture 66 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test1.c 67 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test2.c 68 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test3.c 69 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test4.c 70 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test5.c 71 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test6.c 72 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test7.c 73 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test8.c 74 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test9.c 75 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test_autoext.c 76 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test_async.c 77 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test_backup.c 78 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test_bestindex.c 79 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test_blob.c 80 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test_btree.c 81 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test_config.c 82 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test_delete.c 83 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test_demovfs.c 84 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test_devsym.c 85 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test_fs.c 86 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test_func.c 87 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test_hexio.c 88 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test_init.c 89 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test_intarray.c 90 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test_journal.c 91 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test_malloc.c 92 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test_md5.c 93 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test_multiplex.c 94 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test_mutex.c 95 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test_onefile.c 96 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test_osinst.c 97 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test_pcache.c 98 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test_quota.c 99 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test_rtree.c 100 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test_schema.c 101 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test_server.c 102 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test_superlock.c 103 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test_syscall.c 104 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test_tclsh.c 105 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test_tclvar.c 106 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test_thread.c 107 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test_vdbecov.c 108 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test_vfs.c 109 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test_windirent.c 110 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test_window.c 111 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/test_wsd.c 112 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/ext/fts3/fts3_term.c 113 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/ext/fts3/fts3_test.c 114 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/ext/session/test_session.c 115 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/ext/rbu/test_rbu.c 116 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/ext/expert/sqlite3expert.c 117 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/ext/expert/test_expert.c 118 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/ext/misc/amatch.c 119 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/ext/misc/carray.c 120 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/ext/misc/closure.c 121 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/ext/misc/csv.c 122 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/ext/misc/decimal.c 123 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/ext/misc/eval.c 124 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/ext/misc/explain.c 125 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/ext/misc/fileio.c 126 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/ext/misc/fuzzer.c 127 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/ext/fts5/fts5_tcl.c 128 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/ext/fts5/fts5_test_mi.c 129 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/ext/fts5/fts5_test_tok.c 130 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/ext/misc/ieee754.c 131 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/ext/misc/mmapwarm.c 132 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/ext/misc/nextchar.c 133 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/ext/misc/normalize.c 134 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/ext/misc/percentile.c 135 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/ext/misc/prefixes.c 136 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/ext/misc/regexp.c 137 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/ext/misc/remember.c 138 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/ext/misc/series.c 139 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/ext/misc/spellfix.c 140 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/ext/misc/totype.c 141 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/ext/misc/unionvtab.c 142 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/ext/misc/wholenumber.c 143 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/ext/misc/zipfile.c 144 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/ext/userauth/userauth.c 145 // /home/jnml/src/modernc.org/sqlite/testdata/SQLite-3c5e63c2/src/tclsqlite.c 146 // sqlite3.c 147 // -L/usr/lib/x86_64-linux-gnu 148 // -ltcl8.6 149 // -ldl 150 // -lz 151 // -lpthread 152 153 const ( 154 volatiles = "-volatile=sqlite3_io_error_pending,sqlite3_open_file_count,sqlite3_pager_readdb_count,sqlite3_pager_writedb_count,sqlite3_pager_writej_count,sqlite3_search_count,sqlite3_sort_count,saved_cnt,randomnessPid" 155 ) 156 157 // 2022-11-27 Removing -DSQLITE_ENABLE_SNAPSHOT from configTest. This #define 158 // makes a single test fail on linux/ppc64le. That test is run only when the 159 // -DSQLITE_ENABLE_SNAPSHOT is present when compiling the testfixture. When 160 // investigating the failure it turns out this #define is actually NOT present 161 // when doing '$ ./configure && make tcltest' in sqlite-src-3400000, ie. in the 162 // original C code. 163 // 164 // libtool: link: gcc -g -O2 -DSQLITE_OS_UNIX=1 -I. -I/home/jnml/sqlite-src-3400000/src -I/home/jnml/sqlite-src-3400000/ext/rtree -I/home/jnml/sqlite-src-3400000/ext/icu -I/home/jnml/sqlite-src-3400000/ext/fts3 -I/home/jnml/sqlite-src-3400000/ext/async -I/home/jnml/sqlite-src-3400000/ext/session -I/home/jnml/sqlite-src-3400000/ext/userauth -D_HAVE_SQLITE_CONFIG_H -DBUILD_sqlite -DNDEBUG -I/usr/include/tcl8.6 -DSQLITE_THREADSAFE=1 -DSQLITE_ENABLE_MATH_FUNCTIONS -DSQLITE_HAVE_ZLIB=1 -DSQLITE_NO_SYNC=1 -DSQLITE_TEMP_STORE=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1 -DTCLSH_INIT_PROC=sqlite3TestInit -DSQLITE_SERVER=1 -DSQLITE_PRIVATE= -DSQLITE_CORE -DBUILD_sqlite -DSQLITE_SERIES_CONSTRAINT_VERIFY=1 -DSQLITE_DEFAULT_PAGE_SIZE=1024 -DSQLITE_ENABLE_STMTVTAB -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_BYTECODE_VTAB -DSQLITE_CKSUMVFS_STATIC -o testfixture ... 165 166 var ( 167 configProduction = []string{ 168 "-DHAVE_USLEEP", 169 "-DLONGDOUBLE_TYPE=double", 170 "-DSQLITE_CORE", 171 "-DSQLITE_DEFAULT_MEMSTATUS=0", 172 "-DSQLITE_ENABLE_COLUMN_METADATA", 173 "-DSQLITE_ENABLE_FTS5", 174 "-DSQLITE_ENABLE_GEOPOLY", 175 "-DSQLITE_ENABLE_MATH_FUNCTIONS", 176 "-DSQLITE_ENABLE_MEMORY_MANAGEMENT", 177 "-DSQLITE_ENABLE_OFFSET_SQL_FUNC", 178 "-DSQLITE_ENABLE_PREUPDATE_HOOK", 179 "-DSQLITE_ENABLE_RBU", 180 "-DSQLITE_ENABLE_RTREE", 181 "-DSQLITE_ENABLE_SESSION", 182 "-DSQLITE_ENABLE_SNAPSHOT", 183 "-DSQLITE_ENABLE_STAT4", 184 "-DSQLITE_ENABLE_UNLOCK_NOTIFY", // Adds sqlite3_unlock_notify(). 185 "-DSQLITE_LIKE_DOESNT_MATCH_BLOBS", 186 "-DSQLITE_MUTEX_APPDEF=1", 187 "-DSQLITE_MUTEX_NOOP", 188 "-DSQLITE_SOUNDEX", 189 "-DSQLITE_THREADSAFE=1", 190 //DONT "-DNDEBUG", // To enable GO_GENERATE=-DSQLITE_DEBUG 191 //DONT "-DSQLITE_DQS=0", // testfixture 192 //DONT "-DSQLITE_NO_SYNC=1", 193 //DONT "-DSQLITE_OMIT_DECLTYPE", // testfixture 194 //DONT "-DSQLITE_OMIT_DEPRECATED", // mptest 195 //DONT "-DSQLITE_OMIT_LOAD_EXTENSION", // mptest 196 //DONT "-DSQLITE_OMIT_SHARED_CACHE", 197 //DONT "-DSQLITE_USE_ALLOCA", 198 //TODO "-DHAVE_MALLOC_USABLE_SIZE" 199 //TODO "-DSQLITE_DEFAULT_WAL_SYNCHRONOUS=1", //TODO report bug 200 //TODO "-DSQLITE_ENABLE_FTS3", 201 //TODO "-DSQLITE_ENABLE_FTS3_PARENTHESIS", 202 //TODO "-DSQLITE_ENABLE_FTS3_TOKENIZER", 203 //TODO "-DSQLITE_ENABLE_FTS4", 204 //TODO "-DSQLITE_ENABLE_ICU", 205 //TODO "-DSQLITE_MAX_EXPR_DEPTH=0", // bug reported https://sqlite.org/forum/forumpost/87b9262f66, fixed in https://sqlite.org/src/info/5f58dd3a19605b6f 206 //TODO "-DSQLITE_MAX_MMAP_SIZE=8589934592", // testfixture, bug reported https://sqlite.org/forum/forumpost/34380589f7, fixed in https://sqlite.org/src/info/d8e47382160e98be 207 //TODO- "-DSQLITE_DEBUG", 208 //TODO- "-DSQLITE_ENABLE_API_ARMOR", 209 //TODO- "-DSQLITE_MEMDEBUG", 210 } 211 212 configTest = []string{ 213 "-DHAVE_USLEEP", 214 "-DLONGDOUBLE_TYPE=double", 215 "-DSQLITE_CKSUMVFS_STATIC", 216 "-DSQLITE_CORE", // testfixture 217 "-DSQLITE_DEFAULT_MEMSTATUS=1", 218 "-DSQLITE_DEFAULT_PAGE_SIZE=1024", // testfixture, hardcoded. See file_pages in autovacuum.test. 219 "-DSQLITE_ENABLE_BYTECODE_VTAB", // testfixture 220 "-DSQLITE_ENABLE_COLUMN_METADATA", 221 "-DSQLITE_ENABLE_DBPAGE_VTAB", // testfixture 222 "-DSQLITE_ENABLE_DBSTAT_VTAB", 223 "-DSQLITE_ENABLE_DESERIALIZE", // testfixture 224 "-DSQLITE_ENABLE_EXPLAIN_COMMENTS", 225 "-DSQLITE_ENABLE_FTS5", 226 "-DSQLITE_ENABLE_GEOPOLY", 227 "-DSQLITE_ENABLE_MATH_FUNCTIONS", 228 "-DSQLITE_ENABLE_MEMORY_MANAGEMENT", 229 "-DSQLITE_ENABLE_OFFSET_SQL_FUNC", 230 "-DSQLITE_ENABLE_PREUPDATE_HOOK", 231 "-DSQLITE_ENABLE_RBU", 232 "-DSQLITE_ENABLE_RTREE", 233 "-DSQLITE_ENABLE_SESSION", 234 "-DSQLITE_ENABLE_STAT4", 235 "-DSQLITE_ENABLE_STMTVTAB", // testfixture 236 "-DSQLITE_ENABLE_UNLOCK_NOTIFY", // Adds sqlite3_unlock_notify(). 237 "-DSQLITE_LIKE_DOESNT_MATCH_BLOBS", 238 "-DSQLITE_MUTEX_APPDEF=1", 239 "-DSQLITE_MUTEX_NOOP", 240 "-DSQLITE_SOUNDEX", 241 "-DSQLITE_TEMP_STORE=1", // testfixture 242 "-DSQLITE_TEST", 243 "-DSQLITE_THREADSAFE=1", 244 //DONT "-DNDEBUG", // To enable GO_GENERATE=-DSQLITE_DEBUG 245 //DONT "-DSQLITE_DQS=0", // testfixture 246 //DONT "-DSQLITE_ENABLE_SNAPSHOT", 247 //DONT "-DSQLITE_NO_SYNC=1", 248 //DONT "-DSQLITE_OMIT_DECLTYPE", // testfixture 249 //DONT "-DSQLITE_OMIT_DEPRECATED", // mptest 250 //DONT "-DSQLITE_OMIT_LOAD_EXTENSION", // mptest 251 //DONT "-DSQLITE_OMIT_SHARED_CACHE", 252 //DONT "-DSQLITE_USE_ALLOCA", 253 //TODO "-DHAVE_MALLOC_USABLE_SIZE" 254 //TODO "-DSQLITE_DEFAULT_WAL_SYNCHRONOUS=1", //TODO report bug 255 //TODO "-DSQLITE_ENABLE_FTS3", 256 //TODO "-DSQLITE_ENABLE_FTS3_PARENTHESIS", 257 //TODO "-DSQLITE_ENABLE_FTS3_TOKENIZER", 258 //TODO "-DSQLITE_ENABLE_FTS4", 259 //TODO "-DSQLITE_ENABLE_ICU", 260 //TODO "-DSQLITE_MAX_EXPR_DEPTH=0", // bug reported https://sqlite.org/forum/forumpost/87b9262f66, fixed in https://sqlite.org/src/info/5f58dd3a19605b6f 261 //TODO "-DSQLITE_MAX_MMAP_SIZE=8589934592", // testfixture, bug reported https://sqlite.org/forum/forumpost/34380589f7, fixed in https://sqlite.org/src/info/d8e47382160e98be 262 //TODO- "-DSQLITE_DEBUG", 263 //TODO- "-DSQLITE_ENABLE_API_ARMOR", 264 //TODO- "-DSQLITE_MEMDEBUG", 265 } 266 267 downloads = []struct { 268 dir, url string 269 sz int 270 dev bool 271 }{ 272 {sqliteDir, "https://www.sqlite.org/2023/sqlite-amalgamation-3410200.zip", 2457, false}, 273 {sqliteSrcDir, "https://www.sqlite.org/2023/sqlite-src-3410200.zip", 12814, false}, 274 } 275 276 sqliteDir = filepath.FromSlash("testdata/sqlite-amalgamation-3410200") 277 sqliteSrcDir = filepath.FromSlash("testdata/sqlite-src-3410200") 278 ) 279 280 func download() { 281 tmp, err := os.MkdirTemp("", "") 282 if err != nil { 283 fmt.Fprintf(os.Stderr, "%s\n", err) 284 return 285 } 286 287 defer os.RemoveAll(tmp) 288 289 for _, v := range downloads { 290 dir := filepath.FromSlash(v.dir) 291 root := filepath.Dir(v.dir) 292 fi, err := os.Stat(dir) 293 switch { 294 case err == nil: 295 if !fi.IsDir() { 296 fmt.Fprintf(os.Stderr, "expected %s to be a directory\n", dir) 297 } 298 continue 299 default: 300 if !os.IsNotExist(err) { 301 fmt.Fprintf(os.Stderr, "%s", err) 302 continue 303 } 304 } 305 306 if err := func() error { 307 fmt.Printf("Downloading %v MB from %s\n", float64(v.sz)/1000, v.url) 308 resp, err := http.Get(v.url) 309 if err != nil { 310 return err 311 } 312 313 defer resp.Body.Close() 314 315 base := filepath.Base(v.url) 316 name := filepath.Join(tmp, base) 317 f, err := os.Create(name) 318 if err != nil { 319 return err 320 } 321 322 defer os.Remove(name) 323 324 n, err := io.Copy(f, resp.Body) 325 if err != nil { 326 return err 327 } 328 329 if _, err := f.Seek(0, io.SeekStart); err != nil { 330 return err 331 } 332 333 switch { 334 case strings.HasSuffix(base, ".zip"): 335 r, err := zip.NewReader(f, n) 336 if err != nil { 337 return err 338 } 339 340 for _, f := range r.File { 341 fi := f.FileInfo() 342 if fi.IsDir() { 343 if err := os.MkdirAll(filepath.Join(root, f.Name), 0770); err != nil { 344 return err 345 } 346 347 continue 348 } 349 350 if err := func() error { 351 rc, err := f.Open() 352 if err != nil { 353 return err 354 } 355 356 defer rc.Close() 357 358 file, err := os.OpenFile(filepath.Join(root, f.Name), os.O_CREATE|os.O_WRONLY, fi.Mode()) 359 if err != nil { 360 return err 361 } 362 363 w := bufio.NewWriter(file) 364 if _, err = io.Copy(w, rc); err != nil { 365 return err 366 } 367 368 if err := w.Flush(); err != nil { 369 return err 370 } 371 372 return file.Close() 373 }(); err != nil { 374 return err 375 } 376 } 377 return nil 378 } 379 panic("internal error") //TODOOK 380 }(); err != nil { 381 fmt.Fprintln(os.Stderr, err) 382 } 383 } 384 } 385 386 func fail(s string, args ...interface{}) { 387 fmt.Fprintf(os.Stderr, s, args...) 388 os.Exit(1) 389 } 390 391 var ( 392 oFullPathComments = flag.Bool("full-path-comments", false, "") 393 ) 394 395 func main() { 396 flag.Parse() 397 fmt.Printf("Running on %s/%s.\n", runtime.GOOS, runtime.GOARCH) 398 env := os.Getenv("GO_GENERATE") 399 goarch := runtime.GOARCH 400 goos := runtime.GOOS 401 if s := os.Getenv("TARGET_GOOS"); s != "" { 402 goos = s 403 } 404 if s := os.Getenv("TARGET_GOARCH"); s != "" { 405 goarch = s 406 } 407 var more []string 408 if env != "" { 409 more = strings.Split(env, ",") 410 } 411 ndebug := []string{"-DNDEBUG"} 412 for _, v := range more { 413 if v == "-DSQLITE_DEBUG" { 414 ndebug = nil 415 } 416 } 417 more = append(more, ndebug...) 418 download() 419 switch goos { 420 case "linux", "freebsd", "openbsd": 421 configProduction = append(configProduction, "-DSQLITE_OS_UNIX=1") 422 case "netbsd": 423 configProduction = append(configProduction, []string{ 424 "-DSQLITE_OS_UNIX=1", 425 "-D__libc_cond_broadcast=pthread_cond_broadcast", 426 "-D__libc_cond_destroy=pthread_cond_destroy", 427 "-D__libc_cond_init=pthread_cond_init", 428 "-D__libc_cond_signal=pthread_cond_signal", 429 "-D__libc_cond_wait=pthread_cond_wait", 430 "-D__libc_mutex_destroy=pthread_mutex_destroy", 431 "-D__libc_mutex_init=pthread_mutex_init", 432 "-D__libc_mutex_lock=pthread_mutex_lock", 433 "-D__libc_mutex_trylock=pthread_mutex_trylock", 434 "-D__libc_mutex_unlock=pthread_mutex_unlock", 435 "-D__libc_mutexattr_destroy=pthread_mutexattr_destroy", 436 "-D__libc_mutexattr_init=pthread_mutexattr_init", 437 "-D__libc_mutexattr_settype=pthread_mutexattr_settype", 438 "-D__libc_thr_yield=sched_yield", 439 }...) 440 case "darwin": 441 configProduction = append(configProduction, 442 "-DSQLITE_OS_UNIX=1", 443 "-DSQLITE_WITHOUT_ZONEMALLOC", 444 ) 445 configTest = append(configTest, 446 "-DSQLITE_OS_UNIX=1", 447 "-DSQLITE_WITHOUT_ZONEMALLOC", 448 ) 449 case "windows": 450 configProduction = append(configProduction, 451 "-DSQLITE_OS_WIN=1", 452 "-D_MSC_VER=1900", 453 ) 454 configTest = append(configTest, 455 "-DSQLITE_OS_WIN=1", 456 "-D_MSC_VER=1900", 457 ) 458 default: 459 fail("unknows/unsupported os: %s\n", goos) 460 } 461 makeSqliteProduction(goos, goarch, more) 462 makeSqliteTest(goos, goarch, more) 463 makeMpTest(goos, goarch, more) 464 makeSpeedTest(goos, goarch, more) 465 makeTestfixture(goos, goarch, more) 466 ccgo.MustCopyDir(true, "testdata/tcl", sqliteSrcDir+"/test", nil) 467 ccgo.MustCopyDir(true, "testdata/tcl", "testdata/overlay", nil) 468 } 469 470 func configure(goos, goarch string) { 471 wd, err := os.Getwd() 472 if err != nil { 473 fail("%s", err) 474 } 475 476 defer os.Chdir(wd) 477 478 if err := os.Chdir(sqliteSrcDir); err != nil { 479 fail("%s", err) 480 } 481 482 cmd := newCmd("make", "distclean") 483 cmd.Run() 484 var args []string 485 switch goos { 486 case "linux", "freebsd", "netbsd", "openbsd": 487 // nop 488 case "darwin": 489 args = append(args, "--with-tcl=/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/System/Library/Frameworks/Tcl.framework") 490 case "windows": 491 switch goarch { 492 case "amd64": 493 args = append(args, "--host=x86_64-w64-mingw32") 494 case "386": 495 args = append(args, "--host=i686-w64-mingw32") 496 default: 497 fail("unknown/unsupported os/arch: %s/%s\n", goos, goarch) 498 } 499 default: 500 fail("unknown/unsupported os/arch: %s/%s\n", goos, goarch) 501 } 502 cmd = newCmd("./configure", args...) 503 if err = cmd.Run(); err != nil { 504 fail("%s\n", err) 505 } 506 507 cmd = newCmd("make", "parse.h", "opcodes.h") 508 if err = cmd.Run(); err != nil { 509 fail("%s\n", err) 510 } 511 } 512 513 func newCmd(bin string, args ...string) *exec.Cmd { 514 fmt.Printf("==== newCmd %s\n", bin) 515 for _, v := range args { 516 fmt.Printf("\t%v\n", v) 517 } 518 r := exec.Command(bin, args...) 519 r.Stdout = os.Stdout 520 r.Stderr = os.Stderr 521 return r 522 } 523 524 func makeTestfixture(goos, goarch string, more []string) { 525 dir := filepath.FromSlash(fmt.Sprintf("internal/testfixture")) 526 files := []string{ 527 "ext/expert/sqlite3expert.c", 528 "ext/expert/test_expert.c", 529 "ext/fts3/fts3_term.c", 530 "ext/fts3/fts3_test.c", 531 "ext/fts5/fts5_tcl.c", 532 "ext/fts5/fts5_test_mi.c", 533 "ext/fts5/fts5_test_tok.c", 534 "ext/misc/amatch.c", 535 "ext/misc/appendvfs.c", 536 "ext/misc/basexx.c", 537 "ext/misc/carray.c", 538 "ext/misc/cksumvfs.c", 539 "ext/misc/closure.c", 540 "ext/misc/csv.c", 541 "ext/misc/decimal.c", 542 "ext/misc/eval.c", 543 "ext/misc/explain.c", 544 "ext/misc/fileio.c", 545 "ext/misc/fuzzer.c", 546 "ext/misc/ieee754.c", 547 "ext/misc/mmapwarm.c", 548 "ext/misc/nextchar.c", 549 "ext/misc/normalize.c", 550 "ext/misc/percentile.c", 551 "ext/misc/prefixes.c", 552 "ext/misc/qpvtab.c", 553 "ext/misc/regexp.c", 554 "ext/misc/remember.c", 555 "ext/misc/series.c", 556 "ext/misc/spellfix.c", 557 "ext/misc/totype.c", 558 "ext/misc/unionvtab.c", 559 "ext/misc/wholenumber.c", 560 "ext/rbu/test_rbu.c", 561 "ext/recover/dbdata.c", 562 "ext/recover/sqlite3recover.c", 563 "ext/recover/test_recover.c", 564 "ext/rtree/test_rtreedoc.c", 565 "ext/session/test_session.c", 566 "ext/userauth/userauth.c", 567 "src/tclsqlite.c", 568 "src/test1.c", 569 "src/test2.c", 570 "src/test3.c", 571 "src/test4.c", 572 "src/test5.c", 573 "src/test6.c", 574 "src/test8.c", 575 "src/test9.c", 576 "src/test_async.c", 577 "src/test_autoext.c", 578 "src/test_backup.c", 579 "src/test_bestindex.c", 580 "src/test_blob.c", 581 "src/test_btree.c", 582 "src/test_config.c", 583 "src/test_delete.c", 584 "src/test_demovfs.c", 585 "src/test_devsym.c", 586 "src/test_fs.c", 587 "src/test_func.c", 588 "src/test_hexio.c", 589 "src/test_init.c", 590 "src/test_intarray.c", 591 "src/test_journal.c", 592 "src/test_malloc.c", 593 "src/test_md5.c", 594 "src/test_multiplex.c", 595 "src/test_mutex.c", 596 "src/test_onefile.c", 597 "src/test_osinst.c", 598 "src/test_pcache.c", 599 "src/test_quota.c", 600 "src/test_rtree.c", 601 "src/test_schema.c", 602 "src/test_superlock.c", 603 "src/test_syscall.c", 604 "src/test_tclsh.c", 605 "src/test_tclvar.c", 606 "src/test_thread.c", 607 "src/test_vdbecov.c", 608 "src/test_vfs.c", 609 "src/test_windirent.c", 610 "src/test_window.c", 611 "src/test_wsd.c", 612 } 613 for i, v := range files { 614 files[i] = filepath.Join(sqliteSrcDir, filepath.FromSlash(v)) 615 } 616 configure(goos, goarch) 617 618 var defines, includes []string 619 switch goos { 620 case "freebsd", "openbsd": 621 includes = []string{"-I/usr/local/include/tcl8.6"} 622 case "linux": 623 includes = []string{"-I/usr/include/tcl8.6"} 624 case "windows": 625 includes = []string{"-I/usr/include/tcl8.6"} 626 case "netbsd": 627 includes = []string{"-I/usr/pkg/include"} 628 defines = []string{ 629 "-D__libc_cond_broadcast=pthread_cond_broadcast", 630 "-D__libc_cond_destroy=pthread_cond_destroy", 631 "-D__libc_cond_init=pthread_cond_init", 632 "-D__libc_cond_signal=pthread_cond_signal", 633 "-D__libc_cond_wait=pthread_cond_wait", 634 "-D__libc_mutex_destroy=pthread_mutex_destroy", 635 "-D__libc_mutex_init=pthread_mutex_init", 636 "-D__libc_mutex_lock=pthread_mutex_lock", 637 "-D__libc_mutex_trylock=pthread_mutex_trylock", 638 "-D__libc_mutex_unlock=pthread_mutex_unlock", 639 "-D__libc_mutexattr_destroy=pthread_mutexattr_destroy", 640 "-D__libc_mutexattr_init=pthread_mutexattr_init", 641 "-D__libc_mutexattr_settype=pthread_mutexattr_settype", 642 "-D__libc_thr_yield=sched_yield", 643 } 644 } 645 646 args := join( 647 []string{ 648 "ccgo", 649 "-DBUILD_sqlite", 650 "-DNDEBUG", 651 "-DSQLITE_CKSUMVFS_STATIC", 652 "-DSQLITE_CORE", 653 "-DSQLITE_CRASH_TEST=1", 654 "-DSQLITE_DEFAULT_PAGE_SIZE=1024", 655 "-DSQLITE_ENABLE_BYTECODE_VTAB", 656 "-DSQLITE_ENABLE_DBPAGE_VTAB", 657 "-DSQLITE_ENABLE_MATH_FUNCTIONS", 658 "-DSQLITE_ENABLE_STMTVTAB", 659 "-DSQLITE_NO_SYNC=1", 660 "-DSQLITE_OMIT_LOAD_EXTENSION", 661 "-DSQLITE_PRIVATE=\"\"", 662 "-DSQLITE_SERIES_CONSTRAINT_VERIFY=1", 663 "-DSQLITE_SERVER=1", 664 "-DSQLITE_TEMP_STORE=1", 665 "-DSQLITE_TEST=1", 666 "-DSQLITE_THREADSAFE=1", 667 "-DTCLSH_INIT_PROC=sqlite3TestInit", 668 "-D_HAVE_SQLITE_CONFIG_H", 669 }, 670 defines, 671 includes, 672 []string{ 673 "-export-defines", "", 674 "-export-fields", "F", 675 "-ignore-unsupported-alignment", 676 "-trace-translation-units", 677 volatiles, 678 "-lmodernc.org/sqlite/libtest", 679 "-lmodernc.org/tcl/lib", 680 "-lmodernc.org/z/lib", 681 "-o", filepath.Join(dir, fmt.Sprintf("testfixture_%s_%s.go", goos, goarch)), 682 fmt.Sprintf("-I%s", filepath.Join(sqliteSrcDir, filepath.FromSlash("ext/async"))), 683 fmt.Sprintf("-I%s", filepath.Join(sqliteSrcDir, filepath.FromSlash("ext/fts3"))), 684 fmt.Sprintf("-I%s", filepath.Join(sqliteSrcDir, filepath.FromSlash("ext/icu"))), 685 fmt.Sprintf("-I%s", filepath.Join(sqliteSrcDir, filepath.FromSlash("ext/rtree"))), 686 fmt.Sprintf("-I%s", filepath.Join(sqliteSrcDir, filepath.FromSlash("ext/session"))), 687 fmt.Sprintf("-I%s", filepath.Join(sqliteSrcDir, filepath.FromSlash("ext/userauth"))), 688 fmt.Sprintf("-I%s", filepath.Join(sqliteSrcDir, filepath.FromSlash("src"))), 689 fmt.Sprintf("-I%s", sqliteDir), 690 fmt.Sprintf("-I%s", sqliteSrcDir), 691 }, 692 otherOpts(), 693 files, 694 more, 695 configTest, 696 ) 697 task := ccgo.NewTask(args, nil, nil) 698 if err := task.Main(); err != nil { 699 fail("%s\n", err) 700 } 701 } 702 703 func otherOpts() (r []string) { 704 if *oFullPathComments { 705 r = append(r, "-full-path-comments") 706 } 707 return r 708 } 709 710 func makeSpeedTest(goos, goarch string, more []string) { 711 task := ccgo.NewTask( 712 join( 713 []string{ 714 "ccgo", 715 "-export-defines", "", 716 "-ignore-unsupported-alignment", 717 "-o", filepath.FromSlash(fmt.Sprintf("speedtest1/main_%s_%s.go", goos, goarch)), 718 "-trace-translation-units", 719 filepath.Join(sqliteSrcDir, "test", "speedtest1.c"), 720 fmt.Sprintf("-I%s", sqliteDir), 721 "-l", "modernc.org/sqlite/lib", 722 }, 723 otherOpts(), 724 more, 725 configProduction, 726 ), 727 nil, 728 nil, 729 ) 730 if err := task.Main(); err != nil { 731 fail("%s\n", err) 732 } 733 } 734 735 func makeMpTest(goos, goarch string, more []string) { 736 task := ccgo.NewTask( 737 join( 738 []string{ 739 "ccgo", 740 "-export-defines", "", 741 "-ignore-unsupported-alignment", 742 "-o", filepath.FromSlash(fmt.Sprintf("internal/mptest/main_%s_%s.go", goos, goarch)), 743 "-trace-translation-units", 744 // filepath.Join(sqliteSrcDir, "mptest", "mptest.c"), 745 filepath.Join("testdata", "mptest.c"), 746 fmt.Sprintf("-I%s", sqliteDir), 747 "-l", "modernc.org/sqlite/lib", 748 }, 749 otherOpts(), 750 more, 751 configProduction, 752 ), 753 nil, 754 nil, 755 ) 756 if err := task.Main(); err != nil { 757 fail("%s\n", err) 758 } 759 } 760 761 func makeSqliteProduction(goos, goarch string, more []string) { 762 fn := filepath.FromSlash(fmt.Sprintf("lib/sqlite_%s_%s.go", goos, goarch)) 763 task := ccgo.NewTask( 764 join( 765 []string{ 766 "ccgo", 767 "-DSQLITE_PRIVATE=", 768 "-export-defines", "", 769 "-export-enums", "", 770 "-export-externs", "X", 771 "-export-fields", "F", 772 "-export-typedefs", "", 773 "-ignore-unsupported-alignment", 774 "-pkgname", "sqlite3", 775 volatiles, 776 "-o", fn, 777 "-trace-translation-units", 778 filepath.Join(sqliteDir, "sqlite3.c"), 779 }, 780 otherOpts(), 781 more, 782 configProduction, 783 ), 784 nil, 785 nil, 786 ) 787 if err := task.Main(); err != nil { 788 fail("%s\n", err) 789 } 790 791 if err := patchXsqlite3_initialize(fn); err != nil { 792 fail("%s\n", err) 793 } 794 } 795 796 func makeSqliteTest(goos, goarch string, more []string) { 797 fn := filepath.FromSlash(fmt.Sprintf("libtest/sqlite_%s_%s.go", goos, goarch)) 798 task := ccgo.NewTask( 799 join( 800 []string{ 801 "ccgo", 802 "-DSQLITE_PRIVATE=", 803 "-export-defines", "", 804 "-export-enums", "", 805 "-export-externs", "X", 806 "-export-fields", "F", 807 "-export-typedefs", "", 808 "-ignore-unsupported-alignment", 809 "-pkgname", "sqlite3", 810 volatiles, 811 "-o", fn, 812 "-trace-translation-units", 813 volatiles, 814 filepath.Join(sqliteDir, "sqlite3.c"), 815 }, 816 otherOpts(), 817 more, 818 configTest, 819 ), 820 nil, 821 nil, 822 ) 823 if err := task.Main(); err != nil { 824 fail("%s\n", err) 825 } 826 827 if err := patchXsqlite3_initialize(fn); err != nil { 828 fail("%s\n", err) 829 } 830 } 831 832 func join(a ...[]string) (r []string) { 833 n := 0 834 for _, v := range a { 835 n += len(v) 836 } 837 r = make([]string, 0, n) 838 for _, v := range a { 839 r = append(r, v...) 840 } 841 return r 842 } 843 844 func patchXsqlite3_initialize(fn string) error { 845 const s = "func Xsqlite3_initialize(tls *libc.TLS) int32 {" 846 return patch(fn, func(b []byte) []diff { 847 x := bytes.Index(b, []byte(s)) 848 return []diff{{x, x + len(s), ` 849 var mu mutex 850 851 func init() { mu.recursive = true } 852 853 func Xsqlite3_initialize(tls *libc.TLS) int32 { 854 mu.enter(tls.ID) 855 defer mu.leave(tls.ID) 856 857 `}} 858 }) 859 } 860 861 type diff struct { 862 from, to int // byte offsets 863 replace string // replaces b[from:to] 864 } 865 866 func patch(fn string, f func([]byte) []diff) error { 867 b, err := os.ReadFile(fn) 868 if err != nil { 869 return err 870 } 871 872 diffs := f(b) 873 sort.Slice(diffs, func(i, j int) bool { return diffs[i].from < diffs[j].from }) 874 var patched [][]byte 875 off := 0 876 for _, diff := range diffs { 877 from := diff.from - off 878 to := diff.to - off 879 patched = append(patched, b[:from]) 880 patched = append(patched, []byte(diff.replace)) 881 b = b[to:] 882 off += to 883 } 884 patched = append(patched, b) 885 return os.WriteFile(fn, bytes.Join(patched, nil), 0660) 886 }