From 6a113896d1991f8e1c9b40812730090add60a2cc Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Wed, 22 Jun 2022 15:15:02 +0200 Subject: [PATCH 1/8] Add SQLite port (sqlite3) The SQLite port adds `-sUSE_SQLITE3` to Emscripten. Internally it uses sqlite amalagation (single file distribution). The ports creates `libsqlite3.a` (non-threaded) or `libsqlite3-mt.a` (multi-threaded). SQLite is used by CPython and Pyodide. A port in Emscripten would make our lives a bit easier and could be useful for other projects, too. --- src/settings.js | 4 ++ tools/ports/sqlite3.py | 92 ++++++++++++++++++++++++++++++++++++++++++ tools/settings.py | 1 + 3 files changed, 97 insertions(+) create mode 100644 tools/ports/sqlite3.py diff --git a/src/settings.js b/src/settings.js index 7c1b49c6e11eb..2e412ce7c9c6a 100644 --- a/src/settings.js +++ b/src/settings.js @@ -1498,6 +1498,10 @@ var SDL2_IMAGE_FORMATS = []; // [link] var SDL2_MIXER_FORMATS = ["ogg"]; +// 1 = use sqlite3 from emscripten-ports +// [link] +var USE_SQLITE3 = false; + // If true, the current build is performed for the Emscripten test harness. // [other] var IN_TEST_HARNESS = false; diff --git a/tools/ports/sqlite3.py b/tools/ports/sqlite3.py new file mode 100644 index 0000000000000..2d66de8717aa0 --- /dev/null +++ b/tools/ports/sqlite3.py @@ -0,0 +1,92 @@ +# Copyright 2022 The Emscripten Authors. All rights reserved. +# Emscripten is available under two separate licenses, the MIT license and the +# University of Illinois/NCSA Open Source License. Both these licenses can be +# found in the LICENSE file. + +import os +import shutil +import logging + +TAG = '3380500' +HASH = '4fc2992e4c1ca1664a9b01e07c9b944003c4ed0612978e471eff262a7114e4a0699244d91e71ae4ad2b5e1fc9829917cd7d5f0313aa5859036905f974548d94a' + +deps = [] + + +def needed(settings): + return settings.USE_SQLITE3 + + +def get_lib_name(settings): + return 'libsqlite3' + ('-mt' if settings.USE_PTHREADS else '') + '.a' + + +def get(ports, settings, shared): + # TODO: This is an emscripten-hosted mirror of the sqlite repo from sqlite.prg. + ports.fetch_project('sqlite3', 'https://www.sqlite.org/2022/sqlite-amalgamation-' + TAG + '.zip', 'sqlite-amalgamation-' + TAG, sha512hash=HASH) + + def create(final): + logging.info('building port: libsqlite3') + + source_path = os.path.join(ports.get_dir(), 'sqlite3', 'sqlite-amalgamation-' + TAG) + dest_path = os.path.join(ports.get_build_dir(), 'sqlite3') + + shutil.rmtree(dest_path, ignore_errors=True) + shutil.copytree(source_path, dest_path) + + ports.install_headers(dest_path) + + # flags are based on sqlite-autoconf output. + # SQLITE_HAVE_ZLIB is only used by shell.c + flags = [ + '-DSTDC_HEADERS=1', + '-DHAVE_SYS_TYPES_H=1', + '-DHAVE_SYS_STAT_H=1', + '-DHAVE_STDLIB_H=1', + '-DHAVE_STRING_H=1', + '-DHAVE_MEMORY_H=1', + '-DHAVE_STRINGS_H=1', + '-DHAVE_INTTYPES_H=1', + '-DHAVE_STDINT_H=1', + '-DHAVE_UNISTD_H=1', + '-DHAVE_FDATASYNC=1', + '-DHAVE_USLEEP=1', + '-DHAVE_LOCALTIME_R=1', + '-DHAVE_GMTIME_R=1', + '-DHAVE_DECL_STRERROR_R=1', + '-DHAVE_STRERROR_R=1', + '-DHAVE_POSIX_FALLOCATE=1', + '-DSQLITE_OMIT_LOAD_EXTENSION=1', + '-DSQLITE_ENABLE_MATH_FUNCTIONS=1', + '-DSQLITE_ENABLE_FTS4=1', + '-DSQLITE_ENABLE_FTS5=1', + '-DSQLITE_ENABLE_RTREE=1', + '-DSQLITE_ENABLE_GEOPOLY=1', + '-DSQLITE_OMIT_POPEN=1' + ] + if settings.USE_PTHREADS: + flags += ['-sUSE_PTHREADS=1'] + flags += ['-DSQLITE_THREADSAFE=1'] + flags += ['-D_REENTRANT=1'] + else: + flags += ['-DSQLITE_THREADSAFE=0'] + + ports.build_port(dest_path, final, flags=flags, exclude_files=['shell.c']) + + return [shared.Cache.get_lib(get_lib_name(settings), create, what='port')] + + +def clear(ports, settings, shared): + shared.Cache.erase_lib(get_lib_name(settings)) + + +def process_dependencies(settings): + pass + + +def process_args(ports): + return [] + + +def show(): + return 'sqlite (USE_SQLITE3=1); public domain)' diff --git a/tools/settings.py b/tools/settings.py index 1003f5c7b31f5..9c8480811161f 100644 --- a/tools/settings.py +++ b/tools/settings.py @@ -47,6 +47,7 @@ 'USE_FREETYPE', 'SDL2_MIXER_FORMATS', 'SDL2_IMAGE_FORMATS', + 'USE_SQLITE3', } # Subset of settings that apply at compile time. From e47669dc1fa6e0dffcc201ac9a1033f0eb387647 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Thu, 23 Jun 2022 17:13:00 +0200 Subject: [PATCH 2/8] Address review - drop unnecessary function - use version instead of tag name - example version and version year --- tools/ports/sqlite3.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/tools/ports/sqlite3.py b/tools/ports/sqlite3.py index 2d66de8717aa0..72743ef1b098d 100644 --- a/tools/ports/sqlite3.py +++ b/tools/ports/sqlite3.py @@ -7,7 +7,10 @@ import shutil import logging -TAG = '3380500' +# sqlite amalgamation download URL uses relase year and tag +# 2022 and (3, 38, 5) -> '/2022/sqlite-amalgamation-3380500.zip' +VERSION = (3, 38, 5) +VERSION_YEAR = 2022 HASH = '4fc2992e4c1ca1664a9b01e07c9b944003c4ed0612978e471eff262a7114e4a0699244d91e71ae4ad2b5e1fc9829917cd7d5f0313aa5859036905f974548d94a' deps = [] @@ -22,13 +25,14 @@ def get_lib_name(settings): def get(ports, settings, shared): - # TODO: This is an emscripten-hosted mirror of the sqlite repo from sqlite.prg. - ports.fetch_project('sqlite3', 'https://www.sqlite.org/2022/sqlite-amalgamation-' + TAG + '.zip', 'sqlite-amalgamation-' + TAG, sha512hash=HASH) + release = f'sqlite-amalgamation-{VERSION[0]}{VERSION[1]:02}{VERSION[2]:02}00' + # TODO: Fetch the file from an emscripten-hosted mirror. + ports.fetch_project('sqlite3', f'https://www.sqlite.org/{VERSION_YEAR}/{release}.zip', release, sha512hash=HASH) def create(final): logging.info('building port: libsqlite3') - source_path = os.path.join(ports.get_dir(), 'sqlite3', 'sqlite-amalgamation-' + TAG) + source_path = os.path.join(ports.get_dir(), 'sqlite3', release) dest_path = os.path.join(ports.get_build_dir(), 'sqlite3') shutil.rmtree(dest_path, ignore_errors=True) @@ -62,12 +66,14 @@ def create(final): '-DSQLITE_ENABLE_FTS5=1', '-DSQLITE_ENABLE_RTREE=1', '-DSQLITE_ENABLE_GEOPOLY=1', - '-DSQLITE_OMIT_POPEN=1' + '-DSQLITE_OMIT_POPEN=1', ] if settings.USE_PTHREADS: - flags += ['-sUSE_PTHREADS=1'] - flags += ['-DSQLITE_THREADSAFE=1'] - flags += ['-D_REENTRANT=1'] + flags += [ + '-sUSE_PTHREADS=1', + '-DSQLITE_THREADSAFE=1', + '-D_REENTRANT=1', + ] else: flags += ['-DSQLITE_THREADSAFE=0'] @@ -80,10 +86,6 @@ def clear(ports, settings, shared): shared.Cache.erase_lib(get_lib_name(settings)) -def process_dependencies(settings): - pass - - def process_args(ports): return [] From dbe450683e12a6b3392eb65a7d0e434595102e20 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Wed, 29 Jun 2022 15:22:56 +0200 Subject: [PATCH 3/8] update to 3.39, address review --- tools/ports/sqlite3.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/ports/sqlite3.py b/tools/ports/sqlite3.py index 72743ef1b098d..4e35bbb828459 100644 --- a/tools/ports/sqlite3.py +++ b/tools/ports/sqlite3.py @@ -9,9 +9,9 @@ # sqlite amalgamation download URL uses relase year and tag # 2022 and (3, 38, 5) -> '/2022/sqlite-amalgamation-3380500.zip' -VERSION = (3, 38, 5) +VERSION = (3, 39, 0) VERSION_YEAR = 2022 -HASH = '4fc2992e4c1ca1664a9b01e07c9b944003c4ed0612978e471eff262a7114e4a0699244d91e71ae4ad2b5e1fc9829917cd7d5f0313aa5859036905f974548d94a' +HASH = 'cbaf4adb3e404d9aa403b34f133c5beca5f641ae1e23f84dbb021da1fb9efdc7c56b5922eb533ae5cb6d26410ac60cb3f026085591bc83ebc1c225aed0cf37ca' deps = [] @@ -70,9 +70,8 @@ def create(final): ] if settings.USE_PTHREADS: flags += [ - '-sUSE_PTHREADS=1', + '-sUSE_PTHREADS', '-DSQLITE_THREADSAFE=1', - '-D_REENTRANT=1', ] else: flags += ['-DSQLITE_THREADSAFE=0'] From f51d3809c53b962a093a34fe5c1185908fffb09b Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Wed, 29 Jun 2022 15:38:38 +0200 Subject: [PATCH 4/8] Use sqlite3 port in test_sqlite --- tests/test_core.py | 34 ++++++++++------------------------ 1 file changed, 10 insertions(+), 24 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index 0a4a1c00c93ed..53065cfd168fd 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -6665,37 +6665,23 @@ def test_freetype(self): @no_asan('local count too large for VMs') @no_ubsan('local count too large for VMs') @is_slow_test - def test_sqlite(self): + @parameterized({ + 'single': (False,), + 'pthreads': (True,), + }) + def test_sqlite(self, use_pthreads): + if use_pthreads: + self.set_setting('USE_PTHREADS') + self.setup_node_pthreads() self.set_setting('EXPORTED_FUNCTIONS', ['_main', '_sqlite3_open', '_sqlite3_close', '_sqlite3_exec', '_sqlite3_free']) if '-g' in self.emcc_args: print("disabling inlining") # without registerize (which -g disables), we generate huge amounts of code self.set_setting('INLINING_LIMIT') - # newer clang has a warning for implicit conversions that lose information, - # which happens in sqlite (see #9138) - self.emcc_args += ['-Wno-implicit-int-float-conversion'] - # newer clang warns about "suspicious concatenation of string literals in an - # array initialization; did you mean to separate the elements with a comma?" - self.emcc_args += ['-Wno-string-concatenation'] - # ignore unknown flags, which lets the above flags be used on github CI - # before the LLVM change rolls in (the same LLVM change that adds the - # warning also starts to warn on it) - self.emcc_args += ['-Wno-unknown-warning-option'] - self.emcc_args += ['-Wno-pointer-bool-conversion'] - - self.emcc_args += ['-I' + test_file('third_party/sqlite')] - - src = ''' - #define SQLITE_DISABLE_LFS - #define LONGDOUBLE_TYPE double - #define SQLITE_INT64_TYPE long long int - #define SQLITE_THREADSAFE 0 - ''' - src += read_file(test_file('third_party/sqlite/sqlite3.c')) - src += read_file(test_file('sqlite/benchmark.c')) + self.emcc_args += ['-sUSE_SQLITE3'] + src = read_file(test_file('sqlite/benchmark.c')) self.do_run(src, read_file(test_file('sqlite/benchmark.txt')), - includes=[test_file('sqlite')], force_c=True) @needs_make('mingw32-make') From 04b9bf8fbb35391a680936bc8a594a8691df610d Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Wed, 29 Jun 2022 16:17:06 +0200 Subject: [PATCH 5/8] register port variant --- embuilder.py | 1 + 1 file changed, 1 insertion(+) diff --git a/embuilder.py b/embuilder.py index c7d3783114f3e..6a41b31fef7e0 100755 --- a/embuilder.py +++ b/embuilder.py @@ -103,6 +103,7 @@ 'sdl2_image_png': ('sdl2_image', {'SDL2_IMAGE_FORMATS': ["png"]}), 'sdl2_image_jpg': ('sdl2_image', {'SDL2_IMAGE_FORMATS': ["jpg"]}), 'libpng-mt': ('libpng', {'USE_PTHREADS': 1}), + 'sqlite3-mt': ('sqlite3', {'USE_PTHREADS': 1}), } PORTS = sorted(list(ports.ports_by_name.keys()) + list(PORT_VARIANTS.keys())) From 27d7337f6a975d97b4f7ea087d496909ea7a3978 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Wed, 29 Jun 2022 17:04:09 +0200 Subject: [PATCH 6/8] simplify test case even further --- tests/test_core.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/tests/test_core.py b/tests/test_core.py index 53065cfd168fd..c01b0b7422d95 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -6673,16 +6673,12 @@ def test_sqlite(self, use_pthreads): if use_pthreads: self.set_setting('USE_PTHREADS') self.setup_node_pthreads() - self.set_setting('EXPORTED_FUNCTIONS', ['_main', '_sqlite3_open', '_sqlite3_close', '_sqlite3_exec', '_sqlite3_free']) - if '-g' in self.emcc_args: - print("disabling inlining") # without registerize (which -g disables), we generate huge amounts of code - self.set_setting('INLINING_LIMIT') - self.emcc_args += ['-sUSE_SQLITE3'] - src = read_file(test_file('sqlite/benchmark.c')) - self.do_run(src, - read_file(test_file('sqlite/benchmark.txt')), - force_c=True) + self.do_run_from_file( + test_file('sqlite/benchmark.c'), + test_file('sqlite/benchmark.txt'), + force_c=True + ) @needs_make('mingw32-make') @is_slow_test From 926b15a022249b70af3417aa1e5c39ec85b3b7bb Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Wed, 29 Jun 2022 17:14:52 +0200 Subject: [PATCH 7/8] drop suffix for single thread test case --- tests/test_core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_core.py b/tests/test_core.py index c01b0b7422d95..f36d4f0f54e36 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -6666,7 +6666,7 @@ def test_freetype(self): @no_ubsan('local count too large for VMs') @is_slow_test @parameterized({ - 'single': (False,), + '': (False,), 'pthreads': (True,), }) def test_sqlite(self, use_pthreads): From f1f330a5aeb25757e702d85f6fb539f26ea1ffd7 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Wed, 29 Jun 2022 17:15:04 +0200 Subject: [PATCH 8/8] build sqlite3 for CI --- embuilder.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/embuilder.py b/embuilder.py index 6a41b31fef7e0..6121fa11783a9 100755 --- a/embuilder.py +++ b/embuilder.py @@ -61,6 +61,8 @@ 'crt1_proxy_main', 'libunwind-except', 'libnoexit', + 'sqlite3', + 'sqlite3-mt', ] # Additional tasks on top of MINIMAL_TASKS that are necessary for PIC testing on