From d52fc09c69a9f8cb2487ff60f29b2d984f603111 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 1 Oct 2019 16:35:25 -0700 Subject: [PATCH 01/26] wip [ci skip] --- src/preamble.js | 2 +- tests/test_browser.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/preamble.js b/src/preamble.js index cb07d015c736d..425721713e7ed 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -407,7 +407,7 @@ if (ENVIRONMENT_IS_WEB) { #if USE_PTHREADS if (typeof SharedArrayBuffer === 'undefined' || typeof Atomics === 'undefined') { - xhr = new XMLHttpRequest(); + var xhr = new XMLHttpRequest(); xhr.open('GET', 'http://localhost:8888/report_result?skipped:%20SharedArrayBuffer%20is%20not%20supported!'); xhr.send(); setTimeout(function() { window.close() }, 2000); diff --git a/tests/test_browser.py b/tests/test_browser.py index 3e3e380c54670..e333287d80da1 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -3595,7 +3595,7 @@ def prep_no_SAB(self): # Test that the emscripten_ atomics api functions work. @requires_threads def test_pthread_atomics(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_atomics.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_atomics.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '--closure', '1', '-g1']) # Test 64-bit atomics. @requires_threads From ef259987f528839b99e426989958a19be343ac67 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Tue, 1 Oct 2019 17:27:24 -0700 Subject: [PATCH 02/26] wip but test hangs [ci skip] --- src/library_pthread.js | 5 ++++- src/preamble.js | 6 ++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/library_pthread.js b/src/library_pthread.js index 21e00b5a606f0..fe8eb3d7662bf 100644 --- a/src/library_pthread.js +++ b/src/library_pthread.js @@ -5,7 +5,10 @@ var LibraryPThread = { $PThread__postset: 'if (!ENVIRONMENT_IS_PTHREAD) PThread.initMainThreadBlock();', - $PThread__deps: ['$PROCINFO', '_register_pthread_ptr', 'emscripten_main_thread_process_queued_calls', '$ERRNO_CODES', 'emscripten_futex_wake'], + $PThread__deps: ['$PROCINFO', '_register_pthread_ptr', + 'emscripten_main_thread_process_queued_calls', + '$ERRNO_CODES', 'emscripten_futex_wake', '_kill_thread', + '_cancel_thread'], $PThread: { MAIN_THREAD_ID: 1, // A special constant that identifies the main JS thread ID. mainThreadInfo: { diff --git a/src/preamble.js b/src/preamble.js index 425721713e7ed..b04c94bad0e54 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -62,6 +62,12 @@ var wasmTable = new WebAssembly.Table({ #if USE_PTHREADS // For sending to workers. var wasmModule; +// Only workers actually use these field, but we refer to them from +// library_pthread (which exists on all threads) so this definition is useful +// to avoid accessing the global scope. +var threadInfoStruct = 0; +var selfThreadId = 0; +var __performance_now_clock_drift = 0; #endif // USE_PTHREADS //======================================== From 5c4ec01459c9866c4ae43aa74a6c3a982770f419 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 2 Oct 2019 11:15:22 -0700 Subject: [PATCH 03/26] escaping [ci skip] --- src/library_pthread.js | 109 +++++++++++++++++++++-------------------- 1 file changed, 55 insertions(+), 54 deletions(-) diff --git a/src/library_pthread.js b/src/library_pthread.js index 48e52dc95a594..5714314c46f67 100644 --- a/src/library_pthread.js +++ b/src/library_pthread.js @@ -181,7 +181,7 @@ var LibraryPThread = { if (ENVIRONMENT_IS_PTHREAD) { // Note: in theory we would like to return any offscreen canvases back to the main thread, // but if we ever fetched a rendering context for them that would not be valid, so we don't try. - postMessage({ cmd: 'exit' }); + postMessage({ 'cmd': 'exit' }); } } }, @@ -193,7 +193,7 @@ var LibraryPThread = { _emscripten_futex_wake(threadInfoStruct + {{{ C_STRUCTS.pthread.threadStatus }}}, {{{ cDefine('INT_MAX') }}}); // wake all threads threadInfoStruct = selfThreadId = 0; // Not hosting a pthread anymore in this worker, reset the info structures to null. __register_pthread_ptr(0, 0, 0); // Unregister the thread block also inside the asm.js scope. - postMessage({ cmd: 'cancelDone' }); + postMessage({ 'cmd': 'cancelDone' }); }, terminateAllThreads: function() { @@ -309,32 +309,32 @@ var LibraryPThread = { // Ask the new worker to load up the Emscripten-compiled page. This is a heavy operation. worker.postMessage({ - cmd: 'load', + 'cmd': 'load', // If the application main .js file was loaded from a Blob, then it is not possible // to access the URL of the current script that could be passed to a Web Worker so that // it could load up the same file. In that case, developer must either deliver the Blob // object in Module['mainScriptUrlOrBlob'], or a URL to it, so that pthread Workers can // independently load up the same main application file. - urlOrBlob: Module['mainScriptUrlOrBlob'] || _scriptDir, + 'urlOrBlob': Module['mainScriptUrlOrBlob'] || _scriptDir, #if WASM - wasmMemory: wasmMemory, - wasmModule: wasmModule, + 'wasmMemory': wasmMemory, + 'wasmModule': wasmModule, #if LOAD_SOURCE_MAP - wasmSourceMap: wasmSourceMap, + 'wasmSourceMap': wasmSourceMap, #endif #if USE_OFFSET_CONVERTER - wasmOffsetConverter: wasmOffsetConverter, + 'wasmOffsetConverter': wasmOffsetConverter, #endif #else - buffer: HEAPU8.buffer, - asmJsUrlOrBlob: Module["asmJsUrlOrBlob"], + 'buffer': HEAPU8.buffer, + 'asmJsUrlOrBlob': Module["asmJsUrlOrBlob"], #endif #if !WASM_BACKEND - tempDoublePtr: tempDoublePtr, + 'tempDoublePtr': tempDoublePtr, #endif - DYNAMIC_BASE: DYNAMIC_BASE, - DYNAMICTOP_PTR: DYNAMICTOP_PTR, - PthreadWorkerInit: PthreadWorkerInit + 'DYNAMIC_BASE': DYNAMIC_BASE, + 'DYNAMICTOP_PTR': DYNAMICTOP_PTR, + 'PthreadWorkerInit': PthreadWorkerInit }); PThread.unusedWorkers.push(worker); } @@ -350,34 +350,35 @@ var LibraryPThread = { var worker = workers[i]; (function(worker) { worker.onmessage = function(e) { - var d = e.data; + var d = e['data']; + var cmd = d['cmd']; // Sometimes we need to backproxy events to the calling thread (e.g. HTML5 DOM events handlers such as emscripten_set_mousemove_callback()), so keep track in a globally accessible variable about the thread that initiated the proxying. if (worker.pthread) PThread.currentProxiedOperationCallerThread = worker.pthread.threadInfoStruct; // If this message is intended to a recipient that is not the main thread, forward it to the target thread. - if (d.targetThread && d.targetThread != _pthread_self()) { + if (d['targetThread'] && d['targetThread'] != _pthread_self()) { var thread = PThread.pthreads[d.targetThread]; if (thread) { - thread.worker.postMessage(e.data, d.transferList); + thread.worker.postMessage(e.data, d['transferList']); } else { - console.error('Internal error! Worker sent a message "' + d.cmd + '" to target pthread ' + d.targetThread + ', but that thread no longer exists!'); + console.error('Internal error! Worker sent a message "' + cmd + '" to target pthread ' + d['targetThread'] + ', but that thread no longer exists!'); } PThread.currentProxiedOperationCallerThread = undefined; return; } - if (d.cmd === 'processQueuedMainThreadWork') { + if (cmd === 'processQueuedMainThreadWork') { // TODO: Must post message to main Emscripten thread in PROXY_TO_WORKER mode. _emscripten_main_thread_process_queued_calls(); - } else if (d.cmd === 'spawnThread') { + } else if (cmd === 'spawnThread') { __spawn_thread(e.data); - } else if (d.cmd === 'cleanupThread') { - __cleanup_thread(d.thread); - } else if (d.cmd === 'killThread') { - __kill_thread(d.thread); - } else if (d.cmd === 'cancelThread') { - __cancel_thread(d.thread); - } else if (d.cmd === 'loaded') { + } else if (cmd === 'cleanupThread') { + __cleanup_thread(d['thread']); + } else if (cmd === 'killThread') { + __kill_thread(d['thread']); + } else if (cmd === 'cancelThread') { + __cancel_thread(d['thread']); + } else if (cmd === 'loaded') { worker.loaded = true; // If this Worker is already pending to start running a thread, launch the thread now if (worker.runPthread) { @@ -388,34 +389,34 @@ var LibraryPThread = { if (numWorkersLoaded === numWorkers && onFinishedLoading) { onFinishedLoading(); } - } else if (d.cmd === 'print') { - out('Thread ' + d.threadId + ': ' + d.text); - } else if (d.cmd === 'printErr') { - err('Thread ' + d.threadId + ': ' + d.text); - } else if (d.cmd === 'alert') { - alert('Thread ' + d.threadId + ': ' + d.text); - } else if (d.cmd === 'exit') { + } else if (cmd === 'print') { + out('Thread ' + d['threadId'] + ': ' + d['text']); + } else if (cmd === 'printErr') { + err('Thread ' + d['threadId'] + ': ' + d['text']); + } else if (cmd === 'alert') { + alert('Thread ' + d['threadId'] + ': ' + d['text']); + } else if (cmd === 'exit') { var detached = worker.pthread && Atomics.load(HEAPU32, (worker.pthread.thread + {{{ C_STRUCTS.pthread.detached }}}) >> 2); if (detached) { PThread.returnWorkerToPool(worker); } - } else if (d.cmd === 'exitProcess') { + } else if (cmd === 'exitProcess') { // A pthread has requested to exit the whole application process (runtime). noExitRuntime = false; try { - exit(d.returnCode); + exit(d['returnCode']); } catch (e) { if (e instanceof ExitStatus) return; throw e; } - } else if (d.cmd === 'cancelDone') { + } else if (cmd === 'cancelDone') { PThread.returnWorkerToPool(worker); - } else if (d.cmd === 'objectTransfer') { + } else if (cmd === 'objectTransfer') { PThread.receiveObjectTransfer(e.data); } else if (e.data.target === 'setimmediate') { worker.postMessage(e.data); // Worker wants to postMessage() to itself to implement setImmediate() emulation. } else { - err("worker sent an unknown command " + d.cmd); + err("worker sent an unknown command " + cmd); } PThread.currentProxiedOperationCallerThread = undefined; }; @@ -487,7 +488,7 @@ var LibraryPThread = { if (ENVIRONMENT_IS_PTHREAD) throw 'Internal Error! _cancel_thread() can only ever be called from main application thread!'; if (!pthread_ptr) throw 'Internal Error! Null pthread_ptr in _cancel_thread!'; var pthread = PThread.pthreads[pthread_ptr]; - pthread.worker.postMessage({ cmd: 'cancel' }); + pthread.worker.postMessage({ 'cmd': 'cancel' }); }, _spawn_thread: function(threadParams) { @@ -541,17 +542,17 @@ var LibraryPThread = { worker.pthread = pthread; var msg = { - cmd: 'run', - start_routine: threadParams.startRoutine, - arg: threadParams.arg, - threadInfoStruct: threadParams.pthread_ptr, - selfThreadId: threadParams.pthread_ptr, // TODO: Remove this since thread ID is now the same as the thread address. - parentThreadId: threadParams.parent_pthread_ptr, - stackBase: threadParams.stackBase, - stackSize: threadParams.stackSize, + 'cmd': 'run', + 'start_routine': threadParams.startRoutine, + 'arg': threadParams.arg, + 'threadInfoStruct': threadParams.pthread_ptr, + 'selfThreadId': threadParams.pthread_ptr, // TODO: Remove this since thread ID is now the same as the thread address. + 'parentThreadId': threadParams.parent_pthread_ptr, + 'stackBase': threadParams.stackBase, + 'stackSize': threadParams.stackSize, #if OFFSCREENCANVAS_SUPPORT - moduleCanvasId: threadParams.moduleCanvasId, - offscreenCanvases: threadParams.offscreenCanvases, + 'moduleCanvasId': threadParams.moduleCanvasId, + 'offscreenCanvases': threadParams.offscreenCanvases, #endif }; worker.runPthread = function() { @@ -826,7 +827,7 @@ var LibraryPThread = { Atomics.store(HEAPU32, (thread + {{{ C_STRUCTS.pthread.detached }}} ) >> 2, 1); // Mark the thread as detached. if (!ENVIRONMENT_IS_PTHREAD) __cleanup_thread(thread); - else postMessage({ cmd: 'cleanupThread', thread: thread }); + else postMessage({ 'cmd': 'cleanupThread', 'thread': thread }); return 0; } // TODO HACK! Replace the _js variant with just _pthread_testcancel: @@ -858,7 +859,7 @@ var LibraryPThread = { } if (signal != 0) { if (!ENVIRONMENT_IS_PTHREAD) __kill_thread(thread); - else postMessage({ cmd: 'killThread', thread: thread}); + else postMessage({ 'cmd': 'killThread', 'thread': thread}); } return 0; }, @@ -880,7 +881,7 @@ var LibraryPThread = { } Atomics.compareExchange(HEAPU32, (thread + {{{ C_STRUCTS.pthread.threadStatus }}} ) >> 2, 0, 2); // Signal the thread that it needs to cancel itself. if (!ENVIRONMENT_IS_PTHREAD) __cancel_thread(thread); - else postMessage({ cmd: 'cancelThread', thread: thread}); + else postMessage({ 'cmd': 'cancelThread', 'thread': thread}); return 0; }, @@ -1163,7 +1164,7 @@ var LibraryPThread = { __call_main: function(argc, argv) { var returnCode = _main(argc, argv); - if (!noExitRuntime) postMessage({ cmd: 'exitProcess', returnCode: returnCode }); + if (!noExitRuntime) postMessage({ 'cmd': 'exitProcess', 'returnCode': returnCode }); return returnCode; }, From 70bf88c02d0225b94bc9061013cf28adf58e17e4 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 2 Oct 2019 12:26:12 -0700 Subject: [PATCH 04/26] wip [ci skip] --- src/modules.js | 26 +++++++++++++++----------- src/parseTools.js | 10 ---------- src/preamble.js | 1 + src/worker.js | 34 +++++++++++++++++----------------- tests/test_browser.py | 2 +- 5 files changed, 34 insertions(+), 39 deletions(-) diff --git a/src/modules.js b/src/modules.js index 9ee5ea0d0c1ee..ffb58a4dd4c0e 100644 --- a/src/modules.js +++ b/src/modules.js @@ -447,17 +447,21 @@ function exportRuntime() { runtimeElements.push('warnOnce'); } - if (MODULARIZE) { - // In MODULARIZE=1 mode, the following functions need to be exported out to Module for worker.js to access. - if (STACK_OVERFLOW_CHECK) { - runtimeElements.push('writeStackCookie'); - runtimeElements.push('checkStackCookie'); - runtimeElements.push('abortStackOverflow'); - } - if (USE_PTHREADS) { - runtimeElements.push('PThread'); - runtimeElements.push('ExitStatus'); - } + if (STACK_OVERFLOW_CHECK) { + runtimeElements.push('writeStackCookie'); + runtimeElements.push('checkStackCookie'); + runtimeElements.push('abortStackOverflow'); + } + + if (USE_PTHREADS) { + // In pthreads mode, the following functions always need to be exported to + // Module for closure compiler, and also for MODULARIZE (so worker.js can + // access them). + ['PThread', 'ExitStatus', 'tempDoublePtr', 'wasmMemory', '_pthread_self', + 'ExitStatus', 'tempDoublePtr'].forEach(function(x) { + EXPORTED_RUNTIME_METHODS_SET[x] = 1; + runtimeElements.push(x); + }); } if (SUPPORT_BASE64_EMBEDDING) { diff --git a/src/parseTools.js b/src/parseTools.js index 0ded3f2a325f6..967afda5b66aa 100644 --- a/src/parseTools.js +++ b/src/parseTools.js @@ -1530,16 +1530,6 @@ function makeAsmExportAccessInPthread(variable) { } } -// Generates access to a JS global scope variable in pthreads worker.js. In MODULARIZE mode the JS scope is not directly accessible, so all the relevant variables -// are exported via Module. In non-MODULARIZE mode, we can directly access the variables in global scope. -function makeAsmGlobalAccessInPthread(variable) { - if (MODULARIZE) { - return "Module['" + variable + "']"; // 'Module' is defined in worker.js local scope, so not EXPORT_NAME in this case. - } else { - return variable; - } -} - // Generates access to both global scope variable and exported Module variable, e.g. "Module['foo'] = foo" or just plain "foo" depending on if we are MODULARIZEing. // Used the be able to initialize both variables at the same time in scenarios where a variable exists in both global scope and in Module. function makeAsmExportAndGlobalAssignTargetInPthread(variable) { diff --git a/src/preamble.js b/src/preamble.js index d059a9802c637..6d8d8e5f969d7 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -68,6 +68,7 @@ var wasmModule; var threadInfoStruct = 0; var selfThreadId = 0; var __performance_now_clock_drift = 0; +var tempDoublePtr = 0; #endif // USE_PTHREADS //======================================== diff --git a/src/worker.js b/src/worker.js index 9ca39b432d649..5ed0c0d1f6198 100644 --- a/src/worker.js +++ b/src/worker.js @@ -114,11 +114,11 @@ var wasmOffsetData; #endif this.onmessage = function(e) { - try { + //try { if (e.data.cmd === 'load') { // Preload command that is called once per worker to parse and load the Emscripten code. #if !WASM_BACKEND // Initialize the thread-local field(s): - {{{ makeAsmGlobalAccessInPthread('tempDoublePtr') }}} = e.data.tempDoublePtr; + Module['tempDoublePtr'] = e.data.tempDoublePtr; #endif // Initialize the global "process"-wide fields: @@ -146,7 +146,7 @@ this.onmessage = function(e) { #if USE_OFFSET_CONVERTER wasmOffsetData = e.data.wasmOffsetConverter; #endif - {{{ makeAsmExportAndGlobalAssignTargetInPthread('buffer') }}} = {{{ makeAsmGlobalAccessInPthread('wasmMemory') }}}.buffer; + {{{ makeAsmExportAndGlobalAssignTargetInPthread('buffer') }}} = Module['wasmMemory'].buffer; #else {{{ makeAsmExportAndGlobalAssignTargetInPthread('buffer') }}} = e.data.buffer; @@ -203,7 +203,7 @@ this.onmessage = function(e) { } else if (e.data.cmd === 'run') { // This worker was idle, and now should start executing its pthread entry point. __performance_now_clock_drift = performance.now() - e.data.time; // Sync up to the clock of the main thread. threadInfoStruct = e.data.threadInfoStruct; - {{{ makeAsmGlobalAccessInPthread('__register_pthread_ptr') }}}(threadInfoStruct, /*isMainBrowserThread=*/0, /*isMainRuntimeThread=*/0); // Pass the thread address inside the asm.js scope to store it for fast access that avoids the need for a FFI out. + Module['__register_pthread_ptr'](threadInfoStruct, /*isMainBrowserThread=*/0, /*isMainRuntimeThread=*/0); // Pass the thread address inside the asm.js scope to store it for fast access that avoids the need for a FFI out. selfThreadId = e.data.selfThreadId; parentThreadId = e.data.parentThreadId; // Establish the stack frame for this thread in global scope @@ -245,11 +245,11 @@ this.onmessage = function(e) { Module['___set_stack_limit'](STACK_MAX); #endif #if STACK_OVERFLOW_CHECK - {{{ makeAsmGlobalAccessInPthread('writeStackCookie') }}}(); + Module['writeStackCookie'](); #endif PThread.receiveObjectTransfer(e.data); - PThread.setThreadStatus({{{ makeAsmGlobalAccessInPthread('_pthread_self') }}}(), 1/*EM_THREAD_STATUS_RUNNING*/); + PThread.setThreadStatus(Module['_pthread_self'](), 1/*EM_THREAD_STATUS_RUNNING*/); try { // pthread entry points are always of signature 'void *ThreadMain(void *arg)' @@ -262,7 +262,7 @@ this.onmessage = function(e) { var result = Module['dynCall_ii'](e.data.start_routine, e.data.arg); #if STACK_OVERFLOW_CHECK - {{{ makeAsmGlobalAccessInPthread('checkStackCookie') }}}(); + Module['checkStackCookie'](); #endif } catch(e) { @@ -272,16 +272,16 @@ this.onmessage = function(e) { } else if (e === 'SimulateInfiniteLoop' || e === 'pthread_exit') { return; } else { - Atomics.store(HEAPU32, (threadInfoStruct + 4 /*C_STRUCTS.pthread.threadExitCode*/ ) >> 2, (e instanceof {{{ makeAsmGlobalAccessInPthread('ExitStatus') }}}) ? e.status : -2 /*A custom entry specific to Emscripten denoting that the thread crashed.*/); + Atomics.store(HEAPU32, (threadInfoStruct + 4 /*C_STRUCTS.pthread.threadExitCode*/ ) >> 2, (e instanceof Module['ExitStatus']) ? e.status : -2 /*A custom entry specific to Emscripten denoting that the thread crashed.*/); Atomics.store(HEAPU32, (threadInfoStruct + 0 /*C_STRUCTS.pthread.threadStatus*/ ) >> 2, 1); // Mark the thread as no longer running. #if ASSERTIONS - if (typeof({{{ makeAsmGlobalAccessInPthread('_emscripten_futex_wake') }}}) !== "function") { + if (typeof(Module['_emscripten_futex_wake']) !== "function") { err("Thread Initialisation failed."); throw e; } #endif - {{{ makeAsmGlobalAccessInPthread('_emscripten_futex_wake') }}}(threadInfoStruct + 0 /*C_STRUCTS.pthread.threadStatus*/, 0x7FFFFFFF/*INT_MAX*/); // Wake all threads waiting on this thread to finish. - if (!(e instanceof {{{ makeAsmGlobalAccessInPthread('ExitStatus') }}})) throw e; + Module['_emscripten_futex_wake'](threadInfoStruct + 0 /*C_STRUCTS.pthread.threadStatus*/, 0x7FFFFFFF/*INT_MAX*/); // Wake all threads waiting on this thread to finish. + if (!(e instanceof Module['ExitStatus'])) throw e; } } // The thread might have finished without calling pthread_exit(). If so, then perform the exit operation ourselves. @@ -295,15 +295,15 @@ this.onmessage = function(e) { // no-op } else if (e.data.cmd === 'processThreadQueue') { if (threadInfoStruct) { // If this thread is actually running? - {{{ makeAsmGlobalAccessInPthread('_emscripten_current_thread_process_queued_calls') }}}(); + Module['_emscripten_current_thread_process_queued_calls'](); } } else { err('worker.js received unknown command ' + e.data.cmd); console.error(e.data); } - } catch(e) { - console.error('worker.js onmessage() captured an uncaught exception: ' + e); - console.error(e.stack); - throw e; - } + //} catch(e) { + // console.error('worker.js onmessage() captured an uncaught exception: ' + e); + // console.error(e.stack); + // throw e; + // } }; diff --git a/tests/test_browser.py b/tests/test_browser.py index 84eb7887e5ea2..ccae6cb185a36 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -3595,7 +3595,7 @@ def prep_no_SAB(self): # Test that the emscripten_ atomics api functions work. @requires_threads def test_pthread_atomics(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_atomics.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '--closure', '1', '-g1']) + self.btest(path_from_root('tests', 'pthread', 'test_pthread_atomics.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-g1'])#, '--closure', '1', '-g1']) # Test 64-bit atomics. @requires_threads From a1ab547db63c0bb8d3809dedf737b1356e997858 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 2 Oct 2019 12:30:30 -0700 Subject: [PATCH 05/26] cleanup --- src/worker.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/worker.js b/src/worker.js index 5ed0c0d1f6198..60beb96255932 100644 --- a/src/worker.js +++ b/src/worker.js @@ -114,7 +114,7 @@ var wasmOffsetData; #endif this.onmessage = function(e) { - //try { + try { if (e.data.cmd === 'load') { // Preload command that is called once per worker to parse and load the Emscripten code. #if !WASM_BACKEND // Initialize the thread-local field(s): @@ -146,7 +146,7 @@ this.onmessage = function(e) { #if USE_OFFSET_CONVERTER wasmOffsetData = e.data.wasmOffsetConverter; #endif - {{{ makeAsmExportAndGlobalAssignTargetInPthread('buffer') }}} = Module['wasmMemory'].buffer; + {{{ makeAsmExportAndGlobalAssignTargetInPthread('buffer') }}} = wasmMemory.buffer; #else {{{ makeAsmExportAndGlobalAssignTargetInPthread('buffer') }}} = e.data.buffer; @@ -301,9 +301,9 @@ this.onmessage = function(e) { err('worker.js received unknown command ' + e.data.cmd); console.error(e.data); } - //} catch(e) { - // console.error('worker.js onmessage() captured an uncaught exception: ' + e); - // console.error(e.stack); - // throw e; - // } + } catch(e) { + console.error('worker.js onmessage() captured an uncaught exception: ' + e); + console.error(e.stack); + throw e; + } }; From 21790b0eb4619bafa82aae3a9abae12015512632 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 2 Oct 2019 12:52:41 -0700 Subject: [PATCH 06/26] work [ci skip] --- src/shell_pthreads.js | 2 +- src/worker.js | 4 +--- tests/test_browser.py | 8 ++++++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/shell_pthreads.js b/src/shell_pthreads.js index e4f9802aa9de6..4f8256b3dc26f 100644 --- a/src/shell_pthreads.js +++ b/src/shell_pthreads.js @@ -5,7 +5,7 @@ #if USE_PTHREADS // ENVIRONMENT_IS_PTHREAD=true will have been preset in worker.js. Make it false in the main runtime thread. -var ENVIRONMENT_IS_PTHREAD = Module.ENVIRONMENT_IS_PTHREAD || false; +var ENVIRONMENT_IS_PTHREAD = Module['ENVIRONMENT_IS_PTHREAD'] || false; if (!ENVIRONMENT_IS_PTHREAD) { var PthreadWorkerInit = {}; // Collects together variables that are needed at initialization time for the web workers that host pthreads. } diff --git a/src/worker.js b/src/worker.js index 60beb96255932..99a0d3d3dc084 100644 --- a/src/worker.js +++ b/src/worker.js @@ -185,13 +185,11 @@ this.onmessage = function(e) { importScripts(objectUrl); URL.revokeObjectURL(objectUrl); } -#if MODULARIZE -#if !MODULARIZE_INSTANCE +#if MODULARIZE && !MODULARIZE_INSTANCE Module = {{{ EXPORT_NAME }}}(Module); #endif PThread = Module['PThread']; HEAPU32 = Module['HEAPU32']; -#endif #if !ASMFS if (typeof FS !== 'undefined' && typeof FS.createStandardStreams === 'function') FS.createStandardStreams(); diff --git a/tests/test_browser.py b/tests/test_browser.py index ccae6cb185a36..774319a2446ad 100644 --- a/tests/test_browser.py +++ b/tests/test_browser.py @@ -3593,9 +3593,13 @@ def prep_no_SAB(self): ''')) # Test that the emscripten_ atomics api functions work. + @parameterized({ + 'normal': ([],), + 'closure': (['--closure', '1'],), + }) @requires_threads - def test_pthread_atomics(self): - self.btest(path_from_root('tests', 'pthread', 'test_pthread_atomics.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-g1'])#, '--closure', '1', '-g1']) + def test_pthread_atomics(self, args=[]): + self.btest(path_from_root('tests', 'pthread', 'test_pthread_atomics.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-O3', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-g1'] + args) # Test 64-bit atomics. @requires_threads From 82d64bdfe0393f02e4b65ea979aa213dc0c7820a Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 2 Oct 2019 13:01:17 -0700 Subject: [PATCH 07/26] more [ci skip] --- src/library.js | 14 +++++++------- src/library_fetch.js | 2 +- src/library_pthread.js | 4 ++-- src/shell_pthreads.js | 12 ++++++------ 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/library.js b/src/library.js index d9947563a9dda..bb05f1da150b7 100644 --- a/src/library.js +++ b/src/library.js @@ -25,8 +25,8 @@ LibraryManager.library = { // keep this low in memory, because we flatten arrays with them in them #if USE_PTHREADS - _impure_ptr: '; if (ENVIRONMENT_IS_PTHREAD) __impure_ptr = PthreadWorkerInit.__impure_ptr; else PthreadWorkerInit.__impure_ptr __impure_ptr = {{{ makeStaticAlloc(4) }}}', - __dso_handle: '; if (ENVIRONMENT_IS_PTHREAD) ___dso_handle = PthreadWorkerInit.___dso_handle; else PthreadWorkerInit.___dso_handle = ___dso_handle = {{{ makeStaticAlloc(4) }}}', + _impure_ptr: '; if (ENVIRONMENT_IS_PTHREAD) __impure_ptr = PthreadWorkerInit["__impure_ptr"]; else PthreadWorkerInit["__impure_ptr"] = {{{ makeStaticAlloc(4) }}}', + __dso_handle: '; if (ENVIRONMENT_IS_PTHREAD) ___dso_handle = PthreadWorkerInit["___dso_handle"]; else PthreadWorkerInit["___dso_handle"] = ___dso_handle = {{{ makeStaticAlloc(4) }}}', #else _impure_ptr: '{{{ makeStaticAlloc(1) }}}', __dso_handle: '{{{ makeStaticAlloc(1) }}}', @@ -1864,9 +1864,9 @@ LibraryManager.library = { // Statically allocated time struct. #if USE_PTHREADS - __tm_current: '; if (ENVIRONMENT_IS_PTHREAD) ___tm_current = PthreadWorkerInit.___tm_current; else PthreadWorkerInit.___tm_current = ___tm_current = {{{ makeStaticAlloc(C_STRUCTS.tm.__size__) }}}', - __tm_timezone: '; if (ENVIRONMENT_IS_PTHREAD) ___tm_timezone = PthreadWorkerInit.___tm_timezone; else PthreadWorkerInit.___tm_timezone = ___tm_timezone = {{{ makeStaticString("GMT") }}}', - __tm_formatted: '; if (ENVIRONMENT_IS_PTHREAD) ___tm_formatted = PthreadWorkerInit.___tm_formatted; else PthreadWorkerInit.___tm_formatted = ___tm_formatted = {{{ makeStaticAlloc(C_STRUCTS.tm.__size__) }}}', + __tm_current: '; if (ENVIRONMENT_IS_PTHREAD) ___tm_current = PthreadWorkerInit["___tm_current"]; else PthreadWorkerInit["___tm_current"] = ___tm_current = {{{ makeStaticAlloc(C_STRUCTS.tm.__size__) }}}', + __tm_timezone: '; if (ENVIRONMENT_IS_PTHREAD) ___tm_timezone = PthreadWorkerInit["___tm_timezone"]; else PthreadWorkerInit["___tm_timezone"] = ___tm_timezone = {{{ makeStaticString("GMT") }}}', + __tm_formatted: '; if (ENVIRONMENT_IS_PTHREAD) ___tm_formatted = PthreadWorkerInit["___tm_formatted"]; else PthreadWorkerInit["___tm_formatted"] = ___tm_formatted = {{{ makeStaticAlloc(C_STRUCTS.tm.__size__) }}}', #else __tm_current: '{{{ makeStaticAlloc(C_STRUCTS.tm.__size__) }}}', // Statically allocated copy of the string "GMT" for gmtime() to point to @@ -3223,8 +3223,8 @@ LibraryManager.library = { // ========================================================================== #if USE_PTHREADS - in6addr_any: '; if (ENVIRONMENT_IS_PTHREAD) _in6addr_any = PthreadWorkerInit._in6addr_any; else PthreadWorkerInit._in6addr_any = _in6addr_any = {{{ makeStaticAlloc(16) }}}', - in6addr_loopback: '; if (ENVIRONMENT_IS_PTHREAD) _in6addr_loopback = PthreadWorkerInit._in6addr_loopback; else PthreadWorkerInit._in6addr_loopback = _in6addr_loopback = {{{ makeStaticAlloc(16) }}}', + in6addr_any: '; if (ENVIRONMENT_IS_PTHREAD) _in6addr_any = PthreadWorkerInit["_in6addr_any"]; else PthreadWorkerInit["_in6addr_any"] = _in6addr_any = {{{ makeStaticAlloc(16) }}}', + in6addr_loopback: '; if (ENVIRONMENT_IS_PTHREAD) _in6addr_loopback = PthreadWorkerInit["_in6addr_loopback"]; else PthreadWorkerInit["_in6addr_loopback"] = _in6addr_loopback = {{{ makeStaticAlloc(16) }}}', #else in6addr_any: '{{{ makeStaticAlloc(16) }}}', diff --git a/src/library_fetch.js b/src/library_fetch.js index abdd288daf569..90c86088d8f60 100644 --- a/src/library_fetch.js +++ b/src/library_fetch.js @@ -8,7 +8,7 @@ var LibraryFetch = { #if USE_PTHREADS $Fetch__postset: 'if (!ENVIRONMENT_IS_PTHREAD) Fetch.staticInit();', - fetch_work_queue: '; if (ENVIRONMENT_IS_PTHREAD) _fetch_work_queue = PthreadWorkerInit._fetch_work_queue; else PthreadWorkerInit._fetch_work_queue = _fetch_work_queue = {{{ makeStaticAlloc(12) }}}', + fetch_work_queue: '; if (ENVIRONMENT_IS_PTHREAD) _fetch_work_queue = PthreadWorkerInit["_fetch_work_queue"]; else PthreadWorkerInit["_fetch_work_queue"] = _fetch_work_queue = {{{ makeStaticAlloc(12) }}}', #else $Fetch__postset: 'Fetch.staticInit();', fetch_work_queue: '{{{ makeStaticAlloc(12) }}}', diff --git a/src/library_pthread.js b/src/library_pthread.js index 5714314c46f67..5beb631b4af5f 100644 --- a/src/library_pthread.js +++ b/src/library_pthread.js @@ -567,7 +567,7 @@ var LibraryPThread = { }, _num_logical_cores__deps: ['emscripten_force_num_logical_cores'], - _num_logical_cores: '; if (ENVIRONMENT_IS_PTHREAD) __num_logical_cores = PthreadWorkerInit.__num_logical_cores; else { PthreadWorkerInit.__num_logical_cores = __num_logical_cores = {{{ makeStaticAlloc(4) }}}; HEAPU32[__num_logical_cores>>2] = navigator["hardwareConcurrency"] || ' + {{{ PTHREAD_HINT_NUM_CORES }}} + '; }', + _num_logical_cores: '; if (ENVIRONMENT_IS_PTHREAD) __num_logical_cores = PthreadWorkerInit["__num_logical_cores"]; else { PthreadWorkerInit["__num_logical_cores"] = __num_logical_cores = {{{ makeStaticAlloc(4) }}}; HEAPU32[__num_logical_cores>>2] = navigator["hardwareConcurrency"] || ' + {{{ PTHREAD_HINT_NUM_CORES }}} + '; }', emscripten_has_threading_support: function() { return typeof SharedArrayBuffer !== 'undefined'; @@ -1073,7 +1073,7 @@ var LibraryPThread = { }, // Stores the memory address that the main thread is waiting on, if any. - _main_thread_futex_wait_address: '; if (ENVIRONMENT_IS_PTHREAD) __main_thread_futex_wait_address = PthreadWorkerInit.__main_thread_futex_wait_address; else PthreadWorkerInit.__main_thread_futex_wait_address = __main_thread_futex_wait_address = {{{ makeStaticAlloc(4) }}}', + _main_thread_futex_wait_address: '; if (ENVIRONMENT_IS_PTHREAD) __main_thread_futex_wait_address = PthreadWorkerInit["__main_thread_futex_wait_address"]; else PthreadWorkerInit["__main_thread_futex_wait_address"] = __main_thread_futex_wait_address = {{{ makeStaticAlloc(4) }}}', // Returns 0 on success, or one of the values -ETIMEDOUT, -EWOULDBLOCK or -EINVAL on error. emscripten_futex_wait__deps: ['_main_thread_futex_wait_address', 'emscripten_main_thread_process_queued_calls'], diff --git a/src/shell_pthreads.js b/src/shell_pthreads.js index 4f8256b3dc26f..c621ae2d4e23f 100644 --- a/src/shell_pthreads.js +++ b/src/shell_pthreads.js @@ -12,12 +12,12 @@ if (!ENVIRONMENT_IS_PTHREAD) { #if MODULARIZE else { // Grab imports from the pthread to local scope. - var buffer = {{{EXPORT_NAME}}}.buffer; - var tempDoublePtr = {{{EXPORT_NAME}}}.tempDoublePtr; - var STATICTOP = {{{EXPORT_NAME}}}.STATICTOP; - var DYNAMIC_BASE = {{{EXPORT_NAME}}}.DYNAMIC_BASE; - var DYNAMICTOP_PTR = {{{EXPORT_NAME}}}.DYNAMICTOP_PTR; - var PthreadWorkerInit = {{{EXPORT_NAME}}}.PthreadWorkerInit; + var buffer = {{{EXPORT_NAME}}}['buffer']; + var tempDoublePtr = {{{EXPORT_NAME}}}['tempDoublePtr']; + var STATICTOP = {{{EXPORT_NAME}}}['STATICTOP']; + var DYNAMIC_BASE = {{{EXPORT_NAME}}}['DYNAMIC_BASE']; + var DYNAMICTOP_PTR = {{{EXPORT_NAME}}}['DYNAMICTOP_PTR']; + var PthreadWorkerInit = {{{EXPORT_NAME}}}['PthreadWorkerInit']; // Note that not all runtime fields are imported above. Values for STACK_BASE, STACKTOP and STACK_MAX are not yet known at worker.js load time. // These will be filled in at pthread startup time (the 'run' message for a pthread - pthread start establishes the stack frame) } From d997f8784a9d06cbb1d7a3cfdef11d19c5024423 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 2 Oct 2019 13:23:34 -0700 Subject: [PATCH 08/26] fix [ci skip] --- src/shell_pthreads.js | 6 +++++- src/worker.js | 4 +--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/shell_pthreads.js b/src/shell_pthreads.js index c621ae2d4e23f..e166514341130 100644 --- a/src/shell_pthreads.js +++ b/src/shell_pthreads.js @@ -6,9 +6,13 @@ // ENVIRONMENT_IS_PTHREAD=true will have been preset in worker.js. Make it false in the main runtime thread. var ENVIRONMENT_IS_PTHREAD = Module['ENVIRONMENT_IS_PTHREAD'] || false; +var PthreadWorkerInit; if (!ENVIRONMENT_IS_PTHREAD) { - var PthreadWorkerInit = {}; // Collects together variables that are needed at initialization time for the web workers that host pthreads. + PthreadWorkerInit = {}; // Collects together variables that are needed at initialization time for the web workers that host pthreads. +} else { + PthreadWorkerInit = Module['PthreadWorkerInit']; } + #if MODULARIZE else { // Grab imports from the pthread to local scope. diff --git a/src/worker.js b/src/worker.js index 99a0d3d3dc084..66ca421a74607 100644 --- a/src/worker.js +++ b/src/worker.js @@ -27,8 +27,6 @@ var DYNAMIC_BASE = 0; var noExitRuntime; -var PthreadWorkerInit = {}; - // performance.now() is specced to return a wallclock time in msecs since that Web Worker/main thread launched. However for pthreads this can cause // subtle problems in emscripten_get_now() as this essentially would measure time from pthread_create(), meaning that the clocks between each threads // would be wildly out of sync. Therefore sync all pthreads to the clock on the main browser thread, so that different threads see a somewhat @@ -164,7 +162,7 @@ this.onmessage = function(e) { #endif - {{{ makeAsmExportAndGlobalAssignTargetInPthread('PthreadWorkerInit') }}} = e.data.PthreadWorkerInit; + Module['PthreadWorkerInit'] = e.data.PthreadWorkerInit; Module['ENVIRONMENT_IS_PTHREAD'] = true; #if MODULARIZE && EXPORT_ES6 From db8bb49238c22e250fa1b16b85e27df6ab938f85 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 2 Oct 2019 13:31:03 -0700 Subject: [PATCH 09/26] more [ci skip] --- src/runtime_init_memory.js | 5 ----- src/worker.js | 24 +++++++++++------------- 2 files changed, 11 insertions(+), 18 deletions(-) diff --git a/src/runtime_init_memory.js b/src/runtime_init_memory.js index 93682b73d6d79..40f3c05f22889 100644 --- a/src/runtime_init_memory.js +++ b/src/runtime_init_memory.js @@ -2,12 +2,7 @@ // memory is created in the wasm, not in JS.) #if USE_PTHREADS if (ENVIRONMENT_IS_PTHREAD) { -#if MODULARIZE && WASM - // In pthreads mode the wasmMemory and others are received in an onmessage, and that - // onmessage then loadScripts us, sending wasmMemory etc. on Module. Here we recapture - // it to a local so it can be used normally. wasmMemory = Module['wasmMemory']; -#endif } else { #endif // USE_PTHREADS #if WASM diff --git a/src/worker.js b/src/worker.js index 66ca421a74607..757194dd5421c 100644 --- a/src/worker.js +++ b/src/worker.js @@ -88,9 +88,9 @@ function resetPrototype(constructor, attrs) { Module['instantiateWasm'] = function(info, receiveInstance) { // Instantiate from the module posted from the main thread. // We can just use sync instantiation in the worker. - var instance = new WebAssembly.Instance(wasmModule, info); + var instance = new WebAssembly.Instance(Module['wasmModule'], info); // We don't need the module anymore; new threads will be spawned from the main thread. - wasmModule = null; + Module['wasmModule'] = null; #if LOAD_SOURCE_MAP wasmSourceMap = resetPrototype(WasmSourceMap, wasmSourceMapData); #endif @@ -102,8 +102,6 @@ Module['instantiateWasm'] = function(info, receiveInstance) { }; #endif -var wasmModule; -var wasmMemory; #if LOAD_SOURCE_MAP var wasmSourceMapData; #endif @@ -120,8 +118,8 @@ this.onmessage = function(e) { #endif // Initialize the global "process"-wide fields: - {{{ makeAsmExportAndGlobalAssignTargetInPthread('DYNAMIC_BASE') }}} = e.data.DYNAMIC_BASE; - {{{ makeAsmExportAndGlobalAssignTargetInPthread('DYNAMICTOP_PTR') }}} = e.data.DYNAMICTOP_PTR; + Module['DYNAMIC_BASE'] = e.data.DYNAMIC_BASE; + Module['DYNAMICTOP_PTR'] = e.data.DYNAMICTOP_PTR; #if WASM // The Wasm module will have import fields for STACKTOP and STACK_MAX. At 'load' stage of Worker startup, we are just @@ -136,17 +134,17 @@ this.onmessage = function(e) { {{{ makeAsmExportAccessInPthread('STACK_MAX') }}} = {{{ makeAsmExportAccessInPthread('STACKTOP') }}} = 0x7FFFFFFF; // Module and memory were sent from main thread - {{{ makeAsmExportAndGlobalAssignTargetInPthread('wasmModule') }}} = e.data.wasmModule; - {{{ makeAsmExportAndGlobalAssignTargetInPthread('wasmMemory') }}} = e.data.wasmMemory; + Module['wasmModule'] = e.data.wasmModule; + Module['wasmMemory'] = e.data.wasmMemory; #if LOAD_SOURCE_MAP wasmSourceMapData = e.data.wasmSourceMap; #endif #if USE_OFFSET_CONVERTER wasmOffsetData = e.data.wasmOffsetConverter; #endif - {{{ makeAsmExportAndGlobalAssignTargetInPthread('buffer') }}} = wasmMemory.buffer; + Module['buffer'] = Module['wasmMemory'].buffer; #else - {{{ makeAsmExportAndGlobalAssignTargetInPthread('buffer') }}} = e.data.buffer; + Module['buffer'] = e.data.buffer; #if SEPARATE_ASM // load the separated-out asm.js @@ -211,9 +209,9 @@ this.onmessage = function(e) { var max = e.data.stackBase + e.data.stackSize; var top = e.data.stackBase; #endif - {{{ makeAsmExportAndGlobalAssignTargetInPthread('STACK_BASE') }}} = top; - {{{ makeAsmExportAndGlobalAssignTargetInPthread('STACKTOP') }}} = top; - {{{ makeAsmExportAndGlobalAssignTargetInPthread('STACK_MAX') }}} = max; + Module['STACK_BASE'] = top; + Module['STACKTOP'] = top; + Module['STACK_MAX'] = max; #if ASSERTIONS assert(threadInfoStruct); assert(selfThreadId); From 66e443ef8e9cc5d57ab8034dfcda76dc3864c2f8 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 2 Oct 2019 14:13:04 -0700 Subject: [PATCH 10/26] fix [ci skip] --- src/library_pthread.js | 7 +++++++ src/worker.js | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/library_pthread.js b/src/library_pthread.js index 5beb631b4af5f..78ca2218d94d8 100644 --- a/src/library_pthread.js +++ b/src/library_pthread.js @@ -64,6 +64,13 @@ var LibraryPThread = { Atomics.store(HEAPU32, (PThread.mainThreadBlock + {{{ C_STRUCTS.pthread.tid }}} ) >> 2, PThread.mainThreadBlock); // Main thread ID. Atomics.store(HEAPU32, (PThread.mainThreadBlock + {{{ C_STRUCTS.pthread.pid }}} ) >> 2, PROCINFO.pid); // Process ID. +#if USE_CLOSURE_COMPILER + PThread['receiveObjectTransfer'] = PThread.receiveObjectTransfer; + PThread['setThreadStatus'] = PThread.setThreadStatus; + PThread['threadCancel'] = PThread.threadCancel; + PThread['threadExit'] = PThread.threadExit; +#endif + #if PTHREADS_PROFILING PThread.createProfilerBlock(PThread.mainThreadBlock); PThread.setThreadName(PThread.mainThreadBlock, "Browser main thread"); diff --git a/src/worker.js b/src/worker.js index 757194dd5421c..8a5c29005b725 100644 --- a/src/worker.js +++ b/src/worker.js @@ -282,7 +282,7 @@ this.onmessage = function(e) { // (This is a no-op if explicit pthread_exit() had been called prior.) if (!noExitRuntime) PThread.threadExit(result); } else if (e.data.cmd === 'cancel') { // Main thread is asking for a pthread_cancel() on this thread. - if (threadInfoStruct && PThread.thisThreadCancelState == 0/*PTHREAD_CANCEL_ENABLE*/) { + if (threadInfoStruct) { PThread.threadCancel(); } } else if (e.data.target === 'setimmediate') { From 79b982cc97700e274d8bb8abac3e0e8c8910780c Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 2 Oct 2019 14:32:42 -0700 Subject: [PATCH 11/26] fix [ci skip] --- src/library_pthread.js | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/library_pthread.js b/src/library_pthread.js index 78ca2218d94d8..d408e771fda5d 100644 --- a/src/library_pthread.js +++ b/src/library_pthread.js @@ -4,7 +4,7 @@ // found in the LICENSE file. var LibraryPThread = { - $PThread__postset: 'if (!ENVIRONMENT_IS_PTHREAD) PThread.initMainThreadBlock();', + $PThread__postset: 'if (!ENVIRONMENT_IS_PTHREAD) PThread.initMainThreadBlock(); else PThread.initWorker();', $PThread__deps: ['$PROCINFO', '_register_pthread_ptr', 'emscripten_main_thread_process_queued_calls', '$ERRNO_CODES', 'emscripten_futex_wake', '_kill_thread', @@ -64,18 +64,21 @@ var LibraryPThread = { Atomics.store(HEAPU32, (PThread.mainThreadBlock + {{{ C_STRUCTS.pthread.tid }}} ) >> 2, PThread.mainThreadBlock); // Main thread ID. Atomics.store(HEAPU32, (PThread.mainThreadBlock + {{{ C_STRUCTS.pthread.pid }}} ) >> 2, PROCINFO.pid); // Process ID. +#if PTHREADS_PROFILING + PThread.createProfilerBlock(PThread.mainThreadBlock); + PThread.setThreadName(PThread.mainThreadBlock, "Browser main thread"); + PThread.setThreadStatus(PThread.mainThreadBlock, {{{ cDefine('EM_THREAD_STATUS_RUNNING') }}}); +#endif + }, + initWorker: function() { #if USE_CLOSURE_COMPILER + // worker.js is not compiled together with us, and must access certain + // things. PThread['receiveObjectTransfer'] = PThread.receiveObjectTransfer; PThread['setThreadStatus'] = PThread.setThreadStatus; PThread['threadCancel'] = PThread.threadCancel; PThread['threadExit'] = PThread.threadExit; #endif - -#if PTHREADS_PROFILING - PThread.createProfilerBlock(PThread.mainThreadBlock); - PThread.setThreadName(PThread.mainThreadBlock, "Browser main thread"); - PThread.setThreadStatus(PThread.mainThreadBlock, {{{ cDefine('EM_THREAD_STATUS_RUNNING') }}}); -#endif }, // Maps pthread_t to pthread info objects pthreads: {}, From 082844bcac51d8231d5fbeefe3e645d19059b006 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 2 Oct 2019 14:36:58 -0700 Subject: [PATCH 12/26] fix --- emcc.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/emcc.py b/emcc.py index 21c332309e6d2..1dac514dfd8b4 100755 --- a/emcc.py +++ b/emcc.py @@ -1379,7 +1379,10 @@ def is_supported_link_flag(f): newargs += ['-pthread'] # some pthreads code is in asm.js library functions, which are auto-exported; for the wasm backend, we must # manually export them - shared.Settings.EXPORTED_FUNCTIONS += ['_emscripten_get_global_libc', '___pthread_tsd_run_dtors', '__register_pthread_ptr', '_pthread_self', '___emscripten_pthread_data_constructor'] + shared.Settings.EXPORTED_FUNCTIONS += [ + '_emscripten_get_global_libc', '___pthread_tsd_run_dtors', + '__register_pthread_ptr', '_pthread_self', + '___emscripten_pthread_data_constructor', '_emscripten_futex_wake'] # set location of worker.js shared.Settings.PTHREAD_WORKER_FILE = unsuffixed(os.path.basename(target)) + '.worker.js' From dacf03cde0fd983f26cfd24bc6cc8663399d22c3 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 2 Oct 2019 14:40:25 -0700 Subject: [PATCH 13/26] fix --- src/closure-externs.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/closure-externs.js b/src/closure-externs.js index eb7e25d31d773..291e02fb0834d 100644 --- a/src/closure-externs.js +++ b/src/closure-externs.js @@ -48,6 +48,18 @@ Math.max = function() {}; Math.clz32 = function() {}; Math.trunc = function() {}; +/** + * Atomics + */ + +var Atomics = {}; +Atomics.compareExchange = function() {}; +Atomics.exchange = function() {}; +Atomics.wait = function() {}; +Atomics.notify = function() {}; +Atomics.load = function() {}; +Atomics.store = function() {}; + /** * SIMD.js support (not in upstream closure yet). */ From 9cca50231eeebd3c4205135743b7a3f94979b9f6 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 2 Oct 2019 14:43:06 -0700 Subject: [PATCH 14/26] even simpler --- src/parseTools.js | 21 --------------------- src/worker.js | 2 +- 2 files changed, 1 insertion(+), 22 deletions(-) diff --git a/src/parseTools.js b/src/parseTools.js index 967afda5b66aa..e9284f0daa6fa 100644 --- a/src/parseTools.js +++ b/src/parseTools.js @@ -1519,27 +1519,6 @@ function addAtExit(code) { } } -// Generates access to module exports variable in pthreads worker.js. Depending on whether main code is built with MODULARIZE -// or not, asm module exports need to either be accessed via a local exports object obtained from instantiating the module (in src/worker.js), or via -// the global Module exports object. -function makeAsmExportAccessInPthread(variable) { - if (MODULARIZE) { - return "Module['" + variable + "']"; // 'Module' is defined in worker.js local scope, so not EXPORT_NAME in this case. - } else { - return EXPORT_NAME + "['" + variable + "']"; - } -} - -// Generates access to both global scope variable and exported Module variable, e.g. "Module['foo'] = foo" or just plain "foo" depending on if we are MODULARIZEing. -// Used the be able to initialize both variables at the same time in scenarios where a variable exists in both global scope and in Module. -function makeAsmExportAndGlobalAssignTargetInPthread(variable) { - if (MODULARIZE) { - return "Module['" + variable + "'] = " + variable; // 'Module' is defined in worker.js local scope, so not EXPORT_NAME in this case. - } else { - return variable; - } -} - // Some things, like the dynamic and stack bases, will be computed later and // applied. Return them as {{{ STR }}} for that replacing later. diff --git a/src/worker.js b/src/worker.js index 8a5c29005b725..6f9fb90dd84c5 100644 --- a/src/worker.js +++ b/src/worker.js @@ -131,7 +131,7 @@ this.onmessage = function(e) { // immediately at Wasm Module instantiation time. The values of these will not get used until pthread is actually running some code, so // we'll proceed to set up temporary invalid values for these fields for import purposes. Then whenever a pthread is launched at 'run' stage // below, these values are rewritten to establish proper stack area for the particular pthread. - {{{ makeAsmExportAccessInPthread('STACK_MAX') }}} = {{{ makeAsmExportAccessInPthread('STACKTOP') }}} = 0x7FFFFFFF; + Module['STACK_MAX'] = Module['STACKTOP'] = 0x7FFFFFFF; // Module and memory were sent from main thread Module['wasmModule'] = e.data.wasmModule; From 3ba9b8383ad8c6446f97961c28b0513f90a74816 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 2 Oct 2019 16:22:58 -0700 Subject: [PATCH 15/26] cleanup --- emcc.py | 3 +++ src/worker.js | 9 ++------- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/emcc.py b/emcc.py index 1dac514dfd8b4..1a260ca9da37b 100755 --- a/emcc.py +++ b/emcc.py @@ -1215,6 +1215,9 @@ def is_supported_link_flag(f): # These runtime methods are called from worker.js shared.Settings.EXPORTED_RUNTIME_METHODS += ['establishStackSpace', 'dynCall_ii'] + if shared.Settings.STACK_OVERFLOW_CHECK: + shared.Settings.EXPORTED_RUNTIME_METHODS += ['writeStackCookie', 'checkStackCookie', 'abortStackOverflow'] + if shared.Settings.MODULARIZE_INSTANCE: shared.Settings.MODULARIZE = 1 diff --git a/src/worker.js b/src/worker.js index 6f9fb90dd84c5..035e99193eade 100644 --- a/src/worker.js +++ b/src/worker.js @@ -15,11 +15,6 @@ var parentThreadId = 0; // The ID of the parent pthread that launched this threa var tempDoublePtr = 0; // A temporary memory area for global float and double marshalling operations. #endif -// Thread-local: Each thread has its own allocated stack space. -var STACK_BASE = 0; -var STACKTOP = 0; -var STACK_MAX = 0; - // These are system-wide memory area parameters that are set at main runtime startup in main thread, and stay constant throughout the application. var buffer; // All pthreads share the same Emscripten HEAP as SharedArrayBuffer with the main execution thread. var DYNAMICTOP_PTR = 0; @@ -216,7 +211,7 @@ this.onmessage = function(e) { assert(threadInfoStruct); assert(selfThreadId); assert(parentThreadId); - assert(STACK_BASE != 0); + assert(Module['STACK_BASE'] != 0); #if WASM_BACKEND assert(max === e.data.stackBase); assert(top > max); @@ -236,7 +231,7 @@ this.onmessage = function(e) { Module['_emscripten_tls_init'](); #endif #if SAFE_STACK - Module['___set_stack_limit'](STACK_MAX); + Module['___set_stack_limit'](Module['STACK_MAX']); #endif #if STACK_OVERFLOW_CHECK Module['writeStackCookie'](); From 12e3ffc84aa3c0c0c2ef158f6a82c3853bef1c4c Mon Sep 17 00:00:00 2001 From: "Alon Zakai (kripken)" Date: Wed, 2 Oct 2019 18:38:28 -0700 Subject: [PATCH 16/26] fix --- src/shell.js | 2 ++ src/shell_pthreads.js | 7 +------ 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/shell.js b/src/shell.js index 9b410c4903514..b8fd7a84c0c52 100644 --- a/src/shell.js +++ b/src/shell.js @@ -86,7 +86,9 @@ if (Module['ENVIRONMENT']) { } #endif +#if USE_PTHREADS #include "shell_pthreads.js" +#endif #if USE_PTHREADS && (!MODULARIZE || MODULARIZE_INSTANCE) // In MODULARIZE mode _scriptDir needs to be captured already at the very top of the page immediately when the page is parsed, so it is generated there diff --git a/src/shell_pthreads.js b/src/shell_pthreads.js index e166514341130..30ffda3cd4de0 100644 --- a/src/shell_pthreads.js +++ b/src/shell_pthreads.js @@ -2,7 +2,6 @@ // 1) We could be the application main() thread running in the main JS UI thread. (ENVIRONMENT_IS_WORKER == false and ENVIRONMENT_IS_PTHREAD == false) // 2) We could be the application main() thread proxied to worker. (with Emscripten -s PROXY_TO_WORKER=1) (ENVIRONMENT_IS_WORKER == true, ENVIRONMENT_IS_PTHREAD == false) // 3) We could be an application pthread running in a worker. (ENVIRONMENT_IS_WORKER == true and ENVIRONMENT_IS_PTHREAD == true) -#if USE_PTHREADS // ENVIRONMENT_IS_PTHREAD=true will have been preset in worker.js. Make it false in the main runtime thread. var ENVIRONMENT_IS_PTHREAD = Module['ENVIRONMENT_IS_PTHREAD'] || false; @@ -11,10 +10,7 @@ if (!ENVIRONMENT_IS_PTHREAD) { PthreadWorkerInit = {}; // Collects together variables that are needed at initialization time for the web workers that host pthreads. } else { PthreadWorkerInit = Module['PthreadWorkerInit']; -} - #if MODULARIZE -else { // Grab imports from the pthread to local scope. var buffer = {{{EXPORT_NAME}}}['buffer']; var tempDoublePtr = {{{EXPORT_NAME}}}['tempDoublePtr']; @@ -24,7 +20,6 @@ else { var PthreadWorkerInit = {{{EXPORT_NAME}}}['PthreadWorkerInit']; // Note that not all runtime fields are imported above. Values for STACK_BASE, STACKTOP and STACK_MAX are not yet known at worker.js load time. // These will be filled in at pthread startup time (the 'run' message for a pthread - pthread start establishes the stack frame) +#endif // MODULARIZE } -#endif -#endif From c702690828f54525886205441d45dc31abd6a03a Mon Sep 17 00:00:00 2001 From: "Alon Zakai (kripken)" Date: Wed, 2 Oct 2019 19:44:45 -0700 Subject: [PATCH 17/26] TO_REVERT --- .circleci/config.yml | 110 +------------------------------------------ tests/runner.py | 4 +- 2 files changed, 3 insertions(+), 111 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a27a7b9bf44ee..7caed17ea25e3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -37,30 +37,6 @@ commands: # Remove any system libs the emsdk installed: we want to rebuild # them all from source here, to check whether new changes work. rm -Rf ~/.emscripten_cache - - run: - name: embuilder build ALL - command: | - python3 ./embuilder.py build ALL - python3 tests/runner.py test_hello_world - - run: - name: embuilder (LTO) - command: | - python3 ./embuilder.py build libcompiler_rt libc libc++abi libc++abi-noexcept libc++ libc++-noexcept libal libdlmalloc libpthread_stub libc_rt_wasm struct_info libc-wasm --lto - python3 tests/runner.py test_hello_world - - run: - name: embuilder (PIC) - command: | - python3 ./embuilder.py build SYSTEM --pic - python3 tests/runner.py test_hello_world - - run: - name: embuilder (PIC+LTO) - command: | - python3 ./embuilder.py build libcompiler_rt libc libc++abi libc++abi-noexcept libc++ libc++-noexcept libal libdlmalloc libpthread_stub libc_rt_wasm struct_info libc-wasm --pic --lto - python3 tests/runner.py test_hello_world - - run: - name: freeze cache - command: | - echo "FROZEN_CACHE=True" >> ~/.emscripten - persist_to_workspace: # Must be an absolute path, or relative path from working_directory root: ~/ @@ -285,15 +261,6 @@ jobs: # Remove any system libs the emsdk installed: we want to rebuild # them all from source here, to check whether new changes work. rm -Rf ~/.emscripten_cache - - run: - name: embuilder build ALL - command: | - python3 ./embuilder.py build ALL - python3 tests/runner.py test_hello_world - - run: - name: freeze cache - command: | - echo "FROZEN_CACHE=True" >> ~/.emscripten - persist_to_workspace: # Must be an absolute path, or relative path from working_directory root: ~/ @@ -495,83 +462,8 @@ jobs: workflows: build-test: jobs: - - flake8 - - build-docs - - build - - test-other: - requires: - - build - - test-browser-firefox: - requires: - - build - - test-browser-chrome: - requires: - - build - - test-ab: - requires: - - build - - test-c: - requires: - - build - - test-d: - requires: - - build - - test-e: - requires: - - build - - test-f: - requires: - - build - - test-ghi: - requires: - - build - - test-jklmno: - requires: - - build - - test-p: - requires: - - build - - test-qrst: - requires: - - build - - test-uvwxyz: - requires: - - build - - test-wasm0: - requires: - - build - - test-wasm2: - requires: - - build - - test-wasm3: - requires: - - build - - test-sanity: - requires: - - build - build-upstream-linux - - test-upstream-wasm0: - requires: - - build-upstream-linux - - test-upstream-wasm2: - requires: - - build-upstream-linux - - test-upstream-wasm3: - requires: - - build-upstream-linux - - test-upstream-wasm2js1: - requires: - - build-upstream-linux - - test-upstream-other: - requires: - - build-upstream-linux - test-upstream-browser-chrome: requires: - build-upstream-linux - - test-upstream-browser-firefox: - requires: - - build-upstream-linux - - build-upstream-mac - - test-upstream-other-mac: - requires: - - build-upstream-mac + diff --git a/tests/runner.py b/tests/runner.py index 9b048bbf79f77..742f7d4e35f22 100755 --- a/tests/runner.py +++ b/tests/runner.py @@ -1336,7 +1336,7 @@ class BrowserCore(RunnerCore): # happen, likely something is broken and it is best to abort the test # suite early, as otherwise we will wait for the timeout on every # single test (hundreds of minutes) - MAX_UNRESPONSIVE_TESTS = 10 + MAX_UNRESPONSIVE_TESTS = 100 unresponsive_tests = 0 @@ -1391,7 +1391,7 @@ def assert_out_queue_empty(self, who): # synchronously, so we have a timeout, which can be hit if the VM # we run on stalls temporarily), so we let each test try more than # once by default - def run_browser(self, html_file, message, expectedResult=None, timeout=None, tries_left=1): + def run_browser(self, html_file, message, expectedResult=None, timeout=None, tries_left=0): if not has_browser(): return if BrowserCore.unresponsive_tests >= BrowserCore.MAX_UNRESPONSIVE_TESTS: From bbb3d0fb182d9dc1d6fcfe18446121180c608422 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 3 Oct 2019 08:57:37 -0700 Subject: [PATCH 18/26] cleanup --- src/worker.js | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/worker.js b/src/worker.js index 035e99193eade..6eb224e665910 100644 --- a/src/worker.js +++ b/src/worker.js @@ -15,11 +15,6 @@ var parentThreadId = 0; // The ID of the parent pthread that launched this threa var tempDoublePtr = 0; // A temporary memory area for global float and double marshalling operations. #endif -// These are system-wide memory area parameters that are set at main runtime startup in main thread, and stay constant throughout the application. -var buffer; // All pthreads share the same Emscripten HEAP as SharedArrayBuffer with the main execution thread. -var DYNAMICTOP_PTR = 0; -var DYNAMIC_BASE = 0; - var noExitRuntime; // performance.now() is specced to return a wallclock time in msecs since that Web Worker/main thread launched. However for pthreads this can cause From 1a80c35f7d719e3ab94e65c5198755ab0fa41932 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 3 Oct 2019 09:09:53 -0700 Subject: [PATCH 19/26] fix --- src/preamble.js | 21 +++++++++++++++++---- src/worker.js | 15 +-------------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/preamble.js b/src/preamble.js index 6d8d8e5f969d7..7f25a98e13e89 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -356,10 +356,6 @@ function updateGlobalBufferAndViews(buf) { Module['HEAPF64'] = HEAPF64 = new Float64Array(buf); } -#if USE_PTHREADS -if (!ENVIRONMENT_IS_PTHREAD) { // Pthreads have already initialized these variables in src/worker.js, where they were passed to the thread worker at startup time -#endif - var STATIC_BASE = {{{ GLOBAL_BASE }}}, STACK_BASE = {{{ getQuoted('STACK_BASE') }}}, STACKTOP = STACK_BASE, @@ -373,6 +369,23 @@ assert(DYNAMIC_BASE % 16 === 0, 'heap must start aligned'); #endif #if USE_PTHREADS +if (ENVIRONMENT_IS_PTHREAD) { + // Pthread workers compute these in worker.js when they run, which is after + // this script loads in the worker initially. Set some fake values to + // catch bugs, then set the real values at load time. + STACK_MAX = STACKTOP = STACK_MAX = 0x7FFFFFFF; + + Module['applyStackValues'] = function(stackBase, stackTop, stackMax) { + STACK_BASE = stackBase; + STACKTOP = stackTop; + STACK_MAX = stackMax; + }; + + // TODO DYNAMIC_BASE = Module['DYNAMIC_BASE']; + // TODO DYNAMICTOP_PTR = Module['DYNAMICTOP_PTR']; + // TODO tempDoublePtr = Module['tempDoublePtr']; + // TODO wasmModule = Module['wasmModule']; + // TODO buffer = Module['buffer']; } #endif diff --git a/src/worker.js b/src/worker.js index 6eb224e665910..856c0187ab5b1 100644 --- a/src/worker.js +++ b/src/worker.js @@ -112,17 +112,6 @@ this.onmessage = function(e) { Module['DYNAMICTOP_PTR'] = e.data.DYNAMICTOP_PTR; #if WASM - // The Wasm module will have import fields for STACKTOP and STACK_MAX. At 'load' stage of Worker startup, we are just - // spawning this Web Worker to act as a host for future created pthreads, i.e. we do not have a pthread to start up here yet. - // (A single Worker can also host multiple pthreads throughout its lifetime, shutting down a pthread will not shut down its hosting Worker, - // but the Worker is reused for later spawned pthreads). The 'run' stage below will actually start running a pthread. - // The stack space for a pthread is allocated and deallocated when a pthread is actually run, not yet at Worker 'load' stage. - // However, the WebAssembly module we are loading up here has import fields for STACKTOP and STACK_MAX, which it needs to get filled in - // immediately at Wasm Module instantiation time. The values of these will not get used until pthread is actually running some code, so - // we'll proceed to set up temporary invalid values for these fields for import purposes. Then whenever a pthread is launched at 'run' stage - // below, these values are rewritten to establish proper stack area for the particular pthread. - Module['STACK_MAX'] = Module['STACKTOP'] = 0x7FFFFFFF; - // Module and memory were sent from main thread Module['wasmModule'] = e.data.wasmModule; Module['wasmMemory'] = e.data.wasmMemory; @@ -199,9 +188,7 @@ this.onmessage = function(e) { var max = e.data.stackBase + e.data.stackSize; var top = e.data.stackBase; #endif - Module['STACK_BASE'] = top; - Module['STACKTOP'] = top; - Module['STACK_MAX'] = max; + Module['applyStackValues'](top, top, max); #if ASSERTIONS assert(threadInfoStruct); assert(selfThreadId); From d359916704ce63b89290125629496f724566ad79 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 3 Oct 2019 10:26:30 -0700 Subject: [PATCH 20/26] fixes --- src/preamble.js | 3 +++ src/worker.js | 5 +---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/preamble.js b/src/preamble.js index 7f25a98e13e89..ec496199714c1 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -379,6 +379,9 @@ if (ENVIRONMENT_IS_PTHREAD) { STACK_BASE = stackBase; STACKTOP = stackTop; STACK_MAX = stackMax; +#if SAFE_STACK + Module['___set_stack_limit'](STACK_MAX); +#endif }; // TODO DYNAMIC_BASE = Module['DYNAMIC_BASE']; diff --git a/src/worker.js b/src/worker.js index 856c0187ab5b1..1a5839c246f57 100644 --- a/src/worker.js +++ b/src/worker.js @@ -193,7 +193,7 @@ this.onmessage = function(e) { assert(threadInfoStruct); assert(selfThreadId); assert(parentThreadId); - assert(Module['STACK_BASE'] != 0); + assert(top != 0); #if WASM_BACKEND assert(max === e.data.stackBase); assert(top > max); @@ -212,9 +212,6 @@ this.onmessage = function(e) { #if WASM_BACKEND Module['_emscripten_tls_init'](); #endif -#if SAFE_STACK - Module['___set_stack_limit'](Module['STACK_MAX']); -#endif #if STACK_OVERFLOW_CHECK Module['writeStackCookie'](); #endif From 942027e43ce32c754e9f4ec29aa9a130dd1fb885 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 3 Oct 2019 10:34:46 -0700 Subject: [PATCH 21/26] Revert "TO_REVERT" This reverts commit c702690828f54525886205441d45dc31abd6a03a. --- .circleci/config.yml | 110 ++++++++++++++++++++++++++++++++++++++++++- tests/runner.py | 4 +- 2 files changed, 111 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7caed17ea25e3..a27a7b9bf44ee 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -37,6 +37,30 @@ commands: # Remove any system libs the emsdk installed: we want to rebuild # them all from source here, to check whether new changes work. rm -Rf ~/.emscripten_cache + - run: + name: embuilder build ALL + command: | + python3 ./embuilder.py build ALL + python3 tests/runner.py test_hello_world + - run: + name: embuilder (LTO) + command: | + python3 ./embuilder.py build libcompiler_rt libc libc++abi libc++abi-noexcept libc++ libc++-noexcept libal libdlmalloc libpthread_stub libc_rt_wasm struct_info libc-wasm --lto + python3 tests/runner.py test_hello_world + - run: + name: embuilder (PIC) + command: | + python3 ./embuilder.py build SYSTEM --pic + python3 tests/runner.py test_hello_world + - run: + name: embuilder (PIC+LTO) + command: | + python3 ./embuilder.py build libcompiler_rt libc libc++abi libc++abi-noexcept libc++ libc++-noexcept libal libdlmalloc libpthread_stub libc_rt_wasm struct_info libc-wasm --pic --lto + python3 tests/runner.py test_hello_world + - run: + name: freeze cache + command: | + echo "FROZEN_CACHE=True" >> ~/.emscripten - persist_to_workspace: # Must be an absolute path, or relative path from working_directory root: ~/ @@ -261,6 +285,15 @@ jobs: # Remove any system libs the emsdk installed: we want to rebuild # them all from source here, to check whether new changes work. rm -Rf ~/.emscripten_cache + - run: + name: embuilder build ALL + command: | + python3 ./embuilder.py build ALL + python3 tests/runner.py test_hello_world + - run: + name: freeze cache + command: | + echo "FROZEN_CACHE=True" >> ~/.emscripten - persist_to_workspace: # Must be an absolute path, or relative path from working_directory root: ~/ @@ -462,8 +495,83 @@ jobs: workflows: build-test: jobs: + - flake8 + - build-docs + - build + - test-other: + requires: + - build + - test-browser-firefox: + requires: + - build + - test-browser-chrome: + requires: + - build + - test-ab: + requires: + - build + - test-c: + requires: + - build + - test-d: + requires: + - build + - test-e: + requires: + - build + - test-f: + requires: + - build + - test-ghi: + requires: + - build + - test-jklmno: + requires: + - build + - test-p: + requires: + - build + - test-qrst: + requires: + - build + - test-uvwxyz: + requires: + - build + - test-wasm0: + requires: + - build + - test-wasm2: + requires: + - build + - test-wasm3: + requires: + - build + - test-sanity: + requires: + - build - build-upstream-linux + - test-upstream-wasm0: + requires: + - build-upstream-linux + - test-upstream-wasm2: + requires: + - build-upstream-linux + - test-upstream-wasm3: + requires: + - build-upstream-linux + - test-upstream-wasm2js1: + requires: + - build-upstream-linux + - test-upstream-other: + requires: + - build-upstream-linux - test-upstream-browser-chrome: requires: - build-upstream-linux - + - test-upstream-browser-firefox: + requires: + - build-upstream-linux + - build-upstream-mac + - test-upstream-other-mac: + requires: + - build-upstream-mac diff --git a/tests/runner.py b/tests/runner.py index 742f7d4e35f22..9b048bbf79f77 100755 --- a/tests/runner.py +++ b/tests/runner.py @@ -1336,7 +1336,7 @@ class BrowserCore(RunnerCore): # happen, likely something is broken and it is best to abort the test # suite early, as otherwise we will wait for the timeout on every # single test (hundreds of minutes) - MAX_UNRESPONSIVE_TESTS = 100 + MAX_UNRESPONSIVE_TESTS = 10 unresponsive_tests = 0 @@ -1391,7 +1391,7 @@ def assert_out_queue_empty(self, who): # synchronously, so we have a timeout, which can be hit if the VM # we run on stalls temporarily), so we let each test try more than # once by default - def run_browser(self, html_file, message, expectedResult=None, timeout=None, tries_left=0): + def run_browser(self, html_file, message, expectedResult=None, timeout=None, tries_left=1): if not has_browser(): return if BrowserCore.unresponsive_tests >= BrowserCore.MAX_UNRESPONSIVE_TESTS: From b6427c4ad533f30ab72d1c17650fe07997a9e6b1 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 3 Oct 2019 12:49:39 -0700 Subject: [PATCH 22/26] fastcomp fix --- src/preamble.js | 4 ++-- src/runtime_init_memory.js | 1 + src/shell_pthreads.js | 14 +++++--------- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/src/preamble.js b/src/preamble.js index ec496199714c1..8b854d62734ad 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -68,7 +68,9 @@ var wasmModule; var threadInfoStruct = 0; var selfThreadId = 0; var __performance_now_clock_drift = 0; +#if WASM_BACKEND var tempDoublePtr = 0; +#endif #endif // USE_PTHREADS //======================================== @@ -387,8 +389,6 @@ if (ENVIRONMENT_IS_PTHREAD) { // TODO DYNAMIC_BASE = Module['DYNAMIC_BASE']; // TODO DYNAMICTOP_PTR = Module['DYNAMICTOP_PTR']; // TODO tempDoublePtr = Module['tempDoublePtr']; - // TODO wasmModule = Module['wasmModule']; - // TODO buffer = Module['buffer']; } #endif diff --git a/src/runtime_init_memory.js b/src/runtime_init_memory.js index 40f3c05f22889..004a4a007c72a 100644 --- a/src/runtime_init_memory.js +++ b/src/runtime_init_memory.js @@ -3,6 +3,7 @@ #if USE_PTHREADS if (ENVIRONMENT_IS_PTHREAD) { wasmMemory = Module['wasmMemory']; + buffer = Module['buffer']; } else { #endif // USE_PTHREADS #if WASM diff --git a/src/shell_pthreads.js b/src/shell_pthreads.js index 30ffda3cd4de0..135a452eeac8a 100644 --- a/src/shell_pthreads.js +++ b/src/shell_pthreads.js @@ -9,17 +9,13 @@ var PthreadWorkerInit; if (!ENVIRONMENT_IS_PTHREAD) { PthreadWorkerInit = {}; // Collects together variables that are needed at initialization time for the web workers that host pthreads. } else { - PthreadWorkerInit = Module['PthreadWorkerInit']; -#if MODULARIZE // Grab imports from the pthread to local scope. - var buffer = {{{EXPORT_NAME}}}['buffer']; - var tempDoublePtr = {{{EXPORT_NAME}}}['tempDoublePtr']; - var STATICTOP = {{{EXPORT_NAME}}}['STATICTOP']; - var DYNAMIC_BASE = {{{EXPORT_NAME}}}['DYNAMIC_BASE']; - var DYNAMICTOP_PTR = {{{EXPORT_NAME}}}['DYNAMICTOP_PTR']; - var PthreadWorkerInit = {{{EXPORT_NAME}}}['PthreadWorkerInit']; + buffer = {{{EXPORT_NAME}}}['buffer']; + tempDoublePtr = {{{EXPORT_NAME}}}['tempDoublePtr']; + DYNAMIC_BASE = {{{EXPORT_NAME}}}['DYNAMIC_BASE']; + DYNAMICTOP_PTR = {{{EXPORT_NAME}}}['DYNAMICTOP_PTR']; + PthreadWorkerInit = {{{EXPORT_NAME}}}['PthreadWorkerInit']; // Note that not all runtime fields are imported above. Values for STACK_BASE, STACKTOP and STACK_MAX are not yet known at worker.js load time. // These will be filled in at pthread startup time (the 'run' message for a pthread - pthread start establishes the stack frame) -#endif // MODULARIZE } From 5ea6a10420559569c5fdb910763676dd23d44bd5 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Thu, 3 Oct 2019 15:17:47 -0700 Subject: [PATCH 23/26] Use chrome stable on CI for now as there is some breakage --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a27a7b9bf44ee..104741908afac 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -239,7 +239,8 @@ commands: - run: name: download chrome command: | - wget -O ~/chrome.deb https://dl.google.com/linux/direct/google-chrome-beta_current_amd64.deb + # TODO: return to using beta once it stabilizes + wget -O ~/chrome.deb https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb dpkg -i ~/chrome.deb - run: name: run tests From 800376cf2477078078f697baef3067793f034410 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 7 Oct 2019 09:25:28 -0700 Subject: [PATCH 24/26] ifdef --- src/preamble.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/preamble.js b/src/preamble.js index 8b854d62734ad..7ad65a1c8305d 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -375,7 +375,9 @@ if (ENVIRONMENT_IS_PTHREAD) { // Pthread workers compute these in worker.js when they run, which is after // this script loads in the worker initially. Set some fake values to // catch bugs, then set the real values at load time. +#if ASSERTIONS || SAFE_STACK STACK_MAX = STACKTOP = STACK_MAX = 0x7FFFFFFF; +#endif Module['applyStackValues'] = function(stackBase, stackTop, stackMax) { STACK_BASE = stackBase; From 2100a647d25fcde8cb8917c562453a02dde50737 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Mon, 7 Oct 2019 09:34:24 -0700 Subject: [PATCH 25/26] missing dep --- src/library_pthread.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/library_pthread.js b/src/library_pthread.js index d408e771fda5d..8e27c99e19c17 100644 --- a/src/library_pthread.js +++ b/src/library_pthread.js @@ -8,7 +8,7 @@ var LibraryPThread = { $PThread__deps: ['$PROCINFO', '_register_pthread_ptr', 'emscripten_main_thread_process_queued_calls', '$ERRNO_CODES', 'emscripten_futex_wake', '_kill_thread', - '_cancel_thread'], + '_cancel_thread', '_cleanup_thread'], $PThread: { MAIN_THREAD_ID: 1, // A special constant that identifies the main JS thread ID. mainThreadInfo: { From e6c0d740daab09029acb423410add0c23b1c2a35 Mon Sep 17 00:00:00 2001 From: Alon Zakai Date: Wed, 16 Oct 2019 13:10:55 -0700 Subject: [PATCH 26/26] better comments --- src/preamble.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/preamble.js b/src/preamble.js index 8317d63dad8d8..ec61ee6609219 100644 --- a/src/preamble.js +++ b/src/preamble.js @@ -372,9 +372,11 @@ assert(DYNAMIC_BASE % 16 === 0, 'heap must start aligned'); #if USE_PTHREADS if (ENVIRONMENT_IS_PTHREAD) { - // Pthread workers compute these in worker.js when they run, which is after - // this script loads in the worker initially. Set some fake values to - // catch bugs, then set the real values at load time. + + // At the 'load' stage of Worker startup, we are just loading this script + // but not ready to run yet. At 'run' we receive proper values for the stack + // etc. and can launch a pthread. Set some fake values there meanwhile to + // catch bugs, then set the real values in applyStackValues later. #if ASSERTIONS || SAFE_STACK STACK_MAX = STACKTOP = STACK_MAX = 0x7FFFFFFF; #endif