From 7669b988a9adcaf2123a3294840941d7c0c9358a Mon Sep 17 00:00:00 2001
From: Tayeb Al Karim <tay@google.com>
Date: Thu, 5 Sep 2019 10:28:40 -0400
Subject: [PATCH 01/25] Adding support for pre-allocating worker threads before
 the webassembly module is compiled. This puts the work of loading and
 starting up the javascript in parallel with the wasm compile, reducing
 overall load times.

---
 src/library_pthread.js | 147 +++++++++++++++++++++++++++++------------
 src/settings.js        |  11 ++-
 2 files changed, 113 insertions(+), 45 deletions(-)

diff --git a/src/library_pthread.js b/src/library_pthread.js
index 288fe32e89c36..5410da6b92a41 100644
--- a/src/library_pthread.js
+++ b/src/library_pthread.js
@@ -12,6 +12,10 @@ var LibraryPThread = {
       schedPolicy: 0/*SCHED_OTHER*/,
       schedPrio: 0
     },
+    // Workers that have been created but uninitialized. These have already been
+    // parsed, but the wabassembly model has not yet been loaded, making it
+    // distinct from the unusedWorkers below.
+    preallocatedWorkers: [],
     // Contains all Workers that are idle/unused and not currently hosting an executing pthread.
     // Unused Workers can either be pooled up before page startup, but also when a pthread quits, its hosting
     // Worker is not terminated, but is returned to this pool as an optimization so that starting the next thread is faster.
@@ -28,6 +32,14 @@ var LibraryPThread = {
     },
     initMainThreadBlock: function() {
       if (ENVIRONMENT_IS_PTHREAD) return undefined;
+
+#if PREWARM_PTHREAD_POOL_WORKERS_SIZE > 0
+#if PTHREADS_DEBUG
+      out('Preallocating ' + {{{ PREWARM_PTHREAD_POOL_WORKERS_SIZE }}} + ' workers.');
+#endif
+      PThread.preallocatedWorkers = PThread.createNewWorkers({{{ PREWARM_PTHREAD_POOL_WORKERS_SIZE }}});
+#endif
+
       PThread.mainThreadBlock = {{{ makeStaticAlloc(C_STRUCTS.pthread.__size__) }}};
 
       for (var i = 0; i < {{{ C_STRUCTS.pthread.__size__ }}}/4; ++i) HEAPU32[PThread.mainThreadBlock/4+i] = 0;
@@ -191,6 +203,15 @@ var LibraryPThread = {
       }
       PThread.pthreads = {};
 
+      for (var i = 0; i < PThread.preallocatedWorkers.length; ++i) {
+        var worker = PThread.preallocatedWorkers[i];
+#if ASSERTIONS
+        assert(!worker.pthread); // This Worker should not be hosting a pthread at this time.
+#endif
+        worker.terminate();
+      }
+      PThread.preallocatedWorkers = [];
+
       for (var i = 0; i < PThread.unusedWorkers.length; ++i) {
         var worker = PThread.unusedWorkers[i];
 #if ASSERTIONS
@@ -252,19 +273,78 @@ var LibraryPThread = {
     allocateUnusedWorkers: function(numWorkers, onFinishedLoading) {
       if (typeof SharedArrayBuffer === 'undefined') return; // No multithreading support, no-op.
 #if PTHREADS_DEBUG
-      out('Preallocating ' + numWorkers + ' workers for a pthread spawn pool.');
+      out('Allocating ' + numWorkers + ' workers for a pthread spawn pool.');
 #endif
 
-      var numWorkersLoaded = 0;
-      var pthreadMainJs = "{{{ PTHREAD_WORKER_FILE }}}";
-      // Allow HTML module to configure the location where the 'worker.js' file will be loaded from,
-      // via Module.locateFile() function. If not specified, then the default URL 'worker.js' relative
-      // to the main html file is loaded.
-      pthreadMainJs = locateFile(pthreadMainJs);
+      var workers = [];
 
+      var createNumNewWorkers = numWorkers;
+      if (PThread.preallocatedWorkerPool.length > 0) {
+        var workersUsed = Math.min(PThread.preallocatedWorkerPool.length, numWorkers);
+#if PTHREADS_DEBUG
+        out('using ' + workersUsed + 'preallocated workers');
+#endif
+        workers.push(...PThread.preallocatedWorkerPool.splice(0, workersUsed));
+        createNumNewWorkers -= workersUsed;
+      }
+      if (createNumNewWorkers > 0) {
+        workers.push(...PThread.createNewWorkers(createNumNewWorkers));
+      }
+
+      // Add the listeners.
+      PThread.attachListenerToWorkers(workers, onFinishedLoading);
+
+      // Load the wasm module into the worker.
       for (var i = 0; i < numWorkers; ++i) {
-        var worker = new Worker(pthreadMainJs);
+        var worker = workers[i];
+
+#if !WASM_BACKEND
+        // Allocate tempDoublePtr for the worker. This is done here on the worker's behalf, since we may need to do this statically
+        // if the runtime has not been loaded yet, etc. - so we just use getMemory, which is main-thread only.
+        var tempDoublePtr = getMemory(8); // TODO: leaks. Cleanup after worker terminates.
+#endif
+
+        // Ask the new worker to load up the Emscripten-compiled page. This is a heavy operation.
+        worker.postMessage({
+          cmd: 'load',
+          // If the application main .js file was loaded from a Blob, then it is not possible
+          // to access the URL of the current script that could be passed to a Web Worker so that
+          // it could load up the same file. In that case, developer must either deliver the Blob
+          // object in Module['mainScriptUrlOrBlob'], or a URL to it, so that pthread Workers can
+          // independently load up the same main application file.
+          urlOrBlob: Module['mainScriptUrlOrBlob'] || _scriptDir,
+#if WASM
+          wasmMemory: wasmMemory,
+          wasmModule: wasmModule,
+#if LOAD_SOURCE_MAP
+          wasmSourceMap: wasmSourceMap,
+#endif
+#if USE_OFFSET_CONVERTER
+          wasmOffsetConverter: wasmOffsetConverter,
+#endif
+#else
+          buffer: HEAPU8.buffer,
+          asmJsUrlOrBlob: Module["asmJsUrlOrBlob"],
+#endif
+#if !WASM_BACKEND
+          tempDoublePtr: tempDoublePtr,
+#endif
+          DYNAMIC_BASE: DYNAMIC_BASE,
+          DYNAMICTOP_PTR: DYNAMICTOP_PTR,
+          PthreadWorkerInit: PthreadWorkerInit
+        });
+        PThread.unusedWorkers.push(worker);
+      }
+    },
 
+    // Attaches the listeners to the given workers. If onFinishedLoading is provided,
+    // will call that function when all workers have been loaded. It is assumed that no worker
+    // is yet loaded.
+    attachListenerToWorkers: function(workers, onFinishedLoading) {
+      var numWorkersLoaded = 0;
+      var numWorkers = workers.length;
+      for (var i = 0; i < numWorkers; ++i) {
+        var worker = workers[i];
         (function(worker) {
           worker.onmessage = function(e) {
             var d = e.data;
@@ -341,44 +421,25 @@ var LibraryPThread = {
             err('pthread sent an error! ' + e.filename + ':' + e.lineno + ': ' + e.message);
           };
         }(worker));
+      }  // for each worker
+    },
 
-#if !WASM_BACKEND
-        // Allocate tempDoublePtr for the worker. This is done here on the worker's behalf, since we may need to do this statically
-        // if the runtime has not been loaded yet, etc. - so we just use getMemory, which is main-thread only.
-        var tempDoublePtr = getMemory(8); // TODO: leaks. Cleanup after worker terminates.
-#endif
-
-        // Ask the new worker to load up the Emscripten-compiled page. This is a heavy operation.
-        worker.postMessage({
-          cmd: 'load',
-          // If the application main .js file was loaded from a Blob, then it is not possible
-          // to access the URL of the current script that could be passed to a Web Worker so that
-          // it could load up the same file. In that case, developer must either deliver the Blob
-          // object in Module['mainScriptUrlOrBlob'], or a URL to it, so that pthread Workers can
-          // independently load up the same main application file.
-          urlOrBlob: Module['mainScriptUrlOrBlob'] || _scriptDir,
-#if WASM
-          wasmMemory: wasmMemory,
-          wasmModule: wasmModule,
-#if LOAD_SOURCE_MAP
-          wasmSourceMap: wasmSourceMap,
-#endif
-#if USE_OFFSET_CONVERTER
-          wasmOffsetConverter: wasmOffsetConverter,
-#endif
-#else
-          buffer: HEAPU8.buffer,
-          asmJsUrlOrBlob: Module["asmJsUrlOrBlob"],
-#endif
-#if !WASM_BACKEND
-          tempDoublePtr: tempDoublePtr,
+    createNewWorkers: function(numWorkers) {
+      // Creates new workers with the discovered pthread worker file.
+      if (typeof SharedArrayBuffer === 'undefined') return; // No multithreading support, no-op.
+#if PTHREADS_DEBUG
+      out('Creating ' + numWorkers + ' workers for a pthread spawn pool.');
 #endif
-          DYNAMIC_BASE: DYNAMIC_BASE,
-          DYNAMICTOP_PTR: DYNAMICTOP_PTR,
-          PthreadWorkerInit: PthreadWorkerInit
-        });
-        PThread.unusedWorkers.push(worker);
+      var pthreadMainJs = "{{{ PTHREAD_WORKER_FILE }}}";
+      // Allow HTML module to configure the location where the 'worker.js' file will be loaded from,
+      // via Module.locateFile() function. If not specified, then the default URL 'worker.js' relative
+      // to the main html file is loaded.
+      pthreadMainJs = locateFile(pthreadMainJs);
+      var newWorkers = [];
+      for (var i = 0; i < numWorkers; ++i) {
+        newWorkers.push(new Worker(pthreadMainJs));
       }
+      return newWorkers;
     },
 
     getNewWorker: function() {
diff --git a/src/settings.js b/src/settings.js
index 0547acf0bb3f7..1b027c716fa06 100644
--- a/src/settings.js
+++ b/src/settings.js
@@ -1235,10 +1235,17 @@ var IN_TEST_HARNESS = 0;
 // If true, enables support for pthreads.
 var USE_PTHREADS = 0;
 
-// Specifies the number of web workers that are preallocated before runtime is
-// initialized. If 0, workers are created on demand.
+// Specifies the number of web workers that are preallocated with the
+// webassembly module before runtime is initialized. If 0, workers are
+// created on demand.
 var PTHREAD_POOL_SIZE = 0;
 
+// Specifies the number of web workers that are preallocated before the
+// webassembly module is compiled. If 0, workers are created on demand. You
+// likely want this to be >= PTHREAD_POOL_SIZE. Note that this starts the
+// webworkers in a state where they have yet to load the webassembly module.
+var PREWARM_PTHREAD_POOL_WORKERS_SIZE = 0
+
 // If not explicitly specified, this is the stack size to use for newly created
 // pthreads.  According to
 // http://man7.org/linux/man-pages/man3/pthread_create.3.html, default stack

From a75159351e59c1574c2e32a73b1bb9511424502e Mon Sep 17 00:00:00 2001
From: Tayeb Al Karim <tay@google.com>
Date: Thu, 5 Sep 2019 10:55:19 -0400
Subject: [PATCH 02/25] Fix bad naming of the preallocated worker list.

---
 src/library_pthread.js | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/library_pthread.js b/src/library_pthread.js
index 5410da6b92a41..89eeb470d3b09 100644
--- a/src/library_pthread.js
+++ b/src/library_pthread.js
@@ -279,12 +279,12 @@ var LibraryPThread = {
       var workers = [];
 
       var createNumNewWorkers = numWorkers;
-      if (PThread.preallocatedWorkerPool.length > 0) {
-        var workersUsed = Math.min(PThread.preallocatedWorkerPool.length, numWorkers);
+      if (PThread.preallocatedWorkers.length > 0) {
+        var workersUsed = Math.min(PThread.preallocatedWorkers.length, numWorkers);
 #if PTHREADS_DEBUG
         out('using ' + workersUsed + 'preallocated workers');
 #endif
-        workers.push(...PThread.preallocatedWorkerPool.splice(0, workersUsed));
+        workers.push(...PThread.preallocatedWorkers.splice(0, workersUsed));
         createNumNewWorkers -= workersUsed;
       }
       if (createNumNewWorkers > 0) {

From 78264f4c8baa843fcade03b4d56bb57f15aff9ac Mon Sep 17 00:00:00 2001
From: Tayeb Al Karim <tay@google.com>
Date: Thu, 5 Sep 2019 11:38:27 -0400
Subject: [PATCH 03/25] add tay@google.com to AUTHORS

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index 52dd49361f26b..4c82facecfcbc 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -426,3 +426,4 @@ a license to everyone to use it as detailed in LICENSE.)
 * Ajay Patel <patel.ajay285@gmail.com>
 * Adrien Devresse <adev@adev.name>
 * Petr Penzin (petr.penzin@intel.com) (copyright owned by Intel Corporation)
+* Tayeb Al Karim <tay@google.com> (copyright owned by Google, Inc.)

From 07c3155be6f8e139128d90975957246802abefe4 Mon Sep 17 00:00:00 2001
From: Tayeb Al Karim <tay@google.com>
Date: Thu, 5 Sep 2019 14:31:10 -0400
Subject: [PATCH 04/25] Start adding test for the preallocation.

---
 src/library_pthread.js | 6 +++---
 tests/runner.py        | 2 ++
 tests/test_browser.py  | 8 ++++++++
 3 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/src/library_pthread.js b/src/library_pthread.js
index 89eeb470d3b09..dcdac3f61c558 100644
--- a/src/library_pthread.js
+++ b/src/library_pthread.js
@@ -194,6 +194,7 @@ var LibraryPThread = {
     },
 
     terminateAllThreads: function() {
+      out('TERMINATING');
       for (var t in PThread.pthreads) {
         var pthread = PThread.pthreads[t];
         if (pthread) {
@@ -282,7 +283,7 @@ var LibraryPThread = {
       if (PThread.preallocatedWorkers.length > 0) {
         var workersUsed = Math.min(PThread.preallocatedWorkers.length, numWorkers);
 #if PTHREADS_DEBUG
-        out('using ' + workersUsed + 'preallocated workers');
+        out('Using ' + workersUsed + ' preallocated workers');
 #endif
         workers.push(...PThread.preallocatedWorkers.splice(0, workersUsed));
         createNumNewWorkers -= workersUsed;
@@ -428,7 +429,7 @@ var LibraryPThread = {
       // Creates new workers with the discovered pthread worker file.
       if (typeof SharedArrayBuffer === 'undefined') return; // No multithreading support, no-op.
 #if PTHREADS_DEBUG
-      out('Creating ' + numWorkers + ' workers for a pthread spawn pool.');
+      out('Creating ' + numWorkers + ' workers.');
 #endif
       var pthreadMainJs = "{{{ PTHREAD_WORKER_FILE }}}";
       // Allow HTML module to configure the location where the 'worker.js' file will be loaded from,
@@ -489,7 +490,6 @@ var LibraryPThread = {
 
   _spawn_thread: function(threadParams) {
     if (ENVIRONMENT_IS_PTHREAD) throw 'Internal Error! _spawn_thread() can only ever be called from main application thread!';
-
     var worker = PThread.getNewWorker();
     if (worker.pthread !== undefined) throw 'Internal error!';
     if (!threadParams.pthread_ptr) throw 'Internal error, no pthread ptr!';
diff --git a/tests/runner.py b/tests/runner.py
index 3e3b64e2bb570..fe473cf564227 100755
--- a/tests/runner.py
+++ b/tests/runner.py
@@ -1275,6 +1275,8 @@ def do_GET(self):
           # the browser must keep polling
           self.wfile.write(b'(wait)')
       else:
+        print('[simple HTTP serving:', unquote_plus(self.path), ']')
+
         # Use SimpleHTTPServer default file serving operation for GET.
         if DEBUG:
           print('[simple HTTP serving:', unquote_plus(self.path), ']')
diff --git a/tests/test_browser.py b/tests/test_browser.py
index 9189222955918..6ec2788ad5535 100644
--- a/tests/test_browser.py
+++ b/tests/test_browser.py
@@ -3646,6 +3646,14 @@ def test(args):
     test([])
     test(['-O3'])
     test(['-s', 'MODULARIZE_INSTANCE=1'])
+    test(['-s', 'PREWARM_PTHREAD_POOL_WORKERS_SIZE=4'])
+    test(['-s', 'PREWARM_PTHREAD_POOL_WORKERS_SIZE=8'])
+    test(['-s', 'PREWARM_PTHREAD_POOL_WORKERS_SIZE=12'])
+
+  # Test that preallocating worker threads work.
+  @requires_threads
+  def test_pthread_preallocates_workers(self):
+    self.btest(path_from_root('tests', 'pthread', 'test_pthread_preallocates_workers.cpp'), expected='6765', args=['-O3', '-s', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'PREWARM_PTHREAD_POOL_WORKERS_SIZE=8', '-s', 'PTHREADS_DEBUG=1'])
 
   # Tests the -s PROXY_TO_PTHREAD=1 option.
   @requires_threads

From 9ec2778661a2442bf1fda0a08b417c46f0431a57 Mon Sep 17 00:00:00 2001
From: Tayeb Al Karim <tay@google.com>
Date: Thu, 5 Sep 2019 14:59:28 -0400
Subject: [PATCH 05/25] Change parameter for prewarm to a flag instead of a
 count. You can't join a newly created thread!

---
 src/library_pthread.js                        |  10 +-
 src/settings.js                               |   9 +-
 .../test_pthread_preallocates_workers.cpp     | 101 ++++++++++++++++++
 tests/test_browser.py                         |   6 +-
 4 files changed, 112 insertions(+), 14 deletions(-)
 create mode 100644 tests/pthread/test_pthread_preallocates_workers.cpp

diff --git a/src/library_pthread.js b/src/library_pthread.js
index dcdac3f61c558..065d73bed5da4 100644
--- a/src/library_pthread.js
+++ b/src/library_pthread.js
@@ -33,11 +33,11 @@ var LibraryPThread = {
     initMainThreadBlock: function() {
       if (ENVIRONMENT_IS_PTHREAD) return undefined;
 
-#if PREWARM_PTHREAD_POOL_WORKERS_SIZE > 0
+#if PREWARM_POOL_WORKERS && PTHREAD_POOL_SIZE > 0
 #if PTHREADS_DEBUG
-      out('Preallocating ' + {{{ PREWARM_PTHREAD_POOL_WORKERS_SIZE }}} + ' workers.');
+      out('Preallocating ' + {{{ PTHREAD_POOL_SIZE }}} + ' workers.');
 #endif
-      PThread.preallocatedWorkers = PThread.createNewWorkers({{{ PREWARM_PTHREAD_POOL_WORKERS_SIZE }}});
+      PThread.preallocatedWorkers = PThread.createNewWorkers({{{ PTHREAD_POOL_SIZE }}});
 #endif
 
       PThread.mainThreadBlock = {{{ makeStaticAlloc(C_STRUCTS.pthread.__size__) }}};
@@ -194,7 +194,6 @@ var LibraryPThread = {
     },
 
     terminateAllThreads: function() {
-      out('TERMINATING');
       for (var t in PThread.pthreads) {
         var pthread = PThread.pthreads[t];
         if (pthread) {
@@ -304,7 +303,6 @@ var LibraryPThread = {
         // if the runtime has not been loaded yet, etc. - so we just use getMemory, which is main-thread only.
         var tempDoublePtr = getMemory(8); // TODO: leaks. Cleanup after worker terminates.
 #endif
-
         // Ask the new worker to load up the Emscripten-compiled page. This is a heavy operation.
         worker.postMessage({
           cmd: 'load',
@@ -347,7 +345,9 @@ var LibraryPThread = {
       for (var i = 0; i < numWorkers; ++i) {
         var worker = workers[i];
         (function(worker) {
+          console.log('attaching listener...');
           worker.onmessage = function(e) {
+            console.log(e);
             var d = e.data;
             // Sometimes we need to backproxy events to the calling thread (e.g. HTML5 DOM events handlers such as emscripten_set_mousemove_callback()), so keep track in a globally accessible variable about the thread that initiated the proxying.
             if (worker.pthread) PThread.currentProxiedOperationCallerThread = worker.pthread.threadInfoStruct;
diff --git a/src/settings.js b/src/settings.js
index 1b027c716fa06..ad919866f8f64 100644
--- a/src/settings.js
+++ b/src/settings.js
@@ -1240,11 +1240,10 @@ var USE_PTHREADS = 0;
 // created on demand.
 var PTHREAD_POOL_SIZE = 0;
 
-// Specifies the number of web workers that are preallocated before the
-// webassembly module is compiled. If 0, workers are created on demand. You
-// likely want this to be >= PTHREAD_POOL_SIZE. Note that this starts the
-// webworkers in a state where they have yet to load the webassembly module.
-var PREWARM_PTHREAD_POOL_WORKERS_SIZE = 0
+// If true, PTHREAD_POOL_SIZE web workers are preallocated before the
+// webassembly module is compiled. If 0, workers are created on after the
+// module is compiled. This flag does not do anything if PTHREAD_POOL_SIZE <= 0
+var PREWARM_POOL_WORKERS = 0
 
 // If not explicitly specified, this is the stack size to use for newly created
 // pthreads.  According to
diff --git a/tests/pthread/test_pthread_preallocates_workers.cpp b/tests/pthread/test_pthread_preallocates_workers.cpp
new file mode 100644
index 0000000000000..2fc2d000710b1
--- /dev/null
+++ b/tests/pthread/test_pthread_preallocates_workers.cpp
@@ -0,0 +1,101 @@
+// Copyright 2019 The Emscripten Authors.  All rights reserved.
+// Emscripten is available under two separate licenses, the MIT license and the
+// University of Illinois/NCSA Open Source License.  Both these licenses can be
+// found in the LICENSE file.
+
+#include <pthread.h>
+#include <sys/types.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <unistd.h>
+#include <errno.h>
+#include <emscripten.h>
+#include <emscripten/threading.h>
+#include <vector>
+
+int fib(int n)
+{
+  if (n <= 0) return 0;
+  if (n == 1) return 1;
+  return fib(n-1) + fib(n-2);
+}
+
+static void *thread_start(void *arg)
+{
+  int n = 20;
+  EM_ASM(out('Thread: Computing fib('+$0+')...'), n);
+  int fibn = fib(n);
+  EM_ASM(out('Thread: Computation done. fib('+$0+') = '+$1+'.'), n, fibn);
+  pthread_exit((void*)fibn);
+}
+
+pthread_t threads[4];
+
+void CreateThread(int i) {
+  int rc = pthread_create(&threads[i], NULL, thread_start, NULL);
+  assert(rc == 0);
+}
+
+
+int main()
+{
+  if (!emscripten_has_threading_support())
+  {
+#ifdef REPORT_RESULT
+    REPORT_RESULT(6765);
+#endif
+    printf("Skipped: Threading is not supported.\n");
+    return 0;
+  }
+
+  // assert(EM_ASM_INT(return PThread.runningWorkers.length) == 0);
+  // // This test should be run with a prewarmed pool of size 8. 4 of the
+  // // preallocated workers should have been used already.
+  // assert(EM_ASM_INT(return PThread.preallocatedWorkers.length) == 4);
+  // // This test should be run with a prepopulated pool of size 4.
+  // assert(EM_ASM_INT(return PThread.unusedWorkers.length) == 4);
+
+  int n = 20;
+  EM_ASM(out('Main: Spawning thread to compute fib('+$0+')...'), n);
+  CreateThread(0);
+  EM_ASM(out('Main: Waiting for thread to join.'));
+  int result = 0;
+
+  // assert(EM_ASM_INT(return PThread.preallocatedWorkers.length) == 4);
+  // assert(EM_ASM_INT(return PThread.runningWorkers.length) == 1);
+  // assert(EM_ASM_INT(return PThread.unusedWorkers.length) == 3);
+
+  int s = pthread_join(threads[0], (void**)&result);
+  assert(s == 0);
+  EM_ASM(out('Main: Thread joined with result: '+$0+'.'), result);
+
+  // assert(EM_ASM_INT(return PThread.preallocatedWorkers.length) == 0);
+  // assert(EM_ASM_INT(return PThread.runningWorkers.length) == 0);
+  // assert(EM_ASM_INT(return PThread.unusedWorkers.length) == 4);
+
+  for (int i = 0; i < 4; ++i) {
+    EM_ASM(out('Main: Spawning thread '+$0+' to compute fib('+$1+')...'), i, n);
+    CreateThread(i);
+  }
+  sleep(5);
+
+  // assert(EM_ASM_INT(return PThread.preallocatedWorkers.length) == 3);
+  // assert(EM_ASM_INT(return PThread.runningWorkers.length) == 5);
+  // assert(EM_ASM_INT(return PThread.unusedWorkers.length) == 0);
+
+  // Joining them all should result in 4 in the unused state.
+  for (int i = 0; i < 4; ++i) {
+    EM_ASM(out('Main: Waiting for thread '+$0), i, n);
+    s = pthread_join(threads[i], (void**)&result);
+    assert(s == 0);
+  }
+
+  // assert(EM_ASM_INT(return PThread.preallocatedWorkers.length) == 3);
+  // assert(EM_ASM_INT(return PThread.runningWorkers.length) == 0);
+  // assert(EM_ASM_INT(return PThread.unusedWorkers.length) == 5);
+
+#ifdef REPORT_RESULT
+  REPORT_RESULT(result);
+#endif
+}
diff --git a/tests/test_browser.py b/tests/test_browser.py
index 6ec2788ad5535..d3e9dfb5655da 100644
--- a/tests/test_browser.py
+++ b/tests/test_browser.py
@@ -3646,14 +3646,12 @@ def test(args):
     test([])
     test(['-O3'])
     test(['-s', 'MODULARIZE_INSTANCE=1'])
-    test(['-s', 'PREWARM_PTHREAD_POOL_WORKERS_SIZE=4'])
-    test(['-s', 'PREWARM_PTHREAD_POOL_WORKERS_SIZE=8'])
-    test(['-s', 'PREWARM_PTHREAD_POOL_WORKERS_SIZE=12'])
+    test(['-s', 'PREWARM_POOL_WORKERS=1'])
 
   # Test that preallocating worker threads work.
   @requires_threads
   def test_pthread_preallocates_workers(self):
-    self.btest(path_from_root('tests', 'pthread', 'test_pthread_preallocates_workers.cpp'), expected='6765', args=['-O3', '-s', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'PREWARM_PTHREAD_POOL_WORKERS_SIZE=8', '-s', 'PTHREADS_DEBUG=1'])
+    self.btest(path_from_root('tests', 'pthread', 'test_pthread_preallocates_workers.cpp'), expected='6765', args=['-O3', '-s', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=4', '-s', 'PREWARM_POOL_WORKERS=1', '-s', 'PTHREADS_DEBUG=1'])
 
   # Tests the -s PROXY_TO_PTHREAD=1 option.
   @requires_threads

From 96f4a8ef29e6728714ddfcbef72301e3319bbe39 Mon Sep 17 00:00:00 2001
From: Tayeb Al Karim <tay@google.com>
Date: Thu, 5 Sep 2019 15:01:11 -0400
Subject: [PATCH 06/25] Remove extra logging.

---
 tests/runner.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tests/runner.py b/tests/runner.py
index fe473cf564227..3e3b64e2bb570 100755
--- a/tests/runner.py
+++ b/tests/runner.py
@@ -1275,8 +1275,6 @@ def do_GET(self):
           # the browser must keep polling
           self.wfile.write(b'(wait)')
       else:
-        print('[simple HTTP serving:', unquote_plus(self.path), ']')
-
         # Use SimpleHTTPServer default file serving operation for GET.
         if DEBUG:
           print('[simple HTTP serving:', unquote_plus(self.path), ']')

From eae735368cdadd1f011f12f69210d25dca9446c9 Mon Sep 17 00:00:00 2001
From: Tayeb Al Karim <tay@google.com>
Date: Thu, 5 Sep 2019 15:02:03 -0400
Subject: [PATCH 07/25] Remove debugging logs in library_pthread.

---
 src/library_pthread.js | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/library_pthread.js b/src/library_pthread.js
index 065d73bed5da4..7c6e89589c09f 100644
--- a/src/library_pthread.js
+++ b/src/library_pthread.js
@@ -345,9 +345,7 @@ var LibraryPThread = {
       for (var i = 0; i < numWorkers; ++i) {
         var worker = workers[i];
         (function(worker) {
-          console.log('attaching listener...');
           worker.onmessage = function(e) {
-            console.log(e);
             var d = e.data;
             // Sometimes we need to backproxy events to the calling thread (e.g. HTML5 DOM events handlers such as emscripten_set_mousemove_callback()), so keep track in a globally accessible variable about the thread that initiated the proxying.
             if (worker.pthread) PThread.currentProxiedOperationCallerThread = worker.pthread.threadInfoStruct;

From 233271a3b8e39e014ba4c55592f8ef5671659908 Mon Sep 17 00:00:00 2001
From: Tayeb Al Karim <tay@google.com>
Date: Thu, 5 Sep 2019 15:04:48 -0400
Subject: [PATCH 08/25] Update test due to new flag.

---
 src/library_pthread.js                        |  1 +
 .../test_pthread_preallocates_workers.cpp     | 37 +++++++++----------
 2 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/src/library_pthread.js b/src/library_pthread.js
index 7c6e89589c09f..d92209ee062b2 100644
--- a/src/library_pthread.js
+++ b/src/library_pthread.js
@@ -488,6 +488,7 @@ var LibraryPThread = {
 
   _spawn_thread: function(threadParams) {
     if (ENVIRONMENT_IS_PTHREAD) throw 'Internal Error! _spawn_thread() can only ever be called from main application thread!';
+
     var worker = PThread.getNewWorker();
     if (worker.pthread !== undefined) throw 'Internal error!';
     if (!threadParams.pthread_ptr) throw 'Internal error, no pthread ptr!';
diff --git a/tests/pthread/test_pthread_preallocates_workers.cpp b/tests/pthread/test_pthread_preallocates_workers.cpp
index 2fc2d000710b1..a95baa6b88273 100644
--- a/tests/pthread/test_pthread_preallocates_workers.cpp
+++ b/tests/pthread/test_pthread_preallocates_workers.cpp
@@ -49,12 +49,12 @@ int main()
     return 0;
   }
 
-  // assert(EM_ASM_INT(return PThread.runningWorkers.length) == 0);
-  // // This test should be run with a prewarmed pool of size 8. 4 of the
-  // // preallocated workers should have been used already.
-  // assert(EM_ASM_INT(return PThread.preallocatedWorkers.length) == 4);
-  // // This test should be run with a prepopulated pool of size 4.
-  // assert(EM_ASM_INT(return PThread.unusedWorkers.length) == 4);
+  assert(EM_ASM_INT(return PThread.runningWorkers.length) == 0);
+  // This test should be run with a prewarmed pool of size 4. all 4 of the
+  // preallocated workers should have been used already.
+  assert(EM_ASM_INT(return PThread.preallocatedWorkers.length) == 0);
+  // This test should be run with a prepopulated pool of size 4.
+  assert(EM_ASM_INT(return PThread.unusedWorkers.length) == 4);
 
   int n = 20;
   EM_ASM(out('Main: Spawning thread to compute fib('+$0+')...'), n);
@@ -62,27 +62,26 @@ int main()
   EM_ASM(out('Main: Waiting for thread to join.'));
   int result = 0;
 
-  // assert(EM_ASM_INT(return PThread.preallocatedWorkers.length) == 4);
-  // assert(EM_ASM_INT(return PThread.runningWorkers.length) == 1);
-  // assert(EM_ASM_INT(return PThread.unusedWorkers.length) == 3);
+  assert(EM_ASM_INT(return PThread.preallocatedWorkers.length) == 0);
+  assert(EM_ASM_INT(return PThread.runningWorkers.length) == 1);
+  assert(EM_ASM_INT(return PThread.unusedWorkers.length) == 3);
 
   int s = pthread_join(threads[0], (void**)&result);
   assert(s == 0);
   EM_ASM(out('Main: Thread joined with result: '+$0+'.'), result);
 
-  // assert(EM_ASM_INT(return PThread.preallocatedWorkers.length) == 0);
-  // assert(EM_ASM_INT(return PThread.runningWorkers.length) == 0);
-  // assert(EM_ASM_INT(return PThread.unusedWorkers.length) == 4);
+  assert(EM_ASM_INT(return PThread.preallocatedWorkers.length) == 0);
+  assert(EM_ASM_INT(return PThread.runningWorkers.length) == 0);
+  assert(EM_ASM_INT(return PThread.unusedWorkers.length) == 4);
 
   for (int i = 0; i < 4; ++i) {
     EM_ASM(out('Main: Spawning thread '+$0+' to compute fib('+$1+')...'), i, n);
     CreateThread(i);
   }
-  sleep(5);
 
-  // assert(EM_ASM_INT(return PThread.preallocatedWorkers.length) == 3);
-  // assert(EM_ASM_INT(return PThread.runningWorkers.length) == 5);
-  // assert(EM_ASM_INT(return PThread.unusedWorkers.length) == 0);
+  assert(EM_ASM_INT(return PThread.preallocatedWorkers.length) == 0);
+  assert(EM_ASM_INT(return PThread.runningWorkers.length) == 4);
+  assert(EM_ASM_INT(return PThread.unusedWorkers.length) == 0);
 
   // Joining them all should result in 4 in the unused state.
   for (int i = 0; i < 4; ++i) {
@@ -91,9 +90,9 @@ int main()
     assert(s == 0);
   }
 
-  // assert(EM_ASM_INT(return PThread.preallocatedWorkers.length) == 3);
-  // assert(EM_ASM_INT(return PThread.runningWorkers.length) == 0);
-  // assert(EM_ASM_INT(return PThread.unusedWorkers.length) == 5);
+  assert(EM_ASM_INT(return PThread.preallocatedWorkers.length) == 0);
+  assert(EM_ASM_INT(return PThread.runningWorkers.length) == 0);
+  assert(EM_ASM_INT(return PThread.unusedWorkers.length) == 4);
 
 #ifdef REPORT_RESULT
   REPORT_RESULT(result);

From a4107ba0968f5d875a1286173f260ac760ea3268 Mon Sep 17 00:00:00 2001
From: Tayeb Al Karim <tay@google.com>
Date: Thu, 5 Sep 2019 15:05:53 -0400
Subject: [PATCH 09/25] Remove debugging flags from the test runner.

---
 tests/test_browser.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_browser.py b/tests/test_browser.py
index d3e9dfb5655da..3e1cf81bd845f 100644
--- a/tests/test_browser.py
+++ b/tests/test_browser.py
@@ -3651,7 +3651,7 @@ def test(args):
   # Test that preallocating worker threads work.
   @requires_threads
   def test_pthread_preallocates_workers(self):
-    self.btest(path_from_root('tests', 'pthread', 'test_pthread_preallocates_workers.cpp'), expected='6765', args=['-O3', '-s', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=4', '-s', 'PREWARM_POOL_WORKERS=1', '-s', 'PTHREADS_DEBUG=1'])
+    self.btest(path_from_root('tests', 'pthread', 'test_pthread_preallocates_workers.cpp'), expected='6765', args=['-O3', '-s', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=4', '-s', 'PREWARM_POOL_WORKERS=1'])
 
   # Tests the -s PROXY_TO_PTHREAD=1 option.
   @requires_threads

From d5f62ae99707babb9c77ffc2d10c362ef51e5a48 Mon Sep 17 00:00:00 2001
From: Tayeb Al Karim <tay@google.com>
Date: Thu, 5 Sep 2019 15:20:03 -0400
Subject: [PATCH 10/25] Revert "Change parameter for prewarm to a flag instead
 of a count. You can't join a newly created thread!"

This reverts commit 9ec2778661a2442bf1fda0a08b417c46f0431a57.
---
 src/library_pthread.js                        |  10 +-
 src/settings.js                               |   9 +-
 .../test_pthread_preallocates_workers.cpp     | 101 ------------------
 tests/test_browser.py                         |   6 +-
 4 files changed, 14 insertions(+), 112 deletions(-)
 delete mode 100644 tests/pthread/test_pthread_preallocates_workers.cpp

diff --git a/src/library_pthread.js b/src/library_pthread.js
index 065d73bed5da4..dcdac3f61c558 100644
--- a/src/library_pthread.js
+++ b/src/library_pthread.js
@@ -33,11 +33,11 @@ var LibraryPThread = {
     initMainThreadBlock: function() {
       if (ENVIRONMENT_IS_PTHREAD) return undefined;
 
-#if PREWARM_POOL_WORKERS && PTHREAD_POOL_SIZE > 0
+#if PREWARM_PTHREAD_POOL_WORKERS_SIZE > 0
 #if PTHREADS_DEBUG
-      out('Preallocating ' + {{{ PTHREAD_POOL_SIZE }}} + ' workers.');
+      out('Preallocating ' + {{{ PREWARM_PTHREAD_POOL_WORKERS_SIZE }}} + ' workers.');
 #endif
-      PThread.preallocatedWorkers = PThread.createNewWorkers({{{ PTHREAD_POOL_SIZE }}});
+      PThread.preallocatedWorkers = PThread.createNewWorkers({{{ PREWARM_PTHREAD_POOL_WORKERS_SIZE }}});
 #endif
 
       PThread.mainThreadBlock = {{{ makeStaticAlloc(C_STRUCTS.pthread.__size__) }}};
@@ -194,6 +194,7 @@ var LibraryPThread = {
     },
 
     terminateAllThreads: function() {
+      out('TERMINATING');
       for (var t in PThread.pthreads) {
         var pthread = PThread.pthreads[t];
         if (pthread) {
@@ -303,6 +304,7 @@ var LibraryPThread = {
         // if the runtime has not been loaded yet, etc. - so we just use getMemory, which is main-thread only.
         var tempDoublePtr = getMemory(8); // TODO: leaks. Cleanup after worker terminates.
 #endif
+
         // Ask the new worker to load up the Emscripten-compiled page. This is a heavy operation.
         worker.postMessage({
           cmd: 'load',
@@ -345,9 +347,7 @@ var LibraryPThread = {
       for (var i = 0; i < numWorkers; ++i) {
         var worker = workers[i];
         (function(worker) {
-          console.log('attaching listener...');
           worker.onmessage = function(e) {
-            console.log(e);
             var d = e.data;
             // Sometimes we need to backproxy events to the calling thread (e.g. HTML5 DOM events handlers such as emscripten_set_mousemove_callback()), so keep track in a globally accessible variable about the thread that initiated the proxying.
             if (worker.pthread) PThread.currentProxiedOperationCallerThread = worker.pthread.threadInfoStruct;
diff --git a/src/settings.js b/src/settings.js
index ad919866f8f64..1b027c716fa06 100644
--- a/src/settings.js
+++ b/src/settings.js
@@ -1240,10 +1240,11 @@ var USE_PTHREADS = 0;
 // created on demand.
 var PTHREAD_POOL_SIZE = 0;
 
-// If true, PTHREAD_POOL_SIZE web workers are preallocated before the
-// webassembly module is compiled. If 0, workers are created on after the
-// module is compiled. This flag does not do anything if PTHREAD_POOL_SIZE <= 0
-var PREWARM_POOL_WORKERS = 0
+// Specifies the number of web workers that are preallocated before the
+// webassembly module is compiled. If 0, workers are created on demand. You
+// likely want this to be >= PTHREAD_POOL_SIZE. Note that this starts the
+// webworkers in a state where they have yet to load the webassembly module.
+var PREWARM_PTHREAD_POOL_WORKERS_SIZE = 0
 
 // If not explicitly specified, this is the stack size to use for newly created
 // pthreads.  According to
diff --git a/tests/pthread/test_pthread_preallocates_workers.cpp b/tests/pthread/test_pthread_preallocates_workers.cpp
deleted file mode 100644
index 2fc2d000710b1..0000000000000
--- a/tests/pthread/test_pthread_preallocates_workers.cpp
+++ /dev/null
@@ -1,101 +0,0 @@
-// Copyright 2019 The Emscripten Authors.  All rights reserved.
-// Emscripten is available under two separate licenses, the MIT license and the
-// University of Illinois/NCSA Open Source License.  Both these licenses can be
-// found in the LICENSE file.
-
-#include <pthread.h>
-#include <sys/types.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <assert.h>
-#include <unistd.h>
-#include <errno.h>
-#include <emscripten.h>
-#include <emscripten/threading.h>
-#include <vector>
-
-int fib(int n)
-{
-  if (n <= 0) return 0;
-  if (n == 1) return 1;
-  return fib(n-1) + fib(n-2);
-}
-
-static void *thread_start(void *arg)
-{
-  int n = 20;
-  EM_ASM(out('Thread: Computing fib('+$0+')...'), n);
-  int fibn = fib(n);
-  EM_ASM(out('Thread: Computation done. fib('+$0+') = '+$1+'.'), n, fibn);
-  pthread_exit((void*)fibn);
-}
-
-pthread_t threads[4];
-
-void CreateThread(int i) {
-  int rc = pthread_create(&threads[i], NULL, thread_start, NULL);
-  assert(rc == 0);
-}
-
-
-int main()
-{
-  if (!emscripten_has_threading_support())
-  {
-#ifdef REPORT_RESULT
-    REPORT_RESULT(6765);
-#endif
-    printf("Skipped: Threading is not supported.\n");
-    return 0;
-  }
-
-  // assert(EM_ASM_INT(return PThread.runningWorkers.length) == 0);
-  // // This test should be run with a prewarmed pool of size 8. 4 of the
-  // // preallocated workers should have been used already.
-  // assert(EM_ASM_INT(return PThread.preallocatedWorkers.length) == 4);
-  // // This test should be run with a prepopulated pool of size 4.
-  // assert(EM_ASM_INT(return PThread.unusedWorkers.length) == 4);
-
-  int n = 20;
-  EM_ASM(out('Main: Spawning thread to compute fib('+$0+')...'), n);
-  CreateThread(0);
-  EM_ASM(out('Main: Waiting for thread to join.'));
-  int result = 0;
-
-  // assert(EM_ASM_INT(return PThread.preallocatedWorkers.length) == 4);
-  // assert(EM_ASM_INT(return PThread.runningWorkers.length) == 1);
-  // assert(EM_ASM_INT(return PThread.unusedWorkers.length) == 3);
-
-  int s = pthread_join(threads[0], (void**)&result);
-  assert(s == 0);
-  EM_ASM(out('Main: Thread joined with result: '+$0+'.'), result);
-
-  // assert(EM_ASM_INT(return PThread.preallocatedWorkers.length) == 0);
-  // assert(EM_ASM_INT(return PThread.runningWorkers.length) == 0);
-  // assert(EM_ASM_INT(return PThread.unusedWorkers.length) == 4);
-
-  for (int i = 0; i < 4; ++i) {
-    EM_ASM(out('Main: Spawning thread '+$0+' to compute fib('+$1+')...'), i, n);
-    CreateThread(i);
-  }
-  sleep(5);
-
-  // assert(EM_ASM_INT(return PThread.preallocatedWorkers.length) == 3);
-  // assert(EM_ASM_INT(return PThread.runningWorkers.length) == 5);
-  // assert(EM_ASM_INT(return PThread.unusedWorkers.length) == 0);
-
-  // Joining them all should result in 4 in the unused state.
-  for (int i = 0; i < 4; ++i) {
-    EM_ASM(out('Main: Waiting for thread '+$0), i, n);
-    s = pthread_join(threads[i], (void**)&result);
-    assert(s == 0);
-  }
-
-  // assert(EM_ASM_INT(return PThread.preallocatedWorkers.length) == 3);
-  // assert(EM_ASM_INT(return PThread.runningWorkers.length) == 0);
-  // assert(EM_ASM_INT(return PThread.unusedWorkers.length) == 5);
-
-#ifdef REPORT_RESULT
-  REPORT_RESULT(result);
-#endif
-}
diff --git a/tests/test_browser.py b/tests/test_browser.py
index d3e9dfb5655da..6ec2788ad5535 100644
--- a/tests/test_browser.py
+++ b/tests/test_browser.py
@@ -3646,12 +3646,14 @@ def test(args):
     test([])
     test(['-O3'])
     test(['-s', 'MODULARIZE_INSTANCE=1'])
-    test(['-s', 'PREWARM_POOL_WORKERS=1'])
+    test(['-s', 'PREWARM_PTHREAD_POOL_WORKERS_SIZE=4'])
+    test(['-s', 'PREWARM_PTHREAD_POOL_WORKERS_SIZE=8'])
+    test(['-s', 'PREWARM_PTHREAD_POOL_WORKERS_SIZE=12'])
 
   # Test that preallocating worker threads work.
   @requires_threads
   def test_pthread_preallocates_workers(self):
-    self.btest(path_from_root('tests', 'pthread', 'test_pthread_preallocates_workers.cpp'), expected='6765', args=['-O3', '-s', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=4', '-s', 'PREWARM_POOL_WORKERS=1', '-s', 'PTHREADS_DEBUG=1'])
+    self.btest(path_from_root('tests', 'pthread', 'test_pthread_preallocates_workers.cpp'), expected='6765', args=['-O3', '-s', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'PREWARM_PTHREAD_POOL_WORKERS_SIZE=8', '-s', 'PTHREADS_DEBUG=1'])
 
   # Tests the -s PROXY_TO_PTHREAD=1 option.
   @requires_threads

From ca2251760331774791540b0e7373a214d2731288 Mon Sep 17 00:00:00 2001
From: Tayeb Al Karim <tay@google.com>
Date: Thu, 5 Sep 2019 15:58:32 -0400
Subject: [PATCH 11/25] Revert "Change parameter for prewarm to a flag instead
 of a count. You can't join a newly created thread!"

This reverts commit 9ec2778661a2442bf1fda0a08b417c46f0431a57.

I realized that the issue was with the test  - you can't join() in main() on a thread emscripten might immediately create. This can be fixed in production by proxying the main thread on a pthread, but that would elide giving access to the PThread.foo arrays in the test.
---
 tests/runner.py       | 2 --
 tests/test_browser.py | 2 +-
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/tests/runner.py b/tests/runner.py
index fe473cf564227..3e3b64e2bb570 100755
--- a/tests/runner.py
+++ b/tests/runner.py
@@ -1275,8 +1275,6 @@ def do_GET(self):
           # the browser must keep polling
           self.wfile.write(b'(wait)')
       else:
-        print('[simple HTTP serving:', unquote_plus(self.path), ']')
-
         # Use SimpleHTTPServer default file serving operation for GET.
         if DEBUG:
           print('[simple HTTP serving:', unquote_plus(self.path), ']')
diff --git a/tests/test_browser.py b/tests/test_browser.py
index 6ec2788ad5535..4e99d4ea619a9 100644
--- a/tests/test_browser.py
+++ b/tests/test_browser.py
@@ -3653,7 +3653,7 @@ def test(args):
   # Test that preallocating worker threads work.
   @requires_threads
   def test_pthread_preallocates_workers(self):
-    self.btest(path_from_root('tests', 'pthread', 'test_pthread_preallocates_workers.cpp'), expected='6765', args=['-O3', '-s', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8', '-s', 'PREWARM_PTHREAD_POOL_WORKERS_SIZE=8', '-s', 'PTHREADS_DEBUG=1'])
+    self.btest(path_from_root('tests', 'pthread', 'test_pthread_preallocates_workers.cpp'), expected='0', args=['-O3', '-s', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=4', '-s', 'PREWARM_PTHREAD_POOL_WORKERS_SIZE=8'])
 
   # Tests the -s PROXY_TO_PTHREAD=1 option.
   @requires_threads

From 802759680dd106f499fa81b093d6fb2895305e9a Mon Sep 17 00:00:00 2001
From: Tayeb Al Karim <tay@google.com>
Date: Thu, 5 Sep 2019 16:06:17 -0400
Subject: [PATCH 12/25] Merged changes from head.

---
 tests/test_browser.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/tests/test_browser.py b/tests/test_browser.py
index 61410b258cd3f..4e99d4ea619a9 100644
--- a/tests/test_browser.py
+++ b/tests/test_browser.py
@@ -3653,11 +3653,7 @@ def test(args):
   # Test that preallocating worker threads work.
   @requires_threads
   def test_pthread_preallocates_workers(self):
-<<<<<<< HEAD
     self.btest(path_from_root('tests', 'pthread', 'test_pthread_preallocates_workers.cpp'), expected='0', args=['-O3', '-s', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=4', '-s', 'PREWARM_PTHREAD_POOL_WORKERS_SIZE=8'])
-=======
-    self.btest(path_from_root('tests', 'pthread', 'test_pthread_preallocates_workers.cpp'), expected='6765', args=['-O3', '-s', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=4', '-s', 'PREWARM_POOL_WORKERS=1'])
->>>>>>> a4107ba0968f5d875a1286173f260ac760ea3268
 
   # Tests the -s PROXY_TO_PTHREAD=1 option.
   @requires_threads

From 5ad937fa40fa8bfceb17b423994a83526ace0d99 Mon Sep 17 00:00:00 2001
From: Tayeb Al Karim <tay@google.com>
Date: Thu, 5 Sep 2019 16:17:05 -0400
Subject: [PATCH 13/25] Removed extra debugging.

---
 src/library_pthread.js | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/library_pthread.js b/src/library_pthread.js
index 016b76d015243..fc6fb53f39b21 100644
--- a/src/library_pthread.js
+++ b/src/library_pthread.js
@@ -194,7 +194,6 @@ var LibraryPThread = {
     },
 
     terminateAllThreads: function() {
-      out('TERMINATING');
       for (var t in PThread.pthreads) {
         var pthread = PThread.pthreads[t];
         if (pthread) {

From 4a0b3a8ee3852cc066887164f951bcb4fbf08c42 Mon Sep 17 00:00:00 2001
From: Tayeb Al Karim <tay@google.com>
Date: Thu, 5 Sep 2019 20:32:51 -0400
Subject: [PATCH 14/25] add missing semicolon.

---
 src/settings.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/settings.js b/src/settings.js
index 326d48a83460d..78d7169112186 100644
--- a/src/settings.js
+++ b/src/settings.js
@@ -1241,7 +1241,7 @@ var PTHREAD_POOL_SIZE = 0;
 // webassembly module is compiled. If 0, workers are created on demand. You
 // likely want this to be >= PTHREAD_POOL_SIZE. Note that this starts the
 // webworkers in a state where they have yet to load the webassembly module.
-var PREWARM_PTHREAD_POOL_WORKERS_SIZE = 0
+var PREWARM_PTHREAD_POOL_WORKERS_SIZE = 0;
 
 // If not explicitly specified, this is the stack size to use for newly created
 // pthreads.  According to

From b346f366c060e850d78e4d707980cfd1b421e4c2 Mon Sep 17 00:00:00 2001
From: Tayeb Al Karim <tay@google.com>
Date: Thu, 5 Sep 2019 21:02:44 -0400
Subject: [PATCH 15/25] Various fixes from code review. Comments, renames, and
 make the test less flaky.

---
 src/library_pthread.js                        | 13 +++++-----
 src/settings.js                               |  7 +++---
 .../test_pthread_preallocates_workers.cpp     | 24 +++++++++++++------
 3 files changed, 28 insertions(+), 16 deletions(-)

diff --git a/src/library_pthread.js b/src/library_pthread.js
index fc6fb53f39b21..b521e91c39329 100644
--- a/src/library_pthread.js
+++ b/src/library_pthread.js
@@ -13,8 +13,9 @@ var LibraryPThread = {
       schedPrio: 0
     },
     // Workers that have been created but uninitialized. These have already been
-    // parsed, but the wabassembly model has not yet been loaded, making it
-    // distinct from the unusedWorkers below.
+    // parsed, but the wasm file has not yet been loaded, making it
+    // distinct from the unusedWorkers below. These workers will be created before
+    // loading wasm on the main thread.
     preallocatedWorkers: [],
     // Contains all Workers that are idle/unused and not currently hosting an executing pthread.
     // Unused Workers can either be pooled up before page startup, but also when a pthread quits, its hosting
@@ -278,17 +279,17 @@ var LibraryPThread = {
 
       var workers = [];
 
-      var createNumNewWorkers = numWorkers;
+      var numWorkersToCreate = numWorkers;
       if (PThread.preallocatedWorkers.length > 0) {
         var workersUsed = Math.min(PThread.preallocatedWorkers.length, numWorkers);
 #if PTHREADS_DEBUG
         out('Using ' + workersUsed + ' preallocated workers');
 #endif
         workers.push(...PThread.preallocatedWorkers.splice(0, workersUsed));
-        createNumNewWorkers -= workersUsed;
+        numWorkersToCreate -= workersUsed;
       }
-      if (createNumNewWorkers > 0) {
-        workers.push(...PThread.createNewWorkers(createNumNewWorkers));
+      if (numWorkersToCreate > 0) {
+        workers.push(...PThread.createNewWorkers(numWorkersToCreate));
       }
 
       // Add the listeners.
diff --git a/src/settings.js b/src/settings.js
index 78d7169112186..d3740c4e65def 100644
--- a/src/settings.js
+++ b/src/settings.js
@@ -1237,9 +1237,10 @@ var USE_PTHREADS = 0;
 // created on demand.
 var PTHREAD_POOL_SIZE = 0;
 
-// Specifies the number of web workers that are preallocated before the
-// webassembly module is compiled. If 0, workers are created on demand. You
-// likely want this to be >= PTHREAD_POOL_SIZE. Note that this starts the
+// Specifies the number of web workers that are created before the
+// webassembly module is compiled on the main thread.
+// If 0, workers are created on demand. You likely want this to
+// be >= PTHREAD_POOL_SIZE. Note that this starts the
 // webworkers in a state where they have yet to load the webassembly module.
 var PREWARM_PTHREAD_POOL_WORKERS_SIZE = 0;
 
diff --git a/tests/pthread/test_pthread_preallocates_workers.cpp b/tests/pthread/test_pthread_preallocates_workers.cpp
index 18d7101a94cfc..7d157ad8a128a 100644
--- a/tests/pthread/test_pthread_preallocates_workers.cpp
+++ b/tests/pthread/test_pthread_preallocates_workers.cpp
@@ -14,10 +14,16 @@
 #include <emscripten/threading.h>
 #include <vector>
 
+bool alive[5] = {0};
+pthread_t threads[5];
+
 static void *thread_start(void *arg)
 {
   // This should be long enough for threads to pile up.
-  sleep(2);
+  int idx = (int)arg;
+  while (alive[idx]) {
+    sleep(1);
+  }
   pthread_exit((void*)0);
 }
 
@@ -39,10 +45,9 @@ int main()
   // This test should be run with a prepopulated pool of size 4.
   assert(EM_ASM_INT(return PThread.unusedWorkers.length) == 4);
 
-  pthread_t threads[5];
-  int n = 20;
-  EM_ASM(out('Main: Spawning thread to compute fib('+$0+')...'), n);
-  int rc = pthread_create(&threads[0], NULL, thread_start, NULL);
+  EM_ASM(out('Main: Spawning thread...'), NULL);
+  alive[0] = true;
+  int rc = pthread_create(&threads[0], NULL, thread_start, (void*)0);
   assert(rc == 0);
   EM_ASM(out('Main: Waiting for thread to join.'));
   int result = 0;
@@ -52,6 +57,7 @@ int main()
   assert(EM_ASM_INT(return PThread.unusedWorkers.length) == 3);
 
   // We can join here because 4 threads were preallocated.
+  alive[0] = false;
   rc = pthread_join(threads[0], NULL);
   assert(rc == 0);
   EM_ASM(out('Main: Thread joined with result: '+$0+'.'), result);
@@ -61,8 +67,9 @@ int main()
   assert(EM_ASM_INT(return PThread.unusedWorkers.length) == 4);
 
   for (int i = 0; i < 5; ++i) {
-    EM_ASM(out('Main: Spawning thread '+$0+' to compute fib('+$1+')...'), i, n);
-    int rc = pthread_create(&threads[i], NULL, thread_start, NULL);
+    alive[i] = true;
+    EM_ASM(out('Main: Spawning thread '+$0+'...'), i);
+    int rc = pthread_create(&threads[i], NULL, thread_start, (void*)i);
     assert(rc == 0);
   }
 
@@ -72,8 +79,11 @@ int main()
 
   // We can join the first 4 threads, as they were preallocated.
   for (int i = 0; i < 4; ++i) {
+    alive[i] = false;
     rc = pthread_join(threads[i], NULL);
   }
+  // Mark the last thread to be killed.
+  alive[4] = false;
 
   // We can't join the last one or we'll hang forever. The main thread
   // won't give up the thread to let the 5th thread be created. This is

From e5e2051f4067a4f0765a85fea6fefd06a995274c Mon Sep 17 00:00:00 2001
From: Tayeb Al Karim <tay@google.com>
Date: Mon, 9 Sep 2019 21:08:02 -0400
Subject: [PATCH 16/25] Make the worker pool size the max of the requested size
 or the threadpool size.

---
 src/library_pthread.js | 7 ++++---
 src/settings.js        | 8 +++++---
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/src/library_pthread.js b/src/library_pthread.js
index b521e91c39329..87760cd589d51 100644
--- a/src/library_pthread.js
+++ b/src/library_pthread.js
@@ -34,11 +34,12 @@ var LibraryPThread = {
     initMainThreadBlock: function() {
       if (ENVIRONMENT_IS_PTHREAD) return undefined;
 
-#if PREWARM_PTHREAD_POOL_WORKERS_SIZE > 0
+#if PREWARM_PTHREAD_POOL_WORKERS_SIZE > 0 || PTHREAD_POOL_SIZE > 0
+      var requestedPoolSize = Math.max({{{ PTHREAD_POOL_SIZE }}}, {{{ PREWARM_PTHREAD_POOL_WORKERS_SIZE }}});
 #if PTHREADS_DEBUG
-      out('Preallocating ' + {{{ PREWARM_PTHREAD_POOL_WORKERS_SIZE }}} + ' workers.');
+      out('Preallocating ' + requestedPoolSize + ' workers.');
 #endif
-      PThread.preallocatedWorkers = PThread.createNewWorkers({{{ PREWARM_PTHREAD_POOL_WORKERS_SIZE }}});
+      PThread.preallocatedWorkers = PThread.createNewWorkers(requestedPoolSize);
 #endif
 
       PThread.mainThreadBlock = {{{ makeStaticAlloc(C_STRUCTS.pthread.__size__) }}};
diff --git a/src/settings.js b/src/settings.js
index d3740c4e65def..5f6bc99ace22a 100644
--- a/src/settings.js
+++ b/src/settings.js
@@ -1239,9 +1239,11 @@ var PTHREAD_POOL_SIZE = 0;
 
 // Specifies the number of web workers that are created before the
 // webassembly module is compiled on the main thread.
-// If 0, workers are created on demand. You likely want this to
-// be >= PTHREAD_POOL_SIZE. Note that this starts the
-// webworkers in a state where they have yet to load the webassembly module.
+// If PTHREAD_POOL_SIZE and this value are both 0, workers are created on demand.
+// If PTHREAD_POOL_SIZE > 0, then the minimum value of this setting is that
+// pool size.
+// Note that this starts the webworkers in a state where they have yet to
+// load the webassembly module.
 var PREWARM_PTHREAD_POOL_WORKERS_SIZE = 0;
 
 // If not explicitly specified, this is the stack size to use for newly created

From 865ef8721bb1168f7f7118ef1426ef13877c215e Mon Sep 17 00:00:00 2001
From: Tayeb Al Karim <tay@google.com>
Date: Thu, 12 Sep 2019 10:35:17 -0400
Subject: [PATCH 17/25] Change PREWARM_PTHREAD_POOL_WORKERS_SIZE to be
 PTHREAD_POOL_ONLY_PREWARM. I had to weaken the test, since the main thread
 can't join() on threads that are loaded dynamically. Left comments that it
 _could_ join if PROXY_TO_PTHREAD=1 is set, but then the main thread doesn't
 have the information needed (workerpool sizes) to actually test the feature.

---
 src/library_pthread.js                        |  4 +-
 src/preamble.js                               |  2 +-
 src/settings.js                               | 19 ++---
 .../test_pthread_preallocates_workers.cpp     | 69 ++++++-------------
 tests/test_browser.py                         |  7 +-
 5 files changed, 35 insertions(+), 66 deletions(-)

diff --git a/src/library_pthread.js b/src/library_pthread.js
index 87760cd589d51..f5c0f6d49fbec 100644
--- a/src/library_pthread.js
+++ b/src/library_pthread.js
@@ -34,8 +34,8 @@ var LibraryPThread = {
     initMainThreadBlock: function() {
       if (ENVIRONMENT_IS_PTHREAD) return undefined;
 
-#if PREWARM_PTHREAD_POOL_WORKERS_SIZE > 0 || PTHREAD_POOL_SIZE > 0
-      var requestedPoolSize = Math.max({{{ PTHREAD_POOL_SIZE }}}, {{{ PREWARM_PTHREAD_POOL_WORKERS_SIZE }}});
+#if PTHREAD_POOL_SIZE > 0
+      var requestedPoolSize = {{{ PTHREAD_POOL_SIZE }}};
 #if PTHREADS_DEBUG
       out('Preallocating ' + requestedPoolSize + ' workers.');
 #endif
diff --git a/src/preamble.js b/src/preamble.js
index e19643cee8a18..56c9193957788 100644
--- a/src/preamble.js
+++ b/src/preamble.js
@@ -816,7 +816,7 @@ if (!ENVIRONMENT_IS_PTHREAD) addOnPreRun(function() {
 });
 #endif
 
-#if PTHREAD_POOL_SIZE > 0
+#if PTHREAD_POOL_SIZE > 0 && PTHREAD_POOL_ONLY_PREWARM == 0
 // To work around https://bugzilla.mozilla.org/show_bug.cgi?id=1049079, warm up a worker pool before starting up the application.
 if (!ENVIRONMENT_IS_PTHREAD) addOnPreRun(function() { if (typeof SharedArrayBuffer !== 'undefined') { addRunDependency('pthreads'); PThread.allocateUnusedWorkers({{{PTHREAD_POOL_SIZE}}}, function() { removeRunDependency('pthreads'); }); }});
 #endif
diff --git a/src/settings.js b/src/settings.js
index 5f6bc99ace22a..1cc8de20f1302 100644
--- a/src/settings.js
+++ b/src/settings.js
@@ -1232,19 +1232,14 @@ var IN_TEST_HARNESS = 0;
 // If true, enables support for pthreads.
 var USE_PTHREADS = 0;
 
-// Specifies the number of web workers that are preallocated with the
-// webassembly module before runtime is initialized. If 0, workers are
-// created on demand.
+// PTHREAD_POOL_SIZE specifies the number of web workers that are created
+// before the main runtime is initialized. If 0, workers are created on
+// demand. If PTHREAD_POOL_ONLY_PREWARM = 0, then the workers will be fully
+// loaded (available for use) prior to the main runtime being initialized. If
+// PTHREAD_POOL_ONLY_PREWARM = 1, then the workers will only be created and
+// have their runtimes loaded on demand after the main runtime is initialized.
 var PTHREAD_POOL_SIZE = 0;
-
-// Specifies the number of web workers that are created before the
-// webassembly module is compiled on the main thread.
-// If PTHREAD_POOL_SIZE and this value are both 0, workers are created on demand.
-// If PTHREAD_POOL_SIZE > 0, then the minimum value of this setting is that
-// pool size.
-// Note that this starts the webworkers in a state where they have yet to
-// load the webassembly module.
-var PREWARM_PTHREAD_POOL_WORKERS_SIZE = 0;
+var PTHREAD_POOL_ONLY_PREWARM = 0;
 
 // If not explicitly specified, this is the stack size to use for newly created
 // pthreads.  According to
diff --git a/tests/pthread/test_pthread_preallocates_workers.cpp b/tests/pthread/test_pthread_preallocates_workers.cpp
index 7d157ad8a128a..2040f356c0784 100644
--- a/tests/pthread/test_pthread_preallocates_workers.cpp
+++ b/tests/pthread/test_pthread_preallocates_workers.cpp
@@ -14,19 +14,26 @@
 #include <emscripten/threading.h>
 #include <vector>
 
-bool alive[5] = {0};
 pthread_t threads[5];
 
 static void *thread_start(void *arg)
 {
   // This should be long enough for threads to pile up.
   int idx = (int)arg;
-  while (alive[idx]) {
+  printf("Starting thread %d\n", idx);
+  while (true) {
     sleep(1);
   }
+  printf("Finishing thread %d\n", idx);
   pthread_exit((void*)0);
 }
 
+void CreateThread(int idx) {
+  EM_ASM(out('Main: Spawning thread '+$0+'...'), idx);
+  int rc = pthread_create(&threads[idx], NULL, thread_start, (void*)idx);
+  assert(rc == 0);
+}
+
 int main()
 {
   if (!emscripten_has_threading_support())
@@ -38,63 +45,31 @@ int main()
     return 0;
   }
 
-  assert(EM_ASM_INT(return PThread.runningWorkers.length) == 0);
-  // This test should be run with a prewarmed pool of size 8. 4 of the
-  // preallocated workers should have been used already.
-  assert(EM_ASM_INT(return PThread.preallocatedWorkers.length) == 4);
-  // This test should be run with a prepopulated pool of size 4.
-  assert(EM_ASM_INT(return PThread.unusedWorkers.length) == 4);
-
-  EM_ASM(out('Main: Spawning thread...'), NULL);
-  alive[0] = true;
-  int rc = pthread_create(&threads[0], NULL, thread_start, (void*)0);
-  assert(rc == 0);
-  EM_ASM(out('Main: Waiting for thread to join.'));
-  int result = 0;
-
-  assert(EM_ASM_INT(return PThread.preallocatedWorkers.length) == 4);
-  assert(EM_ASM_INT(return PThread.runningWorkers.length) == 1);
-  assert(EM_ASM_INT(return PThread.unusedWorkers.length) == 3);
-
-  // We can join here because 4 threads were preallocated.
-  alive[0] = false;
-  rc = pthread_join(threads[0], NULL);
-  assert(rc == 0);
-  EM_ASM(out('Main: Thread joined with result: '+$0+'.'), result);
-
+  // This test should be run with a prewarmed pool of size 4. None
+  // of the threads are allocated yet.
   assert(EM_ASM_INT(return PThread.preallocatedWorkers.length) == 4);
+  assert(EM_ASM_INT(return PThread.unusedWorkers.length) == 0);
   assert(EM_ASM_INT(return PThread.runningWorkers.length) == 0);
-  assert(EM_ASM_INT(return PThread.unusedWorkers.length) == 4);
 
-  for (int i = 0; i < 5; ++i) {
-    alive[i] = true;
-    EM_ASM(out('Main: Spawning thread '+$0+'...'), i);
-    int rc = pthread_create(&threads[i], NULL, thread_start, (void*)i);
-    assert(rc == 0);
-  }
+  CreateThread(0);
 
+  // We have one running thread, allocated on demand.
   assert(EM_ASM_INT(return PThread.preallocatedWorkers.length) == 3);
-  assert(EM_ASM_INT(return PThread.runningWorkers.length) == 5);
   assert(EM_ASM_INT(return PThread.unusedWorkers.length) == 0);
+  assert(EM_ASM_INT(return PThread.runningWorkers.length) == 1);
 
-  // We can join the first 4 threads, as they were preallocated.
-  for (int i = 0; i < 4; ++i) {
-    alive[i] = false;
-    rc = pthread_join(threads[i], NULL);
+  for (int i = 1; i < 5; ++i) {
+    CreateThread(i);
   }
-  // Mark the last thread to be killed.
-  alive[4] = false;
 
-  // We can't join the last one or we'll hang forever. The main thread
+  // All the preallocated workers should be used.
+  // We can't join the threads or we'll hang forever. The main thread
   // won't give up the thread to let the 5th thread be created. This is
   // solved in non-test cases by using PROXY_TO_PTHREAD, but we can't
   // do that here since we need to eval the length of the various pthread
-  // arrays. We don't know if the last thread is done or not when we are
-  // finished.
-  assert(EM_ASM_INT(return PThread.preallocatedWorkers.length) == 3);
-  assert(EM_ASM_INT(return PThread.runningWorkers.length) <= 1);
-  int unused_workers = EM_ASM_INT(return PThread.unusedWorkers.length);
-  assert(unused_workers == 4 || unused_workers == 5);
+  // arrays.
+  assert(EM_ASM_INT(return PThread.runningWorkers.length) == 5);
+  assert(EM_ASM_INT(return PThread.unusedWorkers.length) == 0);
 
 #ifdef REPORT_RESULT
   REPORT_RESULT(0);
diff --git a/tests/test_browser.py b/tests/test_browser.py
index 4e99d4ea619a9..874f3450f5353 100644
--- a/tests/test_browser.py
+++ b/tests/test_browser.py
@@ -3646,14 +3646,13 @@ def test(args):
     test([])
     test(['-O3'])
     test(['-s', 'MODULARIZE_INSTANCE=1'])
-    test(['-s', 'PREWARM_PTHREAD_POOL_WORKERS_SIZE=4'])
-    test(['-s', 'PREWARM_PTHREAD_POOL_WORKERS_SIZE=8'])
-    test(['-s', 'PREWARM_PTHREAD_POOL_WORKERS_SIZE=12'])
+    test(['-s', 'PTHREAD_POOL_ONLY_PREWARM=1'])
+
 
   # Test that preallocating worker threads work.
   @requires_threads
   def test_pthread_preallocates_workers(self):
-    self.btest(path_from_root('tests', 'pthread', 'test_pthread_preallocates_workers.cpp'), expected='0', args=['-O3', '-s', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=4', '-s', 'PREWARM_PTHREAD_POOL_WORKERS_SIZE=8'])
+    self.btest(path_from_root('tests', 'pthread', 'test_pthread_preallocates_workers.cpp'), expected='0', args=['-O3', '-s', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=4', '-s', 'PTHREAD_POOL_ONLY_PREWARM=1'])
 
   # Tests the -s PROXY_TO_PTHREAD=1 option.
   @requires_threads

From 01fc229b194394294688e67edf906b4f77b69953 Mon Sep 17 00:00:00 2001
From: Tayeb Al Karim <tay@google.com>
Date: Mon, 16 Sep 2019 12:48:02 -0400
Subject: [PATCH 18/25] Fix style error in comment.

---
 tests/test_browser.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/test_browser.py b/tests/test_browser.py
index 874f3450f5353..d0fc16b445699 100644
--- a/tests/test_browser.py
+++ b/tests/test_browser.py
@@ -3648,7 +3648,6 @@ def test(args):
     test(['-s', 'MODULARIZE_INSTANCE=1'])
     test(['-s', 'PTHREAD_POOL_ONLY_PREWARM=1'])
 
-
   # Test that preallocating worker threads work.
   @requires_threads
   def test_pthread_preallocates_workers(self):

From 9569d322155a341c87c26a00c5e9a316d2ff53d2 Mon Sep 17 00:00:00 2001
From: Tayeb Al Karim <tay@google.com>
Date: Wed, 18 Sep 2019 14:33:24 -0400
Subject: [PATCH 19/25] We don't want to use PTHREAD_POOL_ONLY_PREWARM on the
 pthread create test since it joins the created thread. This had worked
 previously when the option was to define the number of prewarmed threads;
 prewarming more threads than asked for in the threadpool would allow joins to
 work on threads that were created.

---
 tests/test_browser.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/test_browser.py b/tests/test_browser.py
index 9ab06e260c36a..69d40526985c1 100644
--- a/tests/test_browser.py
+++ b/tests/test_browser.py
@@ -3650,7 +3650,6 @@ def test(args):
     test([])
     test(['-O3'])
     test(['-s', 'MODULARIZE_INSTANCE=1'])
-    test(['-s', 'PTHREAD_POOL_ONLY_PREWARM=1'])
 
   # Test that preallocating worker threads work.
   @requires_threads

From 810c85a82beff912cb3ebaed007ed905ffed2081 Mon Sep 17 00:00:00 2001
From: Tayeb Al Karim <tay@google.com>
Date: Wed, 18 Sep 2019 15:55:38 -0400
Subject: [PATCH 20/25] Ensure we return an empty array if no workers were
 required. Switch to concat syntax for concat'ing arrays.

---
 src/library_pthread.js | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/library_pthread.js b/src/library_pthread.js
index f5c0f6d49fbec..c68d584cf0414 100644
--- a/src/library_pthread.js
+++ b/src/library_pthread.js
@@ -286,11 +286,11 @@ var LibraryPThread = {
 #if PTHREADS_DEBUG
         out('Using ' + workersUsed + ' preallocated workers');
 #endif
-        workers.push(...PThread.preallocatedWorkers.splice(0, workersUsed));
+        workers = workers.concat(PThread.preallocatedWorkers.splice(0, workersUsed));
         numWorkersToCreate -= workersUsed;
       }
       if (numWorkersToCreate > 0) {
-        workers.push(...PThread.createNewWorkers(numWorkersToCreate));
+        workers = workers.concat(PThread.createNewWorkers(numWorkersToCreate));
       }
 
       // Add the listeners.
@@ -428,7 +428,7 @@ var LibraryPThread = {
 
     createNewWorkers: function(numWorkers) {
       // Creates new workers with the discovered pthread worker file.
-      if (typeof SharedArrayBuffer === 'undefined') return; // No multithreading support, no-op.
+      if (typeof SharedArrayBuffer === 'undefined') return []; // No multithreading support, no-op.
 #if PTHREADS_DEBUG
       out('Creating ' + numWorkers + ' workers.');
 #endif

From d76c8d65c72690308cad246f4975fa8f2d3edb98 Mon Sep 17 00:00:00 2001
From: Tayeb Al Karim <tay@google.com>
Date: Thu, 19 Sep 2019 13:56:41 -0400
Subject: [PATCH 21/25] Replace PTHREAD_POOL_ONLY_PREWARM with
 PTHREAD_POOL_PREALLOCATE

---
 src/preamble.js       | 2 +-
 src/settings.js       | 6 +++---
 tests/test_browser.py | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/preamble.js b/src/preamble.js
index 84014c370753e..26669dadd8268 100644
--- a/src/preamble.js
+++ b/src/preamble.js
@@ -865,7 +865,7 @@ if (!ENVIRONMENT_IS_PTHREAD) addOnPreRun(function() {
 });
 #endif
 
-#if PTHREAD_POOL_SIZE > 0 && PTHREAD_POOL_ONLY_PREWARM == 0
+#if PTHREAD_POOL_SIZE > 0 && PTHREAD_POOL_PREALLOCATE == 0
 // To work around https://bugzilla.mozilla.org/show_bug.cgi?id=1049079, warm up a worker pool before starting up the application.
 if (!ENVIRONMENT_IS_PTHREAD) addOnPreRun(function() { if (typeof SharedArrayBuffer !== 'undefined') { addRunDependency('pthreads'); PThread.allocateUnusedWorkers({{{PTHREAD_POOL_SIZE}}}, function() { removeRunDependency('pthreads'); }); }});
 #endif
diff --git a/src/settings.js b/src/settings.js
index 9a6e0a574ff8c..d8676a418948b 100644
--- a/src/settings.js
+++ b/src/settings.js
@@ -1237,12 +1237,12 @@ var USE_PTHREADS = 0;
 
 // PTHREAD_POOL_SIZE specifies the number of web workers that are created
 // before the main runtime is initialized. If 0, workers are created on
-// demand. If PTHREAD_POOL_ONLY_PREWARM = 0, then the workers will be fully
+// demand. If PTHREAD_POOL_PREALLOCATE = 0, then the workers will be fully
 // loaded (available for use) prior to the main runtime being initialized. If
-// PTHREAD_POOL_ONLY_PREWARM = 1, then the workers will only be created and
+// PTHREAD_POOL_PREALLOCATE = 1, then the workers will only be created and
 // have their runtimes loaded on demand after the main runtime is initialized.
 var PTHREAD_POOL_SIZE = 0;
-var PTHREAD_POOL_ONLY_PREWARM = 0;
+var PTHREAD_POOL_PREALLOCATE = 0;
 
 // If not explicitly specified, this is the stack size to use for newly created
 // pthreads.  According to
diff --git a/tests/test_browser.py b/tests/test_browser.py
index 69d40526985c1..019cce54c861c 100644
--- a/tests/test_browser.py
+++ b/tests/test_browser.py
@@ -3654,7 +3654,7 @@ def test(args):
   # Test that preallocating worker threads work.
   @requires_threads
   def test_pthread_preallocates_workers(self):
-    self.btest(path_from_root('tests', 'pthread', 'test_pthread_preallocates_workers.cpp'), expected='0', args=['-O3', '-s', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=4', '-s', 'PTHREAD_POOL_ONLY_PREWARM=1'])
+    self.btest(path_from_root('tests', 'pthread', 'test_pthread_preallocates_workers.cpp'), expected='0', args=['-O3', '-s', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=4', '-s', 'PTHREAD_POOL_PREALLOCATE=1'])
 
   # Tests the -s PROXY_TO_PTHREAD=1 option.
   @requires_threads

From b44d4302185103497e1bb01eca58244f0b2873c3 Mon Sep 17 00:00:00 2001
From: Tayeb Al Karim <tay@google.com>
Date: Thu, 19 Sep 2019 14:01:45 -0400
Subject: [PATCH 22/25] Change the sense of PTHREAD_POOL_PREALLOCATE. 1 means
 "on" and is the default.

---
 src/preamble.js       | 2 +-
 src/settings.js       | 8 +++++---
 tests/test_browser.py | 2 +-
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/preamble.js b/src/preamble.js
index 26669dadd8268..d27b6e917c5c2 100644
--- a/src/preamble.js
+++ b/src/preamble.js
@@ -865,7 +865,7 @@ if (!ENVIRONMENT_IS_PTHREAD) addOnPreRun(function() {
 });
 #endif
 
-#if PTHREAD_POOL_SIZE > 0 && PTHREAD_POOL_PREALLOCATE == 0
+#if PTHREAD_POOL_SIZE > 0 && PTHREAD_POOL_PREALLOCATE != 0
 // To work around https://bugzilla.mozilla.org/show_bug.cgi?id=1049079, warm up a worker pool before starting up the application.
 if (!ENVIRONMENT_IS_PTHREAD) addOnPreRun(function() { if (typeof SharedArrayBuffer !== 'undefined') { addRunDependency('pthreads'); PThread.allocateUnusedWorkers({{{PTHREAD_POOL_SIZE}}}, function() { removeRunDependency('pthreads'); }); }});
 #endif
diff --git a/src/settings.js b/src/settings.js
index d8676a418948b..029279f7411fb 100644
--- a/src/settings.js
+++ b/src/settings.js
@@ -1237,12 +1237,14 @@ var USE_PTHREADS = 0;
 
 // PTHREAD_POOL_SIZE specifies the number of web workers that are created
 // before the main runtime is initialized. If 0, workers are created on
-// demand. If PTHREAD_POOL_PREALLOCATE = 0, then the workers will be fully
+// demand. If PTHREAD_POOL_PREALLOCATE = 1, then the workers will be fully
 // loaded (available for use) prior to the main runtime being initialized. If
-// PTHREAD_POOL_PREALLOCATE = 1, then the workers will only be created and
+// PTHREAD_POOL_PREALLOCATE = 0, then the workers will only be created and
 // have their runtimes loaded on demand after the main runtime is initialized.
+// Note that this means that the workers cannot be joined from the main thread
+// unless PROXY_TO_PTHREAD is used.
 var PTHREAD_POOL_SIZE = 0;
-var PTHREAD_POOL_PREALLOCATE = 0;
+var PTHREAD_POOL_PREALLOCATE = 1;
 
 // If not explicitly specified, this is the stack size to use for newly created
 // pthreads.  According to
diff --git a/tests/test_browser.py b/tests/test_browser.py
index 019cce54c861c..2f671d2129139 100644
--- a/tests/test_browser.py
+++ b/tests/test_browser.py
@@ -3654,7 +3654,7 @@ def test(args):
   # Test that preallocating worker threads work.
   @requires_threads
   def test_pthread_preallocates_workers(self):
-    self.btest(path_from_root('tests', 'pthread', 'test_pthread_preallocates_workers.cpp'), expected='0', args=['-O3', '-s', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=4', '-s', 'PTHREAD_POOL_PREALLOCATE=1'])
+    self.btest(path_from_root('tests', 'pthread', 'test_pthread_preallocates_workers.cpp'), expected='0', args=['-O3', '-s', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=4', '-s', 'PTHREAD_POOL_PREALLOCATE=0'])
 
   # Tests the -s PROXY_TO_PTHREAD=1 option.
   @requires_threads

From 129aaedc9d9271d4be223910a33005cbcfff2fc4 Mon Sep 17 00:00:00 2001
From: Tayeb Al Karim <tay@google.com>
Date: Fri, 20 Sep 2019 16:50:44 -0400
Subject: [PATCH 23/25] Add test that allocates 50 threads 10x times. Use the
 default parameter set for preallocated threads. I checked this with my change
 and without my change.

With my change, the initial allocation takes ~27.66 ms.
Without my change, the initial allocation takes 28.67 ms.

As we are talking about a delta of ~1ms, this isn't much of a difference.

Note that this is the version of the test that checks that there are no unexpected changes when the defaults are used when allocating threads. I'm not sure how to measure startup time in a test setting...
---
 tests/test_browser.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tests/test_browser.py b/tests/test_browser.py
index 2f671d2129139..48ce71735407c 100644
--- a/tests/test_browser.py
+++ b/tests/test_browser.py
@@ -3656,6 +3656,11 @@ def test(args):
   def test_pthread_preallocates_workers(self):
     self.btest(path_from_root('tests', 'pthread', 'test_pthread_preallocates_workers.cpp'), expected='0', args=['-O3', '-s', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=4', '-s', 'PTHREAD_POOL_PREALLOCATE=0'])
 
+  # Test that allocating a lot of threads doesn't regress. This needs to be checked manually!
+  @requires_threads
+  def test_pthread_large_pthread_allocation(self):
+    self.btest(path_from_root('tests', 'pthread', 'test_large_pthread_allocation.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=128MB', '-O3', '-s', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=50'], message='Check output from test to ensure that a regression in time it takes to allocate the threads has not occurred.')
+
   # Tests the -s PROXY_TO_PTHREAD=1 option.
   @requires_threads
   def test_pthread_proxy_to_pthread(self):

From ba3f827320d01e5cdae88e7d8c7abde912c31f69 Mon Sep 17 00:00:00 2001
From: Tayeb Al Karim <tay@google.com>
Date: Fri, 20 Sep 2019 16:53:19 -0400
Subject: [PATCH 24/25] Add missing test file.

---
 .../pthread/test_large_pthread_allocation.cpp | 69 +++++++++++++++++++
 1 file changed, 69 insertions(+)
 create mode 100644 tests/pthread/test_large_pthread_allocation.cpp

diff --git a/tests/pthread/test_large_pthread_allocation.cpp b/tests/pthread/test_large_pthread_allocation.cpp
new file mode 100644
index 0000000000000..c7acc0c6dd811
--- /dev/null
+++ b/tests/pthread/test_large_pthread_allocation.cpp
@@ -0,0 +1,69 @@
+// Copyright 2019 The Emscripten Authors.  All rights reserved.
+// Emscripten is available under two separate licenses, the MIT license and the
+// University of Illinois/NCSA Open Source License.  Both these licenses can be
+// found in the LICENSE file.
+
+#include <pthread.h>
+#include <sys/types.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <unistd.h>
+#include <errno.h>
+#include <emscripten.h>
+#include <emscripten/threading.h>
+#include <vector>
+
+pthread_t threads[50];
+
+static void *thread_start(void *arg)
+{
+  // This thread quits immediately...
+  pthread_exit((void*)0);
+}
+
+void CreateThread(int idx) {
+  int rc = pthread_create(&threads[idx], NULL, thread_start, (void*)idx);
+  assert(rc == 0);
+}
+
+void JoinThread(int idx) {
+  int rc = pthread_join(threads[idx], nullptr);
+  assert(rc == 0);
+}
+
+int main()
+{
+  if (!emscripten_has_threading_support())
+  {
+#ifdef REPORT_RESULT
+    REPORT_RESULT(0);
+#endif
+    printf("Skipped: Threading is not supported.\n");
+    return 0;
+  }
+
+  // This test should be run with a prewarmed pool of size 50. They should be fully allocated.
+  assert(EM_ASM_INT(return PThread.unusedWorkers.length) == 50);
+
+  double total = 0;
+  for (int i = 0; i < 10; ++i) {
+    double t1 = emscripten_get_now();
+    for (int j = 0; j < 50; ++j) {
+      CreateThread(j);
+    }
+    double t2 = emscripten_get_now();
+    printf("Took %f ms to allocate 50 threads.\n", t2 - t1);
+    total += (t2 - t1);
+    // Join all the threads to clear the queue..
+    for (int j = 0; j < 50; ++j) {
+      JoinThread(j);
+    }
+  }
+
+  printf("Final average %f ms.\n", total / 10.0);
+
+#ifdef REPORT_RESULT
+  REPORT_RESULT(0);
+#endif
+}

From e74a7124f65927548118d84ab13c5540b8324c36 Mon Sep 17 00:00:00 2001
From: Tayeb Al Karim <tay@google.com>
Date: Tue, 24 Sep 2019 21:59:19 -0400
Subject: [PATCH 25/25] Rename PTHREAD_POOL_PREALLOCATE (default 1) to
 PTHREAD_POOL_DELAY_LOAD (default 0)

---
 src/preamble.js       | 2 +-
 src/settings.js       | 6 +++---
 tests/test_browser.py | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/preamble.js b/src/preamble.js
index d27b6e917c5c2..7686a9c7484f0 100644
--- a/src/preamble.js
+++ b/src/preamble.js
@@ -865,7 +865,7 @@ if (!ENVIRONMENT_IS_PTHREAD) addOnPreRun(function() {
 });
 #endif
 
-#if PTHREAD_POOL_SIZE > 0 && PTHREAD_POOL_PREALLOCATE != 0
+#if PTHREAD_POOL_SIZE > 0 && PTHREAD_POOL_DELAY_LOAD != 1
 // To work around https://bugzilla.mozilla.org/show_bug.cgi?id=1049079, warm up a worker pool before starting up the application.
 if (!ENVIRONMENT_IS_PTHREAD) addOnPreRun(function() { if (typeof SharedArrayBuffer !== 'undefined') { addRunDependency('pthreads'); PThread.allocateUnusedWorkers({{{PTHREAD_POOL_SIZE}}}, function() { removeRunDependency('pthreads'); }); }});
 #endif
diff --git a/src/settings.js b/src/settings.js
index 029279f7411fb..f6c6523c0d7a2 100644
--- a/src/settings.js
+++ b/src/settings.js
@@ -1237,14 +1237,14 @@ var USE_PTHREADS = 0;
 
 // PTHREAD_POOL_SIZE specifies the number of web workers that are created
 // before the main runtime is initialized. If 0, workers are created on
-// demand. If PTHREAD_POOL_PREALLOCATE = 1, then the workers will be fully
+// demand. If PTHREAD_POOL_DELAY_LOAD = 0, then the workers will be fully
 // loaded (available for use) prior to the main runtime being initialized. If
-// PTHREAD_POOL_PREALLOCATE = 0, then the workers will only be created and
+// PTHREAD_POOL_DELAY_LOAD = 1, then the workers will only be created and
 // have their runtimes loaded on demand after the main runtime is initialized.
 // Note that this means that the workers cannot be joined from the main thread
 // unless PROXY_TO_PTHREAD is used.
 var PTHREAD_POOL_SIZE = 0;
-var PTHREAD_POOL_PREALLOCATE = 1;
+var PTHREAD_POOL_DELAY_LOAD = 0;
 
 // If not explicitly specified, this is the stack size to use for newly created
 // pthreads.  According to
diff --git a/tests/test_browser.py b/tests/test_browser.py
index 48ce71735407c..530e9bd78e762 100644
--- a/tests/test_browser.py
+++ b/tests/test_browser.py
@@ -3654,7 +3654,7 @@ def test(args):
   # Test that preallocating worker threads work.
   @requires_threads
   def test_pthread_preallocates_workers(self):
-    self.btest(path_from_root('tests', 'pthread', 'test_pthread_preallocates_workers.cpp'), expected='0', args=['-O3', '-s', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=4', '-s', 'PTHREAD_POOL_PREALLOCATE=0'])
+    self.btest(path_from_root('tests', 'pthread', 'test_pthread_preallocates_workers.cpp'), expected='0', args=['-O3', '-s', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=4', '-s', 'PTHREAD_POOL_DELAY_LOAD=1'])
 
   # Test that allocating a lot of threads doesn't regress. This needs to be checked manually!
   @requires_threads