Support Pioneer pipeline + merging ideas from pioneer-utils

.circleci/config.yml

@@ -64,6 +64,10 @@ jobs:
           path: "test-addon/dist"
           destination: "test-addon/dist"
 
+      - run:
+          name: Import and build the Pioneer opt-in add-on
+          command: npm run import-pioneer-opt-in
+
       # Needs signed add-on to work on branded releases
       #- run:
       #    name: Test with Firefox Release

bin/import-pioneer-opt-in.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+echo "$@"
+
+set -eu
+#set -o xtrace
+
+BASE_DIR="$(dirname "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)")"
+
+# download and build xpi for https://github.com/mozilla/pioneer-opt-in.git
+if [ ! -d "pioneer-opt-in" ]; then
+    git clone https://github.com/mozilla/pioneer-opt-in.git
+fi
+cd pioneer-opt-in
+bin/make-xpi.sh .
+cd -
+
+echo
+echo "SUCCESS: pioneer-opt-in xpi available at pioneer-opt-in/pioneer-opt-in.xpi"
+echo

examples/small-study/src/.eslintrc.js

@@ -7,4 +7,7 @@ module.exports = {
     es6: true,
     webextensions: true,
   },
+  rules: {
+    "no-console": "off",
+  },
 };

examples/small-study/src/studySetup.js

@@ -22,15 +22,17 @@ const baseStudySetup = {
   // used for activeExperiments tagging (telemetryEnvironment.setActiveExperiment)
   activeExperimentName: browser.runtime.id,
 
-  // uses shield sampling and telemetry semantics.  Future: will support "pioneer"
+  // use either "shield" or "pioneer" telemetry semantics and data pipelines
   studyType: "shield",
 
   // telemetry
   telemetry: {
-    // default false. Actually send pings.
+    // Actually submit the pings to Telemetry. [default if omitted: false]
     send: true,
-    // Marks pings with testing=true.  Set flag to `true` before final release
+    // Marks pings with testing=true. Set flag to `true` for pings are meant to be seen by analysts [default if omitted: false]
     removeTestingFlag: false,
+    // Keep an internal telemetry archive. Useful for verifying payloads of Pioneer studies without risking actually sending any unencrypted payloads [default if omitted: false]
+    internalTelemetryArchive: false,
   },
 
   // endings with urls
@@ -99,10 +101,11 @@ const baseStudySetup = {
  *
  * This implementation caches in local storage to speed up second run.
  *
+ * @param {object} studySetup A complete study setup object
  * @returns {Promise<boolean>} answer An boolean answer about whether the user should be
  *       allowed to enroll in the study
  */
-async function cachingFirstRunShouldAllowEnroll() {
+async function cachingFirstRunShouldAllowEnroll(studySetup) {
   // Cached answer.  Used on 2nd run
   let allowed = await browser.storage.local.get("allowedEnrollOnFirstRun");
   if (allowed.allowedEnrollOnFirstRun === true) return true;
@@ -113,7 +116,13 @@ async function cachingFirstRunShouldAllowEnroll() {
   */
 
   // could have other reasons to be eligible, such add-ons, prefs
-  allowed = true;
+  const dataPermissions = await browser.study.getDataPermissions();
+  if (studySetup.studyType === "shield") {
+    allowed = dataPermissions.shield;
+  }
+  if (studySetup.studyType === "pioneer") {
+    allowed = dataPermissions.pioneer;
+  }
 
   // cache the answer
   await browser.storage.local.set({ allowedEnrollOnFirstRun: allowed });
@@ -129,13 +138,14 @@ async function getStudySetup() {
   // shallow copy
   const studySetup = Object.assign({}, baseStudySetup);
 
-  studySetup.allowEnroll = await cachingFirstRunShouldAllowEnroll();
+  studySetup.allowEnroll = await cachingFirstRunShouldAllowEnroll(studySetup);
 
   const testingOverrides = await browser.study.getTestingOverrides();
   studySetup.testing = {
     variationName: testingOverrides.variationName,
     firstRunTimestamp: testingOverrides.firstRunTimestamp,
     expired: testingOverrides.expired,
   };
+
   return studySetup;
 }

package.json

@@ -4,7 +4,8 @@
   "version": "5.1.1",
   "author": "Mozilla",
   "bin": {
-    "copyStudyUtils": "bin/copyStudyUtils.js"
+    "copyStudyUtils": "bin/copyStudyUtils.js",
+    "importPioneerOptIn": "bin/import-pioneer-opt-in.sh"
   },
   "bugs": {
     "url": "https://github.com/mozilla/shield-studies-addon-utils/issues"
@@ -13,6 +14,7 @@
     "ajv": "^6.5.0",
     "commander": "^2.15.1",
     "fs-extra": "^6.0.1",
+    "jose-jwe-jws": "0.1.6",
     "shield-study-schemas": "^0.8.3"
   },
   "devDependencies": {
@@ -41,6 +43,7 @@
   },
   "files": [
     "bin/copyStudyUtils.js",
+    "bin/import-pioneer-opt-in.sh",
     "testUtils",
     "webExtensionApis/study/api.js",
     "webExtensionApis/study/schema.json",
@@ -80,14 +83,15 @@
     "generate:generateSchema:study": "cd webExtensionApis/study && yaml2json schema.yaml -p > schema.json",
     "generate:generateStubApi:study": "cd webExtensionApis/study && generateStubApi ./schema.json > stubApi.js",
     "generate:verifyWeeSchema:study": "cd webExtensionApis/study && verifyWeeSchema schema.json",
+    "import-pioneer-opt-in": "bin/import-pioneer-opt-in.sh",
     "lint": "npm-run-all lint:*",
     "lint:eslint": "npm run eslint",
     "lint:fixpack": "fixpack  # cleans up package.json",
     "postbuild": "if [ -z ${SKIPLINT} ]; then npm run format; fi",
     "postformat": "run-p lint:fixpack eslint-fix",
     "prebuild": "if [ -z ${SKIPLINT} ]; then npm run lint; fi",
     "prepare": "export SKIPLINT=1 && fixpack && npm run build",
-    "pretest": "npm run build && npm run test-addon:bundle-utils && npm run test-addon:build",
+    "pretest": "npm run build && npm run test-addon:bundle-utils && npm run test-addon:build && npm run import-pioneer-opt-in",
     "pretest-addon": "npm run pretest",
     "small-study": "cd examples/small-study && npm run rebuild && npm start",
     "test": "npm run test:func",

test-addon/src/studySetup.js

@@ -22,15 +22,17 @@ const baseStudySetup = {
   // used for activeExperiments tagging (telemetryEnvironment.setActiveExperiment)
   activeExperimentName: browser.runtime.id,
 
-  // uses shield sampling and telemetry semantics.  Future: will support "pioneer"
-  studyType: "shield",
+  // use either "shield" or "pioneer" telemetry semantics and data pipelines
+  studyType: null, // set by internal test override below in getStudySetup()
 
   // telemetry
   telemetry: {
-    // default false. Actually send pings.
+    // Actually submit the pings to Telemetry. [default if omitted: false]
     send: true,
-    // Marks pings with testing=true.  Set flag to `true` before final release
+    // Marks pings with testing=true. Set flag to `true` for pings are meant to be seen by analysts [default if omitted: false]
     removeTestingFlag: false,
+    // Keep an internal telemetry archive. Useful for verifying payloads of Pioneer studies without risking actually sending any unencrypted payloads [default if omitted: false]
+    internalTelemetryArchive: true,
   },
 
   // endings with urls
@@ -99,10 +101,11 @@ const baseStudySetup = {
  *
  * This implementation caches in local storage to speed up second run.
  *
+ * @param {object} studySetup A complete study setup object
  * @returns {Promise<boolean>} answer An boolean answer about whether the user should be
  *       allowed to enroll in the study
  */
-async function cachingFirstRunShouldAllowEnroll() {
+async function cachingFirstRunShouldAllowEnroll(studySetup) {
   // Cached answer.  Used on 2nd run
   let allowed = await browser.storage.local.get("allowedEnrollOnFirstRun");
   if (allowed.allowedEnrollOnFirstRun === true) return true;
@@ -113,7 +116,13 @@ async function cachingFirstRunShouldAllowEnroll() {
   */
 
   // could have other reasons to be eligible, such add-ons, prefs
-  allowed = true;
+  const dataPermissions = await browser.study.getDataPermissions();
+  if (studySetup.studyType === "shield") {
+    allowed = dataPermissions.shield;
+  }
+  if (studySetup.studyType === "pioneer") {
+    allowed = dataPermissions.pioneer;
+  }
 
   // cache the answer
   await browser.storage.local.set({ allowedEnrollOnFirstRun: allowed });
@@ -129,13 +138,18 @@ async function getStudySetup() {
   // shallow copy
   const studySetup = Object.assign({}, baseStudySetup);
 
-  studySetup.allowEnroll = await cachingFirstRunShouldAllowEnroll();
+  // internal testing override necessary to be able to test all study types
+  const internalTestingOverrides = await browser.studyDebug.getInternalTestingOverrides();
+  studySetup.studyType = internalTestingOverrides.studyType;
+
+  studySetup.allowEnroll = await cachingFirstRunShouldAllowEnroll(studySetup);
 
   const testingOverrides = await browser.study.getTestingOverrides();
   studySetup.testing = {
     variationName: testingOverrides.variationName,
     firstRunTimestamp: testingOverrides.firstRunTimestamp,
     expired: testingOverrides.expired,
   };
+
   return studySetup;
 }