Source: pageTransition.js

  1. /**
  2.  * This module enables observing webpage transitions, synthesizing a range of
  3.  * transition data that may be valuable for browser-based studies. See the
  4.  * `onPageTransitionData` event for details.
  5.  * 
  6.  * ## Types of Page Transition Data
  7.  * This module supports several types of page transition data. Some types are
  8.  * supported and recommended, because the data is consistently available, has
  9.  * consistent meaning, and reflects discrete categories of user interactions.
  10.  * Other types of transition data are supported because they appear in prior
  11.  * academic literature, but we  do not recommend them because of significant
  12.  * limitations.
  13.  *   * Supported and Recommended Types of Page Transition Data
  14.  *     * WebExtensions Transitions - This module reports the same webpage
  15.  *       transition data provided by the WebExtensions `webNavigation` API. There
  16.  *       are two types of transition data: `TransitionType` (e.g., "link" or
  17.  *       "typed") and `TransitionQualifier` (e.g., "from_address_bar" or
  18.  *       "forward_back"). Note that Firefox's support for these values is mostly
  19.  *       but not entirely complete and defaults to a "link" transition type. The 
  20.  *       MDN documentation for Firefox's implementation is also currently out of
  21.  *       date, see: https://github.com/mdn/browser-compat-data/issues/9019. We
  22.  *       recommend checking click transition data to confirm whether the user
  23.  *       clicked on a link.
  24.  *     * Tab-based Transitions - This module reports the webpage that was
  25.  *       previously loaded in a new webpage's tab. If the webpage is loading in a
  26.  *       newly created tab, this module reports the webpage that was open in
  27.  *       the opener tab. We recommend using tab-based transition data when the user
  28.  *       has clicked a link (according to both WebExtensions and click data), when
  29.  *       the user has navigated with forward and back buttons, and when the page
  30.  *       has refreshed (due to user action or automatically). In these situations,
  31.  *       there is a clear causal relationship between the previous and current
  32.  *       pages. We do not otherwise recommend using tab-based transition data,
  33.  *       because the user might be reusing a tab for reasons unrelated to the page
  34.  *       loaded in the tab.
  35.  *     * Click Transitions - This module reports when a click on a webpage is
  36.  *       immediately followed by a new webpage loading in the same tab (or a
  37.  *       newly opened tab were that tab is the opener). This activity indicates
  38.  *       the user likely clicked a link, and it compensates for limitations in
  39.  *       how browsers detect link clicks for the `webNavigation` API.
  40.  *   * Supported But Not Recommended Types of Page Transition Data   
  41.  *     * Referrers - This module reports the HTTP referrer for each new page. While
  42.  *       referrers have long been a method for associating webpage loads with
  43.  *       prior pages, they are not consistently available (webpages and browsers
  44.  *       are increasingly limiting when referrers are sent), do not have consistent
  45.  *       content (similarly, webpages and browsers are increasingly limiting
  46.  *       referrers to just origins), and do not have consistent meaning (the rules
  47.  *       for setting referrers are notoriously complex and can have nonintuitive
  48.  *       semantics). Be especially careful with referrers for webpage loads via
  49.  *       the History API---because there is no new document-level HTTP request, the
  50.  *       referrer will not change when the URL changes.
  51.  *     * Time-based Transitions - This module reports the most recent webpage that
  52.  *       loaded in any tab. We do not recommend relying on this data, because a
  53.  *       chronological ordering of webpage loads may have no relation to user
  54.  *       activity or perception (e.g., a webpage might automatically reload in the
  55.  *       background before a user navigates to a new page).
  56.  *  
  57.  * ## Page Transition Data Sources
  58.  * This module builds on the page tracking provided by the `pageManager`
  59.  * module and uses browser events, DOM events, and a set of heuristics to
  60.  * associate transition information with each page visit. The module relies on
  61.  * the following sources of data about page transitions, in addition to the
  62.  * page visit tracking, attention tracking, and URL normalization provided by
  63.  * `pageManager`:
  64.  *   * Background Script Data Sources
  65.  *     * `webNavigation.onCommitted` - provides tab ID, url,
  66.  *       `webNavigation.TransitionType`, and `webNavigation.TransitionQualifier`
  67.  *       values when a new page is loading in a tab.
  68.  *     * `webNavigation.onDOMContentLoaded` - provides tab ID, url, and a
  69.  *       timestamp approximating when the `DOMContentLoaded` event fired on a
  70.  *       page.
  71.  *     * `webNavigation.onHistoryStateUpdated` - provides tab ID, url,
  72.  *       `webNavigation.TransitionType`, and `webNavigation.TransitionQualifier`
  73.  *       values when a new page loads in a tab via the History API.
  74.  *     * `webNavigation.onCreatedNavigationTarget` - provides tab ID, source
  75.  *       tab ID, and url when a page loads in a tab newly created by another
  76.  *       tab. Because of a regression, this event does not currently fire
  77.  *       in Firefox for a click on a link with the target="_blank" attribute.
  78.  *     * `tabs.onCreated` - provides tab ID and source tab ID when a page
  79.  *       loads in a tab newly created by another tab, except if the new
  80.  *       tab is in a different window.
  81.  *   * Content Script Data Sources
  82.  *     * The `click` event on the `document` element - detects possible link
  83.  *       clicks via the mouse (e.g., left click).
  84.  *     * The `contextmenu` event on the `document` element - detects possible
  85.  *       link clicks via the mouse (e.g., right click or control + click).
  86.  *     * The `keyup` event on the document element - detects possible link
  87.  *       clicks via the keyboard.
  88.  * 
  89.  * ## Combining Data Sources into a Page Transition
  90.  * Merging these data sources into a page transition event poses several
  91.  * challenges.
  92.  *   * We have to sync background script `webNavigation` events with content
  93.  *     scripts. As with `pageManager`, we have to account for the possibility
  94.  *     of race conditions between the background script and content script
  95.  *     environments. We use the same general approach in this module as in
  96.  *     `pageManager`, converting background script events into messages posted
  97.  *     to content scripts. We have to be a bit more careful about race
  98.  *     conditions than in `pageManager`, though, because if a tab property
  99.  *     event handled in that module goes to the wrong content script the
  100.  *     consequences are minimal (because correct event data will quickly
  101.  *     arrive afterward). In this module, by contrast, an error could mean 
  102.  *     incorrectly associating a pair of pages. We further account for the
  103.  *     possibility of race conditions by matching the `webNavigation` URL and
  104.  *     DOMContentLoaded timestamp with the content script's URL and
  105.  *     DOMContentLoaded timestamp.
  106.  *   * We have to sync background script `webNavigation` events for different
  107.  *     stages in the webpage loading lifecycle, because we want properties of
  108.  *     both `webNavigation.onCommitted` and `webNavigation.onDOMContentLoaded`:
  109.  *     the former has transition types and qualifiers, while the latter has a
  110.  *     timestamp that is comparable to an event in the content script and does
  111.  *     not have the risk of firing before the content script is ready to
  112.  *     receive messages. Unlike `webRequest` events, `webNavigation` events are
  113.  *     not associated with unique identifiers. We accomplish syncing across
  114.  *     events by assuming that when the `webNavigation.onDOMContentLoaded` event
  115.  *     fires for a tab, it is part of the same navigation lifecycle as the most
  116.  *     recent `webNavigation.onCommitted` event in the tab.
  117.  *   * We have to sync content script data for a page with content script
  118.  *     data for a prior page (either loaded in the same tab, loaded in an
  119.  *     opener tab, or loaded immediately before in time). We accomplish this for
  120.  *     ordinary page loads by maintaining a cache of page visit data in the
  121.  *     in the background script. We accomplish this for History API page loads
  122.  *     by passing information in the content script environment.
  123.  *   * We have to account for a regression in Firefox where
  124.  *     `webNavigation.onCreatedNavigationTarget` does not currently fire for
  125.  *     a click on a link with the target="_blank" attribute. We accomplish this
  126.  *     by using `tabs.onCreated` event data when
  127.  *     `webNavigation.onCreatedNavigationTarget` event data is not available.
  128.  *  
  129.  * @see {@link https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/API/webNavigation/onCommitted}
  130.  * @see {@link https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/API/webNavigation/TransitionType}
  131.  * @see {@link https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/API/webNavigation/TransitionQualifier}
  132.  * @see {@link https://github.com/mdn/browser-compat-data/issues/9019}
  133.  * @see {@link https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/API/tabs/onCreated}
  134.  * @module pageTransition
  135.  */
  137. import * as events from "./events.js";
  138. import * as permissions from "./permissions.js";
  139. import * as messaging from "./messaging.js";
  140. import * as matching from "./matching.js";
  141. import * as timing from "./timing.js";
  142. import * as pageManager from "./pageManager.js";
  143. import pageTransitionEventContentScript from "include:./content-scripts/pageTransition.event.content.js";
  144. import pageTransitionClickContentScript from "include:./content-scripts/pageTransition.click.content.js";
  146. /**
  147.  * A listener for the `onPageTransitionData` event.
  148.  * @callback pageTransitionDataListener
  149.  * @memberof module:pageTransition.onPageTransitionData
  150.  * @param {Object} details - Additional information about the page transition data event.
  151.  * @param {string} details.pageId - The ID for the page, unique across browsing sessions.
  152.  * @param {string} details.url - The URL of the page, without any hash.
  153.  * @param {string} details.referrer - The referrer URL for the page, or `""` if there is no referrer. Note that we
  154.  * recommend against using referrers for analyzing page transitions.
  155.  * @param {number} details.tabId - The ID for the tab containing the page, unique to the browsing session. Note that if
  156.  * you send a message to the content script in the tab, there is a possible race condition where the page in 
  157.  * the tab changes before your message arrives. You should specify a page ID (e.g., `pageId`) in your message to the
  158.  * content script, and the content script should check that page ID against its current page ID to ensure that the 
  159.  * message was received by the intended page.
  160.  * @param {boolean} details.isHistoryChange - Whether the page transition was caused by a URL change via the History API.
  161.  * @param {boolean} details.isOpenedTab - Whether the page is loading in a tab that was newly opened from another tab.
  162.  * @param {number} details.openerTabId - If the page is loading in a tab that was newly opened from another tab
  163.  * (i.e., `isOpenedTab` is `true`), the tab ID of the opener tab. Otherwise, `tabs.TAB_ID_NONE`. Note that if
  164.  * you send a message to the content script in the tab, there is a possible race condition where the page in 
  165.  * the tab changes before your message arrives. You should specify a page ID (e.g., `tabSourcePageId`) in your
  166.  * message to the content script, and the content script should check that page ID against its current page ID to
  167.  * ensure that the message was received by the intended page.
  168.  * @param {string} details.transitionType - The transition type, from `webNavigation.onCommitted` or
  169.  * `webNavigation.onHistoryStateUpdated`.
  170.  * @param {string[]} details.transitionQualifiers - The transition qualifiers, from `webNavigation.onCommitted` or
  171.  * `webNavigation.onHistoryStateUpdated`.
  172.  * @param {string} details.tabSourcePageId - The ID for the most recent page in the same tab. If the page is opening
  173.  * in a new tab, then the ID of the most recent page in the opener tab. The value is `""` if there is no such page.
  174.  * @param {string} details.tabSourceUrl - The URL, without any hash, for the most recent page in the same tab. If the page
  175.  * is opening in a new tab, then the URL of the most recent page in the opener tab. The value is `""` if there is no
  176.  * such page.
  177.  * @param {boolean} details.tabSourceClick - Whether the user recently clicked or pressed enter/return on the most recent
  178.  * page in the same tab. If the page is loading in a tab that was newly opened by another tab, then whether the user
  179.  * recently clicked or pressed enter/return on the most recent page in the opener tab. The value is `false` if there
  180.  * is no such page.
  181.  * @param {string} details.timeSourcePageId - The ID for the most recent page that loaded into any tab. If this is the
  182.  * first page visit after the extension starts, the value is "". Note that we recommend against using time-based
  183.  * page transition data.
  184.  * @param {string} details.timeSourceUrl - The URL for the most recent page that loaded into any tab. If this is the
  185.  * first page visit after the extension starts, the value is "". Note that we recommend against using time-based
  186.  * page transition data.
  187.  */
  189. /**
  190.  * @typedef {Object} PageTransitionDataListenerRecord
  191.  * @property {matching.MatchPatternSet} matchPatternSet - Match patterns for pages where the listener should be
  192.  * notified about transition data.
  193.  * @property {boolean} privateWindows - Whether to notify the listener about page transitions in
  194.  * private windows and whether to consider pages loaded in private windows when generating
  195.  * time-based transition information.
  196.  * @property {browser.contentScripts.RegisteredContentScript} contentScript - The content
  197.  * script associated with the listener.
  198.  * @private
  199.  */
  201. /**
  202.  * A map where each key is a listener and each value is a record for that listener.
  203.  * @constant {Map<pageTransitionDataListener, PageTransitionDataListenerRecord>}
  204.  * @private
  205.  */
  206. const pageTransitionDataListeners = new Map();
  208. /**
  209.  * Add a listener for the `onPageTransitionData` event.
  210.  * @function addListener
  211.  * @memberof module:pageTransition.onPageTransitionData
  212.  * @param {pageTransitionDataListener} listener - The listener to add.
  213.  * @param {Object} options - Options for the listener.
  214.  * @param {string[]} options.matchPatterns - Match patterns for pages where the listener should be notified about
  215.  * transition data.
  216.  * @param {boolean} [options.privateWindows=false] - Whether to notify the listener about page transitions in
  217.  * private windows and whether to consider pages loaded in private windows when generating time-based
  218.  * transition information.
  219.  */
  221. /**
  222.  * Remove a listener for the `onPageTransitionData` event.
  223.  * @function removeListener
  224.  * @memberof module:pageTransition.onPageTransitionData
  225.  * @param {pageTransitionDataListener} listener - The listener to remove.
  226.  */
  228. /**
  229.  * Whether a specified listener has been added for the `onPageTransitionData` event.
  230.  * @function hasListener
  231.  * @memberof module:pageTransition.onPageTransitionData
  232.  * @param {pageTransitionDataListener} listener - The listener to check.
  233.  * @returns {boolean} Whether the listener has been added for the event.
  234.  */
  236. /**
  237.  * Whether the `onPageTransitionData` event has any listeners.
  238.  * @function hasAnyListeners
  239.  * @memberof module:pageTransition.onPageTransitionData
  240.  * @returns {boolean} Whether the event has any listeners.
  241.  */
  243. /**
  244.  * An event that fires when data about a page transition is available. The event will fire after
  245.  * the `pageManager.onPageVisitStart` event, when DOM content has loaded (for ordinary page loads)
  246.  * or just after the URL changes (for History API page loads).
  247.  * @namespace
  248.  */
  249. export const onPageTransitionData = events.createEvent({
  250.     name: "webScience.pageTransition.onPageTransitionData",
  251.     addListenerCallback: addListener,
  252.     removeListenerCallback: removeListener,
  253.     notifyListenersCallback: () => { return false; }
  254. });
  256. /**
  257.  * A callback function for adding a page transition data listener. The options for the listener must be kept in
  258.  * sync with the public `onPageTransitionData.addListener` type.
  259.  * @param {pageTransitionDataListener} listener - The listener being added.
  260.  * @param {Object} options - Options for the listener.
  261.  * @param {string[]} options.matchPatterns - Match patterns for pages where the listener should be notified about
  262.  * transition data.
  263.  * @param {boolean} [options.privateWindows=false] - Whether to notify the listener about page transitions in
  264.  * private windows and whether to consider pages loaded in private windows when generating time-based
  265.  * transition information.
  266.  * @private
  267.  */
  268.  async function addListener(listener, {
  269.     matchPatterns,
  270.     privateWindows = false
  271. }) {
  272.     await initialize();
  273.     // Store a record for the listener
  274.     pageTransitionDataListeners.set(listener, {
  275.         // Compile the listener's match pattern set
  276.         matchPatternSet: matching.createMatchPatternSet(matchPatterns),
  277.         privateWindows,
  278.         // Register the event content script with the listener's match patterns
  279.         contentScript: await browser.contentScripts.register({
  280.             matches: matchPatterns,
  281.             js: [{
  282.                 file: pageTransitionEventContentScript
  283.             }],
  284.             runAt: "document_start"
  285.         })
  286.     });
  287. }
  289. /**
  290.  * A callback function for removing a page transition data listener.
  291.  * @param {pageTransitionDataListener} listener - The listener that is being removed.
  292.  * @private
  293.  */
  294. function removeListener(listener) {
  295.     const listenerRecord = pageTransitionDataListeners.get(listener);
  296.     if(listenerRecord === undefined) {
  297.         return;
  298.     }
  299.     listenerRecord.contentScript.unregister();
  300.     pageTransitionDataListeners.delete(listenerRecord);
  301. }
  303. /**
  304.  * Whether the module has been initialized.
  305.  * @type {boolean}
  306.  * @private
  307.  */
  308. let initialized = false;
  310. /**
  311.  * Initialize the module, registering event handlers and message schemas.
  312.  * @private
  313.  */
  314. async function initialize() {
  315.     if(initialized) {
  316.         return;
  317.     }
  318.     initialized = true;
  320.     permissions.check({
  321.         module: "webScience.pageTransition",
  322.         requiredPermissions: [ "webNavigation" ],
  323.         suggestedOrigins: [ "<all_urls>" ]
  324.     });
  326.     await pageManager.initialize();
  328.     // Register the click content script for all URLs permitted by the extension manifest
  329.     await browser.contentScripts.register({
  330.         matches: permissions.getManifestOriginMatchPatterns(),
  331.         js: [{
  332.             file: pageTransitionClickContentScript
  333.         }],
  334.         runAt: "document_start"
  335.     });
  337.     // When pageManager.onPageVisitStart fires...
  338.     pageManager.onPageVisitStart.addListener(({ pageId, url, pageVisitStartTime, privateWindow, tabId }) => {
  339.         // Add the page visit's page ID, URL, start time, and private window status to the time-based transition cache
  340.         pageVisitTimeCache[pageId] = { url, pageVisitStartTime, privateWindow };
  342.         // Add the page visit's tab ID, page ID, URL, and start time to the tab-based transition cache
  343.         let cachedPageVisitsForTab = pageVisitTabCache.get(tabId);
  344.         if(cachedPageVisitsForTab === undefined) {
  345.             cachedPageVisitsForTab = { };
  346.             pageVisitTabCache.set(tabId, cachedPageVisitsForTab);
  347.         }
  348.         cachedPageVisitsForTab[pageId] = { url, pageVisitStartTime, clickTimeStamps: [ ] };
  350.         // We can't remove stale pages from the time-based and tab-based caches here, because otherwise we can
  351.         // have a race condition where the most recent page in a cache (from pageManager.onPageVisitStart)
  352.         // is the same page that's about to receive a message from the background script (because of
  353.         // webNavigation.onDOMContentLoaded). In that situation, we might evict an older page from the cache
  354.         // that was the correct page for time-based or tab-based transition information.
  355.     });
  357.     // When webNavigation.onCommitted fires, store the details in the per-tab onCommitted details cache
  358.     browser.webNavigation.onCommitted.addListener(details => {
  359.         // Ignore subframe navigation
  360.         if(details.frameId !== 0) {
  361.             return;
  362.         }
  363.         webNavigationOnCommittedCache.set(details.tabId, details);
  364.     }, {
  365.         url: [ { schemes: [ "http", "https" ] } ]
  366.     });
  368.     // When webNavigation.onDOMContentLoaded fires, pull the webNavigation.onCommitted
  369.     // details from the per-tab cache and notify the content script
  370.     browser.webNavigation.onDOMContentLoaded.addListener(details => {
  371.         // Ignore subframe navigation
  372.         if(details.frameId !== 0) {
  373.             return;
  374.         }
  376.         // Get the cached webNavigation.onCommitted details and expire the cache
  377.         const webNavigationOnCommittedDetails = webNavigationOnCommittedCache.get(details.tabId);
  378.         if(webNavigationOnCommittedDetails === undefined) {
  379.             return;
  380.         }
  381.         webNavigationOnCommittedCache.delete(details.tabId);
  383.         // Confirm that the webNavigation.onCommitted URL matches the webNavigation.onDOMContentLoaded URL
  384.         if(details.url !== webNavigationOnCommittedDetails.url) {
  385.             return;
  386.         }
  388.         // Notify the content script
  389.         sendUpdateToContentScript({
  390.             tabId: details.tabId,
  391.             url: details.url,
  392.             timeStamp: timing.fromSystemClock(details.timeStamp),
  393.             webNavigationTimeStamp: details.timeStamp,
  394.             transitionType: webNavigationOnCommittedDetails.transitionType,
  395.             transitionQualifiers: webNavigationOnCommittedDetails.transitionQualifiers,
  396.             isHistoryChange: false
  397.         });
  398.     }, {
  399.         url: [ { schemes: [ "http", "https" ] } ]
  400.     });
  402.     // When webNavigation.onHistoryStateUpdated fires, notify the content script
  403.     browser.webNavigation.onHistoryStateUpdated.addListener(details => {
  404.         // Ignore subframe navigation
  405.         if(details.frameId !== 0) {
  406.             return;
  407.         }
  409.         // Notify the content script
  410.         sendUpdateToContentScript({
  411.             tabId: details.tabId,
  412.             url: details.url,
  413.             timeStamp: timing.fromSystemClock(details.timeStamp),
  414.             webNavigationTimeStamp: details.timeStamp,
  415.             transitionType: details.transitionType,
  416.             transitionQualifiers: details.transitionQualifiers,
  417.             isHistoryChange: true
  418.         });
  419.     }, {
  420.         url: [ { schemes: [ "http", "https" ] } ]
  421.     });
  423.     // Register the message schemas for background script updates
  424.     messaging.registerSchema("webScience.pageTransition.backgroundScriptEventUpdate", {
  425.         url: "string",
  426.         timeStamp: "number",
  427.         webNavigationTimeStamp: "number",
  428.         transitionType: "string",
  429.         transitionQualifiers: "object",
  430.         pageVisitTimeCache: "object",
  431.         cachedPageVisitsForTab: "object",
  432.         isHistoryChange: "boolean",
  433.         isOpenedTab: "boolean",
  434.         openerTabId: "number",
  435.         tabOpeningTimeStamp: "number"
  436.     });
  438.     // When webNavigation.onCreatedNavigationTarget fires, update the the opener tab cache.
  439.     // This event fires for all opened tabs regardless of window, except for a regression
  440.     // since Firefox 65 where the event does not fire for tabs opened by clicking a link
  441.     // with a target="_blank" attribute. See https://github.com/mdn/content/issues/4507.
  442.     // We observe those tab openings tabs.onCreated, since the tabs are always in the same
  443.     // window. We do not use the URL from webNavigation.onCreatedNavigationTarget, because
  444.     // an HTTP redirect might change the URL before webNavigation.onCommitted and
  445.     // webNavigation.onDOMContentLoaded fire.
  446.     browser.webNavigation.onCreatedNavigationTarget.addListener(details => {
  447.         openerTabCache.set(details.tabId, {
  448.             openerTabId: details.sourceTabId,
  449.             timeStamp: details.timeStamp
  450.         });
  451.     }, {
  452.         url: [ { schemes: [ "http", "https" ] } ]
  453.     });
  455.     // When tabs.onCreated fires, update the opener tab cache. This event fires for all opened
  456.     // tabs in the same window, but not opened tabs in a new window. We observe tabs that open
  457.     // in new windows with webNavigation.onCreatedNavigationTarget.
  458.     browser.tabs.onCreated.addListener(tab => {
  459.         // Ignore non-content tabs
  460.         if(!("id" in tab) || (tab.id === browser.tabs.TAB_ID_NONE)) {
  461.             return;
  462.         }
  463.         // Ignore tabs without content opener tabs
  464.         if(!("openerTabId" in tab) || (tab.openerTabId === browser.tabs.TAB_ID_NONE)) {
  465.             return;
  466.         }
  467.         // If we've already populated the opener tab cache for this tab with data from a more
  468.         // detailed webNavigation.onCreatedNavigationTarget event, ignore this event
  469.         if(openerTabCache.get(tab.id) !== undefined) {
  470.             return;
  471.         }
  472.         openerTabCache.set(tab.id, {
  473.             openerTabId: tab.openerTabId,
  474.             timeStamp: timing.now()
  475.         });
  476.     });
  478.     // When tabs.onRemoved fires, set a timeout to expire the tab-based transition information
  479.     // and opener information for that tab
  480.     browser.tabs.onRemoved.addListener(tabId => {
  481.         setTimeout(() => {
  482.             pageVisitTabCache.delete(tabId);
  483.             openerTabCache.delete(tabId);
  484.         }, tabRemovedExpiry);
  485.     });
  487.     // When the event content script sends an update message, notify the relevant listeners
  488.     messaging.onMessage.addListener((eventUpdateMessage, sender) => {
  489.         for(const [listener, listenerRecord] of pageTransitionDataListeners) {
  490.             if(eventUpdateMessage.privateWindow && !listenerRecord.privateWindows) {
  491.                 continue;
  492.             }
  493.             if(listenerRecord.matchPatternSet.matches(eventUpdateMessage.url)) {
  494.                 listener({
  495.                     pageId: eventUpdateMessage.pageId,
  496.                     url: eventUpdateMessage.url,
  497.                     referrer: eventUpdateMessage.referrer,
  498.                     tabId: sender.tab.id,
  499.                     isHistoryChange: eventUpdateMessage.isHistoryChange,
  500.                     isOpenedTab: eventUpdateMessage.isOpenedTab,
  501.                     openerTabId: eventUpdateMessage.openerTabId,
  502.                     transitionType: eventUpdateMessage.transitionType,
  503.                     transitionQualifiers: eventUpdateMessage.transitionQualifiers.slice(),
  504.                     tabSourcePageId: eventUpdateMessage.tabSourcePageId,
  505.                     tabSourceUrl: eventUpdateMessage.tabSourceUrl,
  506.                     tabSourceClick: eventUpdateMessage.tabSourceClick,
  507.                     timeSourcePageId: listenerRecord.privateWindows ? eventUpdateMessage.timeSourcePageId : eventUpdateMessage.timeSourceNonPrivatePageId,
  508.                     timeSourceUrl: listenerRecord.privateWindows ? eventUpdateMessage.timeSourceUrl : eventUpdateMessage.timeSourceNonPrivateUrl
  509.                 });
  510.             }
  511.         }
  512.     },
  513.     {
  514.         type: "webScience.pageTransition.contentScriptEventUpdate",
  515.         schema: {
  516.             pageId: "string",
  517.             url: "string",
  518.             referrer: "string",
  519.             isHistoryChange: "boolean",
  520.             isOpenedTab: "boolean",
  521.             openerTabId: "number",
  522.             transitionType: "string",
  523.             transitionQualifiers: "object",
  524.             tabSourcePageId: "string",
  525.             tabSourceUrl: "string",
  526.             tabSourceClick: "boolean",
  527.             timeSourcePageId: "string",
  528.             timeSourceUrl: "string",
  529.             timeSourceNonPrivatePageId: "string",
  530.             timeSourceNonPrivateUrl: "string",
  531.             privateWindow: "boolean"
  532.         }
  533.     });
  535.     // When the click content script sends an update message, update the tab-based transition cache
  536.     messaging.onMessage.addListener((clickUpdateMessage, sender) => {
  537.         // There should be a tab ID associated with the message, but might as well make certain
  538.         if(!("tab" in sender) || !("id" in sender.tab)) {
  539.             return;
  540.         }
  542.         // Update the cached link clicks for the page
  543.         const cachedPageVisitsForTab = pageVisitTabCache.get(sender.tab.id);
  544.         if((cachedPageVisitsForTab === undefined) || !(clickUpdateMessage.pageId in cachedPageVisitsForTab)) {
  545.             return;
  546.         }
  547.         cachedPageVisitsForTab[clickUpdateMessage.pageId].clickTimeStamps = cachedPageVisitsForTab[clickUpdateMessage.pageId].clickTimeStamps.concat(clickUpdateMessage.clickTimeStamps);
  548.     },
  549.     {
  550.         type: "webScience.pageTransition.contentScriptClickUpdate",
  551.         schema: {
  552.             pageId: "string",
  553.             clickTimeStamps: "object"
  554.         }
  555.     });
  556. }
  558. /**
  559.  * A map where keys are tab IDs and values are the most recent `webNavigation.onCommitted`
  560.  * details, removed from the map when a subsequent `webNavigation.onDOMContentLoaded` fires
  561.  * for the tab.
  562.  * @constant {Map<number, Object>}
  563.  * @private
  564.  */
  565. const webNavigationOnCommittedCache = new Map();
  567. /**
  568.  * A map, represented as an object, where keys are page IDs and values are objects with
  569.  * `pageVisitStartTime`, `url`, and `privateWindow` properties from `pageManager.onPageVisitStart`.
  570.  * We use an object so that it can be easily serialized. The reason we maintain this cache
  571.  * is to account for possible race conditions between when pages load in the content script
  572.  * environment and when the background script environment learns about page loads.
  573.  * @constant {Object}
  574.  * @private
  575.  */
  576. const pageVisitTimeCache = { };
  578. /**
  579.  * The maximum time, in milliseconds, to consider a page visit in any tab as a possible most
  580.  * recent page visit in the content script environment, even though it's not the most recent
  581.  * page visit in the background script environment.
  582.  * @constant {number}
  583.  * @private
  584.  */
  585. const pageVisitTimeCacheExpiry = 1000;
  587. /**
  588.  * @typedef {Object} PageVisitCachedDetails
  589.  * @property {number} pageVisitStartTime - The page visit start time from `pageManager`.
  590.  * @property {string} url - The URL from `pageManager`.
  591.  * @property {number[]} clickTimeStamps - Timestamps for recent clicks on the page, from
  592.  * the module's click content script.
  593.  * @private
  594.  */
  596. /**
  597.  * A map where keys are tab IDs and values are maps, represented as objects, where keys
  598.  * are page IDs and values are PageVisitCachedDetails objects.
  599.  * @constant {Map<number, Object>}
  600.  * @private
  601.  */
  602. const pageVisitTabCache = new Map();
  604. /**
  605.  * The maximum time, in milliseconds, to consider a page visit in a specific tab as a possible
  606.  * most recent page visit for that tab in the content script environment, even though it's not
  607.  * the most recent page visit for that tab in the background script environment.
  608.  * @constant {number}
  609.  * @private
  610.  */
  611. const pageVisitTabCacheExpiry = 5000;
  613. /**
  614.  * The maximum time, in milliseconds, to consider a click on a page as a possible most recent
  615.  * click on the page in the content script environment, even though it's not the most recent
  616.  * click in the background script environment.
  617.  * @constant {number}
  618.  * @private
  619.  */
  620. const clickCacheExpiry = 5000;
  622. /**
  623.  * The minimum time, in milliseconds, to wait after a tab is removed before expiring the cache
  624.  * of page visits in that tab for tab-based transition information and the cached opener tab
  625.  * for that tab.
  626.  * @constant {number}
  627.  * @private
  628.  */
  629. const tabRemovedExpiry = 10000;
  631. /**
  632.  * A map where keys are tab IDs and values are objects with `openerTabId` and `timeStamp`
  633.  * properties.
  634.  * @constant {Map<number, Object>}
  635.  * @private
  636.  */
  637. const openerTabCache = new Map();
  639. /**
  640.  * Send an update to the content script running on a page, called when a
  641.  * `webNavigation.onDOMContentLoaded` or `webNavigation.onHistoryStateUpdated`
  642.  * event fires.
  643.  * @param {Object} details - Details for the update to the content script.
  644.  * @param {number} details.tabId - The tab ID for the tab where the page is loading.
  645.  * @param {string} details.url - The URL for the page.
  646.  * @param {number} details.timeStamp - The timestamp for the page that is loading, either from
  647.  * `webNavigation.onDOMContentLoaded` or `webNavigation.onHistoryStateUpdated`, adjusted to
  648.  * the shared monotonic clock.
  649.  * @param {number} details.webNavigationTimeStamp - The timestamp for the page that is loading,
  650.  * either from `webNavigation.onDOMContentLoaded` or `webNavigation.onHistoryStateUpdated`.
  651.  * This timestamp, from the event, is on the system clock rather than the shared monotonic
  652.  * clock.
  653.  * @param {string} details.transitionType - The transition type for the page that is loading,
  654.  * `webNavigation.onDOMContentLoaded` or `webNavigation.onHistoryStateUpdated`.
  655.  * @param {string[]} details.transitionQualifiers - The transition qualifiers for the page
  656.  * that is loading, either from `webNavigation.onDOMContentLoaded` or
  657.  * `webNavigation.onHistoryStateUpdated`.
  658.  * @param {boolean} details.isHistoryChange - Whether the update was caused by
  659.  * `webNavigation.onDOMContentLoaded` (`false`) or `webNavigation.onHistoryStateUpdated`
  660.  * (`true`).
  661.  * @private
  662.  */
  663.  function sendUpdateToContentScript({
  664.     tabId,
  665.     url,
  666.     timeStamp,
  667.     webNavigationTimeStamp,
  668.     transitionType,
  669.     transitionQualifiers,
  670.     isHistoryChange
  671. }) {
  672.     // Retrieve cached page visits for this tab if this is not a History API change
  673.     let cachedPageVisitsForTab = { };
  674.     if(!isHistoryChange) {
  675.         cachedPageVisitsForTab = pageVisitTabCache.get(tabId);
  676.     }
  678.     // Get the cached opener tab details if this is not a History API change
  679.     let isOpenedTab = false;
  680.     let openerTabId = browser.tabs.TAB_ID_NONE;
  681.     let tabOpeningTimeStamp = 0;
  682.     if(!isHistoryChange) {
  683.         const openerTabDetails = openerTabCache.get(tabId);
  684.         // If there are cached opener tab details, expire the cache and swap in the cached page
  685.         // visits for the opener tab
  686.         if(openerTabDetails !== undefined) {
  687.             openerTabCache.delete(tabId);
  688.             isOpenedTab = true;
  689.             openerTabId = openerTabDetails.openerTabId;
  690.             tabOpeningTimeStamp = openerTabDetails.timeStamp;
  691.             cachedPageVisitsForTab = pageVisitTabCache.get(openerTabDetails.openerTabId);
  692.         }
  693.     }
  695.     // Send a message to the content script with transition information. The content script will
  696.     // merge this information with its local information to generate a PageTransitionData event.
  697.     messaging.sendMessageToTab(tabId, {
  698.         type: "webScience.pageTransition.backgroundScriptEventUpdate",
  699.         url,
  700.         timeStamp,
  701.         webNavigationTimeStamp,
  702.         transitionType,
  703.         transitionQualifiers,
  704.         isHistoryChange,
  705.         pageVisitTimeCache,
  706.         cachedPageVisitsForTab: (cachedPageVisitsForTab !== undefined) ? cachedPageVisitsForTab : { },
  707.         isOpenedTab,
  708.         openerTabId,
  709.         tabOpeningTimeStamp
  710.     });
  712.     // Remove stale page visits from the time-based transition cache, retaining the most recent page
  713.     // visit in any window and the most recent page visit in only non-private windows. We have to
  714.     // track the most recent non-private page separately, since a listener might only be registered
  715.     // for transitions involving non-private pages. We perform this expiration after sending a
  716.     // message to the content script, for the reasons explained in the pageManager.onPageVisitStart
  717.     // listener.
  718.     const nowTimeStamp = timing.now();
  719.     const expiredCachePageIds = new Set();
  720.     let mostRecentPageId = "";
  721.     let mostRecentPageVisitStartTime = 0;
  722.     let mostRecentNonPrivatePageId = "";
  723.     let mostRecentNonPrivatePageVisitStartTime = 0;
  724.     for(const cachePageId in pageVisitTimeCache) {
  725.         if(pageVisitTimeCache[cachePageId].pageVisitStartTime > mostRecentPageVisitStartTime) {
  726.             mostRecentPageId = cachePageId;
  727.             mostRecentPageVisitStartTime = pageVisitTimeCache[cachePageId].pageVisitStartTime;
  728.         }
  729.         if(!pageVisitTimeCache[cachePageId].privateWindow && (pageVisitTimeCache[cachePageId].pageVisitStartTime > mostRecentNonPrivatePageVisitStartTime)) {
  730.             mostRecentNonPrivatePageId = cachePageId;
  731.             mostRecentNonPrivatePageVisitStartTime = pageVisitTimeCache[cachePageId].pageVisitStartTime;
  732.         }
  733.         if((nowTimeStamp - pageVisitTimeCache[cachePageId].pageVisitStartTime) > pageVisitTimeCacheExpiry) {
  734.             expiredCachePageIds.add(cachePageId);
  735.         }
  736.     }
  737.     expiredCachePageIds.delete(mostRecentPageId);
  738.     expiredCachePageIds.delete(mostRecentNonPrivatePageId);
  739.     for(const expiredCachePageId of expiredCachePageIds) {
  740.         delete pageVisitTimeCache[expiredCachePageId];
  741.     }
  743.     // Remove stale page visits and clicks from the tab-based transition cache. We don't have to
  744.     // handle private and non-private windows separately, because if a tab precedes another tab
  745.     // we know they have the same private window status.
  746.     if(cachedPageVisitsForTab !== undefined) {
  747.         // Expire stale pages, expect for the most recent page if it's also stale
  748.         mostRecentPageId = "";
  749.         mostRecentPageVisitStartTime = 0;
  750.         expiredCachePageIds.clear();
  751.         for(const cachePageId in cachedPageVisitsForTab) {
  752.             if(cachedPageVisitsForTab[cachePageId].pageVisitStartTime > mostRecentPageVisitStartTime) {
  753.                 mostRecentPageId = cachePageId;
  754.                 mostRecentPageVisitStartTime = cachedPageVisitsForTab[cachePageId].pageVisitStartTime;
  755.             }
  756.             if((nowTimeStamp - cachedPageVisitsForTab[cachePageId].pageVisitStartTime) > pageVisitTabCacheExpiry) {
  757.                 expiredCachePageIds.add(cachePageId);
  758.             }
  759.         }
  760.         expiredCachePageIds.delete(mostRecentPageId);
  761.         for(const expiredCachePageId of expiredCachePageIds) {
  762.             delete cachedPageVisitsForTab[expiredCachePageId];
  763.         }
  765.         // Expire stale clicks on the remaining pages, except for the most recent click if it's
  766.         // also stale
  767.         for(const cachePageId in cachedPageVisitsForTab) {
  768.             let mostRecentClickOnPage = 0;
  769.             const clickTimeStamps = [ ];
  770.             for(const clickTimeStamp of cachedPageVisitsForTab[cachePageId].clickTimeStamps) {
  771.                 if((nowTimeStamp - clickTimeStamp) <= clickCacheExpiry) {
  772.                     clickTimeStamps.push(clickTimeStamp);
  773.                 }
  774.                 mostRecentClickOnPage = Math.max(mostRecentClickOnPage, clickTimeStamp);
  775.             }
  776.             if((clickTimeStamps.length === 0) && (mostRecentClickOnPage > 0)) {
  777.                 clickTimeStamps.push(mostRecentClickOnPage);
  778.             }
  779.             cachedPageVisitsForTab[cachePageId].clickTimeStamps = clickTimeStamps;
  780.         }
  781.     }
  782. }