diff options
Diffstat (limited to 'application/basilisk/modules/DirectoryLinksProvider.jsm')
| -rw-r--r-- | application/basilisk/modules/DirectoryLinksProvider.jsm | 1243 |
1 files changed, 0 insertions, 1243 deletions
diff --git a/application/basilisk/modules/DirectoryLinksProvider.jsm b/application/basilisk/modules/DirectoryLinksProvider.jsm deleted file mode 100644 index 5fe9f9fce0..0000000000 --- a/application/basilisk/modules/DirectoryLinksProvider.jsm +++ /dev/null @@ -1,1243 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this file, - * You can obtain one at http://mozilla.org/MPL/2.0/. */ - -"use strict"; - -this.EXPORTED_SYMBOLS = ["DirectoryLinksProvider"]; - -const Ci = Components.interfaces; -const Cc = Components.classes; -const Cu = Components.utils; -const ParserUtils = Cc["@mozilla.org/parserutils;1"].getService(Ci.nsIParserUtils); - -Cu.importGlobalProperties(["XMLHttpRequest"]); - -Cu.import("resource://gre/modules/XPCOMUtils.jsm"); -Cu.import("resource://gre/modules/Services.jsm"); -Cu.import("resource://gre/modules/Task.jsm"); -Cu.import("resource://gre/modules/Timer.jsm"); - -XPCOMUtils.defineLazyModuleGetter(this, "NetUtil", - "resource://gre/modules/NetUtil.jsm"); -XPCOMUtils.defineLazyModuleGetter(this, "NewTabUtils", - "resource://gre/modules/NewTabUtils.jsm"); -XPCOMUtils.defineLazyModuleGetter(this, "OS", - "resource://gre/modules/osfile.jsm") -XPCOMUtils.defineLazyModuleGetter(this, "Promise", - "resource://gre/modules/Promise.jsm"); -XPCOMUtils.defineLazyModuleGetter(this, "UpdateUtils", - "resource://gre/modules/UpdateUtils.jsm"); -XPCOMUtils.defineLazyServiceGetter(this, "eTLD", - "@mozilla.org/network/effective-tld-service;1", - "nsIEffectiveTLDService"); -XPCOMUtils.defineLazyGetter(this, "gTextDecoder", () => { - return new TextDecoder(); -}); -XPCOMUtils.defineLazyGetter(this, "gCryptoHash", function() { - return Cc["@mozilla.org/security/hash;1"].createInstance(Ci.nsICryptoHash); -}); -XPCOMUtils.defineLazyGetter(this, "gUnicodeConverter", function() { - let converter = Cc["@mozilla.org/intl/scriptableunicodeconverter"] - .createInstance(Ci.nsIScriptableUnicodeConverter); - converter.charset = "utf8"; - return converter; -}); - - -// The filename where directory links are stored locally -const DIRECTORY_LINKS_FILE = "directoryLinks.json"; -const DIRECTORY_LINKS_TYPE = "application/json"; - -// The preference that tells whether to match the OS locale -const PREF_MATCH_OS_LOCALE = "intl.locale.matchOS"; - -// The preference that tells what locale the user selected -const PREF_SELECTED_LOCALE = "general.useragent.locale"; - -// The preference that tells where to obtain directory links -const PREF_DIRECTORY_SOURCE = "browser.newtabpage.directory.source"; - -// The preference that tells where to send click/view pings -const PREF_DIRECTORY_PING = "browser.newtabpage.directory.ping"; - -// The preference that tells if newtab is enhanced -const PREF_NEWTAB_ENHANCED = "browser.newtabpage.enhanced"; - -// Only allow link urls that are http(s) -const ALLOWED_LINK_SCHEMES = new Set(["http", "https"]); - -// Only allow link image urls that are https or data -const ALLOWED_IMAGE_SCHEMES = new Set(["https", "data"]); - -// Only allow urls to Mozilla's CDN or empty (for data URIs) -const ALLOWED_URL_BASE = new Set(["mozilla.net", ""]); - -// The frecency of a directory link -const DIRECTORY_FRECENCY = 1000; - -// The frecency of a suggested link -const SUGGESTED_FRECENCY = Infinity; - -// The filename where frequency cap data stored locally -const FREQUENCY_CAP_FILE = "frequencyCap.json"; - -// Default settings for daily and total frequency caps -const DEFAULT_DAILY_FREQUENCY_CAP = 3; -const DEFAULT_TOTAL_FREQUENCY_CAP = 10; - -// Default timeDelta to prune unused frequency cap objects -// currently set to 10 days in milliseconds -const DEFAULT_PRUNE_TIME_DELTA = 10 * 24 * 60 * 60 * 1000; - -// The min number of visible (not blocked) history tiles to have before showing suggested tiles -const MIN_VISIBLE_HISTORY_TILES = 8; - -// The max number of visible (not blocked) history tiles to test for inadjacency -const MAX_VISIBLE_HISTORY_TILES = 15; - -// Allowed ping actions remotely stored as columns: case-insensitive [a-z0-9_] -const PING_ACTIONS = ["block", "click", "pin", "sponsored", "sponsored_link", "unpin", "view"]; - -// Location of inadjacent sites json -const INADJACENCY_SOURCE = "chrome://browser/content/newtab/newTab.inadjacent.json"; - -// Fake URL to keep track of last block of a suggested tile in the frequency cap object -const FAKE_SUGGESTED_BLOCK_URL = "ignore://suggested_block"; - -// Time before suggested tile is allowed to play again after block - default to 1 day -const AFTER_SUGGESTED_BLOCK_DECAY_TIME = 24 * 60 * 60 * 1000; - -/** - * Singleton that serves as the provider of directory links. - * Directory links are a hard-coded set of links shown if a user's link - * inventory is empty. - */ -var DirectoryLinksProvider = { - - __linksURL: null, - - _observers: new Set(), - - // links download deferred, resolved upon download completion - _downloadDeferred: null, - - // download default interval is 24 hours in milliseconds - _downloadIntervalMS: 86400000, - - /** - * A mapping from eTLD+1 to an enhanced link objects - */ - _enhancedLinks: new Map(), - - /** - * A mapping from site to a list of suggested link objects - */ - _suggestedLinks: new Map(), - - /** - * Frequency Cap object - maintains daily and total tile counts, and frequency cap settings - */ - _frequencyCaps: {}, - - /** - * A set of top sites that we can provide suggested links for - */ - _topSitesWithSuggestedLinks: new Set(), - - /** - * lookup Set of inadjacent domains - */ - _inadjacentSites: new Set(), - - /** - * This flag is set if there is a suggested tile configured to avoid - * inadjacent sites in new tab - */ - _avoidInadjacentSites: false, - - /** - * This flag is set if _avoidInadjacentSites is true and there is - * an inadjacent site in the new tab - */ - _newTabHasInadjacentSite: false, - - get _observedPrefs() { - return Object.freeze({ - enhanced: PREF_NEWTAB_ENHANCED, - linksURL: PREF_DIRECTORY_SOURCE, - matchOSLocale: PREF_MATCH_OS_LOCALE, - prefSelectedLocale: PREF_SELECTED_LOCALE, - }); - }, - - get _linksURL() { - if (!this.__linksURL) { - try { - this.__linksURL = Services.prefs.getCharPref(this._observedPrefs["linksURL"]); - this.__linksURLModified = Services.prefs.prefHasUserValue(this._observedPrefs["linksURL"]); - } catch (e) { - Cu.reportError("Error fetching directory links url from prefs: " + e); - } - } - return this.__linksURL; - }, - - /** - * Gets the currently selected locale for display. - * @return the selected locale or "en-US" if none is selected - */ - get locale() { - let matchOS; - try { - matchOS = Services.prefs.getBoolPref(PREF_MATCH_OS_LOCALE); - } catch (e) {} - - if (matchOS) { - return Services.locale.getLocaleComponentForUserAgent(); - } - - try { - let locale = Services.prefs.getComplexValue(PREF_SELECTED_LOCALE, - Ci.nsIPrefLocalizedString); - if (locale) { - return locale.data; - } - } catch (e) {} - - try { - return Services.prefs.getCharPref(PREF_SELECTED_LOCALE); - } catch (e) {} - - return "en-US"; - }, - - /** - * Set appropriate default ping behavior controlled by enhanced pref - */ - _setDefaultEnhanced: function DirectoryLinksProvider_setDefaultEnhanced() { - if (!Services.prefs.prefHasUserValue(PREF_NEWTAB_ENHANCED)) { - let enhanced = Services.prefs.getBoolPref(PREF_NEWTAB_ENHANCED); - try { - // Default to not enhanced if DNT is set to tell websites to not track - if (Services.prefs.getBoolPref("privacy.donottrackheader.enabled")) { - enhanced = false; - } - } catch (ex) {} - Services.prefs.setBoolPref(PREF_NEWTAB_ENHANCED, enhanced); - } - }, - - observe: function DirectoryLinksProvider_observe(aSubject, aTopic, aData) { - if (aTopic == "nsPref:changed") { - switch (aData) { - // Re-set the default in case the user clears the pref - case this._observedPrefs.enhanced: - this._setDefaultEnhanced(); - break; - - case this._observedPrefs.linksURL: - delete this.__linksURL; - // fallthrough - - // Force directory download on changes to fetch related prefs - case this._observedPrefs.matchOSLocale: - case this._observedPrefs.prefSelectedLocale: - this._fetchAndCacheLinksIfNecessary(true); - break; - } - } - }, - - _addPrefsObserver: function DirectoryLinksProvider_addObserver() { - for (let pref in this._observedPrefs) { - let prefName = this._observedPrefs[pref]; - Services.prefs.addObserver(prefName, this, false); - } - }, - - _removePrefsObserver: function DirectoryLinksProvider_removeObserver() { - for (let pref in this._observedPrefs) { - let prefName = this._observedPrefs[pref]; - Services.prefs.removeObserver(prefName, this); - } - }, - - _cacheSuggestedLinks(link) { - // Don't cache links that don't have the expected 'frecent_sites' - if (!link.frecent_sites) { - return; - } - - for (let suggestedSite of link.frecent_sites) { - let suggestedMap = this._suggestedLinks.get(suggestedSite) || new Map(); - suggestedMap.set(link.url, link); - this._setupStartEndTime(link); - this._suggestedLinks.set(suggestedSite, suggestedMap); - } - }, - - _fetchAndCacheLinks: function DirectoryLinksProvider_fetchAndCacheLinks(uri) { - // Replace with the same display locale used for selecting links data - uri = uri.replace("%LOCALE%", this.locale); - uri = uri.replace("%CHANNEL%", UpdateUtils.UpdateChannel); - - return this._downloadJsonData(uri).then(json => { - return OS.File.writeAtomic(this._directoryFilePath, json, {tmpPath: this._directoryFilePath + ".tmp"}); - }); - }, - - /** - * Downloads a links with json content - * @param download uri - * @return promise resolved to json string, "{}" returned if status != 200 - */ - _downloadJsonData: function DirectoryLinksProvider__downloadJsonData(uri) { - let deferred = Promise.defer(); - let xmlHttp = this._newXHR(); - - xmlHttp.onload = function(aResponse) { - let json = this.responseText; - if (this.status && this.status != 200) { - json = "{}"; - } - deferred.resolve(json); - }; - - xmlHttp.onerror = function(e) { - deferred.reject("Fetching " + uri + " results in error code: " + e.target.status); - }; - - try { - xmlHttp.open("GET", uri); - // Override the type so XHR doesn't complain about not well-formed XML - xmlHttp.overrideMimeType(DIRECTORY_LINKS_TYPE); - // Set the appropriate request type for servers that require correct types - xmlHttp.setRequestHeader("Content-Type", DIRECTORY_LINKS_TYPE); - xmlHttp.send(); - } catch (e) { - deferred.reject("Error fetching " + uri); - Cu.reportError(e); - } - return deferred.promise; - }, - - /** - * Downloads directory links if needed - * @return promise resolved immediately if no download needed, or upon completion - */ - _fetchAndCacheLinksIfNecessary: function DirectoryLinksProvider_fetchAndCacheLinksIfNecessary(forceDownload = false) { - if (this._downloadDeferred) { - // fetching links already - just return the promise - return this._downloadDeferred.promise; - } - - if (forceDownload || this._needsDownload) { - this._downloadDeferred = Promise.defer(); - this._fetchAndCacheLinks(this._linksURL).then(() => { - // the new file was successfully downloaded and cached, so update a timestamp - this._lastDownloadMS = Date.now(); - this._downloadDeferred.resolve(); - this._downloadDeferred = null; - this._callObservers("onManyLinksChanged") - }, - error => { - this._downloadDeferred.resolve(); - this._downloadDeferred = null; - this._callObservers("onDownloadFail"); - }); - return this._downloadDeferred.promise; - } - - // download is not needed - return Promise.resolve(); - }, - - /** - * @return true if download is needed, false otherwise - */ - get _needsDownload() { - // fail if last download occured less then 24 hours ago - if ((Date.now() - this._lastDownloadMS) > this._downloadIntervalMS) { - return true; - } - return false; - }, - - /** - * Create a new XMLHttpRequest that is anonymous, i.e., doesn't send cookies - */ - _newXHR() { - return new XMLHttpRequest({mozAnon: true}); - }, - - /** - * Reads directory links file and parses its content - * @return a promise resolved to an object with keys 'directory' and 'suggested', - * each containing a valid list of links, - * or {'directory': [], 'suggested': []} if read or parse fails. - */ - _readDirectoryLinksFile: function DirectoryLinksProvider_readDirectoryLinksFile() { - let emptyOutput = {directory: [], suggested: [], enhanced: []}; - return OS.File.read(this._directoryFilePath).then(binaryData => { - let output; - try { - let json = gTextDecoder.decode(binaryData); - let linksObj = JSON.parse(json); - output = {directory: linksObj.directory || [], - suggested: linksObj.suggested || [], - enhanced: linksObj.enhanced || []}; - } catch (e) { - Cu.reportError(e); - } - return output || emptyOutput; - }, - error => { - Cu.reportError(error); - return emptyOutput; - }); - }, - - /** - * Translates link.time_limits to UTC miliseconds and sets - * link.startTime and link.endTime properties in link object - */ - _setupStartEndTime: function DirectoryLinksProvider_setupStartEndTime(link) { - // set start/end limits. Use ISO_8601 format: '2014-01-10T20:20:20.600Z' - // (details here http://en.wikipedia.org/wiki/ISO_8601) - // Note that if timezone is missing, FX will interpret as local time - // meaning that the server can sepecify any time, but if the capmaign - // needs to start at same time across multiple timezones, the server - // omits timezone indicator - if (!link.time_limits) { - return; - } - - let parsedTime; - if (link.time_limits.start) { - parsedTime = Date.parse(link.time_limits.start); - if (parsedTime && !isNaN(parsedTime)) { - link.startTime = parsedTime; - } - } - if (link.time_limits.end) { - parsedTime = Date.parse(link.time_limits.end); - if (parsedTime && !isNaN(parsedTime)) { - link.endTime = parsedTime; - } - } - }, - - /* - * Handles campaign timeout - */ - _onCampaignTimeout: function DirectoryLinksProvider_onCampaignTimeout() { - // _campaignTimeoutID is invalid here, so just set it to null - this._campaignTimeoutID = null; - this._updateSuggestedTile(); - }, - - /* - * Clears capmpaign timeout - */ - _clearCampaignTimeout: function DirectoryLinksProvider_clearCampaignTimeout() { - if (this._campaignTimeoutID) { - clearTimeout(this._campaignTimeoutID); - this._campaignTimeoutID = null; - } - }, - - /** - * Setup capmpaign timeout to recompute suggested tiles upon - * reaching soonest start or end time for the campaign - * @param timeout in milliseconds - */ - _setupCampaignTimeCheck: function DirectoryLinksProvider_setupCampaignTimeCheck(timeout) { - // sanity check - if (!timeout || timeout <= 0) { - return; - } - this._clearCampaignTimeout(); - // setup next timeout - this._campaignTimeoutID = setTimeout(this._onCampaignTimeout.bind(this), timeout); - }, - - /** - * Test link for campaign time limits: checks if link falls within start/end time - * and returns an object containing a use flag and the timeoutDate milliseconds - * when the link has to be re-checked for campaign start-ready or end-reach - * @param link - * @return object {use: true or false, timeoutDate: milliseconds or null} - */ - _testLinkForCampaignTimeLimits: function DirectoryLinksProvider_testLinkForCampaignTimeLimits(link) { - let currentTime = Date.now(); - // test for start time first - if (link.startTime && link.startTime > currentTime) { - // not yet ready for start - return {use: false, timeoutDate: link.startTime}; - } - // otherwise check for end time - if (link.endTime) { - // passed end time - if (link.endTime <= currentTime) { - return {use: false}; - } - // otherwise link is still ok, but we need to set timeoutDate - return {use: true, timeoutDate: link.endTime}; - } - // if we are here, the link is ok and no timeoutDate needed - return {use: true}; - }, - - /** - * Handles block on suggested tile: updates fake block url with current timestamp - */ - handleSuggestedTileBlock: function DirectoryLinksProvider_handleSuggestedTileBlock() { - this._updateFrequencyCapSettings({url: FAKE_SUGGESTED_BLOCK_URL}); - this._writeFrequencyCapFile(); - this._updateSuggestedTile(); - }, - - /** - * Checks if suggested tile is being blocked for the rest of "decay time" - * @return True if blocked, false otherwise - */ - _isSuggestedTileBlocked: function DirectoryLinksProvider__isSuggestedTileBlocked() { - let capObject = this._frequencyCaps[FAKE_SUGGESTED_BLOCK_URL]; - if (!capObject || !capObject.lastUpdated) { - // user never blocked suggested tile or lastUpdated is missing - return false; - } - // otherwise, make sure that enough time passed after suggested tile was blocked - return (capObject.lastUpdated + AFTER_SUGGESTED_BLOCK_DECAY_TIME) > Date.now(); - }, - - /** - * Report some action on a newtab page (view, click) - * @param sites Array of sites shown on newtab page - * @param action String of the behavior to report - * @param triggeringSiteIndex optional Int index of the site triggering action - * @return download promise - */ - reportSitesAction: function DirectoryLinksProvider_reportSitesAction(sites, action, triggeringSiteIndex) { - // Check if the suggested tile was shown - if (action == "view") { - sites.slice(0, triggeringSiteIndex + 1).filter(s => s).forEach(site => { - let {targetedSite, url} = site.link; - if (targetedSite) { - this._addFrequencyCapView(url); - } - }); - } else { - // any click action on a suggested tile should stop that tile suggestion - // click/block - user either removed a tile or went to a landing page - // pin - tile turned into history tile, should no longer be suggested - // unpin - the tile was pinned before, should not matter - // suggested tile has targetedSite, or frecent_sites if it was pinned - let {frecent_sites, targetedSite, url} = sites[triggeringSiteIndex].link; - if (frecent_sites || targetedSite) { - this._setFrequencyCapClick(url); - } - } - - let newtabEnhanced = false; - let pingEndPoint = ""; - try { - newtabEnhanced = Services.prefs.getBoolPref(PREF_NEWTAB_ENHANCED); - pingEndPoint = Services.prefs.getCharPref(PREF_DIRECTORY_PING); - } catch (ex) {} - - // Bug 1240245 - We no longer send pings, but frequency capping and fetching - // tests depend on the following actions, so references to PING remain. - let invalidAction = PING_ACTIONS.indexOf(action) == -1; - if (!newtabEnhanced || pingEndPoint == "" || invalidAction) { - return Promise.resolve(); - } - - return Task.spawn(function* () { - // since we updated views/clicks we need write _frequencyCaps to disk - yield this._writeFrequencyCapFile(); - // Use this as an opportunity to potentially fetch new links - yield this._fetchAndCacheLinksIfNecessary(); - }.bind(this)); - }, - - /** - * Get the enhanced link object for a link (whether history or directory) - */ - getEnhancedLink: function DirectoryLinksProvider_getEnhancedLink(link) { - // Use the provided link if it's already enhanced - return link.enhancedImageURI && link ? link : - this._enhancedLinks.get(NewTabUtils.extractSite(link.url)); - }, - - /** - * Check if a url's scheme is in a Set of allowed schemes and if the base - * domain is allowed. - * @param url to check - * @param allowed Set of allowed schemes - * @param checkBase boolean to check the base domain - */ - isURLAllowed(url, allowed, checkBase) { - // Assume no url is an allowed url - if (!url) { - return true; - } - - let scheme = "", base = ""; - try { - // A malformed url will not be allowed - let uri = Services.io.newURI(url); - scheme = uri.scheme; - - // URIs without base domains will be allowed - base = Services.eTLD.getBaseDomain(uri); - } catch (ex) {} - // Require a scheme match and the base only if desired - return allowed.has(scheme) && (!checkBase || ALLOWED_URL_BASE.has(base)); - }, - - _escapeChars(text) { - let charMap = { - "&": "&", - "<": "<", - ">": ">", - '"': """, - "'": "'" - }; - - return text.replace(/[&<>"']/g, (character) => charMap[character]); - }, - - /** - * Gets the current set of directory links. - * @param aCallback The function that the array of links is passed to. - */ - getLinks: function DirectoryLinksProvider_getLinks(aCallback) { - this._readDirectoryLinksFile().then(rawLinks => { - // Reset the cache of suggested tiles and enhanced images for this new set of links - this._enhancedLinks.clear(); - this._suggestedLinks.clear(); - this._clearCampaignTimeout(); - this._avoidInadjacentSites = false; - - // Only check base domain for images when using the default pref - let checkBase = !this.__linksURLModified; - let validityFilter = function(link) { - // Make sure the link url is allowed and images too if they exist - return this.isURLAllowed(link.url, ALLOWED_LINK_SCHEMES, false) && - (!link.imageURI || - this.isURLAllowed(link.imageURI, ALLOWED_IMAGE_SCHEMES, checkBase)) && - (!link.enhancedImageURI || - this.isURLAllowed(link.enhancedImageURI, ALLOWED_IMAGE_SCHEMES, checkBase)); - }.bind(this); - - rawLinks.suggested.filter(validityFilter).forEach((link, position) => { - // Suggested sites must have an adgroup name. - if (!link.adgroup_name) { - return; - } - - let sanitizeFlags = ParserUtils.SanitizerCidEmbedsOnly | - ParserUtils.SanitizerDropForms | - ParserUtils.SanitizerDropNonCSSPresentation; - - link.explanation = this._escapeChars(link.explanation ? ParserUtils.convertToPlainText(link.explanation, sanitizeFlags, 0) : ""); - link.targetedName = this._escapeChars(ParserUtils.convertToPlainText(link.adgroup_name, sanitizeFlags, 0)); - link.lastVisitDate = rawLinks.suggested.length - position; - // check if link wants to avoid inadjacent sites - if (link.check_inadjacency) { - this._avoidInadjacentSites = true; - } - - // We cache suggested tiles here but do not push any of them in the links list yet. - // The decision for which suggested tile to include will be made separately. - this._cacheSuggestedLinks(link); - this._updateFrequencyCapSettings(link); - }); - - rawLinks.enhanced.filter(validityFilter).forEach((link, position) => { - link.lastVisitDate = rawLinks.enhanced.length - position; - - // Stash the enhanced image for the site - if (link.enhancedImageURI) { - this._enhancedLinks.set(NewTabUtils.extractSite(link.url), link); - } - }); - - let links = rawLinks.directory.filter(validityFilter).map((link, position) => { - link.lastVisitDate = rawLinks.directory.length - position; - link.frecency = DIRECTORY_FRECENCY; - return link; - }); - - // Allow for one link suggestion on top of the default directory links - this.maxNumLinks = links.length + 1; - - // prune frequency caps of outdated urls - this._pruneFrequencyCapUrls(); - // write frequency caps object to disk asynchronously - this._writeFrequencyCapFile(); - - return links; - }).catch(ex => { - Cu.reportError(ex); - return []; - }).then(links => { - aCallback(links); - this._populatePlacesLinks(); - }); - }, - - init: function DirectoryLinksProvider_init() { - this._setDefaultEnhanced(); - this._addPrefsObserver(); - // setup directory file path and last download timestamp - this._directoryFilePath = OS.Path.join(OS.Constants.Path.localProfileDir, DIRECTORY_LINKS_FILE); - this._lastDownloadMS = 0; - - // setup frequency cap file path - this._frequencyCapFilePath = OS.Path.join(OS.Constants.Path.localProfileDir, FREQUENCY_CAP_FILE); - // setup inadjacent sites URL - this._inadjacentSitesUrl = INADJACENCY_SOURCE; - - NewTabUtils.placesProvider.addObserver(this); - NewTabUtils.links.addObserver(this); - - return Task.spawn(function*() { - // get the last modified time of the links file if it exists - let doesFileExists = yield OS.File.exists(this._directoryFilePath); - if (doesFileExists) { - let fileInfo = yield OS.File.stat(this._directoryFilePath); - this._lastDownloadMS = Date.parse(fileInfo.lastModificationDate); - } - // read frequency cap file - yield this._readFrequencyCapFile(); - // fetch directory on startup without force - yield this._fetchAndCacheLinksIfNecessary(); - // fecth inadjacent sites on startup - yield this._loadInadjacentSites(); - }.bind(this)); - }, - - _handleManyLinksChanged() { - this._topSitesWithSuggestedLinks.clear(); - this._suggestedLinks.forEach((suggestedLinks, site) => { - if (NewTabUtils.isTopPlacesSite(site)) { - this._topSitesWithSuggestedLinks.add(site); - } - }); - this._updateSuggestedTile(); - }, - - /** - * Updates _topSitesWithSuggestedLinks based on the link that was changed. - * - * @return true if _topSitesWithSuggestedLinks was modified, false otherwise. - */ - _handleLinkChanged(aLink) { - let changedLinkSite = NewTabUtils.extractSite(aLink.url); - let linkStored = this._topSitesWithSuggestedLinks.has(changedLinkSite); - - if (!NewTabUtils.isTopPlacesSite(changedLinkSite) && linkStored) { - this._topSitesWithSuggestedLinks.delete(changedLinkSite); - return true; - } - - if (this._suggestedLinks.has(changedLinkSite) && - NewTabUtils.isTopPlacesSite(changedLinkSite) && !linkStored) { - this._topSitesWithSuggestedLinks.add(changedLinkSite); - return true; - } - - // always run _updateSuggestedTile if aLink is inadjacent - // and there are tiles configured to avoid it - if (this._avoidInadjacentSites && this._isInadjacentLink(aLink)) { - return true; - } - - return false; - }, - - _populatePlacesLinks() { - NewTabUtils.links.populateProviderCache(NewTabUtils.placesProvider, () => { - this._handleManyLinksChanged(); - }); - }, - - onDeleteURI(aProvider, aLink) { - let {url} = aLink; - // remove clicked flag for that url and - // call observer upon disk write completion - this._removeTileClick(url).then(() => { - this._callObservers("onDeleteURI", url); - }); - }, - - onClearHistory() { - // remove all clicked flags and call observers upon file write - this._removeAllTileClicks().then(() => { - this._callObservers("onClearHistory"); - }); - }, - - onLinkChanged(aProvider, aLink) { - // Make sure NewTabUtils.links handles the notification first. - setTimeout(() => { - if (this._handleLinkChanged(aLink) || this._shouldUpdateSuggestedTile()) { - this._updateSuggestedTile(); - } - }, 0); - }, - - onManyLinksChanged() { - // Make sure NewTabUtils.links handles the notification first. - setTimeout(() => { - this._handleManyLinksChanged(); - }, 0); - }, - - _getCurrentTopSiteCount() { - let visibleTopSiteCount = 0; - let newTabLinks = NewTabUtils.links.getLinks(); - for (let link of newTabLinks.slice(0, MIN_VISIBLE_HISTORY_TILES)) { - // compute visibleTopSiteCount for suggested tiles - if (link && (link.type == "history" || link.type == "enhanced")) { - visibleTopSiteCount++; - } - } - // since newTabLinks are available, set _newTabHasInadjacentSite here - // note that _shouldUpdateSuggestedTile is called by _updateSuggestedTile - this._newTabHasInadjacentSite = this._avoidInadjacentSites && this._checkForInadjacentSites(newTabLinks); - - return visibleTopSiteCount; - }, - - _shouldUpdateSuggestedTile() { - let sortedLinks = NewTabUtils.getProviderLinks(this); - - let mostFrecentLink = {}; - if (sortedLinks && sortedLinks.length) { - mostFrecentLink = sortedLinks[0] - } - - let currTopSiteCount = this._getCurrentTopSiteCount(); - if ((!mostFrecentLink.targetedSite && currTopSiteCount >= MIN_VISIBLE_HISTORY_TILES) || - (mostFrecentLink.targetedSite && currTopSiteCount < MIN_VISIBLE_HISTORY_TILES)) { - // If mostFrecentLink has a targetedSite then mostFrecentLink is a suggested link. - // If we have enough history links (8+) to show a suggested tile and we are not - // already showing one, then we should update (to *attempt* to add a suggested tile). - // OR if we don't have enough history to show a suggested tile (<8) and we are - // currently showing one, we should update (to remove it). - return true; - } - - return false; - }, - - /** - * Chooses and returns a suggested tile based on a user's top sites - * that we have an available suggested tile for. - * - * @return the chosen suggested tile, or undefined if there isn't one - */ - _updateSuggestedTile() { - let sortedLinks = NewTabUtils.getProviderLinks(this); - - if (!sortedLinks) { - // If NewTabUtils.links.resetCache() is called before getting here, - // sortedLinks may be undefined. - return undefined; - } - - // Delete the current suggested tile, if one exists. - let initialLength = sortedLinks.length; - if (initialLength) { - let mostFrecentLink = sortedLinks[0]; - if (mostFrecentLink.targetedSite) { - this._callObservers("onLinkChanged", { - url: mostFrecentLink.url, - frecency: SUGGESTED_FRECENCY, - lastVisitDate: mostFrecentLink.lastVisitDate, - type: mostFrecentLink.type, - }, 0, true); - } - } - - if (this._topSitesWithSuggestedLinks.size == 0 || - !this._shouldUpdateSuggestedTile() || - this._isSuggestedTileBlocked()) { - // There are no potential suggested links we can show or not - // enough history for a suggested tile, or suggested tile was - // recently blocked and wait time interval has not decayed yet - return undefined; - } - - // Create a flat list of all possible links we can show as suggested. - // Note that many top sites may map to the same suggested links, but we only - // want to count each suggested link once (based on url), thus possibleLinks is a map - // from url to suggestedLink. Thus, each link has an equal chance of being chosen at - // random from flattenedLinks if it appears only once. - let nextTimeout; - let possibleLinks = new Map(); - let targetedSites = new Map(); - this._topSitesWithSuggestedLinks.forEach(topSiteWithSuggestedLink => { - let suggestedLinksMap = this._suggestedLinks.get(topSiteWithSuggestedLink); - suggestedLinksMap.forEach((suggestedLink, url) => { - // Skip this link if we've shown it too many times already - if (!this._testFrequencyCapLimits(url)) { - return; - } - - // as we iterate suggestedLinks, check for campaign start/end - // time limits, and set nextTimeout to the closest timestamp - let {use, timeoutDate} = this._testLinkForCampaignTimeLimits(suggestedLink); - // update nextTimeout is necessary - if (timeoutDate && (!nextTimeout || nextTimeout > timeoutDate)) { - nextTimeout = timeoutDate; - } - // Skip link if it falls outside campaign time limits - if (!use) { - return; - } - - // Skip link if it avoids inadjacent sites and newtab has one - if (suggestedLink.check_inadjacency && this._newTabHasInadjacentSite) { - return; - } - - possibleLinks.set(url, suggestedLink); - - // Keep a map of URL to targeted sites. We later use this to show the user - // what site they visited to trigger this suggestion. - if (!targetedSites.get(url)) { - targetedSites.set(url, []); - } - targetedSites.get(url).push(topSiteWithSuggestedLink); - }) - }); - - // setup timeout check for starting or ending campaigns - if (nextTimeout) { - this._setupCampaignTimeCheck(nextTimeout - Date.now()); - } - - // We might have run out of possible links to show - let numLinks = possibleLinks.size; - if (numLinks == 0) { - return undefined; - } - - let flattenedLinks = [...possibleLinks.values()]; - - // Choose our suggested link at random - let suggestedIndex = Math.floor(Math.random() * numLinks); - let chosenSuggestedLink = flattenedLinks[suggestedIndex]; - - // Add the suggested link to the front with some extra values - this._callObservers("onLinkChanged", Object.assign({ - frecency: SUGGESTED_FRECENCY, - - // Choose the first site a user has visited as the target. In the future, - // this should be the site with the highest frecency. However, we currently - // store frecency by URL not by site. - targetedSite: targetedSites.get(chosenSuggestedLink.url).length ? - targetedSites.get(chosenSuggestedLink.url)[0] : null - }, chosenSuggestedLink)); - return chosenSuggestedLink; - }, - - /** - * Loads inadjacent sites - * @return a promise resolved when lookup Set for sites is built - */ - _loadInadjacentSites: function DirectoryLinksProvider_loadInadjacentSites() { - return this._downloadJsonData(this._inadjacentSitesUrl).then(jsonString => { - let jsonObject = {}; - try { - jsonObject = JSON.parse(jsonString); - } catch (e) { - Cu.reportError(e); - } - - this._inadjacentSites = new Set(jsonObject.domains); - }); - }, - - /** - * Genegrates hash suitable for looking up inadjacent site - * @param value to hsh - * @return hased value, base64-ed - */ - _generateHash: function DirectoryLinksProvider_generateHash(value) { - let byteArr = gUnicodeConverter.convertToByteArray(value); - gCryptoHash.init(gCryptoHash.MD5); - gCryptoHash.update(byteArr, byteArr.length); - return gCryptoHash.finish(true); - }, - - /** - * Checks if link belongs to inadjacent domain - * @param link to check - * @return true for inadjacent domains, false otherwise - */ - _isInadjacentLink: function DirectoryLinksProvider_isInadjacentLink(link) { - let baseDomain = link.baseDomain || NewTabUtils.extractSite(link.url || ""); - if (!baseDomain) { - return false; - } - // check if hashed domain is inadjacent - return this._inadjacentSites.has(this._generateHash(baseDomain)); - }, - - /** - * Checks if new tab has inadjacent site - * @param new tab links (or nothing, in which case NewTabUtils.links.getLinks() is called - * @return true if new tab shows has inadjacent site - */ - _checkForInadjacentSites: function DirectoryLinksProvider_checkForInadjacentSites(newTabLink) { - let links = newTabLink || NewTabUtils.links.getLinks(); - for (let link of links.slice(0, MAX_VISIBLE_HISTORY_TILES)) { - // check links against inadjacent list - specifically include ALL link types - if (this._isInadjacentLink(link)) { - return true; - } - } - return false; - }, - - /** - * Reads json file, parses its content, and returns resulting object - * @param json file path - * @param json object to return in case file read or parse fails - * @return a promise resolved to a valid object or undefined upon error - */ - _readJsonFile: Task.async(function* (filePath, nullObject) { - let jsonObj; - try { - let binaryData = yield OS.File.read(filePath); - let json = gTextDecoder.decode(binaryData); - jsonObj = JSON.parse(json); - } catch (e) {} - return jsonObj || nullObject; - }), - - /** - * Loads frequency cap object from file and parses its content - * @return a promise resolved upon load completion - * on error or non-exstent file _frequencyCaps is set to empty object - */ - _readFrequencyCapFile: Task.async(function* () { - // set _frequencyCaps object to file's content or empty object - this._frequencyCaps = yield this._readJsonFile(this._frequencyCapFilePath, {}); - }), - - /** - * Saves frequency cap object to file - * @return a promise resolved upon file i/o completion - */ - _writeFrequencyCapFile: function DirectoryLinksProvider_writeFrequencyCapFile() { - let json = JSON.stringify(this._frequencyCaps || {}); - return OS.File.writeAtomic(this._frequencyCapFilePath, json, {tmpPath: this._frequencyCapFilePath + ".tmp"}); - }, - - /** - * Clears frequency cap object and writes empty json to file - * @return a promise resolved upon file i/o completion - */ - _clearFrequencyCap: function DirectoryLinksProvider_clearFrequencyCap() { - this._frequencyCaps = {}; - return this._writeFrequencyCapFile(); - }, - - /** - * updates frequency cap configuration for a link - */ - _updateFrequencyCapSettings: function DirectoryLinksProvider_updateFrequencyCapSettings(link) { - let capsObject = this._frequencyCaps[link.url]; - if (!capsObject) { - // create an object with empty counts - capsObject = { - dailyViews: 0, - totalViews: 0, - lastShownDate: 0, - }; - this._frequencyCaps[link.url] = capsObject; - } - // set last updated timestamp - capsObject.lastUpdated = Date.now(); - // check for link configuration - if (link.frequency_caps) { - capsObject.dailyCap = link.frequency_caps.daily || DEFAULT_DAILY_FREQUENCY_CAP; - capsObject.totalCap = link.frequency_caps.total || DEFAULT_TOTAL_FREQUENCY_CAP; - } else { - // fallback to defaults - capsObject.dailyCap = DEFAULT_DAILY_FREQUENCY_CAP; - capsObject.totalCap = DEFAULT_TOTAL_FREQUENCY_CAP; - } - }, - - /** - * Prunes frequency cap objects for outdated links - * @param timeDetla milliseconds - * all cap objects with lastUpdated less than (now() - timeDelta) - * will be removed. This is done to remove frequency cap objects - * for unused tile urls - */ - _pruneFrequencyCapUrls: function DirectoryLinksProvider_pruneFrequencyCapUrls(timeDelta = DEFAULT_PRUNE_TIME_DELTA) { - let timeThreshold = Date.now() - timeDelta; - Object.keys(this._frequencyCaps).forEach(url => { - // remove url if it is not ignorable and wasn't updated for a while - if (!url.startsWith("ignore") && this._frequencyCaps[url].lastUpdated <= timeThreshold) { - delete this._frequencyCaps[url]; - } - }); - }, - - /** - * Checks if supplied timestamp happened today - * @param timestamp in milliseconds - * @return true if the timestamp was made today, false otherwise - */ - _wasToday: function DirectoryLinksProvider_wasToday(timestamp) { - let showOn = new Date(timestamp); - let today = new Date(); - // call timestamps identical if both day and month are same - return showOn.getDate() == today.getDate() && - showOn.getMonth() == today.getMonth() && - showOn.getYear() == today.getYear(); - }, - - /** - * adds some number of views for a url - * @param url String url of the suggested link - */ - _addFrequencyCapView: function DirectoryLinksProvider_addFrequencyCapView(url) { - let capObject = this._frequencyCaps[url]; - // sanity check - if (!capObject) { - return; - } - - // if the day is new: reset the daily counter and lastShownDate - if (!this._wasToday(capObject.lastShownDate)) { - capObject.dailyViews = 0; - // update lastShownDate - capObject.lastShownDate = Date.now(); - } - - // bump both daily and total counters - capObject.totalViews++; - capObject.dailyViews++; - - // if any of the caps is reached - update suggested tiles - if (capObject.totalViews >= capObject.totalCap || - capObject.dailyViews >= capObject.dailyCap) { - this._updateSuggestedTile(); - } - }, - - /** - * Sets clicked flag for link url - * @param url String url of the suggested link - */ - _setFrequencyCapClick(url) { - let capObject = this._frequencyCaps[url]; - // sanity check - if (!capObject) { - return; - } - capObject.clicked = true; - // and update suggested tiles, since current tile became invalid - this._updateSuggestedTile(); - }, - - /** - * Tests frequency cap limits for link url - * @param url String url of the suggested link - * @return true if link is viewable, false otherwise - */ - _testFrequencyCapLimits: function DirectoryLinksProvider_testFrequencyCapLimits(url) { - let capObject = this._frequencyCaps[url]; - // sanity check: if url is missing - do not show this tile - if (!capObject) { - return false; - } - - // check for clicked set or total views reached - if (capObject.clicked || capObject.totalViews >= capObject.totalCap) { - return false; - } - - // otherwise check if link is over daily views limit - if (this._wasToday(capObject.lastShownDate) && - capObject.dailyViews >= capObject.dailyCap) { - return false; - } - - // we passed all cap tests: return true - return true; - }, - - /** - * Removes clicked flag from frequency cap entry for tile landing url - * @param url String url of the suggested link - * @return promise resolved upon disk write completion - */ - _removeTileClick: function DirectoryLinksProvider_removeTileClick(url = "") { - // remove trailing slash, to accomodate Places sending site urls ending with '/' - let noTrailingSlashUrl = url.replace(/\/$/, ""); - let capObject = this._frequencyCaps[url] || this._frequencyCaps[noTrailingSlashUrl]; - // return resolved promise if capObject is not found - if (!capObject) { - return Promise.resolve(); - } - // otherwise remove clicked flag - delete capObject.clicked; - return this._writeFrequencyCapFile(); - }, - - /** - * Removes all clicked flags from frequency cap object - * @return promise resolved upon disk write completion - */ - _removeAllTileClicks: function DirectoryLinksProvider_removeAllTileClicks() { - Object.keys(this._frequencyCaps).forEach(url => { - delete this._frequencyCaps[url].clicked; - }); - return this._writeFrequencyCapFile(); - }, - - /** - * Return the object to its pre-init state - */ - reset: function DirectoryLinksProvider_reset() { - delete this.__linksURL; - this._removePrefsObserver(); - this._removeObservers(); - }, - - addObserver: function DirectoryLinksProvider_addObserver(aObserver) { - this._observers.add(aObserver); - }, - - removeObserver: function DirectoryLinksProvider_removeObserver(aObserver) { - this._observers.delete(aObserver); - }, - - _callObservers(methodName, ...args) { - for (let obs of this._observers) { - if (typeof(obs[methodName]) == "function") { - try { - obs[methodName](this, ...args); - } catch (err) { - Cu.reportError(err); - } - } - } - }, - - _removeObservers() { - this._observers.clear(); - } -}; |