From 9884643742b54b88f94111d62f5e89a12a60e604 Mon Sep 17 00:00:00 2001 From: hackademix Date: Tue, 26 Mar 2019 22:45:37 +0100 Subject: Subdomain wildcard support. --- main_background.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'main_background.js') diff --git a/main_background.js b/main_background.js index 8060ebd..f1c7f09 100644 --- a/main_background.js +++ b/main_background.js @@ -326,7 +326,7 @@ async function connected(p) { if (m[action]) { let [key] = m[action]; if (m.site) { - key = ListStore.siteItem(key); + key = ListStore.siteItem(m.site); } else { key = ListStore.inlineItem(key) || key; } @@ -745,12 +745,12 @@ async function get_script(response, url, tabId = -1, whitelisted = false, index let scriptName = url.split("/").pop(); if (whitelisted) { if (tabId !== -1) { - let site = ListStore.siteItem(url); + let site = ListManager.siteMatch(url, whitelist); // Accept without reading script, it was explicitly whitelisted - let reason = whitelist.contains(site) + let reason = site ? `All ${site} whitelisted by user` : "Address whitelisted by user"; - addReportEntry(tabId, url, {"whitelisted": [url, reason], url}); + addReportEntry(tabId, url, {"whitelisted": [site, reason], url}); } if (response.startsWith("javascript:")) return result(response); @@ -855,7 +855,7 @@ var ResponseHandler = { url = ListStore.urlItem(url); let site = ListStore.siteItem(url); - let blacklistedSite = blacklist.contains(site); + let blacklistedSite = ListManager.siteMatch(site, blacklist); let blacklisted = blacklistedSite || blacklist.contains(url); let topUrl = request.frameAncestors && request.frameAncestors.pop() || documentUrl; @@ -863,7 +863,7 @@ var ResponseHandler = { if (type === "script") { // abort the request before the response gets fetched addReportEntry(tabId, url, {url: topUrl, - "blacklisted": [url, blacklistedSite ? `User blacklisted ${site}` : "Blacklisted by user"]}); + "blacklisted": [url, blacklistedSite ? `User blacklisted ${blacklistedSite}` : "Blacklisted by user"]}); return ResponseProcessor.REJECT; } // use CSP to restrict JavaScript execution in the page @@ -872,13 +872,13 @@ var ResponseHandler = { value: `script-src '${blacklistedSite ? 'self' : 'none'}';` }); } else { - let whitelistedSite = whitelist.contains(site); + let whitelistedSite = ListManager.siteMatch(site, whitelist); let whitelisted = response.whitelisted = whitelistedSite || whitelist.contains(url); if (type === "script") { if (whitelisted) { // accept the script and stop processing addReportEntry(tabId, url, {url: topUrl, - "whitelisted": [url, whitelistedSite ? `User whitelisted ${site}` : "Whitelisted by user"]}); + "whitelisted": [url, whitelistedSite ? `User whitelisted ${whitelistedSite}` : "Whitelisted by user"]}); return ResponseProcessor.ACCEPT; } else { let scriptInfo = await ExternalLicenses.check({url: fullUrl, tabId, frameId, documentUrl}); -- cgit v1.2.3 From a99f7d91acf0fe82a8c82440c28ffc5fe379924a Mon Sep 17 00:00:00 2001 From: hackademix Date: Tue, 26 Mar 2019 22:47:00 +0100 Subject: Automated regression tests for whitelist and blacklist management, including wildcards. --- main_background.js | 1 + test/spec/LibreJSSpec.js | 40 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 1 deletion(-) (limited to 'main_background.js') diff --git a/main_background.js b/main_background.js index f1c7f09..5943fb8 100644 --- a/main_background.js +++ b/main_background.js @@ -1208,6 +1208,7 @@ async function init_addon() { editHtml, handle_script, ExternalLicenses, + ListManager, ListStore, Storage, }; // create or focus the autotest tab if it's a debugging session if ((await browser.management.getSelf()).installType === "development") { diff --git a/test/spec/LibreJSSpec.js b/test/spec/LibreJSSpec.js index 57c7f65..43f37a7 100644 --- a/test/spec/LibreJSSpec.js +++ b/test/spec/LibreJSSpec.js @@ -38,15 +38,53 @@ describe("LibreJS' components", () => { let licensed = `// @license ${license.magnet} ${license.id}\n${nontrivial}\n// @license-end`; let unknownLicensed = `// @license ${unknownLicense.magnet} ${unknownLicense.id}\n${nontrivial}\n// @license-end`; let malformedLicensed = `// @license\n${nontrivial}`; - let tab, documentUrl; beforeAll(async () => { let url = browser.extension.getURL("/test/resources/index.html"); tab = (await browser.tabs.query({url}))[0] || (await browser.tabs.create({url})); documentUrl = url; + }); + describe("The whitelist/blacklist manager", () => { + let {ListManager, ListStore, Storage} = LibreJS; + let lm = new ListManager(new ListStore("_test.whitelist", Storage.CSV), new ListStore("_test.blacklist", Storage.CSV), new Set()); + let forgot = ["http://formerly.whitelist.ed/", "http://formerly.blacklist.ed/"]; + + beforeAll(async () => { + await lm.whitelist("https://fsf.org/*", "https://*.gnu.org/*", forgot[0]); + await lm.blacklist("https://*.evil.gnu.org/*", "https://verybad.com/*", forgot[1]); + }); + + it("Should handle basic CRUD operations", async () => { + expect(lm.getStatus(forgot[0])).toBe("whitelisted"); + expect(lm.getStatus(forgot[1])).toBe("blacklisted"); + + await lm.forget(...forgot); + + for (let url of forgot) { + expect(lm.getStatus(url)).toBe("unknown"); + } + }); + + it("Should support full path wildcards", () => { + expect(lm.getStatus("https://unknown.org")).toBe("unknown"); + expect(lm.getStatus("https://fsf.org/some/path")).toBe("whitelisted"); + expect(lm.getStatus("https://fsf.org/")).toBe("whitelisted"); + expect(lm.getStatus("https://fsf.org")).toBe("whitelisted"); + expect(lm.getStatus("https://subdomain.fsf.org")).toBe("unknown"); + expect(lm.getStatus("https://verybad.com/some/other/path?with=querystring")).toBe("blacklisted"); + }); + it("Should support subdomain wildcards", () => { + expect(lm.getStatus("https://gnu.org")).toBe("whitelisted"); + expect(lm.getStatus("https://www.gnu.org")).toBe("whitelisted"); + expect(lm.getStatus("https://evil.gnu.org")).toBe("blacklisted"); + expect(lm.getStatus("https://more.evil.gnu.org")).toBe("blacklisted"); + expect(lm.getStatus("https://more.evil.gnu.org/some/evil/path?too")).toBe("blacklisted"); + }); + }) + describe("The external script source processor", () => { let url = "https://www.gnu.org/mock-script.js"; -- cgit v1.2.3 From 2bee3260b45b559eb3bc75c6421dc7e573571cdb Mon Sep 17 00:00:00 2001 From: hackademix Date: Mon, 8 Apr 2019 21:37:35 +0200 Subject: Fixed UI inconsistencies when whitelisting/blacklisting through wide wildcard matching. --- html/display_panel/content/display-panel.html | 1 + html/display_panel/content/main_panel.js | 13 +++++++++++-- html/display_panel/content/panel-styles.css | 7 +++++-- main_background.js | 6 +++++- 4 files changed, 22 insertions(+), 5 deletions(-) (limited to 'main_background.js') diff --git a/html/display_panel/content/display-panel.html b/html/display_panel/content/display-panel.html index 5b9f99f..2e705ab 100644 --- a/html/display_panel/content/display-panel.html +++ b/html/display_panel/content/display-panel.html @@ -60,6 +60,7 @@

This whole site

+
diff --git a/html/display_panel/content/main_panel.js b/html/display_panel/content/main_panel.js index 9172265..75de6aa 100644 --- a/html/display_panel/content/main_panel.js +++ b/html/display_panel/content/main_panel.js @@ -173,8 +173,8 @@ function createList(data, group){ */ function refreshUI(report) { currentReport = report; - - document.querySelector("#site").className = report.siteStatus || ""; + let {siteStatus, listedSite} = report; + document.querySelector("#site").className = siteStatus || ""; document.querySelector("#site h2").textContent = `This site ${report.site}`; @@ -198,6 +198,15 @@ function refreshUI(report) { b.disabled = true; } + if (siteStatus && siteStatus !== "unknown") { + let statusLabel = siteStatus; + if (listedSite && listedSite !== report.site) statusLabel += ` via ${listedSite}`; + let status = document.querySelector("#site .status"); + status.classList.add(siteStatus); + document.querySelector("#site .status").textContent = statusLabel; + document.querySelector("#site .forget").disabled = true; + } + let noscript = scriptsCount === 0; document.body.classList.toggle("empty", noscript); } diff --git a/html/display_panel/content/panel-styles.css b/html/display_panel/content/panel-styles.css index 502323f..3257192 100644 --- a/html/display_panel/content/panel-styles.css +++ b/html/display_panel/content/panel-styles.css @@ -94,11 +94,14 @@ ul { display: initial; } +.status { + margin: .2em; +} -button.whitelist { +button.whitelist, .status.whitelisted { color: #080; } -button.blacklist { +button.blacklist, .status.blacklisted { color: #800; } button.forget { diff --git a/main_background.js b/main_background.js index 5943fb8..ba88f15 100644 --- a/main_background.js +++ b/main_background.js @@ -134,6 +134,10 @@ async function createReport(initializer) { template.url = url; template.site = ListStore.siteItem(url); template.siteStatus = listManager.getStatus(template.site); + let list = {"whitelisted": whitelist, "blacklisted": blacklist}[template.siteStatus]; + if (list) { + template.listedSite = ListManager.siteMatch(template.site, list); + } return template; } @@ -750,7 +754,7 @@ async function get_script(response, url, tabId = -1, whitelisted = false, index let reason = site ? `All ${site} whitelisted by user` : "Address whitelisted by user"; - addReportEntry(tabId, url, {"whitelisted": [site, reason], url}); + addReportEntry(tabId, url, {"whitelisted": [site || url, reason], url}); } if (response.startsWith("javascript:")) return result(response); -- cgit v1.2.3 From 0aacfa705e307286aa87861de5442785b1b51f65 Mon Sep 17 00:00:00 2001 From: hackademix Date: Fri, 12 Apr 2019 01:26:11 +0200 Subject: More consistent and efficient blacklisting. --- bg/ResponseProcessor.js | 2 +- html/display_panel/content/main_panel.js | 15 ++++-- main_background.js | 79 +++++++++++++++++--------------- 3 files changed, 52 insertions(+), 44 deletions(-) (limited to 'main_background.js') diff --git a/bg/ResponseProcessor.js b/bg/ResponseProcessor.js index 1aa89de..2c613c9 100644 --- a/bg/ResponseProcessor.js +++ b/bg/ResponseProcessor.js @@ -78,7 +78,7 @@ class ResponseTextFilter { let res = await handler.pre(response); if (res) return res; if (handler.post) handler = handler.post; - if (typeof handler !== "function") ResponseProcessor.ACCEPT; + if (typeof handler !== "function") return ResponseProcessor.ACCEPT; } let {requestId, responseHeaders} = request; diff --git a/html/display_panel/content/main_panel.js b/html/display_panel/content/main_panel.js index 75de6aa..32d1bda 100644 --- a/html/display_panel/content/main_panel.js +++ b/html/display_panel/content/main_panel.js @@ -59,7 +59,7 @@ document.querySelector("#info").addEventListener("click", e => { setTimeout(close, 100); return; } - if (!button.tagName === "BUTTON") button = button.closest("button"); + if (button.tagName !== "BUTTON") button = button.closest("button"); if (button.matches(".toggle-source")) { let parent = button.parentNode; if (!parent.querySelector(".source").textContent) { @@ -199,12 +199,17 @@ function refreshUI(report) { } if (siteStatus && siteStatus !== "unknown") { + let siteContainer = document.querySelector("#site"); let statusLabel = siteStatus; - if (listedSite && listedSite !== report.site) statusLabel += ` via ${listedSite}`; - let status = document.querySelector("#site .status"); + if (listedSite && listedSite !== report.site) { + statusLabel += ` via ${listedSite}`; + siteContainer.querySelector(".forget").disabled = true; + } + let status = siteContainer.querySelector(".status"); status.classList.add(siteStatus); - document.querySelector("#site .status").textContent = statusLabel; - document.querySelector("#site .forget").disabled = true; + status.textContent = statusLabel; + } else { + document.querySelector("#site .status").textContent = ""; } let noscript = scriptsCount === 0; diff --git a/main_background.js b/main_background.js index ba88f15..f9e7521 100644 --- a/main_background.js +++ b/main_background.js @@ -805,42 +805,28 @@ function updateBadge(tabId, report = null, forceRed = false) { } } -/** -* Tests if a request is google analytics or not -*/ -function test_GA(a){ // TODO: DRY me - // This is just an HTML page - if(a.url == 'https://www.google.com/analytics/#?modal_active=none'){ - return false; - } - else if(a.url.match(/https:\/\/www\.google\.com\/analytics\//g)){ - dbg_print("%c Google analytics (1)","color:red"); - return {cancel: true}; - } - else if(a.url == 'https://www.google-analytics.com/analytics.js'){ - dbg_print("%c Google analytics (2)","color:red"); - return {cancel: true}; +function blockGoogleAnalytics(request) { + let {url} = request; + let res = {}; + if (url === 'https://www.google-analytics.com/analytics.js' || + /^https:\/\/www\.google\.com\/analytics\/[^#]/.test(url) + ) { + res.cancel = true; } - else if(a.url == 'https://www.google.com/analytics/js/analytics.min.js'){ - dbg_print("%c Google analytics (3)","color:red"); - return {cancel: true}; - } - else return false; + return res; } -/** -* A callback that every type of request invokes. -*/ -function block_ga(a){ - var GA = test_GA(a); - if(GA != false){ - return GA; - } - else return {}; +async function blockBlacklistedScripts(request) { + let {url, tabId, documentUrl} = request; + url = ListStore.urlItem(url); + let status = listManager.getStatus(url); + if (status !== "blacklisted") return {}; + let blacklistedSite = ListManager.siteMatch(url, blacklist); + await addReportEntry(tabId, url, {url: documentUrl, + "blacklisted": [url, /\*/.test(blacklistedSite) ? `User blacklisted ${blacklistedSite}` : "Blacklisted by user"]}); + return {cancel: true}; } - - /** * This listener gets called as soon as we've got all the HTTP headers, can guess * content type and encoding, and therefore correctly parse HTML documents @@ -861,20 +847,27 @@ var ResponseHandler = { let blacklistedSite = ListManager.siteMatch(site, blacklist); let blacklisted = blacklistedSite || blacklist.contains(url); - let topUrl = request.frameAncestors && request.frameAncestors.pop() || documentUrl; + let topUrl = type === "sub_frame" && request.frameAncestors && request.frameAncestors.pop() || documentUrl; if (blacklisted) { if (type === "script") { - // abort the request before the response gets fetched - addReportEntry(tabId, url, {url: topUrl, - "blacklisted": [url, blacklistedSite ? `User blacklisted ${blacklistedSite}` : "Blacklisted by user"]}); + // this shouldn't happen, because we intercept earlier in blockBlacklistedScripts() return ResponseProcessor.REJECT; } + if (type === "main_frame") { // we handle the page change here too, since we won't call edit_html() + activityReports[tabId] = await createReport({url: fullUrl, tabId}); + // Go on without parsing the page: it was explicitly blacklisted + let reason = blacklistedSite + ? `All ${blacklistedSite} blacklisted by user` + : "Address blacklisted by user"; + await addReportEntry(tabId, url, {"blacklisted": [blacklistedSite || url, reason], url: fullUrl}); + } // use CSP to restrict JavaScript execution in the page request.responseHeaders.unshift({ name: `Content-security-policy`, - value: `script-src '${blacklistedSite ? 'self' : 'none'}';` + value: `script-src 'none';` }); + return {responseHeaders: request.responseHeaders}; // let's skip the inline script parsing, since we block by CSP } else { let whitelistedSite = ListManager.siteMatch(site, whitelist); let whitelisted = response.whitelisted = whitelistedSite || whitelist.contains(url); @@ -1193,11 +1186,21 @@ async function init_addon() { "web_manifest", "websocket", "xbl", "xml_dtd", "xmlhttprequest", "xslt", "other" ]; - browser.webRequest.onBeforeRequest.addListener( - block_ga, + browser.webRequest.onBeforeRequest.addListener(blockGoogleAnalytics, {urls: [""], types: all_types}, ["blocking"] ); + browser.webRequest.onBeforeRequest.addListener(blockBlacklistedScripts, + {urls: [""], types: ["script"]}, + ["blocking"] + ); + browser.webRequest.onResponseStarted.addListener(request => { + let {tabId} = request; + let report = activityReports[tabId]; + if (report) { + updateBadge(tabId, activityReports[tabId]); + } + }, {urls: [""], types: ["main_frame"]}); // Analyzes all the html documents and external scripts as they're loaded ResponseProcessor.install(ResponseHandler); -- cgit v1.2.3