2 files changed, 59 insertions, 21 deletions

diff --git a/bg/ExternalLicenses.js b/bg/ExternalLicenses.js
index 27d9223..33bcbf6 100644
--- a/bg/ExternalLicenses.js
+++ b/bg/ExternalLicenses.js
@@ -26,10 +26,28 @@
 "use strict";
 
 let licensesByLabel = new Map();
+let licensesByUrl = new Map();
 {
   let {licenses} = require("../license_definitions");
-  for (let l of Object.values(licenses).filter(l => l.identifier)) {
-    licensesByLabel.set(l.identifier, l);
+  let mapByLabel = (label, license) => licensesByLabel.set(label.toUpperCase(), license);
+  for (let [id, l] of Object.entries(licenses)) {
+    let {identifier, canonicalUrl, licenseName} = l;
+    if (identifier) {
+      mapByLabel(identifier, l);
+    } else {
+      l.identifier = id;
+    }
+    if (id !== identifier) {
+      mapByLabel(id, l);
+    }
+    if (licenseName) {
+      mapByLabel(licenseName, l);
+    }
+    if (Array.isArray(canonicalUrl)) {
+      for (let url of canonicalUrl) {
+        licensesByUrl.set(url, l);
+      }
+    }
   }
 }
 
@@ -55,23 +73,26 @@ var ExternalLicenses = {
       return null;
     }
     scriptInfo.licenses = new Set();
-    scriptInfo.allFree = true;
     scriptInfo.toString = function() {
       let licenseIds = [...this.licenses].map(l => l.identifier).sort().join(", ");
       return licenseIds
-         ? (this.allFree ? `Free license${this.licenses.length > 1 ? "s" : ""} (${licenseIds})`
-                         : `Mixed free (${licenseIds}) and unknown licenses`)
+         ? `Free license${this.licenses.size > 1 ? "s" : ""} (${licenseIds})`
          : "Unknown license(s)";
     }
-
-    for (let {label} of scriptInfo.licenseLinks) {
-      if (licensesByLabel.has(label)) {
-        scriptInfo.licenses.add(licensesByLabel.get(label));
-      } else {
-        scriptInfo.allFree = false;
-        break;
+    let match = (map, key) => {
+      if (map.has(key)) {
+        scriptInfo.licenses.add(map.get(key));
+        return true;
       }
+      return false;
+    };
+
+    for (let {label, url} of scriptInfo.licenseLinks) {
+      match(licensesByLabel, label = label.trim().toUpperCase()) ||
+        match(licensesByUrl, url) ||
+        match(licensesByLabel, label.replace(/^GNU-|-(?:OR-LATER|ONLY)$/, ''));
     }
+    scriptInfo.free = scriptInfo.licenses.size > 0;
     return scriptInfo;
   },
 
diff --git a/bg/ResponseProcessor.js b/bg/ResponseProcessor.js
index 1af8cd7..4443d90 100644
--- a/bg/ResponseProcessor.js
+++ b/bg/ResponseProcessor.js
@@ -33,7 +33,7 @@ class ResponseProcessor {
 
   static install(handler, types = ["main_frame", "sub_frame", "script"]) {
     if (listeners.has(handler)) return false;
-    let listener = 
+    let listener =
       async request =>  await new ResponseTextFilter(request).process(handler);
     listeners.set(handler, listener);
     webRequestEvent.addListener(
@@ -80,7 +80,7 @@ class ResponseTextFilter {
       if (handler.post) handler = handler.post;
       if (typeof handler !== "function") ResponseProcessor.ACCEPT;
     }
-    
+
     let {requestId, responseHeaders} = request;
     let filter = browser.webRequest.filterResponseData(requestId);
     let buffer = [];
@@ -90,11 +90,29 @@ class ResponseTextFilter {
     };
 
     filter.onstop = async event => {
-      let decoder = metaData.createDecoder();
+
       let params = {stream: true};
-      response.text = buffer.map(
-        chunk => decoder.decode(chunk, params))
-        .join('');
+      // concatenate chunks
+      let size = buffer.reduce((sum, chunk, n) => sum + chunk.byteLength, 0)
+      let allBytes = new Uint8Array(size);
+      let pos = 0;
+      for (let chunk of buffer) {
+        allBytes.set(new Uint8Array(chunk), pos);
+        pos += chunk.byteLength;
+      }
+      buffer = null; // allow garbage collection
+      if (allBytes.indexOf(0) !== -1) {
+        console.debug("Warning: zeroes in bytestream, probable cached encoding mismatch.", request);
+        if (request.type === "script") {
+          console.debug("It's a script, trying to refetch it.");
+          response.text = await (await fetch(request.url, {cache: "reload", credentials: "include"})).text();
+        } else {
+          console.debug("It's a %s, trying to decode it as UTF-16.", request.type);
+          response.text = new TextDecoder("utf-16be").decode(allBytes);
+        }
+      } else {
+        response.text = metaData.createDecoder().decode(allBytes, {stream: true});
+      }
       let editedText = null;
       try {
         editedText = await handler(response);
@@ -108,10 +126,9 @@ class ResponseTextFilter {
         filter.write(new TextEncoder().encode(editedText));
       } else {
         // ... otherwise pass all the raw bytes through
-        for (let chunk of buffer) filter.write(chunk);
+        filter.write(allBytes);
       }
-
-      filter.disconnect();
+      filter.close();
     }
 
     return metaData.forceUTF8() ? {responseHeaders} : ResponseProcessor.ACCEPT;;