Sniffing of charset in <meta> tags and BOM.

2 files changed, 47 insertions, 8 deletions

diff --git a/bg/ResponseMetaData.js b/bg/ResponseMetaData.js
index 5a75a3c..0f768fe 100644
--- a/bg/ResponseMetaData.js
+++ b/bg/ResponseMetaData.js
@@ -25,6 +25,9 @@
   to parse textual data through a decoder.
 */
 
+const BOM = [0xEF, 0xBB, 0xBF];
+const DECODER_PARAMS = {stream: true};
+
 class ResponseMetaData {
   constructor(request) {
     let {responseHeaders} = request;
@@ -37,6 +40,7 @@ class ResponseMetaData {
         this.headers[propertyName] = h;
       }
     }
+    this.computedCharset = "";
   }
 
   get charset() {
@@ -48,19 +52,54 @@ class ResponseMetaData {
       }
     }
     Object.defineProperty(this, "charset", { value: charset, writable: false, configurable: true });
-    return charset;
+    return this.computedCharset = charset;
+  }
+
+  decode(data) {
+    let charset = this.charset;
+    let decoder = this.createDecoder();
+    let text = decoder.decode(data, DECODER_PARAMS);
+    if (!charset && /html/i.test(this.contentType)) {
+      // missing HTTP charset, sniffing in content...
+
+      if (data[0] === BOM[0] && data[1] === BOM[1] && data[2] === BOM[2]) {
+        // forced UTF-8, nothing to do
+        return text;
+      }
+
+      // let's try figuring out the charset from <meta> tags
+      let parser = new DOMParser();
+      let doc = parser.parseFromString(text, "text/html");
+      let meta = doc.querySelectorAll('meta[charset], meta[http-equiv="content-type"], meta[content*="charset"]');
+      for (let m of meta) {
+        charset = m.getAttribute("charset");
+        if (!charset) {
+          let match = m.getAttribute("content").match(/;\s*charset\s*=\s*([\w-]+)/i)
+          if (match) charset = match[1];
+        }
+        if (charset) {
+          decoder = this.createDecoder(charset, null);
+          if (decoder) {
+            this.computedCharset = charset;
+            return decoder.decode(data, DECODER_PARAMS);
+          }
+        }
+      }
+    }
+    return text;
   }
 
-  createDecoder() {
-    if (this.charset) {
+  createDecoder(charset = this.charset, def = "latin1") {
+    if (charset) {
       try {
-        return new TextDecoder(this.charset);
+        return new TextDecoder(charset);
       } catch (e) {
         console.error(e);
       }
     }
-    return new TextDecoder("latin1");
+    return def ? new TextDecoder(def) : null;
   }
 };
+ResponseMetaData.UTF8BOM = new Uint8Array(BOM);
 
 module.exports = { ResponseMetaData };
diff --git a/bg/ResponseProcessor.js b/bg/ResponseProcessor.js
index d10d46d..1aa89de 100644
--- a/bg/ResponseProcessor.js
+++ b/bg/ResponseProcessor.js
@@ -106,10 +106,10 @@ class ResponseTextFilter {
           response.text = await (await fetch(request.url, {cache: "reload", credentials: "include"})).text();
         } else {
           console.debug("It's a %s, trying to decode it as UTF-16.", request.type);
-          response.text = new TextDecoder("utf-16be").decode(allBytes);
+          response.text = new TextDecoder("utf-16be").decode(allBytes, {stream: true});
         }
       } else {
-        response.text = metaData.createDecoder().decode(allBytes, {stream: true});
+        response.text = metaData.decode(allBytes);
       }
       let editedText = null;
       try {
@@ -122,7 +122,7 @@ class ResponseTextFilter {
         let encoded = new TextEncoder().encode(editedText);
         // pre-pending the UTF-8 BOM will force the charset per HTML 5 specs
         allBytes = new Uint8Array(encoded.byteLength + 3);
-        allBytes.set(new Uint8Array([0xEF, 0xBB, 0xBF]), 0); // UTF-8 BOM
+        allBytes.set(ResponseMetaData.UTF8BOM, 0); // UTF-8 BOM
         allBytes.set(encoded, 3);
       }
       filter.write(allBytes);