aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--bg/ResponseMetaData.js49
-rw-r--r--bg/ResponseProcessor.js6
2 files changed, 47 insertions, 8 deletions
diff --git a/bg/ResponseMetaData.js b/bg/ResponseMetaData.js
index 5a75a3c..0f768fe 100644
--- a/bg/ResponseMetaData.js
+++ b/bg/ResponseMetaData.js
@@ -25,6 +25,9 @@
to parse textual data through a decoder.
*/
+const BOM = [0xEF, 0xBB, 0xBF];
+const DECODER_PARAMS = {stream: true};
+
class ResponseMetaData {
constructor(request) {
let {responseHeaders} = request;
@@ -37,6 +40,7 @@ class ResponseMetaData {
this.headers[propertyName] = h;
}
}
+ this.computedCharset = "";
}
get charset() {
@@ -48,19 +52,54 @@ class ResponseMetaData {
}
}
Object.defineProperty(this, "charset", { value: charset, writable: false, configurable: true });
- return charset;
+ return this.computedCharset = charset;
+ }
+
+ decode(data) {
+ let charset = this.charset;
+ let decoder = this.createDecoder();
+ let text = decoder.decode(data, DECODER_PARAMS);
+ if (!charset && /html/i.test(this.contentType)) {
+ // missing HTTP charset, sniffing in content...
+
+ if (data[0] === BOM[0] && data[1] === BOM[1] && data[2] === BOM[2]) {
+ // forced UTF-8, nothing to do
+ return text;
+ }
+
+ // let's try figuring out the charset from <meta> tags
+ let parser = new DOMParser();
+ let doc = parser.parseFromString(text, "text/html");
+ let meta = doc.querySelectorAll('meta[charset], meta[http-equiv="content-type"], meta[content*="charset"]');
+ for (let m of meta) {
+ charset = m.getAttribute("charset");
+ if (!charset) {
+ let match = m.getAttribute("content").match(/;\s*charset\s*=\s*([\w-]+)/i)
+ if (match) charset = match[1];
+ }
+ if (charset) {
+ decoder = this.createDecoder(charset, null);
+ if (decoder) {
+ this.computedCharset = charset;
+ return decoder.decode(data, DECODER_PARAMS);
+ }
+ }
+ }
+ }
+ return text;
}
- createDecoder() {
- if (this.charset) {
+ createDecoder(charset = this.charset, def = "latin1") {
+ if (charset) {
try {
- return new TextDecoder(this.charset);
+ return new TextDecoder(charset);
} catch (e) {
console.error(e);
}
}
- return new TextDecoder("latin1");
+ return def ? new TextDecoder(def) : null;
}
};
+ResponseMetaData.UTF8BOM = new Uint8Array(BOM);
module.exports = { ResponseMetaData };
diff --git a/bg/ResponseProcessor.js b/bg/ResponseProcessor.js
index d10d46d..1aa89de 100644
--- a/bg/ResponseProcessor.js
+++ b/bg/ResponseProcessor.js
@@ -106,10 +106,10 @@ class ResponseTextFilter {
response.text = await (await fetch(request.url, {cache: "reload", credentials: "include"})).text();
} else {
console.debug("It's a %s, trying to decode it as UTF-16.", request.type);
- response.text = new TextDecoder("utf-16be").decode(allBytes);
+ response.text = new TextDecoder("utf-16be").decode(allBytes, {stream: true});
}
} else {
- response.text = metaData.createDecoder().decode(allBytes, {stream: true});
+ response.text = metaData.decode(allBytes);
}
let editedText = null;
try {
@@ -122,7 +122,7 @@ class ResponseTextFilter {
let encoded = new TextEncoder().encode(editedText);
// pre-pending the UTF-8 BOM will force the charset per HTML 5 specs
allBytes = new Uint8Array(encoded.byteLength + 3);
- allBytes.set(new Uint8Array([0xEF, 0xBB, 0xBF]), 0); // UTF-8 BOM
+ allBytes.set(ResponseMetaData.UTF8BOM, 0); // UTF-8 BOM
allBytes.set(encoded, 3);
}
filter.write(allBytes);