diff options
| -rw-r--r-- | bg/ResponseMetaData.js | 49 | ||||
| -rw-r--r-- | bg/ResponseProcessor.js | 6 |
2 files changed, 47 insertions, 8 deletions
diff --git a/bg/ResponseMetaData.js b/bg/ResponseMetaData.js index 5a75a3c..0f768fe 100644 --- a/bg/ResponseMetaData.js +++ b/bg/ResponseMetaData.js @@ -25,6 +25,9 @@ to parse textual data through a decoder. */ +const BOM = [0xEF, 0xBB, 0xBF]; +const DECODER_PARAMS = {stream: true}; + class ResponseMetaData { constructor(request) { let {responseHeaders} = request; @@ -37,6 +40,7 @@ class ResponseMetaData { this.headers[propertyName] = h; } } + this.computedCharset = ""; } get charset() { @@ -48,19 +52,54 @@ class ResponseMetaData { } } Object.defineProperty(this, "charset", { value: charset, writable: false, configurable: true }); - return charset; + return this.computedCharset = charset; + } + + decode(data) { + let charset = this.charset; + let decoder = this.createDecoder(); + let text = decoder.decode(data, DECODER_PARAMS); + if (!charset && /html/i.test(this.contentType)) { + // missing HTTP charset, sniffing in content... + + if (data[0] === BOM[0] && data[1] === BOM[1] && data[2] === BOM[2]) { + // forced UTF-8, nothing to do + return text; + } + + // let's try figuring out the charset from <meta> tags + let parser = new DOMParser(); + let doc = parser.parseFromString(text, "text/html"); + let meta = doc.querySelectorAll('meta[charset], meta[http-equiv="content-type"], meta[content*="charset"]'); + for (let m of meta) { + charset = m.getAttribute("charset"); + if (!charset) { + let match = m.getAttribute("content").match(/;\s*charset\s*=\s*([\w-]+)/i) + if (match) charset = match[1]; + } + if (charset) { + decoder = this.createDecoder(charset, null); + if (decoder) { + this.computedCharset = charset; + return decoder.decode(data, DECODER_PARAMS); + } + } + } + } + return text; } - createDecoder() { - if (this.charset) { + createDecoder(charset = this.charset, def = "latin1") { + if (charset) { try { - return new TextDecoder(this.charset); + return new TextDecoder(charset); } catch (e) { console.error(e); } } - return new TextDecoder("latin1"); + return def ? new TextDecoder(def) : null; } }; +ResponseMetaData.UTF8BOM = new Uint8Array(BOM); module.exports = { ResponseMetaData }; diff --git a/bg/ResponseProcessor.js b/bg/ResponseProcessor.js index d10d46d..1aa89de 100644 --- a/bg/ResponseProcessor.js +++ b/bg/ResponseProcessor.js @@ -106,10 +106,10 @@ class ResponseTextFilter { response.text = await (await fetch(request.url, {cache: "reload", credentials: "include"})).text(); } else { console.debug("It's a %s, trying to decode it as UTF-16.", request.type); - response.text = new TextDecoder("utf-16be").decode(allBytes); + response.text = new TextDecoder("utf-16be").decode(allBytes, {stream: true}); } } else { - response.text = metaData.createDecoder().decode(allBytes, {stream: true}); + response.text = metaData.decode(allBytes); } let editedText = null; try { @@ -122,7 +122,7 @@ class ResponseTextFilter { let encoded = new TextEncoder().encode(editedText); // pre-pending the UTF-8 BOM will force the charset per HTML 5 specs allBytes = new Uint8Array(encoded.byteLength + 3); - allBytes.set(new Uint8Array([0xEF, 0xBB, 0xBF]), 0); // UTF-8 BOM + allBytes.set(ResponseMetaData.UTF8BOM, 0); // UTF-8 BOM allBytes.set(encoded, 3); } filter.write(allBytes); |