1
0
mirror of https://github.com/S2-/minifyfromhtml.git synced 2025-08-03 12:20:04 +02:00

update packages to latest version

This commit is contained in:
s2
2022-08-20 18:51:33 +02:00
parent 09663a35a5
commit 806ebf9a57
4513 changed files with 366205 additions and 92512 deletions

View File

@@ -49,6 +49,48 @@ function DBCSCodec(codecOptions, iconv) {
for (var i = 0; i < mappingTable.length; i++)
this._addDecodeChunk(mappingTable[i]);
// Load & create GB18030 tables when needed.
if (typeof codecOptions.gb18030 === 'function') {
this.gb18030 = codecOptions.gb18030(); // Load GB18030 ranges.
// Add GB18030 common decode nodes.
var commonThirdByteNodeIdx = this.decodeTables.length;
this.decodeTables.push(UNASSIGNED_NODE.slice(0));
var commonFourthByteNodeIdx = this.decodeTables.length;
this.decodeTables.push(UNASSIGNED_NODE.slice(0));
// Fill out the tree
var firstByteNode = this.decodeTables[0];
for (var i = 0x81; i <= 0xFE; i++) {
var secondByteNode = this.decodeTables[NODE_START - firstByteNode[i]];
for (var j = 0x30; j <= 0x39; j++) {
if (secondByteNode[j] === UNASSIGNED) {
secondByteNode[j] = NODE_START - commonThirdByteNodeIdx;
} else if (secondByteNode[j] > NODE_START) {
throw new Error("gb18030 decode tables conflict at byte 2");
}
var thirdByteNode = this.decodeTables[NODE_START - secondByteNode[j]];
for (var k = 0x81; k <= 0xFE; k++) {
if (thirdByteNode[k] === UNASSIGNED) {
thirdByteNode[k] = NODE_START - commonFourthByteNodeIdx;
} else if (thirdByteNode[k] === NODE_START - commonFourthByteNodeIdx) {
continue;
} else if (thirdByteNode[k] > NODE_START) {
throw new Error("gb18030 decode tables conflict at byte 3");
}
var fourthByteNode = this.decodeTables[NODE_START - thirdByteNode[k]];
for (var l = 0x30; l <= 0x39; l++) {
if (fourthByteNode[l] === UNASSIGNED)
fourthByteNode[l] = GB18030_CODE;
}
}
}
}
}
this.defaultCharUnicode = iconv.defaultCharUnicode;
@@ -92,30 +134,6 @@ function DBCSCodec(codecOptions, iconv) {
this.defCharSB = this.encodeTable[0][iconv.defaultCharSingleByte.charCodeAt(0)];
if (this.defCharSB === UNASSIGNED) this.defCharSB = this.encodeTable[0]['?'];
if (this.defCharSB === UNASSIGNED) this.defCharSB = "?".charCodeAt(0);
// Load & create GB18030 tables when needed.
if (typeof codecOptions.gb18030 === 'function') {
this.gb18030 = codecOptions.gb18030(); // Load GB18030 ranges.
// Add GB18030 decode tables.
var thirdByteNodeIdx = this.decodeTables.length;
var thirdByteNode = this.decodeTables[thirdByteNodeIdx] = UNASSIGNED_NODE.slice(0);
var fourthByteNodeIdx = this.decodeTables.length;
var fourthByteNode = this.decodeTables[fourthByteNodeIdx] = UNASSIGNED_NODE.slice(0);
for (var i = 0x81; i <= 0xFE; i++) {
var secondByteNodeIdx = NODE_START - this.decodeTables[0][i];
var secondByteNode = this.decodeTables[secondByteNodeIdx];
for (var j = 0x30; j <= 0x39; j++)
secondByteNode[j] = NODE_START - thirdByteNodeIdx;
}
for (var i = 0x81; i <= 0xFE; i++)
thirdByteNode[i] = NODE_START - fourthByteNodeIdx;
for (var i = 0x30; i <= 0x39; i++)
fourthByteNode[i] = GB18030_CODE
}
}
DBCSCodec.prototype.encoder = DBCSEncoder;
@@ -124,7 +142,7 @@ DBCSCodec.prototype.decoder = DBCSDecoder;
// Decoder helpers
DBCSCodec.prototype._getDecodeTrieNode = function(addr) {
var bytes = [];
for (; addr > 0; addr >>= 8)
for (; addr > 0; addr >>>= 8)
bytes.push(addr & 0xFF);
if (bytes.length == 0)
bytes.push(0);
@@ -249,19 +267,32 @@ DBCSCodec.prototype._setEncodeSequence = function(seq, dbcsCode) {
DBCSCodec.prototype._fillEncodeTable = function(nodeIdx, prefix, skipEncodeChars) {
var node = this.decodeTables[nodeIdx];
var hasValues = false;
var subNodeEmpty = {};
for (var i = 0; i < 0x100; i++) {
var uCode = node[i];
var mbCode = prefix + i;
if (skipEncodeChars[mbCode])
continue;
if (uCode >= 0)
if (uCode >= 0) {
this._setEncodeChar(uCode, mbCode);
else if (uCode <= NODE_START)
this._fillEncodeTable(NODE_START - uCode, mbCode << 8, skipEncodeChars);
else if (uCode <= SEQ_START)
hasValues = true;
} else if (uCode <= NODE_START) {
var subNodeIdx = NODE_START - uCode;
if (!subNodeEmpty[subNodeIdx]) { // Skip empty subtrees (they are too large in gb18030).
var newPrefix = (mbCode << 8) >>> 0; // NOTE: '>>> 0' keeps 32-bit num positive.
if (this._fillEncodeTable(subNodeIdx, newPrefix, skipEncodeChars))
hasValues = true;
else
subNodeEmpty[subNodeIdx] = true;
}
} else if (uCode <= SEQ_START) {
this._setEncodeSequence(this.decodeTableSeq[SEQ_START - uCode], mbCode);
hasValues = true;
}
}
return hasValues;
}
@@ -388,10 +419,15 @@ DBCSEncoder.prototype.write = function(str) {
newBuf[j++] = dbcsCode >> 8; // high byte
newBuf[j++] = dbcsCode & 0xFF; // low byte
}
else {
else if (dbcsCode < 0x1000000) {
newBuf[j++] = dbcsCode >> 16;
newBuf[j++] = (dbcsCode >> 8) & 0xFF;
newBuf[j++] = dbcsCode & 0xFF;
} else {
newBuf[j++] = dbcsCode >>> 24;
newBuf[j++] = (dbcsCode >>> 16) & 0xFF;
newBuf[j++] = (dbcsCode >>> 8) & 0xFF;
newBuf[j++] = dbcsCode & 0xFF;
}
}
@@ -440,7 +476,7 @@ DBCSEncoder.prototype.findIdx = findIdx;
function DBCSDecoder(options, codec) {
// Decoder state
this.nodeIdx = 0;
this.prevBuf = Buffer.alloc(0);
this.prevBytes = [];
// Static data
this.decodeTables = codec.decodeTables;
@@ -452,15 +488,12 @@ function DBCSDecoder(options, codec) {
DBCSDecoder.prototype.write = function(buf) {
var newBuf = Buffer.alloc(buf.length*2),
nodeIdx = this.nodeIdx,
prevBuf = this.prevBuf, prevBufOffset = this.prevBuf.length,
seqStart = -this.prevBuf.length, // idx of the start of current parsed sequence.
prevBytes = this.prevBytes, prevOffset = this.prevBytes.length,
seqStart = -this.prevBytes.length, // idx of the start of current parsed sequence.
uCode;
if (prevBufOffset > 0) // Make prev buf overlap a little to make it easier to slice later.
prevBuf = Buffer.concat([prevBuf, buf.slice(0, 10)]);
for (var i = 0, j = 0; i < buf.length; i++) {
var curByte = (i >= 0) ? buf[i] : prevBuf[i + prevBufOffset];
var curByte = (i >= 0) ? buf[i] : prevBytes[i + prevOffset];
// Lookup in current trie node.
var uCode = this.decodeTables[nodeIdx][curByte];
@@ -470,13 +503,18 @@ DBCSDecoder.prototype.write = function(buf) {
}
else if (uCode === UNASSIGNED) { // Unknown char.
// TODO: Callback with seq.
//var curSeq = (seqStart >= 0) ? buf.slice(seqStart, i+1) : prevBuf.slice(seqStart + prevBufOffset, i+1 + prevBufOffset);
i = seqStart; // Try to parse again, after skipping first byte of the sequence ('i' will be incremented by 'for' cycle).
uCode = this.defaultCharUnicode.charCodeAt(0);
i = seqStart; // Skip one byte ('i' will be incremented by the for loop) and try to parse again.
}
else if (uCode === GB18030_CODE) {
var curSeq = (seqStart >= 0) ? buf.slice(seqStart, i+1) : prevBuf.slice(seqStart + prevBufOffset, i+1 + prevBufOffset);
var ptr = (curSeq[0]-0x81)*12600 + (curSeq[1]-0x30)*1260 + (curSeq[2]-0x81)*10 + (curSeq[3]-0x30);
if (i >= 3) {
var ptr = (buf[i-3]-0x81)*12600 + (buf[i-2]-0x30)*1260 + (buf[i-1]-0x81)*10 + (curByte-0x30);
} else {
var ptr = (prevBytes[i-3+prevOffset]-0x81)*12600 +
(((i-2 >= 0) ? buf[i-2] : prevBytes[i-2+prevOffset])-0x30)*1260 +
(((i-1 >= 0) ? buf[i-1] : prevBytes[i-1+prevOffset])-0x81)*10 +
(curByte-0x30);
}
var idx = findIdx(this.gb18030.gbChars, ptr);
uCode = this.gb18030.uChars[idx] + ptr - this.gb18030.gbChars[idx];
}
@@ -497,13 +535,13 @@ DBCSDecoder.prototype.write = function(buf) {
throw new Error("iconv-lite internal error: invalid decoding table value " + uCode + " at " + nodeIdx + "/" + curByte);
// Write the character to buffer, handling higher planes using surrogate pair.
if (uCode > 0xFFFF) {
if (uCode >= 0x10000) {
uCode -= 0x10000;
var uCodeLead = 0xD800 + Math.floor(uCode / 0x400);
var uCodeLead = 0xD800 | (uCode >> 10);
newBuf[j++] = uCodeLead & 0xFF;
newBuf[j++] = uCodeLead >> 8;
uCode = 0xDC00 + uCode % 0x400;
uCode = 0xDC00 | (uCode & 0x3FF);
}
newBuf[j++] = uCode & 0xFF;
newBuf[j++] = uCode >> 8;
@@ -513,7 +551,10 @@ DBCSDecoder.prototype.write = function(buf) {
}
this.nodeIdx = nodeIdx;
this.prevBuf = (seqStart >= 0) ? buf.slice(seqStart) : prevBuf.slice(seqStart + prevBufOffset);
this.prevBytes = (seqStart >= 0)
? Array.prototype.slice.call(buf, seqStart)
: prevBytes.slice(seqStart + prevOffset).concat(Array.prototype.slice.call(buf));
return newBuf.slice(0, j).toString('ucs2');
}
@@ -521,18 +562,19 @@ DBCSDecoder.prototype.end = function() {
var ret = '';
// Try to parse all remaining chars.
while (this.prevBuf.length > 0) {
while (this.prevBytes.length > 0) {
// Skip 1 character in the buffer.
ret += this.defaultCharUnicode;
var buf = this.prevBuf.slice(1);
var bytesArr = this.prevBytes.slice(1);
// Parse remaining as usual.
this.prevBuf = Buffer.alloc(0);
this.prevBytes = [];
this.nodeIdx = 0;
if (buf.length > 0)
ret += this.write(buf);
if (bytesArr.length > 0)
ret += this.write(bytesArr);
}
this.prevBytes = [];
this.nodeIdx = 0;
return ret;
}
@@ -544,7 +586,7 @@ function findIdx(table, val) {
var l = 0, r = table.length;
while (l < r-1) { // always table[l] <= val < table[r]
var mid = l + Math.floor((r-l+1)/2);
var mid = l + ((r-l+1) >> 1);
if (table[mid] <= val)
l = mid;
else

View File

@@ -167,7 +167,19 @@ module.exports = {
'big5hkscs': {
type: '_dbcs',
table: function() { return require('./tables/cp950.json').concat(require('./tables/big5-added.json')) },
encodeSkipVals: [0xa2cc],
encodeSkipVals: [
// Although Encoding Standard says we should avoid encoding to HKSCS area (See Step 1 of
// https://encoding.spec.whatwg.org/#index-big5-pointer), we still do it to increase compatibility with ICU.
// But if a single unicode point can be encoded both as HKSCS and regular Big5, we prefer the latter.
0x8e69, 0x8e6f, 0x8e7e, 0x8eab, 0x8eb4, 0x8ecd, 0x8ed0, 0x8f57, 0x8f69, 0x8f6e, 0x8fcb, 0x8ffe,
0x906d, 0x907a, 0x90c4, 0x90dc, 0x90f1, 0x91bf, 0x92af, 0x92b0, 0x92b1, 0x92b2, 0x92d1, 0x9447, 0x94ca,
0x95d9, 0x96fc, 0x9975, 0x9b76, 0x9b78, 0x9b7b, 0x9bc6, 0x9bde, 0x9bec, 0x9bf6, 0x9c42, 0x9c53, 0x9c62,
0x9c68, 0x9c6b, 0x9c77, 0x9cbc, 0x9cbd, 0x9cd0, 0x9d57, 0x9d5a, 0x9dc4, 0x9def, 0x9dfb, 0x9ea9, 0x9eef,
0x9efd, 0x9f60, 0x9fcb, 0xa077, 0xa0dc, 0xa0df, 0x8fcc, 0x92c8, 0x9644, 0x96ed,
// Step 2 of https://encoding.spec.whatwg.org/#index-big5-pointer: Use last pointer for U+2550, U+255E, U+2561, U+256A, U+5341, or U+5345
0xa2a4, 0xa2a5, 0xa2a7, 0xa2a6, 0xa2cc, 0xa2ce,
],
},
'cnbig5': 'big5hkscs',

View File

@@ -4,6 +4,7 @@
// We support Browserify by skipping automatic module discovery and requiring modules directly.
var modules = [
require("./internal"),
require("./utf32"),
require("./utf16"),
require("./utf7"),
require("./sbcs-codec"),
@@ -13,7 +14,7 @@ var modules = [
require("./dbcs-data"),
];
// Put all encoding/alias/codec definitions to single object and export it.
// Put all encoding/alias/codec definitions to single object and export it.
for (var i = 0; i < modules.length; i++) {
var module = modules[i];
for (var enc in module)

View File

@@ -53,10 +53,20 @@ if (!StringDecoder.prototype.end) // Node v0.8 doesn't have this method.
function InternalDecoder(options, codec) {
StringDecoder.call(this, codec.enc);
this.decoder = new StringDecoder(codec.enc);
}
InternalDecoder.prototype = StringDecoder.prototype;
InternalDecoder.prototype.write = function(buf) {
if (!Buffer.isBuffer(buf)) {
buf = Buffer.from(buf);
}
return this.decoder.write(buf);
}
InternalDecoder.prototype.end = function() {
return this.decoder.end();
}
//------------------------------------------------------------------------------

View File

@@ -22,6 +22,11 @@ module.exports = {
"chars": "АБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдежзийклмнопрстуфхцчшщъыьэюя└┴┬├─┼╣║╚╔╩╦╠═╬┐░▒▓│┤№§╗╝┘┌█▄▌▐▀αßΓπΣσµτΦΘΩδ∞φε∩≡±≥≤⌠⌡÷≈°∙·√ⁿ²■ "
},
"cp720": {
"type": "_sbcs",
"chars": "\x80\x81éâ\x84à\x86çêëèïî\x8d\x8e\x8f\x90\u0651\u0652ô¤ـûùءآأؤ£إئابةتثجحخدذرزسشص«»░▒▓│┤╡╢╖╕╣║╗╝╜╛┐└┴┬├─┼╞╟╚╔╩╦╠═╬╧╨╤╥╙╘╒╓╫╪┘┌█▄▌▐▀ضطظعغفµقكلمنهوىي≡\u064b\u064c\u064d\u064e\u064f\u0650≈°∙·√ⁿ²■\u00a0"
},
// Aliases of generated encodings.
"ascii8bit": "ascii",
"usascii": "ascii",

View File

@@ -27,7 +27,7 @@
["a7c2","",14],
["a7f2","",12],
["a896","",10],
["a8bc",""],
["a8bc","ḿ"],
["a8bf","ǹ"],
["a8c1",""],
["a8ea","",20],
@@ -51,5 +51,6 @@
["fca1","",93],
["fda1","",93],
["fe50","⺁⺄㑳㑇⺈⺋㖞㘚㘎⺌⺗㥮㤘㧏㧟㩳㧐㭎㱮㳠⺧⺪䁖䅟⺮䌷⺳⺶⺷䎱䎬⺻䏝䓖䙡䙌"],
["fe80","䜣䜩䝼䞍⻊䥇䥺䥽䦂䦃䦅䦆䦟䦛䦷䦶䲣䲟䲠䲡䱷䲢䴓",6,"䶮",93]
["fe80","䜣䜩䝼䞍⻊䥇䥺䥽䦂䦃䦅䦆䦟䦛䦷䦶䲣䲟䲠䲡䱷䲢䴓",6,"䶮",93],
["8135f437",""]
]

View File

@@ -61,6 +61,7 @@ Utf16BEDecoder.prototype.write = function(buf) {
}
Utf16BEDecoder.prototype.end = function() {
this.overflowByte = -1;
}
@@ -103,8 +104,8 @@ Utf16Encoder.prototype.end = function() {
function Utf16Decoder(options, codec) {
this.decoder = null;
this.initialBytes = [];
this.initialBytesLen = 0;
this.initialBufs = [];
this.initialBufsLen = 0;
this.options = options || {};
this.iconv = codec.iconv;
@@ -113,17 +114,22 @@ function Utf16Decoder(options, codec) {
Utf16Decoder.prototype.write = function(buf) {
if (!this.decoder) {
// Codec is not chosen yet. Accumulate initial bytes.
this.initialBytes.push(buf);
this.initialBytesLen += buf.length;
this.initialBufs.push(buf);
this.initialBufsLen += buf.length;
if (this.initialBytesLen < 16) // We need more bytes to use space heuristic (see below)
if (this.initialBufsLen < 16) // We need more bytes to use space heuristic (see below)
return '';
// We have enough bytes -> detect endianness.
var buf = Buffer.concat(this.initialBytes),
encoding = detectEncoding(buf, this.options.defaultEncoding);
var encoding = detectEncoding(this.initialBufs, this.options.defaultEncoding);
this.decoder = this.iconv.getDecoder(encoding, this.options);
this.initialBytes.length = this.initialBytesLen = 0;
var resStr = '';
for (var i = 0; i < this.initialBufs.length; i++)
resStr += this.decoder.write(this.initialBufs[i]);
this.initialBufs.length = this.initialBufsLen = 0;
return resStr;
}
return this.decoder.write(buf);
@@ -131,47 +137,61 @@ Utf16Decoder.prototype.write = function(buf) {
Utf16Decoder.prototype.end = function() {
if (!this.decoder) {
var buf = Buffer.concat(this.initialBytes),
encoding = detectEncoding(buf, this.options.defaultEncoding);
var encoding = detectEncoding(this.initialBufs, this.options.defaultEncoding);
this.decoder = this.iconv.getDecoder(encoding, this.options);
var res = this.decoder.write(buf),
trail = this.decoder.end();
var resStr = '';
for (var i = 0; i < this.initialBufs.length; i++)
resStr += this.decoder.write(this.initialBufs[i]);
return trail ? (res + trail) : res;
var trail = this.decoder.end();
if (trail)
resStr += trail;
this.initialBufs.length = this.initialBufsLen = 0;
return resStr;
}
return this.decoder.end();
}
function detectEncoding(buf, defaultEncoding) {
var enc = defaultEncoding || 'utf-16le';
function detectEncoding(bufs, defaultEncoding) {
var b = [];
var charsProcessed = 0;
var asciiCharsLE = 0, asciiCharsBE = 0; // Number of ASCII chars when decoded as LE or BE.
if (buf.length >= 2) {
// Check BOM.
if (buf[0] == 0xFE && buf[1] == 0xFF) // UTF-16BE BOM
enc = 'utf-16be';
else if (buf[0] == 0xFF && buf[1] == 0xFE) // UTF-16LE BOM
enc = 'utf-16le';
else {
// No BOM found. Try to deduce encoding from initial content.
// Most of the time, the content has ASCII chars (U+00**), but the opposite (U+**00) is uncommon.
// So, we count ASCII as if it was LE or BE, and decide from that.
var asciiCharsLE = 0, asciiCharsBE = 0, // Counts of chars in both positions
_len = Math.min(buf.length - (buf.length % 2), 64); // Len is always even.
outer_loop:
for (var i = 0; i < bufs.length; i++) {
var buf = bufs[i];
for (var j = 0; j < buf.length; j++) {
b.push(buf[j]);
if (b.length === 2) {
if (charsProcessed === 0) {
// Check BOM first.
if (b[0] === 0xFF && b[1] === 0xFE) return 'utf-16le';
if (b[0] === 0xFE && b[1] === 0xFF) return 'utf-16be';
}
for (var i = 0; i < _len; i += 2) {
if (buf[i] === 0 && buf[i+1] !== 0) asciiCharsBE++;
if (buf[i] !== 0 && buf[i+1] === 0) asciiCharsLE++;
if (b[0] === 0 && b[1] !== 0) asciiCharsBE++;
if (b[0] !== 0 && b[1] === 0) asciiCharsLE++;
b.length = 0;
charsProcessed++;
if (charsProcessed >= 100) {
break outer_loop;
}
}
if (asciiCharsBE > asciiCharsLE)
enc = 'utf-16be';
else if (asciiCharsBE < asciiCharsLE)
enc = 'utf-16le';
}
}
return enc;
// Make decisions.
// Most of the time, the content has ASCII chars (U+00**), but the opposite (U+**00) is uncommon.
// So, we count ASCII as if it was LE or BE, and decide from that.
if (asciiCharsBE > asciiCharsLE) return 'utf-16be';
if (asciiCharsBE < asciiCharsLE) return 'utf-16le';
// Couldn't decide (likely all zeros or not enough data).
return defaultEncoding || 'utf-16le';
}

319
node_modules/iconv-lite/encodings/utf32.js generated vendored Normal file
View File

@@ -0,0 +1,319 @@
'use strict';
var Buffer = require('safer-buffer').Buffer;
// == UTF32-LE/BE codec. ==========================================================
exports._utf32 = Utf32Codec;
function Utf32Codec(codecOptions, iconv) {
this.iconv = iconv;
this.bomAware = true;
this.isLE = codecOptions.isLE;
}
exports.utf32le = { type: '_utf32', isLE: true };
exports.utf32be = { type: '_utf32', isLE: false };
// Aliases
exports.ucs4le = 'utf32le';
exports.ucs4be = 'utf32be';
Utf32Codec.prototype.encoder = Utf32Encoder;
Utf32Codec.prototype.decoder = Utf32Decoder;
// -- Encoding
function Utf32Encoder(options, codec) {
this.isLE = codec.isLE;
this.highSurrogate = 0;
}
Utf32Encoder.prototype.write = function(str) {
var src = Buffer.from(str, 'ucs2');
var dst = Buffer.alloc(src.length * 2);
var write32 = this.isLE ? dst.writeUInt32LE : dst.writeUInt32BE;
var offset = 0;
for (var i = 0; i < src.length; i += 2) {
var code = src.readUInt16LE(i);
var isHighSurrogate = (0xD800 <= code && code < 0xDC00);
var isLowSurrogate = (0xDC00 <= code && code < 0xE000);
if (this.highSurrogate) {
if (isHighSurrogate || !isLowSurrogate) {
// There shouldn't be two high surrogates in a row, nor a high surrogate which isn't followed by a low
// surrogate. If this happens, keep the pending high surrogate as a stand-alone semi-invalid character
// (technically wrong, but expected by some applications, like Windows file names).
write32.call(dst, this.highSurrogate, offset);
offset += 4;
}
else {
// Create 32-bit value from high and low surrogates;
var codepoint = (((this.highSurrogate - 0xD800) << 10) | (code - 0xDC00)) + 0x10000;
write32.call(dst, codepoint, offset);
offset += 4;
this.highSurrogate = 0;
continue;
}
}
if (isHighSurrogate)
this.highSurrogate = code;
else {
// Even if the current character is a low surrogate, with no previous high surrogate, we'll
// encode it as a semi-invalid stand-alone character for the same reasons expressed above for
// unpaired high surrogates.
write32.call(dst, code, offset);
offset += 4;
this.highSurrogate = 0;
}
}
if (offset < dst.length)
dst = dst.slice(0, offset);
return dst;
};
Utf32Encoder.prototype.end = function() {
// Treat any leftover high surrogate as a semi-valid independent character.
if (!this.highSurrogate)
return;
var buf = Buffer.alloc(4);
if (this.isLE)
buf.writeUInt32LE(this.highSurrogate, 0);
else
buf.writeUInt32BE(this.highSurrogate, 0);
this.highSurrogate = 0;
return buf;
};
// -- Decoding
function Utf32Decoder(options, codec) {
this.isLE = codec.isLE;
this.badChar = codec.iconv.defaultCharUnicode.charCodeAt(0);
this.overflow = [];
}
Utf32Decoder.prototype.write = function(src) {
if (src.length === 0)
return '';
var i = 0;
var codepoint = 0;
var dst = Buffer.alloc(src.length + 4);
var offset = 0;
var isLE = this.isLE;
var overflow = this.overflow;
var badChar = this.badChar;
if (overflow.length > 0) {
for (; i < src.length && overflow.length < 4; i++)
overflow.push(src[i]);
if (overflow.length === 4) {
// NOTE: codepoint is a signed int32 and can be negative.
// NOTE: We copied this block from below to help V8 optimize it (it works with array, not buffer).
if (isLE) {
codepoint = overflow[i] | (overflow[i+1] << 8) | (overflow[i+2] << 16) | (overflow[i+3] << 24);
} else {
codepoint = overflow[i+3] | (overflow[i+2] << 8) | (overflow[i+1] << 16) | (overflow[i] << 24);
}
overflow.length = 0;
offset = _writeCodepoint(dst, offset, codepoint, badChar);
}
}
// Main loop. Should be as optimized as possible.
for (; i < src.length - 3; i += 4) {
// NOTE: codepoint is a signed int32 and can be negative.
if (isLE) {
codepoint = src[i] | (src[i+1] << 8) | (src[i+2] << 16) | (src[i+3] << 24);
} else {
codepoint = src[i+3] | (src[i+2] << 8) | (src[i+1] << 16) | (src[i] << 24);
}
offset = _writeCodepoint(dst, offset, codepoint, badChar);
}
// Keep overflowing bytes.
for (; i < src.length; i++) {
overflow.push(src[i]);
}
return dst.slice(0, offset).toString('ucs2');
};
function _writeCodepoint(dst, offset, codepoint, badChar) {
// NOTE: codepoint is signed int32 and can be negative. We keep it that way to help V8 with optimizations.
if (codepoint < 0 || codepoint > 0x10FFFF) {
// Not a valid Unicode codepoint
codepoint = badChar;
}
// Ephemeral Planes: Write high surrogate.
if (codepoint >= 0x10000) {
codepoint -= 0x10000;
var high = 0xD800 | (codepoint >> 10);
dst[offset++] = high & 0xff;
dst[offset++] = high >> 8;
// Low surrogate is written below.
var codepoint = 0xDC00 | (codepoint & 0x3FF);
}
// Write BMP char or low surrogate.
dst[offset++] = codepoint & 0xff;
dst[offset++] = codepoint >> 8;
return offset;
};
Utf32Decoder.prototype.end = function() {
this.overflow.length = 0;
};
// == UTF-32 Auto codec =============================================================
// Decoder chooses automatically from UTF-32LE and UTF-32BE using BOM and space-based heuristic.
// Defaults to UTF-32LE. http://en.wikipedia.org/wiki/UTF-32
// Encoder/decoder default can be changed: iconv.decode(buf, 'utf32', {defaultEncoding: 'utf-32be'});
// Encoder prepends BOM (which can be overridden with (addBOM: false}).
exports.utf32 = Utf32AutoCodec;
exports.ucs4 = 'utf32';
function Utf32AutoCodec(options, iconv) {
this.iconv = iconv;
}
Utf32AutoCodec.prototype.encoder = Utf32AutoEncoder;
Utf32AutoCodec.prototype.decoder = Utf32AutoDecoder;
// -- Encoding
function Utf32AutoEncoder(options, codec) {
options = options || {};
if (options.addBOM === undefined)
options.addBOM = true;
this.encoder = codec.iconv.getEncoder(options.defaultEncoding || 'utf-32le', options);
}
Utf32AutoEncoder.prototype.write = function(str) {
return this.encoder.write(str);
};
Utf32AutoEncoder.prototype.end = function() {
return this.encoder.end();
};
// -- Decoding
function Utf32AutoDecoder(options, codec) {
this.decoder = null;
this.initialBufs = [];
this.initialBufsLen = 0;
this.options = options || {};
this.iconv = codec.iconv;
}
Utf32AutoDecoder.prototype.write = function(buf) {
if (!this.decoder) {
// Codec is not chosen yet. Accumulate initial bytes.
this.initialBufs.push(buf);
this.initialBufsLen += buf.length;
if (this.initialBufsLen < 32) // We need more bytes to use space heuristic (see below)
return '';
// We have enough bytes -> detect endianness.
var encoding = detectEncoding(this.initialBufs, this.options.defaultEncoding);
this.decoder = this.iconv.getDecoder(encoding, this.options);
var resStr = '';
for (var i = 0; i < this.initialBufs.length; i++)
resStr += this.decoder.write(this.initialBufs[i]);
this.initialBufs.length = this.initialBufsLen = 0;
return resStr;
}
return this.decoder.write(buf);
};
Utf32AutoDecoder.prototype.end = function() {
if (!this.decoder) {
var encoding = detectEncoding(this.initialBufs, this.options.defaultEncoding);
this.decoder = this.iconv.getDecoder(encoding, this.options);
var resStr = '';
for (var i = 0; i < this.initialBufs.length; i++)
resStr += this.decoder.write(this.initialBufs[i]);
var trail = this.decoder.end();
if (trail)
resStr += trail;
this.initialBufs.length = this.initialBufsLen = 0;
return resStr;
}
return this.decoder.end();
};
function detectEncoding(bufs, defaultEncoding) {
var b = [];
var charsProcessed = 0;
var invalidLE = 0, invalidBE = 0; // Number of invalid chars when decoded as LE or BE.
var bmpCharsLE = 0, bmpCharsBE = 0; // Number of BMP chars when decoded as LE or BE.
outer_loop:
for (var i = 0; i < bufs.length; i++) {
var buf = bufs[i];
for (var j = 0; j < buf.length; j++) {
b.push(buf[j]);
if (b.length === 4) {
if (charsProcessed === 0) {
// Check BOM first.
if (b[0] === 0xFF && b[1] === 0xFE && b[2] === 0 && b[3] === 0) {
return 'utf-32le';
}
if (b[0] === 0 && b[1] === 0 && b[2] === 0xFE && b[3] === 0xFF) {
return 'utf-32be';
}
}
if (b[0] !== 0 || b[1] > 0x10) invalidBE++;
if (b[3] !== 0 || b[2] > 0x10) invalidLE++;
if (b[0] === 0 && b[1] === 0 && (b[2] !== 0 || b[3] !== 0)) bmpCharsBE++;
if ((b[0] !== 0 || b[1] !== 0) && b[2] === 0 && b[3] === 0) bmpCharsLE++;
b.length = 0;
charsProcessed++;
if (charsProcessed >= 100) {
break outer_loop;
}
}
}
}
// Make decisions.
if (bmpCharsBE - invalidBE > bmpCharsLE - invalidLE) return 'utf-32be';
if (bmpCharsBE - invalidBE < bmpCharsLE - invalidLE) return 'utf-32le';
// Couldn't decide (likely all zeros or not enough data).
return defaultEncoding || 'utf-32le';
}

View File

@@ -74,7 +74,7 @@ Utf7Decoder.prototype.write = function(buf) {
if (i == lastI && buf[i] == minusChar) {// "+-" -> "+"
res += "+";
} else {
var b64str = base64Accum + buf.slice(lastI, i).toString();
var b64str = base64Accum + this.iconv.decode(buf.slice(lastI, i), "ascii");
res += this.iconv.decode(Buffer.from(b64str, 'base64'), "utf16-be");
}
@@ -91,7 +91,7 @@ Utf7Decoder.prototype.write = function(buf) {
if (!inBase64) {
res += this.iconv.decode(buf.slice(lastI), "ascii"); // Write direct chars.
} else {
var b64str = base64Accum + buf.slice(lastI).toString();
var b64str = base64Accum + this.iconv.decode(buf.slice(lastI), "ascii");
var canBeDecoded = b64str.length - (b64str.length % 8); // Minimal chunk: 2 quads -> 2x3 bytes -> 3 chars.
base64Accum = b64str.slice(canBeDecoded); // The rest will be decoded in future.
@@ -245,7 +245,7 @@ Utf7IMAPDecoder.prototype.write = function(buf) {
if (i == lastI && buf[i] == minusChar) { // "&-" -> "&"
res += "&";
} else {
var b64str = base64Accum + buf.slice(lastI, i).toString().replace(/,/g, '/');
var b64str = base64Accum + this.iconv.decode(buf.slice(lastI, i), "ascii").replace(/,/g, '/');
res += this.iconv.decode(Buffer.from(b64str, 'base64'), "utf16-be");
}
@@ -262,7 +262,7 @@ Utf7IMAPDecoder.prototype.write = function(buf) {
if (!inBase64) {
res += this.iconv.decode(buf.slice(lastI), "ascii"); // Write direct chars.
} else {
var b64str = base64Accum + buf.slice(lastI).toString().replace(/,/g, '/');
var b64str = base64Accum + this.iconv.decode(buf.slice(lastI), "ascii").replace(/,/g, '/');
var canBeDecoded = b64str.length - (b64str.length % 8); // Minimal chunk: 2 quads -> 2x3 bytes -> 3 chars.
base64Accum = b64str.slice(canBeDecoded); // The rest will be decoded in future.