mirror of
https://github.com/S2-/minifyfromhtml.git
synced 2025-08-03 12:20:04 +02:00
update packages to latest version
This commit is contained in:
144
node_modules/iconv-lite/encodings/dbcs-codec.js
generated
vendored
144
node_modules/iconv-lite/encodings/dbcs-codec.js
generated
vendored
@@ -49,6 +49,48 @@ function DBCSCodec(codecOptions, iconv) {
|
||||
for (var i = 0; i < mappingTable.length; i++)
|
||||
this._addDecodeChunk(mappingTable[i]);
|
||||
|
||||
// Load & create GB18030 tables when needed.
|
||||
if (typeof codecOptions.gb18030 === 'function') {
|
||||
this.gb18030 = codecOptions.gb18030(); // Load GB18030 ranges.
|
||||
|
||||
// Add GB18030 common decode nodes.
|
||||
var commonThirdByteNodeIdx = this.decodeTables.length;
|
||||
this.decodeTables.push(UNASSIGNED_NODE.slice(0));
|
||||
|
||||
var commonFourthByteNodeIdx = this.decodeTables.length;
|
||||
this.decodeTables.push(UNASSIGNED_NODE.slice(0));
|
||||
|
||||
// Fill out the tree
|
||||
var firstByteNode = this.decodeTables[0];
|
||||
for (var i = 0x81; i <= 0xFE; i++) {
|
||||
var secondByteNode = this.decodeTables[NODE_START - firstByteNode[i]];
|
||||
for (var j = 0x30; j <= 0x39; j++) {
|
||||
if (secondByteNode[j] === UNASSIGNED) {
|
||||
secondByteNode[j] = NODE_START - commonThirdByteNodeIdx;
|
||||
} else if (secondByteNode[j] > NODE_START) {
|
||||
throw new Error("gb18030 decode tables conflict at byte 2");
|
||||
}
|
||||
|
||||
var thirdByteNode = this.decodeTables[NODE_START - secondByteNode[j]];
|
||||
for (var k = 0x81; k <= 0xFE; k++) {
|
||||
if (thirdByteNode[k] === UNASSIGNED) {
|
||||
thirdByteNode[k] = NODE_START - commonFourthByteNodeIdx;
|
||||
} else if (thirdByteNode[k] === NODE_START - commonFourthByteNodeIdx) {
|
||||
continue;
|
||||
} else if (thirdByteNode[k] > NODE_START) {
|
||||
throw new Error("gb18030 decode tables conflict at byte 3");
|
||||
}
|
||||
|
||||
var fourthByteNode = this.decodeTables[NODE_START - thirdByteNode[k]];
|
||||
for (var l = 0x30; l <= 0x39; l++) {
|
||||
if (fourthByteNode[l] === UNASSIGNED)
|
||||
fourthByteNode[l] = GB18030_CODE;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.defaultCharUnicode = iconv.defaultCharUnicode;
|
||||
|
||||
|
||||
@@ -92,30 +134,6 @@ function DBCSCodec(codecOptions, iconv) {
|
||||
this.defCharSB = this.encodeTable[0][iconv.defaultCharSingleByte.charCodeAt(0)];
|
||||
if (this.defCharSB === UNASSIGNED) this.defCharSB = this.encodeTable[0]['?'];
|
||||
if (this.defCharSB === UNASSIGNED) this.defCharSB = "?".charCodeAt(0);
|
||||
|
||||
|
||||
// Load & create GB18030 tables when needed.
|
||||
if (typeof codecOptions.gb18030 === 'function') {
|
||||
this.gb18030 = codecOptions.gb18030(); // Load GB18030 ranges.
|
||||
|
||||
// Add GB18030 decode tables.
|
||||
var thirdByteNodeIdx = this.decodeTables.length;
|
||||
var thirdByteNode = this.decodeTables[thirdByteNodeIdx] = UNASSIGNED_NODE.slice(0);
|
||||
|
||||
var fourthByteNodeIdx = this.decodeTables.length;
|
||||
var fourthByteNode = this.decodeTables[fourthByteNodeIdx] = UNASSIGNED_NODE.slice(0);
|
||||
|
||||
for (var i = 0x81; i <= 0xFE; i++) {
|
||||
var secondByteNodeIdx = NODE_START - this.decodeTables[0][i];
|
||||
var secondByteNode = this.decodeTables[secondByteNodeIdx];
|
||||
for (var j = 0x30; j <= 0x39; j++)
|
||||
secondByteNode[j] = NODE_START - thirdByteNodeIdx;
|
||||
}
|
||||
for (var i = 0x81; i <= 0xFE; i++)
|
||||
thirdByteNode[i] = NODE_START - fourthByteNodeIdx;
|
||||
for (var i = 0x30; i <= 0x39; i++)
|
||||
fourthByteNode[i] = GB18030_CODE
|
||||
}
|
||||
}
|
||||
|
||||
DBCSCodec.prototype.encoder = DBCSEncoder;
|
||||
@@ -124,7 +142,7 @@ DBCSCodec.prototype.decoder = DBCSDecoder;
|
||||
// Decoder helpers
|
||||
DBCSCodec.prototype._getDecodeTrieNode = function(addr) {
|
||||
var bytes = [];
|
||||
for (; addr > 0; addr >>= 8)
|
||||
for (; addr > 0; addr >>>= 8)
|
||||
bytes.push(addr & 0xFF);
|
||||
if (bytes.length == 0)
|
||||
bytes.push(0);
|
||||
@@ -249,19 +267,32 @@ DBCSCodec.prototype._setEncodeSequence = function(seq, dbcsCode) {
|
||||
|
||||
DBCSCodec.prototype._fillEncodeTable = function(nodeIdx, prefix, skipEncodeChars) {
|
||||
var node = this.decodeTables[nodeIdx];
|
||||
var hasValues = false;
|
||||
var subNodeEmpty = {};
|
||||
for (var i = 0; i < 0x100; i++) {
|
||||
var uCode = node[i];
|
||||
var mbCode = prefix + i;
|
||||
if (skipEncodeChars[mbCode])
|
||||
continue;
|
||||
|
||||
if (uCode >= 0)
|
||||
if (uCode >= 0) {
|
||||
this._setEncodeChar(uCode, mbCode);
|
||||
else if (uCode <= NODE_START)
|
||||
this._fillEncodeTable(NODE_START - uCode, mbCode << 8, skipEncodeChars);
|
||||
else if (uCode <= SEQ_START)
|
||||
hasValues = true;
|
||||
} else if (uCode <= NODE_START) {
|
||||
var subNodeIdx = NODE_START - uCode;
|
||||
if (!subNodeEmpty[subNodeIdx]) { // Skip empty subtrees (they are too large in gb18030).
|
||||
var newPrefix = (mbCode << 8) >>> 0; // NOTE: '>>> 0' keeps 32-bit num positive.
|
||||
if (this._fillEncodeTable(subNodeIdx, newPrefix, skipEncodeChars))
|
||||
hasValues = true;
|
||||
else
|
||||
subNodeEmpty[subNodeIdx] = true;
|
||||
}
|
||||
} else if (uCode <= SEQ_START) {
|
||||
this._setEncodeSequence(this.decodeTableSeq[SEQ_START - uCode], mbCode);
|
||||
hasValues = true;
|
||||
}
|
||||
}
|
||||
return hasValues;
|
||||
}
|
||||
|
||||
|
||||
@@ -388,10 +419,15 @@ DBCSEncoder.prototype.write = function(str) {
|
||||
newBuf[j++] = dbcsCode >> 8; // high byte
|
||||
newBuf[j++] = dbcsCode & 0xFF; // low byte
|
||||
}
|
||||
else {
|
||||
else if (dbcsCode < 0x1000000) {
|
||||
newBuf[j++] = dbcsCode >> 16;
|
||||
newBuf[j++] = (dbcsCode >> 8) & 0xFF;
|
||||
newBuf[j++] = dbcsCode & 0xFF;
|
||||
} else {
|
||||
newBuf[j++] = dbcsCode >>> 24;
|
||||
newBuf[j++] = (dbcsCode >>> 16) & 0xFF;
|
||||
newBuf[j++] = (dbcsCode >>> 8) & 0xFF;
|
||||
newBuf[j++] = dbcsCode & 0xFF;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -440,7 +476,7 @@ DBCSEncoder.prototype.findIdx = findIdx;
|
||||
function DBCSDecoder(options, codec) {
|
||||
// Decoder state
|
||||
this.nodeIdx = 0;
|
||||
this.prevBuf = Buffer.alloc(0);
|
||||
this.prevBytes = [];
|
||||
|
||||
// Static data
|
||||
this.decodeTables = codec.decodeTables;
|
||||
@@ -452,15 +488,12 @@ function DBCSDecoder(options, codec) {
|
||||
DBCSDecoder.prototype.write = function(buf) {
|
||||
var newBuf = Buffer.alloc(buf.length*2),
|
||||
nodeIdx = this.nodeIdx,
|
||||
prevBuf = this.prevBuf, prevBufOffset = this.prevBuf.length,
|
||||
seqStart = -this.prevBuf.length, // idx of the start of current parsed sequence.
|
||||
prevBytes = this.prevBytes, prevOffset = this.prevBytes.length,
|
||||
seqStart = -this.prevBytes.length, // idx of the start of current parsed sequence.
|
||||
uCode;
|
||||
|
||||
if (prevBufOffset > 0) // Make prev buf overlap a little to make it easier to slice later.
|
||||
prevBuf = Buffer.concat([prevBuf, buf.slice(0, 10)]);
|
||||
|
||||
for (var i = 0, j = 0; i < buf.length; i++) {
|
||||
var curByte = (i >= 0) ? buf[i] : prevBuf[i + prevBufOffset];
|
||||
var curByte = (i >= 0) ? buf[i] : prevBytes[i + prevOffset];
|
||||
|
||||
// Lookup in current trie node.
|
||||
var uCode = this.decodeTables[nodeIdx][curByte];
|
||||
@@ -470,13 +503,18 @@ DBCSDecoder.prototype.write = function(buf) {
|
||||
}
|
||||
else if (uCode === UNASSIGNED) { // Unknown char.
|
||||
// TODO: Callback with seq.
|
||||
//var curSeq = (seqStart >= 0) ? buf.slice(seqStart, i+1) : prevBuf.slice(seqStart + prevBufOffset, i+1 + prevBufOffset);
|
||||
i = seqStart; // Try to parse again, after skipping first byte of the sequence ('i' will be incremented by 'for' cycle).
|
||||
uCode = this.defaultCharUnicode.charCodeAt(0);
|
||||
i = seqStart; // Skip one byte ('i' will be incremented by the for loop) and try to parse again.
|
||||
}
|
||||
else if (uCode === GB18030_CODE) {
|
||||
var curSeq = (seqStart >= 0) ? buf.slice(seqStart, i+1) : prevBuf.slice(seqStart + prevBufOffset, i+1 + prevBufOffset);
|
||||
var ptr = (curSeq[0]-0x81)*12600 + (curSeq[1]-0x30)*1260 + (curSeq[2]-0x81)*10 + (curSeq[3]-0x30);
|
||||
if (i >= 3) {
|
||||
var ptr = (buf[i-3]-0x81)*12600 + (buf[i-2]-0x30)*1260 + (buf[i-1]-0x81)*10 + (curByte-0x30);
|
||||
} else {
|
||||
var ptr = (prevBytes[i-3+prevOffset]-0x81)*12600 +
|
||||
(((i-2 >= 0) ? buf[i-2] : prevBytes[i-2+prevOffset])-0x30)*1260 +
|
||||
(((i-1 >= 0) ? buf[i-1] : prevBytes[i-1+prevOffset])-0x81)*10 +
|
||||
(curByte-0x30);
|
||||
}
|
||||
var idx = findIdx(this.gb18030.gbChars, ptr);
|
||||
uCode = this.gb18030.uChars[idx] + ptr - this.gb18030.gbChars[idx];
|
||||
}
|
||||
@@ -497,13 +535,13 @@ DBCSDecoder.prototype.write = function(buf) {
|
||||
throw new Error("iconv-lite internal error: invalid decoding table value " + uCode + " at " + nodeIdx + "/" + curByte);
|
||||
|
||||
// Write the character to buffer, handling higher planes using surrogate pair.
|
||||
if (uCode > 0xFFFF) {
|
||||
if (uCode >= 0x10000) {
|
||||
uCode -= 0x10000;
|
||||
var uCodeLead = 0xD800 + Math.floor(uCode / 0x400);
|
||||
var uCodeLead = 0xD800 | (uCode >> 10);
|
||||
newBuf[j++] = uCodeLead & 0xFF;
|
||||
newBuf[j++] = uCodeLead >> 8;
|
||||
|
||||
uCode = 0xDC00 + uCode % 0x400;
|
||||
uCode = 0xDC00 | (uCode & 0x3FF);
|
||||
}
|
||||
newBuf[j++] = uCode & 0xFF;
|
||||
newBuf[j++] = uCode >> 8;
|
||||
@@ -513,7 +551,10 @@ DBCSDecoder.prototype.write = function(buf) {
|
||||
}
|
||||
|
||||
this.nodeIdx = nodeIdx;
|
||||
this.prevBuf = (seqStart >= 0) ? buf.slice(seqStart) : prevBuf.slice(seqStart + prevBufOffset);
|
||||
this.prevBytes = (seqStart >= 0)
|
||||
? Array.prototype.slice.call(buf, seqStart)
|
||||
: prevBytes.slice(seqStart + prevOffset).concat(Array.prototype.slice.call(buf));
|
||||
|
||||
return newBuf.slice(0, j).toString('ucs2');
|
||||
}
|
||||
|
||||
@@ -521,18 +562,19 @@ DBCSDecoder.prototype.end = function() {
|
||||
var ret = '';
|
||||
|
||||
// Try to parse all remaining chars.
|
||||
while (this.prevBuf.length > 0) {
|
||||
while (this.prevBytes.length > 0) {
|
||||
// Skip 1 character in the buffer.
|
||||
ret += this.defaultCharUnicode;
|
||||
var buf = this.prevBuf.slice(1);
|
||||
var bytesArr = this.prevBytes.slice(1);
|
||||
|
||||
// Parse remaining as usual.
|
||||
this.prevBuf = Buffer.alloc(0);
|
||||
this.prevBytes = [];
|
||||
this.nodeIdx = 0;
|
||||
if (buf.length > 0)
|
||||
ret += this.write(buf);
|
||||
if (bytesArr.length > 0)
|
||||
ret += this.write(bytesArr);
|
||||
}
|
||||
|
||||
this.prevBytes = [];
|
||||
this.nodeIdx = 0;
|
||||
return ret;
|
||||
}
|
||||
@@ -544,7 +586,7 @@ function findIdx(table, val) {
|
||||
|
||||
var l = 0, r = table.length;
|
||||
while (l < r-1) { // always table[l] <= val < table[r]
|
||||
var mid = l + Math.floor((r-l+1)/2);
|
||||
var mid = l + ((r-l+1) >> 1);
|
||||
if (table[mid] <= val)
|
||||
l = mid;
|
||||
else
|
||||
|
14
node_modules/iconv-lite/encodings/dbcs-data.js
generated
vendored
14
node_modules/iconv-lite/encodings/dbcs-data.js
generated
vendored
@@ -167,7 +167,19 @@ module.exports = {
|
||||
'big5hkscs': {
|
||||
type: '_dbcs',
|
||||
table: function() { return require('./tables/cp950.json').concat(require('./tables/big5-added.json')) },
|
||||
encodeSkipVals: [0xa2cc],
|
||||
encodeSkipVals: [
|
||||
// Although Encoding Standard says we should avoid encoding to HKSCS area (See Step 1 of
|
||||
// https://encoding.spec.whatwg.org/#index-big5-pointer), we still do it to increase compatibility with ICU.
|
||||
// But if a single unicode point can be encoded both as HKSCS and regular Big5, we prefer the latter.
|
||||
0x8e69, 0x8e6f, 0x8e7e, 0x8eab, 0x8eb4, 0x8ecd, 0x8ed0, 0x8f57, 0x8f69, 0x8f6e, 0x8fcb, 0x8ffe,
|
||||
0x906d, 0x907a, 0x90c4, 0x90dc, 0x90f1, 0x91bf, 0x92af, 0x92b0, 0x92b1, 0x92b2, 0x92d1, 0x9447, 0x94ca,
|
||||
0x95d9, 0x96fc, 0x9975, 0x9b76, 0x9b78, 0x9b7b, 0x9bc6, 0x9bde, 0x9bec, 0x9bf6, 0x9c42, 0x9c53, 0x9c62,
|
||||
0x9c68, 0x9c6b, 0x9c77, 0x9cbc, 0x9cbd, 0x9cd0, 0x9d57, 0x9d5a, 0x9dc4, 0x9def, 0x9dfb, 0x9ea9, 0x9eef,
|
||||
0x9efd, 0x9f60, 0x9fcb, 0xa077, 0xa0dc, 0xa0df, 0x8fcc, 0x92c8, 0x9644, 0x96ed,
|
||||
|
||||
// Step 2 of https://encoding.spec.whatwg.org/#index-big5-pointer: Use last pointer for U+2550, U+255E, U+2561, U+256A, U+5341, or U+5345
|
||||
0xa2a4, 0xa2a5, 0xa2a7, 0xa2a6, 0xa2cc, 0xa2ce,
|
||||
],
|
||||
},
|
||||
|
||||
'cnbig5': 'big5hkscs',
|
||||
|
3
node_modules/iconv-lite/encodings/index.js
generated
vendored
3
node_modules/iconv-lite/encodings/index.js
generated
vendored
@@ -4,6 +4,7 @@
|
||||
// We support Browserify by skipping automatic module discovery and requiring modules directly.
|
||||
var modules = [
|
||||
require("./internal"),
|
||||
require("./utf32"),
|
||||
require("./utf16"),
|
||||
require("./utf7"),
|
||||
require("./sbcs-codec"),
|
||||
@@ -13,7 +14,7 @@ var modules = [
|
||||
require("./dbcs-data"),
|
||||
];
|
||||
|
||||
// Put all encoding/alias/codec definitions to single object and export it.
|
||||
// Put all encoding/alias/codec definitions to single object and export it.
|
||||
for (var i = 0; i < modules.length; i++) {
|
||||
var module = modules[i];
|
||||
for (var enc in module)
|
||||
|
14
node_modules/iconv-lite/encodings/internal.js
generated
vendored
14
node_modules/iconv-lite/encodings/internal.js
generated
vendored
@@ -53,10 +53,20 @@ if (!StringDecoder.prototype.end) // Node v0.8 doesn't have this method.
|
||||
|
||||
|
||||
function InternalDecoder(options, codec) {
|
||||
StringDecoder.call(this, codec.enc);
|
||||
this.decoder = new StringDecoder(codec.enc);
|
||||
}
|
||||
|
||||
InternalDecoder.prototype = StringDecoder.prototype;
|
||||
InternalDecoder.prototype.write = function(buf) {
|
||||
if (!Buffer.isBuffer(buf)) {
|
||||
buf = Buffer.from(buf);
|
||||
}
|
||||
|
||||
return this.decoder.write(buf);
|
||||
}
|
||||
|
||||
InternalDecoder.prototype.end = function() {
|
||||
return this.decoder.end();
|
||||
}
|
||||
|
||||
|
||||
//------------------------------------------------------------------------------
|
||||
|
5
node_modules/iconv-lite/encodings/sbcs-data.js
generated
vendored
5
node_modules/iconv-lite/encodings/sbcs-data.js
generated
vendored
@@ -22,6 +22,11 @@ module.exports = {
|
||||
"chars": "АБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдежзийклмнопрстуфхцчшщъыьэюя└┴┬├─┼╣║╚╔╩╦╠═╬┐░▒▓│┤№§╗╝┘┌█▄▌▐▀αßΓπΣσµτΦΘΩδ∞φε∩≡±≥≤⌠⌡÷≈°∙·√ⁿ²■ "
|
||||
},
|
||||
|
||||
"cp720": {
|
||||
"type": "_sbcs",
|
||||
"chars": "\x80\x81éâ\x84à\x86çêëèïî\x8d\x8e\x8f\x90\u0651\u0652ô¤ـûùءآأؤ£إئابةتثجحخدذرزسشص«»░▒▓│┤╡╢╖╕╣║╗╝╜╛┐└┴┬├─┼╞╟╚╔╩╦╠═╬╧╨╤╥╙╘╒╓╫╪┘┌█▄▌▐▀ضطظعغفµقكلمنهوىي≡\u064b\u064c\u064d\u064e\u064f\u0650≈°∙·√ⁿ²■\u00a0"
|
||||
},
|
||||
|
||||
// Aliases of generated encodings.
|
||||
"ascii8bit": "ascii",
|
||||
"usascii": "ascii",
|
||||
|
5
node_modules/iconv-lite/encodings/tables/gbk-added.json
generated
vendored
5
node_modules/iconv-lite/encodings/tables/gbk-added.json
generated
vendored
@@ -27,7 +27,7 @@
|
||||
["a7c2","",14],
|
||||
["a7f2","",12],
|
||||
["a896","",10],
|
||||
["a8bc",""],
|
||||
["a8bc","ḿ"],
|
||||
["a8bf","ǹ"],
|
||||
["a8c1",""],
|
||||
["a8ea","",20],
|
||||
@@ -51,5 +51,6 @@
|
||||
["fca1","",93],
|
||||
["fda1","",93],
|
||||
["fe50","⺁⺄㑳㑇⺈⺋㖞㘚㘎⺌⺗㥮㤘㧏㧟㩳㧐㭎㱮㳠⺧⺪䁖䅟⺮䌷⺳⺶⺷䎱䎬⺻䏝䓖䙡䙌"],
|
||||
["fe80","䜣䜩䝼䞍⻊䥇䥺䥽䦂䦃䦅䦆䦟䦛䦷䦶䲣䲟䲠䲡䱷䲢䴓",6,"䶮",93]
|
||||
["fe80","䜣䜩䝼䞍⻊䥇䥺䥽䦂䦃䦅䦆䦟䦛䦷䦶䲣䲟䲠䲡䱷䲢䴓",6,"䶮",93],
|
||||
["8135f437",""]
|
||||
]
|
||||
|
92
node_modules/iconv-lite/encodings/utf16.js
generated
vendored
92
node_modules/iconv-lite/encodings/utf16.js
generated
vendored
@@ -61,6 +61,7 @@ Utf16BEDecoder.prototype.write = function(buf) {
|
||||
}
|
||||
|
||||
Utf16BEDecoder.prototype.end = function() {
|
||||
this.overflowByte = -1;
|
||||
}
|
||||
|
||||
|
||||
@@ -103,8 +104,8 @@ Utf16Encoder.prototype.end = function() {
|
||||
|
||||
function Utf16Decoder(options, codec) {
|
||||
this.decoder = null;
|
||||
this.initialBytes = [];
|
||||
this.initialBytesLen = 0;
|
||||
this.initialBufs = [];
|
||||
this.initialBufsLen = 0;
|
||||
|
||||
this.options = options || {};
|
||||
this.iconv = codec.iconv;
|
||||
@@ -113,17 +114,22 @@ function Utf16Decoder(options, codec) {
|
||||
Utf16Decoder.prototype.write = function(buf) {
|
||||
if (!this.decoder) {
|
||||
// Codec is not chosen yet. Accumulate initial bytes.
|
||||
this.initialBytes.push(buf);
|
||||
this.initialBytesLen += buf.length;
|
||||
this.initialBufs.push(buf);
|
||||
this.initialBufsLen += buf.length;
|
||||
|
||||
if (this.initialBytesLen < 16) // We need more bytes to use space heuristic (see below)
|
||||
if (this.initialBufsLen < 16) // We need more bytes to use space heuristic (see below)
|
||||
return '';
|
||||
|
||||
// We have enough bytes -> detect endianness.
|
||||
var buf = Buffer.concat(this.initialBytes),
|
||||
encoding = detectEncoding(buf, this.options.defaultEncoding);
|
||||
var encoding = detectEncoding(this.initialBufs, this.options.defaultEncoding);
|
||||
this.decoder = this.iconv.getDecoder(encoding, this.options);
|
||||
this.initialBytes.length = this.initialBytesLen = 0;
|
||||
|
||||
var resStr = '';
|
||||
for (var i = 0; i < this.initialBufs.length; i++)
|
||||
resStr += this.decoder.write(this.initialBufs[i]);
|
||||
|
||||
this.initialBufs.length = this.initialBufsLen = 0;
|
||||
return resStr;
|
||||
}
|
||||
|
||||
return this.decoder.write(buf);
|
||||
@@ -131,47 +137,61 @@ Utf16Decoder.prototype.write = function(buf) {
|
||||
|
||||
Utf16Decoder.prototype.end = function() {
|
||||
if (!this.decoder) {
|
||||
var buf = Buffer.concat(this.initialBytes),
|
||||
encoding = detectEncoding(buf, this.options.defaultEncoding);
|
||||
var encoding = detectEncoding(this.initialBufs, this.options.defaultEncoding);
|
||||
this.decoder = this.iconv.getDecoder(encoding, this.options);
|
||||
|
||||
var res = this.decoder.write(buf),
|
||||
trail = this.decoder.end();
|
||||
var resStr = '';
|
||||
for (var i = 0; i < this.initialBufs.length; i++)
|
||||
resStr += this.decoder.write(this.initialBufs[i]);
|
||||
|
||||
return trail ? (res + trail) : res;
|
||||
var trail = this.decoder.end();
|
||||
if (trail)
|
||||
resStr += trail;
|
||||
|
||||
this.initialBufs.length = this.initialBufsLen = 0;
|
||||
return resStr;
|
||||
}
|
||||
return this.decoder.end();
|
||||
}
|
||||
|
||||
function detectEncoding(buf, defaultEncoding) {
|
||||
var enc = defaultEncoding || 'utf-16le';
|
||||
function detectEncoding(bufs, defaultEncoding) {
|
||||
var b = [];
|
||||
var charsProcessed = 0;
|
||||
var asciiCharsLE = 0, asciiCharsBE = 0; // Number of ASCII chars when decoded as LE or BE.
|
||||
|
||||
if (buf.length >= 2) {
|
||||
// Check BOM.
|
||||
if (buf[0] == 0xFE && buf[1] == 0xFF) // UTF-16BE BOM
|
||||
enc = 'utf-16be';
|
||||
else if (buf[0] == 0xFF && buf[1] == 0xFE) // UTF-16LE BOM
|
||||
enc = 'utf-16le';
|
||||
else {
|
||||
// No BOM found. Try to deduce encoding from initial content.
|
||||
// Most of the time, the content has ASCII chars (U+00**), but the opposite (U+**00) is uncommon.
|
||||
// So, we count ASCII as if it was LE or BE, and decide from that.
|
||||
var asciiCharsLE = 0, asciiCharsBE = 0, // Counts of chars in both positions
|
||||
_len = Math.min(buf.length - (buf.length % 2), 64); // Len is always even.
|
||||
outer_loop:
|
||||
for (var i = 0; i < bufs.length; i++) {
|
||||
var buf = bufs[i];
|
||||
for (var j = 0; j < buf.length; j++) {
|
||||
b.push(buf[j]);
|
||||
if (b.length === 2) {
|
||||
if (charsProcessed === 0) {
|
||||
// Check BOM first.
|
||||
if (b[0] === 0xFF && b[1] === 0xFE) return 'utf-16le';
|
||||
if (b[0] === 0xFE && b[1] === 0xFF) return 'utf-16be';
|
||||
}
|
||||
|
||||
for (var i = 0; i < _len; i += 2) {
|
||||
if (buf[i] === 0 && buf[i+1] !== 0) asciiCharsBE++;
|
||||
if (buf[i] !== 0 && buf[i+1] === 0) asciiCharsLE++;
|
||||
if (b[0] === 0 && b[1] !== 0) asciiCharsBE++;
|
||||
if (b[0] !== 0 && b[1] === 0) asciiCharsLE++;
|
||||
|
||||
b.length = 0;
|
||||
charsProcessed++;
|
||||
|
||||
if (charsProcessed >= 100) {
|
||||
break outer_loop;
|
||||
}
|
||||
}
|
||||
|
||||
if (asciiCharsBE > asciiCharsLE)
|
||||
enc = 'utf-16be';
|
||||
else if (asciiCharsBE < asciiCharsLE)
|
||||
enc = 'utf-16le';
|
||||
}
|
||||
}
|
||||
|
||||
return enc;
|
||||
// Make decisions.
|
||||
// Most of the time, the content has ASCII chars (U+00**), but the opposite (U+**00) is uncommon.
|
||||
// So, we count ASCII as if it was LE or BE, and decide from that.
|
||||
if (asciiCharsBE > asciiCharsLE) return 'utf-16be';
|
||||
if (asciiCharsBE < asciiCharsLE) return 'utf-16le';
|
||||
|
||||
// Couldn't decide (likely all zeros or not enough data).
|
||||
return defaultEncoding || 'utf-16le';
|
||||
}
|
||||
|
||||
|
||||
|
319
node_modules/iconv-lite/encodings/utf32.js
generated
vendored
Normal file
319
node_modules/iconv-lite/encodings/utf32.js
generated
vendored
Normal file
@@ -0,0 +1,319 @@
|
||||
'use strict';
|
||||
|
||||
var Buffer = require('safer-buffer').Buffer;
|
||||
|
||||
// == UTF32-LE/BE codec. ==========================================================
|
||||
|
||||
exports._utf32 = Utf32Codec;
|
||||
|
||||
function Utf32Codec(codecOptions, iconv) {
|
||||
this.iconv = iconv;
|
||||
this.bomAware = true;
|
||||
this.isLE = codecOptions.isLE;
|
||||
}
|
||||
|
||||
exports.utf32le = { type: '_utf32', isLE: true };
|
||||
exports.utf32be = { type: '_utf32', isLE: false };
|
||||
|
||||
// Aliases
|
||||
exports.ucs4le = 'utf32le';
|
||||
exports.ucs4be = 'utf32be';
|
||||
|
||||
Utf32Codec.prototype.encoder = Utf32Encoder;
|
||||
Utf32Codec.prototype.decoder = Utf32Decoder;
|
||||
|
||||
// -- Encoding
|
||||
|
||||
function Utf32Encoder(options, codec) {
|
||||
this.isLE = codec.isLE;
|
||||
this.highSurrogate = 0;
|
||||
}
|
||||
|
||||
Utf32Encoder.prototype.write = function(str) {
|
||||
var src = Buffer.from(str, 'ucs2');
|
||||
var dst = Buffer.alloc(src.length * 2);
|
||||
var write32 = this.isLE ? dst.writeUInt32LE : dst.writeUInt32BE;
|
||||
var offset = 0;
|
||||
|
||||
for (var i = 0; i < src.length; i += 2) {
|
||||
var code = src.readUInt16LE(i);
|
||||
var isHighSurrogate = (0xD800 <= code && code < 0xDC00);
|
||||
var isLowSurrogate = (0xDC00 <= code && code < 0xE000);
|
||||
|
||||
if (this.highSurrogate) {
|
||||
if (isHighSurrogate || !isLowSurrogate) {
|
||||
// There shouldn't be two high surrogates in a row, nor a high surrogate which isn't followed by a low
|
||||
// surrogate. If this happens, keep the pending high surrogate as a stand-alone semi-invalid character
|
||||
// (technically wrong, but expected by some applications, like Windows file names).
|
||||
write32.call(dst, this.highSurrogate, offset);
|
||||
offset += 4;
|
||||
}
|
||||
else {
|
||||
// Create 32-bit value from high and low surrogates;
|
||||
var codepoint = (((this.highSurrogate - 0xD800) << 10) | (code - 0xDC00)) + 0x10000;
|
||||
|
||||
write32.call(dst, codepoint, offset);
|
||||
offset += 4;
|
||||
this.highSurrogate = 0;
|
||||
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (isHighSurrogate)
|
||||
this.highSurrogate = code;
|
||||
else {
|
||||
// Even if the current character is a low surrogate, with no previous high surrogate, we'll
|
||||
// encode it as a semi-invalid stand-alone character for the same reasons expressed above for
|
||||
// unpaired high surrogates.
|
||||
write32.call(dst, code, offset);
|
||||
offset += 4;
|
||||
this.highSurrogate = 0;
|
||||
}
|
||||
}
|
||||
|
||||
if (offset < dst.length)
|
||||
dst = dst.slice(0, offset);
|
||||
|
||||
return dst;
|
||||
};
|
||||
|
||||
Utf32Encoder.prototype.end = function() {
|
||||
// Treat any leftover high surrogate as a semi-valid independent character.
|
||||
if (!this.highSurrogate)
|
||||
return;
|
||||
|
||||
var buf = Buffer.alloc(4);
|
||||
|
||||
if (this.isLE)
|
||||
buf.writeUInt32LE(this.highSurrogate, 0);
|
||||
else
|
||||
buf.writeUInt32BE(this.highSurrogate, 0);
|
||||
|
||||
this.highSurrogate = 0;
|
||||
|
||||
return buf;
|
||||
};
|
||||
|
||||
// -- Decoding
|
||||
|
||||
function Utf32Decoder(options, codec) {
|
||||
this.isLE = codec.isLE;
|
||||
this.badChar = codec.iconv.defaultCharUnicode.charCodeAt(0);
|
||||
this.overflow = [];
|
||||
}
|
||||
|
||||
Utf32Decoder.prototype.write = function(src) {
|
||||
if (src.length === 0)
|
||||
return '';
|
||||
|
||||
var i = 0;
|
||||
var codepoint = 0;
|
||||
var dst = Buffer.alloc(src.length + 4);
|
||||
var offset = 0;
|
||||
var isLE = this.isLE;
|
||||
var overflow = this.overflow;
|
||||
var badChar = this.badChar;
|
||||
|
||||
if (overflow.length > 0) {
|
||||
for (; i < src.length && overflow.length < 4; i++)
|
||||
overflow.push(src[i]);
|
||||
|
||||
if (overflow.length === 4) {
|
||||
// NOTE: codepoint is a signed int32 and can be negative.
|
||||
// NOTE: We copied this block from below to help V8 optimize it (it works with array, not buffer).
|
||||
if (isLE) {
|
||||
codepoint = overflow[i] | (overflow[i+1] << 8) | (overflow[i+2] << 16) | (overflow[i+3] << 24);
|
||||
} else {
|
||||
codepoint = overflow[i+3] | (overflow[i+2] << 8) | (overflow[i+1] << 16) | (overflow[i] << 24);
|
||||
}
|
||||
overflow.length = 0;
|
||||
|
||||
offset = _writeCodepoint(dst, offset, codepoint, badChar);
|
||||
}
|
||||
}
|
||||
|
||||
// Main loop. Should be as optimized as possible.
|
||||
for (; i < src.length - 3; i += 4) {
|
||||
// NOTE: codepoint is a signed int32 and can be negative.
|
||||
if (isLE) {
|
||||
codepoint = src[i] | (src[i+1] << 8) | (src[i+2] << 16) | (src[i+3] << 24);
|
||||
} else {
|
||||
codepoint = src[i+3] | (src[i+2] << 8) | (src[i+1] << 16) | (src[i] << 24);
|
||||
}
|
||||
offset = _writeCodepoint(dst, offset, codepoint, badChar);
|
||||
}
|
||||
|
||||
// Keep overflowing bytes.
|
||||
for (; i < src.length; i++) {
|
||||
overflow.push(src[i]);
|
||||
}
|
||||
|
||||
return dst.slice(0, offset).toString('ucs2');
|
||||
};
|
||||
|
||||
function _writeCodepoint(dst, offset, codepoint, badChar) {
|
||||
// NOTE: codepoint is signed int32 and can be negative. We keep it that way to help V8 with optimizations.
|
||||
if (codepoint < 0 || codepoint > 0x10FFFF) {
|
||||
// Not a valid Unicode codepoint
|
||||
codepoint = badChar;
|
||||
}
|
||||
|
||||
// Ephemeral Planes: Write high surrogate.
|
||||
if (codepoint >= 0x10000) {
|
||||
codepoint -= 0x10000;
|
||||
|
||||
var high = 0xD800 | (codepoint >> 10);
|
||||
dst[offset++] = high & 0xff;
|
||||
dst[offset++] = high >> 8;
|
||||
|
||||
// Low surrogate is written below.
|
||||
var codepoint = 0xDC00 | (codepoint & 0x3FF);
|
||||
}
|
||||
|
||||
// Write BMP char or low surrogate.
|
||||
dst[offset++] = codepoint & 0xff;
|
||||
dst[offset++] = codepoint >> 8;
|
||||
|
||||
return offset;
|
||||
};
|
||||
|
||||
Utf32Decoder.prototype.end = function() {
|
||||
this.overflow.length = 0;
|
||||
};
|
||||
|
||||
// == UTF-32 Auto codec =============================================================
|
||||
// Decoder chooses automatically from UTF-32LE and UTF-32BE using BOM and space-based heuristic.
|
||||
// Defaults to UTF-32LE. http://en.wikipedia.org/wiki/UTF-32
|
||||
// Encoder/decoder default can be changed: iconv.decode(buf, 'utf32', {defaultEncoding: 'utf-32be'});
|
||||
|
||||
// Encoder prepends BOM (which can be overridden with (addBOM: false}).
|
||||
|
||||
exports.utf32 = Utf32AutoCodec;
|
||||
exports.ucs4 = 'utf32';
|
||||
|
||||
function Utf32AutoCodec(options, iconv) {
|
||||
this.iconv = iconv;
|
||||
}
|
||||
|
||||
Utf32AutoCodec.prototype.encoder = Utf32AutoEncoder;
|
||||
Utf32AutoCodec.prototype.decoder = Utf32AutoDecoder;
|
||||
|
||||
// -- Encoding
|
||||
|
||||
function Utf32AutoEncoder(options, codec) {
|
||||
options = options || {};
|
||||
|
||||
if (options.addBOM === undefined)
|
||||
options.addBOM = true;
|
||||
|
||||
this.encoder = codec.iconv.getEncoder(options.defaultEncoding || 'utf-32le', options);
|
||||
}
|
||||
|
||||
Utf32AutoEncoder.prototype.write = function(str) {
|
||||
return this.encoder.write(str);
|
||||
};
|
||||
|
||||
Utf32AutoEncoder.prototype.end = function() {
|
||||
return this.encoder.end();
|
||||
};
|
||||
|
||||
// -- Decoding
|
||||
|
||||
function Utf32AutoDecoder(options, codec) {
|
||||
this.decoder = null;
|
||||
this.initialBufs = [];
|
||||
this.initialBufsLen = 0;
|
||||
this.options = options || {};
|
||||
this.iconv = codec.iconv;
|
||||
}
|
||||
|
||||
Utf32AutoDecoder.prototype.write = function(buf) {
|
||||
if (!this.decoder) {
|
||||
// Codec is not chosen yet. Accumulate initial bytes.
|
||||
this.initialBufs.push(buf);
|
||||
this.initialBufsLen += buf.length;
|
||||
|
||||
if (this.initialBufsLen < 32) // We need more bytes to use space heuristic (see below)
|
||||
return '';
|
||||
|
||||
// We have enough bytes -> detect endianness.
|
||||
var encoding = detectEncoding(this.initialBufs, this.options.defaultEncoding);
|
||||
this.decoder = this.iconv.getDecoder(encoding, this.options);
|
||||
|
||||
var resStr = '';
|
||||
for (var i = 0; i < this.initialBufs.length; i++)
|
||||
resStr += this.decoder.write(this.initialBufs[i]);
|
||||
|
||||
this.initialBufs.length = this.initialBufsLen = 0;
|
||||
return resStr;
|
||||
}
|
||||
|
||||
return this.decoder.write(buf);
|
||||
};
|
||||
|
||||
Utf32AutoDecoder.prototype.end = function() {
|
||||
if (!this.decoder) {
|
||||
var encoding = detectEncoding(this.initialBufs, this.options.defaultEncoding);
|
||||
this.decoder = this.iconv.getDecoder(encoding, this.options);
|
||||
|
||||
var resStr = '';
|
||||
for (var i = 0; i < this.initialBufs.length; i++)
|
||||
resStr += this.decoder.write(this.initialBufs[i]);
|
||||
|
||||
var trail = this.decoder.end();
|
||||
if (trail)
|
||||
resStr += trail;
|
||||
|
||||
this.initialBufs.length = this.initialBufsLen = 0;
|
||||
return resStr;
|
||||
}
|
||||
|
||||
return this.decoder.end();
|
||||
};
|
||||
|
||||
function detectEncoding(bufs, defaultEncoding) {
|
||||
var b = [];
|
||||
var charsProcessed = 0;
|
||||
var invalidLE = 0, invalidBE = 0; // Number of invalid chars when decoded as LE or BE.
|
||||
var bmpCharsLE = 0, bmpCharsBE = 0; // Number of BMP chars when decoded as LE or BE.
|
||||
|
||||
outer_loop:
|
||||
for (var i = 0; i < bufs.length; i++) {
|
||||
var buf = bufs[i];
|
||||
for (var j = 0; j < buf.length; j++) {
|
||||
b.push(buf[j]);
|
||||
if (b.length === 4) {
|
||||
if (charsProcessed === 0) {
|
||||
// Check BOM first.
|
||||
if (b[0] === 0xFF && b[1] === 0xFE && b[2] === 0 && b[3] === 0) {
|
||||
return 'utf-32le';
|
||||
}
|
||||
if (b[0] === 0 && b[1] === 0 && b[2] === 0xFE && b[3] === 0xFF) {
|
||||
return 'utf-32be';
|
||||
}
|
||||
}
|
||||
|
||||
if (b[0] !== 0 || b[1] > 0x10) invalidBE++;
|
||||
if (b[3] !== 0 || b[2] > 0x10) invalidLE++;
|
||||
|
||||
if (b[0] === 0 && b[1] === 0 && (b[2] !== 0 || b[3] !== 0)) bmpCharsBE++;
|
||||
if ((b[0] !== 0 || b[1] !== 0) && b[2] === 0 && b[3] === 0) bmpCharsLE++;
|
||||
|
||||
b.length = 0;
|
||||
charsProcessed++;
|
||||
|
||||
if (charsProcessed >= 100) {
|
||||
break outer_loop;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Make decisions.
|
||||
if (bmpCharsBE - invalidBE > bmpCharsLE - invalidLE) return 'utf-32be';
|
||||
if (bmpCharsBE - invalidBE < bmpCharsLE - invalidLE) return 'utf-32le';
|
||||
|
||||
// Couldn't decide (likely all zeros or not enough data).
|
||||
return defaultEncoding || 'utf-32le';
|
||||
}
|
8
node_modules/iconv-lite/encodings/utf7.js
generated
vendored
8
node_modules/iconv-lite/encodings/utf7.js
generated
vendored
@@ -74,7 +74,7 @@ Utf7Decoder.prototype.write = function(buf) {
|
||||
if (i == lastI && buf[i] == minusChar) {// "+-" -> "+"
|
||||
res += "+";
|
||||
} else {
|
||||
var b64str = base64Accum + buf.slice(lastI, i).toString();
|
||||
var b64str = base64Accum + this.iconv.decode(buf.slice(lastI, i), "ascii");
|
||||
res += this.iconv.decode(Buffer.from(b64str, 'base64'), "utf16-be");
|
||||
}
|
||||
|
||||
@@ -91,7 +91,7 @@ Utf7Decoder.prototype.write = function(buf) {
|
||||
if (!inBase64) {
|
||||
res += this.iconv.decode(buf.slice(lastI), "ascii"); // Write direct chars.
|
||||
} else {
|
||||
var b64str = base64Accum + buf.slice(lastI).toString();
|
||||
var b64str = base64Accum + this.iconv.decode(buf.slice(lastI), "ascii");
|
||||
|
||||
var canBeDecoded = b64str.length - (b64str.length % 8); // Minimal chunk: 2 quads -> 2x3 bytes -> 3 chars.
|
||||
base64Accum = b64str.slice(canBeDecoded); // The rest will be decoded in future.
|
||||
@@ -245,7 +245,7 @@ Utf7IMAPDecoder.prototype.write = function(buf) {
|
||||
if (i == lastI && buf[i] == minusChar) { // "&-" -> "&"
|
||||
res += "&";
|
||||
} else {
|
||||
var b64str = base64Accum + buf.slice(lastI, i).toString().replace(/,/g, '/');
|
||||
var b64str = base64Accum + this.iconv.decode(buf.slice(lastI, i), "ascii").replace(/,/g, '/');
|
||||
res += this.iconv.decode(Buffer.from(b64str, 'base64'), "utf16-be");
|
||||
}
|
||||
|
||||
@@ -262,7 +262,7 @@ Utf7IMAPDecoder.prototype.write = function(buf) {
|
||||
if (!inBase64) {
|
||||
res += this.iconv.decode(buf.slice(lastI), "ascii"); // Write direct chars.
|
||||
} else {
|
||||
var b64str = base64Accum + buf.slice(lastI).toString().replace(/,/g, '/');
|
||||
var b64str = base64Accum + this.iconv.decode(buf.slice(lastI), "ascii").replace(/,/g, '/');
|
||||
|
||||
var canBeDecoded = b64str.length - (b64str.length % 8); // Minimal chunk: 2 quads -> 2x3 bytes -> 3 chars.
|
||||
base64Accum = b64str.slice(canBeDecoded); // The rest will be decoded in future.
|
||||
|
Reference in New Issue
Block a user