function IconvLiteDecoderStream(conv, options) {
this.conv = conv;
options = options || {};
options.encoding = this.encoding = 'utf8'; // We output strings.
Transform.call(this, options);
}n/a
function IconvLiteEncoderStream(conv, options) {
this.conv = conv;
options = options || {};
options.decodeStrings = false; // We accept only strings, so we don't need to decode them.
Transform.call(this, options);
}n/a
_collect = function (cb) {
var res = '';
this.on('error', cb);
this.on('data', function(chunk) { res += chunk; });
this.on('end', function() {
cb(null, res);
});
return this;
}n/a
function decode(buf, encoding, options) {
if (typeof buf === 'string') {
if (!iconv.skipDecodeWarning) {
console.error('Iconv-lite warning: decode()-ing strings is deprecated. Refer to https://github.com/ashtuchkin/iconv-
lite/wiki/Use-Buffers-when-decoding');
iconv.skipDecodeWarning = true;
}
buf = new Buffer("" + (buf || ""), "binary"); // Ensure buffer.
}
var decoder = iconv.getDecoder(encoding, options);
var res = decoder.write(buf);
var trail = decoder.end();
return trail ? (res + trail) : res;
}...
## Usage
### Basic API
```javascript
var iconv = require('iconv-lite');
// Convert from an encoded buffer to js string.
str = iconv.decode(new Buffer([0x68, 0x65, 0x6c, 0x6c, 0x6f]), 'win1251');
// Convert from js string to an encoded buffer.
buf = iconv.encode("Sample input string", 'win1251');
// Check if encoding is supported
iconv.encodingExists("us-ascii")
```
...function decodeStream(encoding, options) {
return new IconvLiteDecoderStream(iconv.getDecoder(encoding, options), options);
}...
```
### Streaming API (Node v0.10+)
```javascript
// Decode stream (from binary stream to js strings)
http.createServer(function(req, res) {
var converterStream = iconv.decodeStream('win1251');
req.pipe(converterStream);
converterStream.on('data', function(str) {
console.log(str); // Do something with decoded strings, chunk-by-chunk.
});
});
...function encode(str, encoding, options) {
str = "" + (str || ""); // Ensure string.
var encoder = iconv.getEncoder(encoding, options);
var res = encoder.write(str);
var trail = encoder.end();
return (trail && trail.length > 0) ? Buffer.concat([res, trail]) : res;
}...
```javascript
var iconv = require('iconv-lite');
// Convert from an encoded buffer to js string.
str = iconv.decode(new Buffer([0x68, 0x65, 0x6c, 0x6c, 0x6f]), 'win1251');
// Convert from js string to an encoded buffer.
buf = iconv.encode("Sample input string", 'win1251');
// Check if encoding is supported
iconv.encodingExists("us-ascii")
```
### Streaming API (Node v0.10+)
```javascript
...function encodeStream(encoding, options) {
return new IconvLiteEncoderStream(iconv.getEncoder(encoding, options), options);
}...
console.log(str); // Do something with decoded strings, chunk-by-chunk.
});
});
// Convert encoding streaming example
fs.createReadStream('file-in-win1251.txt')
.pipe(iconv.decodeStream('win1251'))
.pipe(iconv.encodeStream('ucs2'))
.pipe(fs.createWriteStream('file-in-ucs2.txt'));
// Sugar: all encode/decode streams have .collect(cb) method to accumulate data.
http.createServer(function(req, res) {
req.pipe(iconv.decodeStream('win1251')).collect(function(err, body) {
assert(typeof body == 'string');
console.log(body); // full request body string
...function encodingExists(enc) {
try {
iconv.getCodec(enc);
return true;
} catch (e) {
return false;
}
}...
// Convert from an encoded buffer to js string.
str = iconv.decode(new Buffer([0x68, 0x65, 0x6c, 0x6c, 0x6f]), 'win1251');
// Convert from js string to an encoded buffer.
buf = iconv.encode("Sample input string", 'win1251');
// Check if encoding is supported
iconv.encodingExists("us-ascii")
```
### Streaming API (Node v0.10+)
```javascript
// Decode stream (from binary stream to js strings)
http.createServer(function(req, res) {
...function extendNodeEncodings() {
if (original) return;
original = {};
if (!iconv.supportsNodeEncodingsExtension) {
console.error("ACTION NEEDED: require('iconv-lite').extendNodeEncodings() is not supported in your version of Node");
console.error("See more info at https://github.com/ashtuchkin/iconv-lite/wiki/Node-v4-compatibility");
return;
}
var nodeNativeEncodings = {
'hex': true, 'utf8': true, 'utf-8': true, 'ascii': true, 'binary': true,
'base64': true, 'ucs2': true, 'ucs-2': true, 'utf16le': true, 'utf-16le': true,
};
Buffer.isNativeEncoding = function(enc) {
return enc && nodeNativeEncodings[enc.toLowerCase()];
}
// -- SlowBuffer -----------------------------------------------------------
var SlowBuffer = require('buffer').SlowBuffer;
original.SlowBufferToString = SlowBuffer.prototype.toString;
SlowBuffer.prototype.toString = function(encoding, start, end) {
encoding = String(encoding || 'utf8').toLowerCase();
// Use native conversion when possible
if (Buffer.isNativeEncoding(encoding))
return original.SlowBufferToString.call(this, encoding, start, end);
// Otherwise, use our decoding method.
if (typeof start == 'undefined') start = 0;
if (typeof end == 'undefined') end = this.length;
return iconv.decode(this.slice(start, end), encoding);
}
original.SlowBufferWrite = SlowBuffer.prototype.write;
SlowBuffer.prototype.write = function(string, offset, length, encoding) {
// Support both (string, offset, length, encoding)
// and the legacy (string, encoding, offset, length)
if (isFinite(offset)) {
if (!isFinite(length)) {
encoding = length;
length = undefined;
}
} else { // legacy
var swap = encoding;
encoding = offset;
offset = length;
length = swap;
}
offset = +offset || 0;
var remaining = this.length - offset;
if (!length) {
length = remaining;
} else {
length = +length;
if (length > remaining) {
length = remaining;
}
}
encoding = String(encoding || 'utf8').toLowerCase();
// Use native conversion when possible
if (Buffer.isNativeEncoding(encoding))
return original.SlowBufferWrite.call(this, string, offset, length, encoding);
if (string.length > 0 && (length < 0 || offset < 0))
throw new RangeError('attempt to write beyond buffer bounds');
// Otherwise, use our encoding method.
var buf = iconv.encode(string, encoding);
if (buf.length < length) length = buf.length;
buf.copy(this, offset, 0, length);
return length;
}
// -- Buffer ---------------------------------------------------------------
original.BufferIsEncoding = Buffer.isEncoding;
Buffer.isEncoding = function(encoding) {
return Buffer.isNativeEncoding(encoding) || iconv.encodingExists(encoding);
}
original.BufferByteLength = Buffer.byteLength;
Buffer.byteLength = SlowBuffer.byteLength = function(str, encoding) {
encoding = String(encoding || 'utf8').toLowerCase();
// Use native conversion when possible
if (Buffer.isNativeEncoding(encoding))
return original.BufferByteLength.call(this, str, encoding);
// Slow, I know, but we don't have a better way yet.
return iconv.encode(str, encoding).length;
}
original.BufferToString = Buffer.prototype.toString;
Buffer.prototype.toString = function(encoding, start, end) {
encoding = String(encoding || 'utf8').toLowerCase();
// Use native conversion when possible
if (Buffer.isNativeEncoding(encoding))
return original.BufferToString.call(this, encoding, start, end);
// Otherwise, use our decoding method.
if (typeof start == 'undefined') start = 0; ......
```
### [Deprecated] Extend Node.js own encodings
> NOTE: This doesn't work on latest Node versions. See [details](https://github.com/ashtuchkin/iconv-lite/wiki/Node-v4
-compatibility).
```javascript
// After this call all Node basic primitives will understand iconv-lite encodings.
iconv.extendNodeEncodings();
// Examples:
buf = new Buffer(str, 'win1251');
buf.write(str, 'gbk');
str = buf.toString('latin1');
assert(Buffer.isEncoding('iso-8859-15'));
Buffer.byteLength(str, 'us-ascii');
...function decode(buf, encoding, options) {
if (typeof buf === 'string') {
if (!iconv.skipDecodeWarning) {
console.error('Iconv-lite warning: decode()-ing strings is deprecated. Refer to https://github.com/ashtuchkin/iconv-
lite/wiki/Use-Buffers-when-decoding');
iconv.skipDecodeWarning = true;
}
buf = new Buffer("" + (buf || ""), "binary"); // Ensure buffer.
}
var decoder = iconv.getDecoder(encoding, options);
var res = decoder.write(buf);
var trail = decoder.end();
return trail ? (res + trail) : res;
}n/a
function getCodec(encoding) {
if (!iconv.encodings)
iconv.encodings = require("../encodings"); // Lazy load all encoding definitions.
// Canonicalize encoding name: strip all non-alphanumeric chars and appended year.
var enc = (''+encoding).toLowerCase().replace(/[^0-9a-z]|:\d{4}$/g, "");
// Traverse iconv.encodings to find actual codec.
var codecOptions = {};
while (true) {
var codec = iconv._codecDataCache[enc];
if (codec)
return codec;
var codecDef = iconv.encodings[enc];
switch (typeof codecDef) {
case "string": // Direct alias to other encoding.
enc = codecDef;
break;
case "object": // Alias with options. Can be layered.
for (var key in codecDef)
codecOptions[key] = codecDef[key];
if (!codecOptions.encodingName)
codecOptions.encodingName = enc;
enc = codecDef.type;
break;
case "function": // Codec itself.
if (!codecOptions.encodingName)
codecOptions.encodingName = enc;
// The codec function must load all tables and return object with .encoder and .decoder methods.
// It'll be called only once (for each different options object).
codec = new codecDef(codecOptions, iconv);
iconv._codecDataCache[codecOptions.encodingName] = codec; // Save it to be reused later.
return codec;
default:
throw new Error("Encoding not recognized: '" + encoding + "' (searched as: '"+enc+"')");
}
}
}n/a
function getDecoder(encoding, options) {
var codec = iconv.getCodec(encoding),
decoder = new codec.decoder(options, codec);
if (codec.bomAware && !(options && options.stripBOM === false))
decoder = new bomHandling.StripBOM(decoder, options);
return decoder;
}...
if (this.initialBytesLen < 16) // We need more bytes to use space heuristic (see below)
return '';
// We have enough bytes -> detect endianness.
var buf = Buffer.concat(this.initialBytes),
encoding = detectEncoding(buf, this.options.defaultEncoding);
this.decoder = this.iconv.getDecoder(encoding, this.options);
this.initialBytes.length = this.initialBytesLen = 0;
}
return this.decoder.write(buf);
}
Utf16Decoder.prototype.end = function() {
...function getEncoder(encoding, options) {
var codec = iconv.getCodec(encoding),
encoder = new codec.encoder(options, codec);
if (codec.bomAware && options && options.addBOM)
encoder = new bomHandling.PrependBOM(encoder, options);
return encoder;
}...
// -- Encoding (pass-through)
function Utf16Encoder(options, codec) {
options = options || {};
if (options.addBOM === undefined)
options.addBOM = true;
this.encoder = codec.iconv.getEncoder('utf-16le', options);
}
Utf16Encoder.prototype.write = function(str) {
return this.encoder.write(str);
}
Utf16Encoder.prototype.end = function() {
...function encode(str, encoding, options) {
str = "" + (str || ""); // Ensure string.
var encoder = iconv.getEncoder(encoding, options);
var res = encoder.write(str);
var trail = encoder.end();
return (trail && trail.length > 0) ? Buffer.concat([res, trail]) : res;
}n/a
function undoExtendNodeEncodings() {
if (!iconv.supportsNodeEncodingsExtension)
return;
if (!original)
throw new Error("require('iconv-lite').undoExtendNodeEncodings(): Nothing to undo; extendNodeEncodings() is not called.")
delete Buffer.isNativeEncoding;
var SlowBuffer = require('buffer').SlowBuffer;
SlowBuffer.prototype.toString = original.SlowBufferToString;
SlowBuffer.prototype.write = original.SlowBufferWrite;
Buffer.isEncoding = original.BufferIsEncoding;
Buffer.byteLength = original.BufferByteLength;
Buffer.prototype.toString = original.BufferToString;
Buffer.prototype.write = original.BufferWrite;
if (iconv.supportsStreams) {
var Readable = require('stream').Readable;
Readable.prototype.setEncoding = original.ReadableSetEncoding;
delete Readable.prototype.collect;
}
original = undefined;
}...
request = require('request');
request({
url: "http://github.com/",
encoding: "cp932"
});
// To remove extensions
iconv.undoExtendNodeEncodings();
```
## Supported encodings
* All node.js native encodings: utf8, ucs2 / utf16-le, ascii, binary, base64, hex.
* Additional unicode encodings: utf16, utf16-be, utf-7, utf-7-imap.
* All widespread singlebyte encodings: Windows 125x family, ISO-8859 family,
...function IconvLiteDecoderStream(conv, options) {
this.conv = conv;
options = options || {};
options.encoding = this.encoding = 'utf8'; // We output strings.
Transform.call(this, options);
}n/a
_flush = function (done) {
try {
var res = this.conv.end();
if (res && res.length) this.push(res, this.encoding);
done();
}
catch (e) {
done(e);
}
}n/a
_transform = function (chunk, encoding, done) {
if (!Buffer.isBuffer(chunk))
return done(new Error("Iconv decoding stream needs buffers as its input."));
try {
var res = this.conv.write(chunk);
if (res && res.length) this.push(res, this.encoding);
done();
}
catch (e) {
done(e);
}
}n/a
collect = function (cb) {
var res = '';
this.on('error', cb);
this.on('data', function(chunk) { res += chunk; });
this.on('end', function() {
cb(null, res);
});
return this;
}...
// Convert encoding streaming example
fs.createReadStream('file-in-win1251.txt')
.pipe(iconv.decodeStream('win1251'))
.pipe(iconv.encodeStream('ucs2'))
.pipe(fs.createWriteStream('file-in-ucs2.txt'));
// Sugar: all encode/decode streams have .collect(cb) method to accumulate data.
http.createServer(function(req, res) {
req.pipe(iconv.decodeStream('win1251')).collect(function(err, body) {
assert(typeof body == 'string');
console.log(body); // full request body string
});
});
```
...function IconvLiteEncoderStream(conv, options) {
this.conv = conv;
options = options || {};
options.decodeStrings = false; // We accept only strings, so we don't need to decode them.
Transform.call(this, options);
}n/a
_flush = function (done) {
try {
var res = this.conv.end();
if (res && res.length) this.push(res);
done();
}
catch (e) {
done(e);
}
}n/a
_transform = function (chunk, encoding, done) {
if (typeof chunk != 'string')
return done(new Error("Iconv encoding stream needs strings as its input."));
try {
var res = this.conv.write(chunk);
if (res && res.length) this.push(res);
done();
}
catch (e) {
done(e);
}
}n/a
collect = function (cb) {
var chunks = [];
this.on('error', cb);
this.on('data', function(chunk) { chunks.push(chunk); });
this.on('end', function() {
cb(null, Buffer.concat(chunks));
});
return this;
}...
// Convert encoding streaming example
fs.createReadStream('file-in-win1251.txt')
.pipe(iconv.decodeStream('win1251'))
.pipe(iconv.encodeStream('ucs2'))
.pipe(fs.createWriteStream('file-in-ucs2.txt'));
// Sugar: all encode/decode streams have .collect(cb) method to accumulate data.
http.createServer(function(req, res) {
req.pipe(iconv.decodeStream('win1251')).collect(function(err, body) {
assert(typeof body == 'string');
console.log(body); // full request body string
});
});
```
...function PrependBOMWrapper(encoder, options) {
this.encoder = encoder;
this.addBOM = true;
}n/a
function StripBOMWrapper(decoder, options) {
this.decoder = decoder;
this.pass = false;
this.options = options || {};
}n/a
function DBCSCodec(codecOptions, iconv) {
this.encodingName = codecOptions.encodingName;
if (!codecOptions)
throw new Error("DBCS codec is called without the data.")
if (!codecOptions.table)
throw new Error("Encoding '" + this.encodingName + "' has no data.");
// Load tables.
var mappingTable = codecOptions.table();
// Decode tables: MBCS -> Unicode.
// decodeTables is a trie, encoded as an array of arrays of integers. Internal arrays are trie nodes and all have len = 256.
// Trie root is decodeTables[0].
// Values: >= 0 -> unicode character code. can be > 0xFFFF
// == UNASSIGNED -> unknown/unassigned sequence.
// == GB18030_CODE -> this is the end of a GB18030 4-byte sequence.
// <= NODE_START -> index of the next node in our trie to process next byte.
// <= SEQ_START -> index of the start of a character code sequence, in decodeTableSeq.
this.decodeTables = [];
this.decodeTables[0] = UNASSIGNED_NODE.slice(0); // Create root node.
// Sometimes a MBCS char corresponds to a sequence of unicode chars. We store them as arrays of integers here.
this.decodeTableSeq = [];
// Actual mapping tables consist of chunks. Use them to fill up decode tables.
for (var i = 0; i < mappingTable.length; i++)
this._addDecodeChunk(mappingTable[i]);
this.defaultCharUnicode = iconv.defaultCharUnicode;
// Encode tables: Unicode -> DBCS.
// `encodeTable` is array mapping from unicode char to encoded char. All its values are integers for performance.
// Because it can be sparse, it is represented as array of buckets by 256 chars each. Bucket can be null.
// Values: >= 0 -> it is a normal char. Write the value (if <=256 then 1 byte, if <=65536 then 2 bytes, etc.).
// == UNASSIGNED -> no conversion found. Output a default char.
// <= SEQ_START -> it's an index in encodeTableSeq, see below. The character starts a sequence.
this.encodeTable = [];
// `encodeTableSeq` is used when a sequence of unicode characters is encoded as a single code. We use a tree of
// objects where keys correspond to characters in sequence and leafs are the encoded dbcs values. A special DEF_CHAR key
// means end of sequence (needed when one sequence is a strict subsequence of another).
// Objects are kept separately from encodeTable to increase performance.
this.encodeTableSeq = [];
// Some chars can be decoded, but need not be encoded.
var skipEncodeChars = {};
if (codecOptions.encodeSkipVals)
for (var i = 0; i < codecOptions.encodeSkipVals.length; i++) {
var val = codecOptions.encodeSkipVals[i];
if (typeof val === 'number')
skipEncodeChars[val] = true;
else
for (var j = val.from; j <= val.to; j++)
skipEncodeChars[j] = true;
}
// Use decode trie to recursively fill out encode tables.
this._fillEncodeTable(0, 0, skipEncodeChars);
// Add more encoding pairs when needed.
if (codecOptions.encodeAdd) {
for (var uChar in codecOptions.encodeAdd)
if (Object.prototype.hasOwnProperty.call(codecOptions.encodeAdd, uChar))
this._setEncodeChar(uChar.charCodeAt(0), codecOptions.encodeAdd[uChar]);
}
this.defCharSB = this.encodeTable[0][iconv.defaultCharSingleByte.charCodeAt(0)];
if (this.defCharSB === UNASSIGNED) this.defCharSB = this.encodeTable[0]['?'];
if (this.defCharSB === UNASSIGNED) this.defCharSB = "?".charCodeAt(0);
// Load & create GB18030 tables when needed.
if (typeof codecOptions.gb18030 === 'function') {
this.gb18030 = codecOptions.gb18030(); // Load GB18030 ranges.
// Add GB18030 decode tables.
var thirdByteNodeIdx = this.decodeTables.length;
var thirdByteNode = this.decodeTables[thirdByteNodeIdx] = UNASSIGNED_NODE.slice(0);
var fourthByteNodeIdx = this.decodeTables.length;
var fourthByteNode = this.decod ...n/a
function InternalCodec(codecOptions, iconv) {
this.enc = codecOptions.encodingName;
this.bomAware = codecOptions.bomAware;
if (this.enc === "base64")
this.encoder = InternalEncoderBase64;
else if (this.enc === "cesu8") {
this.enc = "utf8"; // Use utf8 for decoding.
this.encoder = InternalEncoderCesu8;
// Add decoder for versions of Node not supporting CESU-8
if (new Buffer("eda080", 'hex').toString().length == 3) {
this.decoder = InternalDecoderCesu8;
this.defaultCharUnicode = iconv.defaultCharUnicode;
}
}
}n/a
function SBCSCodec(codecOptions, iconv) {
if (!codecOptions)
throw new Error("SBCS codec is called without the data.")
// Prepare char buffer for decoding.
if (!codecOptions.chars || (codecOptions.chars.length !== 128 && codecOptions.chars.length !== 256))
throw new Error("Encoding '"+codecOptions.type+"' has incorrect 'chars' (must be of len 128 or 256)");
if (codecOptions.chars.length === 128) {
var asciiString = "";
for (var i = 0; i < 128; i++)
asciiString += String.fromCharCode(i);
codecOptions.chars = asciiString + codecOptions.chars;
}
this.decodeBuf = new Buffer(codecOptions.chars, 'ucs2');
// Encoding buffer.
var encodeBuf = new Buffer(65536);
encodeBuf.fill(iconv.defaultCharSingleByte.charCodeAt(0));
for (var i = 0; i < codecOptions.chars.length; i++)
encodeBuf[codecOptions.chars.charCodeAt(i)] = i;
this.encodeBuf = encodeBuf;
}n/a
function Utf16Codec(codecOptions, iconv) {
this.iconv = iconv;
}n/a
function Utf16BECodec() {
}n/a
function Utf7Codec(codecOptions, iconv) {
this.iconv = iconv;
}n/a
function Utf7IMAPCodec(codecOptions, iconv) {
this.iconv = iconv;
}n/a