1 // Copyright Joyent, Inc. and other Node contributors.
3 // Permission is hereby granted, free of charge, to any person obtaining a
4 // copy of this software and associated documentation files (the
5 // "Software"), to deal in the Software without restriction, including
6 // without limitation the rights to use, copy, modify, merge, publish,
7 // distribute, sublicense, and/or sell copies of the Software, and to permit
8 // persons to whom the Software is furnished to do so, subject to the
9 // following conditions:
11 // The above copyright notice and this permission notice shall be included
12 // in all copies or substantial portions of the Software.
14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15 // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16 // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
17 // NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
18 // DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
19 // OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
20 // USE OR OTHER DEALINGS IN THE SOFTWARE.
22 module.exports = Readable;
25 var isArray = require('isarray');
30 var Buffer = require('buffer').Buffer;
33 Readable.ReadableState = ReadableState;
35 var EE = require('events').EventEmitter;
38 if (!EE.listenerCount) EE.listenerCount = function(emitter, type) {
39 return emitter.listeners(type).length;
43 var Stream = require('stream');
46 var util = require('core-util-is');
47 util.inherits = require('inherits');
54 var debug = require('util');
55 if (debug && debug.debuglog) {
56 debug = debug.debuglog('stream');
58 debug = function () {};
63 util.inherits(Readable, Stream);
65 function ReadableState(options, stream) {
66 var Duplex = require('./_stream_duplex');
68 options = options || {};
70 // the point at which it stops calling _read() to fill the buffer
71 // Note: 0 is a valid value, means "don't call _read preemptively ever"
72 var hwm = options.highWaterMark;
73 var defaultHwm = options.objectMode ? 16 : 16 * 1024;
74 this.highWaterMark = (hwm || hwm === 0) ? hwm : defaultHwm;
77 this.highWaterMark = ~~this.highWaterMark;
85 this.endEmitted = false;
88 // a flag to be able to tell if the onwrite cb is called immediately,
89 // or on a later tick. We set this to true at first, because any
90 // actions that shouldn't happen until "later" should generally also
91 // not happen before the first write call.
94 // whenever we return null, then we set a flag to say
95 // that we're awaiting a 'readable' event emission.
96 this.needReadable = false;
97 this.emittedReadable = false;
98 this.readableListening = false;
101 // object stream flag. Used to make read(n) ignore n and to
102 // make all the buffer merging and length checks go away
103 this.objectMode = !!options.objectMode;
105 if (stream instanceof Duplex)
106 this.objectMode = this.objectMode || !!options.readableObjectMode;
108 // Crypto is kind of old and crusty. Historically, its default string
109 // encoding is 'binary' so we have to make this configurable.
110 // Everything else in the universe uses 'utf8', though.
111 this.defaultEncoding = options.defaultEncoding || 'utf8';
113 // when piping, we only care about 'readable' events that happen
114 // after read()ing all the bytes and not getting any pushback.
117 // the number of writers that are awaiting a drain event in .pipe()s
120 // if true, a maybeReadMore has been scheduled
121 this.readingMore = false;
124 this.encoding = null;
125 if (options.encoding) {
127 StringDecoder = require('string_decoder/').StringDecoder;
128 this.decoder = new StringDecoder(options.encoding);
129 this.encoding = options.encoding;
133 function Readable(options) {
134 var Duplex = require('./_stream_duplex');
136 if (!(this instanceof Readable))
137 return new Readable(options);
139 this._readableState = new ReadableState(options, this);
142 this.readable = true;
147 // Manually shove something into the read() buffer.
148 // This returns true if the highWaterMark has not been hit yet,
149 // similar to how Writable.write() returns true if you should
150 // write() some more.
151 Readable.prototype.push = function(chunk, encoding) {
152 var state = this._readableState;
154 if (util.isString(chunk) && !state.objectMode) {
155 encoding = encoding || state.defaultEncoding;
156 if (encoding !== state.encoding) {
157 chunk = new Buffer(chunk, encoding);
162 return readableAddChunk(this, state, chunk, encoding, false);
165 // Unshift should *always* be something directly out of read()
166 Readable.prototype.unshift = function(chunk) {
167 var state = this._readableState;
168 return readableAddChunk(this, state, chunk, '', true);
171 function readableAddChunk(stream, state, chunk, encoding, addToFront) {
172 var er = chunkInvalid(state, chunk);
174 stream.emit('error', er);
175 } else if (util.isNullOrUndefined(chunk)) {
176 state.reading = false;
178 onEofChunk(stream, state);
179 } else if (state.objectMode || chunk && chunk.length > 0) {
180 if (state.ended && !addToFront) {
181 var e = new Error('stream.push() after EOF');
182 stream.emit('error', e);
183 } else if (state.endEmitted && addToFront) {
184 var e = new Error('stream.unshift() after end event');
185 stream.emit('error', e);
187 if (state.decoder && !addToFront && !encoding)
188 chunk = state.decoder.write(chunk);
191 state.reading = false;
193 // if we want the data now, just emit it.
194 if (state.flowing && state.length === 0 && !state.sync) {
195 stream.emit('data', chunk);
198 // update the buffer info.
199 state.length += state.objectMode ? 1 : chunk.length;
201 state.buffer.unshift(chunk);
203 state.buffer.push(chunk);
205 if (state.needReadable)
206 emitReadable(stream);
209 maybeReadMore(stream, state);
211 } else if (!addToFront) {
212 state.reading = false;
215 return needMoreData(state);
220 // if it's past the high water mark, we can push in some more.
221 // Also, if we have no data yet, we can stand some
222 // more bytes. This is to work around cases where hwm=0,
223 // such as the repl. Also, if the push() triggered a
224 // readable event, and the user called read(largeNumber) such that
225 // needReadable was set, then we ought to push more, so that another
226 // 'readable' event will be triggered.
227 function needMoreData(state) {
228 return !state.ended &&
229 (state.needReadable ||
230 state.length < state.highWaterMark ||
234 // backwards compatibility.
235 Readable.prototype.setEncoding = function(enc) {
237 StringDecoder = require('string_decoder/').StringDecoder;
238 this._readableState.decoder = new StringDecoder(enc);
239 this._readableState.encoding = enc;
243 // Don't raise the hwm > 128MB
244 var MAX_HWM = 0x800000;
245 function roundUpToNextPowerOf2(n) {
249 // Get the next highest power of 2
251 for (var p = 1; p < 32; p <<= 1) n |= n >> p;
257 function howMuchToRead(n, state) {
258 if (state.length === 0 && state.ended)
261 if (state.objectMode)
262 return n === 0 ? 0 : 1;
264 if (isNaN(n) || util.isNull(n)) {
265 // only flow one buffer at a time
266 if (state.flowing && state.buffer.length)
267 return state.buffer[0].length;
275 // If we're asking for more than the target buffer level,
276 // then raise the water mark. Bump up to the next highest
277 // power of 2, to prevent increasing it excessively in tiny
279 if (n > state.highWaterMark)
280 state.highWaterMark = roundUpToNextPowerOf2(n);
282 // don't have that much. return null, unless we've ended.
283 if (n > state.length) {
285 state.needReadable = true;
294 // you can override either this method, or the async _read(n) below.
295 Readable.prototype.read = function(n) {
297 var state = this._readableState;
300 if (!util.isNumber(n) || n > 0)
301 state.emittedReadable = false;
303 // if we're doing read(0) to trigger a readable event, but we
304 // already have a bunch of data in the buffer, then just trigger
305 // the 'readable' event and move on.
307 state.needReadable &&
308 (state.length >= state.highWaterMark || state.ended)) {
309 debug('read: emitReadable', state.length, state.ended);
310 if (state.length === 0 && state.ended)
317 n = howMuchToRead(n, state);
319 // if we've ended, and we're now clear, then finish it up.
320 if (n === 0 && state.ended) {
321 if (state.length === 0)
326 // All the actual chunk generation logic needs to be
327 // *below* the call to _read. The reason is that in certain
328 // synthetic stream cases, such as passthrough streams, _read
329 // may be a completely synchronous operation which may change
330 // the state of the read buffer, providing enough data when
331 // before there was *not* enough.
333 // So, the steps are:
334 // 1. Figure out what the state of things will be after we do
335 // a read from the buffer.
337 // 2. If that resulting state will trigger a _read, then call _read.
338 // Note that this may be asynchronous, or synchronous. Yes, it is
339 // deeply ugly to write APIs this way, but that still doesn't mean
340 // that the Readable class should behave improperly, as streams are
341 // designed to be sync/async agnostic.
342 // Take note if the _read call is sync or async (ie, if the read call
343 // has returned yet), so that we know whether or not it's safe to emit
346 // 3. Actually pull the requested chunks out of the buffer and return.
348 // if we need a readable event, then we need to do some reading.
349 var doRead = state.needReadable;
350 debug('need readable', doRead);
352 // if we currently have less than the highWaterMark, then also read some
353 if (state.length === 0 || state.length - n < state.highWaterMark) {
355 debug('length less than watermark', doRead);
358 // however, if we've ended, then there's no point, and if we're already
359 // reading, then it's unnecessary.
360 if (state.ended || state.reading) {
362 debug('reading or ended', doRead);
367 state.reading = true;
369 // if the length is currently zero, then we *need* a readable event.
370 if (state.length === 0)
371 state.needReadable = true;
372 // call internal read method
373 this._read(state.highWaterMark);
377 // If _read pushed data synchronously, then `reading` will be false,
378 // and we need to re-evaluate how much data we can return to the user.
379 if (doRead && !state.reading)
380 n = howMuchToRead(nOrig, state);
384 ret = fromList(n, state);
388 if (util.isNull(ret)) {
389 state.needReadable = true;
395 // If we have nothing in the buffer, then we want to know
396 // as soon as we *do* get something into the buffer.
397 if (state.length === 0 && !state.ended)
398 state.needReadable = true;
400 // If we tried to read() past the EOF, then emit end on the next tick.
401 if (nOrig !== n && state.ended && state.length === 0)
404 if (!util.isNull(ret))
405 this.emit('data', ret);
410 function chunkInvalid(state, chunk) {
412 if (!util.isBuffer(chunk) &&
413 !util.isString(chunk) &&
414 !util.isNullOrUndefined(chunk) &&
416 er = new TypeError('Invalid non-string/buffer chunk');
422 function onEofChunk(stream, state) {
423 if (state.decoder && !state.ended) {
424 var chunk = state.decoder.end();
425 if (chunk && chunk.length) {
426 state.buffer.push(chunk);
427 state.length += state.objectMode ? 1 : chunk.length;
432 // emit 'readable' now to make sure it gets picked up.
433 emitReadable(stream);
436 // Don't emit readable right away in sync mode, because this can trigger
437 // another read() call => stack overflow. This way, it might trigger
438 // a nextTick recursion warning, but that's not so bad.
439 function emitReadable(stream) {
440 var state = stream._readableState;
441 state.needReadable = false;
442 if (!state.emittedReadable) {
443 debug('emitReadable', state.flowing);
444 state.emittedReadable = true;
446 process.nextTick(function() {
447 emitReadable_(stream);
450 emitReadable_(stream);
454 function emitReadable_(stream) {
455 debug('emit readable');
456 stream.emit('readable');
461 // at this point, the user has presumably seen the 'readable' event,
462 // and called read() to consume some data. that may have triggered
463 // in turn another _read(n) call, in which case reading = true if
465 // However, if we're not ended, or reading, and the length < hwm,
466 // then go ahead and try to read some more preemptively.
467 function maybeReadMore(stream, state) {
468 if (!state.readingMore) {
469 state.readingMore = true;
470 process.nextTick(function() {
471 maybeReadMore_(stream, state);
476 function maybeReadMore_(stream, state) {
477 var len = state.length;
478 while (!state.reading && !state.flowing && !state.ended &&
479 state.length < state.highWaterMark) {
480 debug('maybeReadMore read 0');
482 if (len === state.length)
483 // didn't get any data, stop spinning.
488 state.readingMore = false;
491 // abstract method. to be overridden in specific implementation classes.
492 // call cb(er, data) where data is <= n in length.
493 // for virtual (non-string, non-buffer) streams, "length" is somewhat
494 // arbitrary, and perhaps not very meaningful.
495 Readable.prototype._read = function(n) {
496 this.emit('error', new Error('not implemented'));
499 Readable.prototype.pipe = function(dest, pipeOpts) {
501 var state = this._readableState;
503 switch (state.pipesCount) {
508 state.pipes = [state.pipes, dest];
511 state.pipes.push(dest);
514 state.pipesCount += 1;
515 debug('pipe count=%d opts=%j', state.pipesCount, pipeOpts);
517 var doEnd = (!pipeOpts || pipeOpts.end !== false) &&
518 dest !== process.stdout &&
519 dest !== process.stderr;
521 var endFn = doEnd ? onend : cleanup;
522 if (state.endEmitted)
523 process.nextTick(endFn);
525 src.once('end', endFn);
527 dest.on('unpipe', onunpipe);
528 function onunpipe(readable) {
530 if (readable === src) {
540 // when the dest drains, it reduces the awaitDrain counter
541 // on the source. This would be more elegant with a .once()
542 // handler in flow(), but adding and removing repeatedly is
544 var ondrain = pipeOnDrain(src);
545 dest.on('drain', ondrain);
549 // cleanup event handlers once the pipe is broken
550 dest.removeListener('close', onclose);
551 dest.removeListener('finish', onfinish);
552 dest.removeListener('drain', ondrain);
553 dest.removeListener('error', onerror);
554 dest.removeListener('unpipe', onunpipe);
555 src.removeListener('end', onend);
556 src.removeListener('end', cleanup);
557 src.removeListener('data', ondata);
559 // if the reader is waiting for a drain event from this
560 // specific writer, then it would cause it to never start
562 // So, if this is awaiting a drain, then we just call it now.
563 // If we don't know, then assume that we are waiting for one.
564 if (state.awaitDrain &&
565 (!dest._writableState || dest._writableState.needDrain))
569 src.on('data', ondata);
570 function ondata(chunk) {
572 var ret = dest.write(chunk);
574 debug('false write response, pause',
575 src._readableState.awaitDrain);
576 src._readableState.awaitDrain++;
581 // if the dest has an error, then stop piping into it.
582 // however, don't suppress the throwing behavior for this.
583 function onerror(er) {
584 debug('onerror', er);
586 dest.removeListener('error', onerror);
587 if (EE.listenerCount(dest, 'error') === 0)
588 dest.emit('error', er);
590 // This is a brutally ugly hack to make sure that our error handler
591 // is attached before any userland ones. NEVER DO THIS.
592 if (!dest._events || !dest._events.error)
593 dest.on('error', onerror);
594 else if (isArray(dest._events.error))
595 dest._events.error.unshift(onerror);
597 dest._events.error = [onerror, dest._events.error];
601 // Both close and finish should trigger unpipe, but only once.
603 dest.removeListener('finish', onfinish);
606 dest.once('close', onclose);
607 function onfinish() {
609 dest.removeListener('close', onclose);
612 dest.once('finish', onfinish);
619 // tell the dest that it's being piped to
620 dest.emit('pipe', src);
622 // start the flow if it hasn't been started already.
623 if (!state.flowing) {
624 debug('pipe resume');
631 function pipeOnDrain(src) {
633 var state = src._readableState;
634 debug('pipeOnDrain', state.awaitDrain);
635 if (state.awaitDrain)
637 if (state.awaitDrain === 0 && EE.listenerCount(src, 'data')) {
638 state.flowing = true;
645 Readable.prototype.unpipe = function(dest) {
646 var state = this._readableState;
648 // if we're not piping anywhere, then do nothing.
649 if (state.pipesCount === 0)
652 // just one destination. most common case.
653 if (state.pipesCount === 1) {
654 // passed in one, but it's not the right one.
655 if (dest && dest !== state.pipes)
663 state.pipesCount = 0;
664 state.flowing = false;
666 dest.emit('unpipe', this);
670 // slow case. multiple pipe destinations.
674 var dests = state.pipes;
675 var len = state.pipesCount;
677 state.pipesCount = 0;
678 state.flowing = false;
680 for (var i = 0; i < len; i++)
681 dests[i].emit('unpipe', this);
685 // try to find the right one.
686 var i = indexOf(state.pipes, dest);
690 state.pipes.splice(i, 1);
691 state.pipesCount -= 1;
692 if (state.pipesCount === 1)
693 state.pipes = state.pipes[0];
695 dest.emit('unpipe', this);
700 // set up data events if they are asked for
701 // Ensure readable listeners eventually get something
702 Readable.prototype.on = function(ev, fn) {
703 var res = Stream.prototype.on.call(this, ev, fn);
705 // If listening to data, and it has not explicitly been paused,
706 // then call resume to start the flow of data on the next tick.
707 if (ev === 'data' && false !== this._readableState.flowing) {
711 if (ev === 'readable' && this.readable) {
712 var state = this._readableState;
713 if (!state.readableListening) {
714 state.readableListening = true;
715 state.emittedReadable = false;
716 state.needReadable = true;
717 if (!state.reading) {
719 process.nextTick(function() {
720 debug('readable nexttick read 0');
723 } else if (state.length) {
724 emitReadable(this, state);
731 Readable.prototype.addListener = Readable.prototype.on;
733 // pause() and resume() are remnants of the legacy readable stream API
734 // If the user uses them, then switch into old mode.
735 Readable.prototype.resume = function() {
736 var state = this._readableState;
737 if (!state.flowing) {
739 state.flowing = true;
740 if (!state.reading) {
741 debug('resume read 0');
749 function resume(stream, state) {
750 if (!state.resumeScheduled) {
751 state.resumeScheduled = true;
752 process.nextTick(function() {
753 resume_(stream, state);
758 function resume_(stream, state) {
759 state.resumeScheduled = false;
760 stream.emit('resume');
762 if (state.flowing && !state.reading)
766 Readable.prototype.pause = function() {
767 debug('call pause flowing=%j', this._readableState.flowing);
768 if (false !== this._readableState.flowing) {
770 this._readableState.flowing = false;
776 function flow(stream) {
777 var state = stream._readableState;
778 debug('flow', state.flowing);
781 var chunk = stream.read();
782 } while (null !== chunk && state.flowing);
786 // wrap an old-style stream as the async data source.
787 // This is *not* part of the readable stream interface.
788 // It is an ugly unfortunate mess of history.
789 Readable.prototype.wrap = function(stream) {
790 var state = this._readableState;
794 stream.on('end', function() {
795 debug('wrapped end');
796 if (state.decoder && !state.ended) {
797 var chunk = state.decoder.end();
798 if (chunk && chunk.length)
805 stream.on('data', function(chunk) {
806 debug('wrapped data');
808 chunk = state.decoder.write(chunk);
809 if (!chunk || !state.objectMode && !chunk.length)
812 var ret = self.push(chunk);
819 // proxy all the other methods.
820 // important when wrapping filters and duplexes.
821 for (var i in stream) {
822 if (util.isFunction(stream[i]) && util.isUndefined(this[i])) {
823 this[i] = function(method) { return function() {
824 return stream[method].apply(stream, arguments);
829 // proxy certain important events.
830 var events = ['error', 'close', 'destroy', 'pause', 'resume'];
831 forEach(events, function(ev) {
832 stream.on(ev, self.emit.bind(self, ev));
835 // when we try to consume some more bytes, simply unpause the
836 // underlying stream.
837 self._read = function(n) {
838 debug('wrapped _read', n);
850 // exposed for testing purposes only.
851 Readable._fromList = fromList;
853 // Pluck off n bytes from an array of buffers.
854 // Length is the combined lengths of all the buffers in the list.
855 function fromList(n, state) {
856 var list = state.buffer;
857 var length = state.length;
858 var stringMode = !!state.decoder;
859 var objectMode = !!state.objectMode;
862 // nothing in the list, definitely empty.
863 if (list.length === 0)
870 else if (!n || n >= length) {
871 // read it all, truncate the array.
875 ret = Buffer.concat(list, length);
878 // read just some of it.
879 if (n < list[0].length) {
880 // just take a part of the first list item.
881 // slice is the same for buffers and strings.
883 ret = buf.slice(0, n);
884 list[0] = buf.slice(n);
885 } else if (n === list[0].length) {
886 // first list is a perfect match
890 // we have enough to cover it, but it spans past the first buffer.
897 for (var i = 0, l = list.length; i < l && c < n; i++) {
899 var cpy = Math.min(n - c, buf.length);
902 ret += buf.slice(0, cpy);
904 buf.copy(ret, c, 0, cpy);
906 if (cpy < buf.length)
907 list[0] = buf.slice(cpy);
919 function endReadable(stream) {
920 var state = stream._readableState;
922 // If we get here before consuming all the bytes, then that is a
923 // bug in node. Should never happen.
924 if (state.length > 0)
925 throw new Error('endReadable called on non-empty stream');
927 if (!state.endEmitted) {
929 process.nextTick(function() {
930 // Check that we didn't get one last unshift.
931 if (!state.endEmitted && state.length === 0) {
932 state.endEmitted = true;
933 stream.readable = false;
940 function forEach (xs, f) {
941 for (var i = 0, l = xs.length; i < l; i++) {
946 function indexOf (xs, x) {
947 for (var i = 0, l = xs.length; i < l; i++) {
948 if (xs[i] === x) return i;