Subprocess.Popen: cloning stdout and stderr both to terminal and variables

To capture and display at the same time both stdout and stderr from a child process line by line in a single thread, you could use asynchronous I/O:

#!/usr/bin/env python3import asyncioimport osimport sysfrom asyncio.subprocess import PIPE@asyncio.coroutinedef read_stream_and_display(stream, display):    """Read from stream line by line until EOF, display, and capture the lines.    """    output = []    while True:        line = yield from stream.readline()        if not line:            break        output.append(line)        display(line) # assume it doesn't block    return b''.join(output)@asyncio.coroutinedef read_and_display(*cmd):    """Capture cmd's stdout, stderr while displaying them as they arrive    (line by line).    """    # start process    process = yield from asyncio.create_subprocess_exec(*cmd,            stdout=PIPE, stderr=PIPE)    # read child's stdout/stderr concurrently (capture and display)    try:        stdout, stderr = yield from asyncio.gather(            read_stream_and_display(process.stdout, sys.stdout.buffer.write),            read_stream_and_display(process.stderr, sys.stderr.buffer.write))    except Exception:        process.kill()        raise    finally:        # wait for the process to exit        rc = yield from process.wait()    return rc, stdout, stderr# run the event loopif os.name == 'nt':    loop = asyncio.ProactorEventLoop() # for subprocess' pipes on Windows    asyncio.set_event_loop(loop)else:    loop = asyncio.get_event_loop()rc, *output = loop.run_until_complete(read_and_display(*cmd))loop.close()

You could spawn threads to read the stdout and stderr pipes, write to a common queue, and append to lists. Then use a third thread to print items from the queue.

import timeimport Queueimport sysimport threadingimport subprocessPIPE = subprocess.PIPEdef read_output(pipe, funcs):    for line in iter(pipe.readline, ''):        for func in funcs:            func(line)            # time.sleep(1)    pipe.close()def write_output(get):    for line in iter(get, None):        sys.stdout.write(line)process = subprocess.Popen(    ['random_print.py'], stdout=PIPE, stderr=PIPE, close_fds=True, bufsize=1)q = Queue.Queue()out, err = [], []tout = threading.Thread(    target=read_output, args=(process.stdout, [q.put, out.append]))terr = threading.Thread(    target=read_output, args=(process.stderr, [q.put, err.append]))twrite = threading.Thread(target=write_output, args=(q.get,))for t in (tout, terr, twrite):    t.daemon = True    t.start()process.wait()for t in (tout, terr):    t.join()q.put(None)print(out)print(err)

The reason for using the third thread -- instead of letting the first two threads both print directly to the terminal -- is to prevent both print statements from occurring concurrently, which can result in sometimes garbled text.

The above calls random_print.py, which prints to stdout and stderr at random:

import sysimport timeimport randomfor i in range(50):    f = random.choice([sys.stdout,sys.stderr])    f.write(str(i)+'\n')    f.flush()    time.sleep(0.1)

This solution borrows code and ideas from J. F. Sebastian, here.

Here is an alternative solution for Unix-like systems, using select.select:

import collectionsimport selectimport fcntlimport osimport timeimport Queueimport sysimport threadingimport subprocessPIPE = subprocess.PIPEdef make_async(fd):    # https://stackoverflow.com/a/7730201/190597    '''add the O_NONBLOCK flag to a file descriptor'''    fcntl.fcntl(        fd, fcntl.F_SETFL, fcntl.fcntl(fd, fcntl.F_GETFL) | os.O_NONBLOCK)def read_async(fd):    # https://stackoverflow.com/a/7730201/190597    '''read some data from a file descriptor, ignoring EAGAIN errors'''    # time.sleep(1)    try:        return fd.read()    except IOError, e:        if e.errno != errno.EAGAIN:            raise e        else:            return ''def write_output(fds, outmap):    for fd in fds:        line = read_async(fd)        sys.stdout.write(line)        outmap[fd.fileno()].append(line)process = subprocess.Popen(    ['random_print.py'], stdout=PIPE, stderr=PIPE, close_fds=True)make_async(process.stdout)make_async(process.stderr)outmap = collections.defaultdict(list)while True:    rlist, wlist, xlist = select.select([process.stdout, process.stderr], [], [])    write_output(rlist, outmap)    if process.poll() is not None:        write_output([process.stdout, process.stderr], outmap)        breakfileno = {'stdout': process.stdout.fileno(),          'stderr': process.stderr.fileno()}print(outmap[fileno['stdout']])print(outmap[fileno['stderr']])

This solution uses code and ideas from Adam Rosenfield's post, here.