subprocess source code learning - posix - 1
Just start from the initialization function Popen. Code like this:
def __init__(self, args, bufsize=0, executable=None,
stdin=None, stdout=None, stderr=None,
preexec_fn=None, close_fds=False, shell=False,
cwd=None, env=None, universal_newlines=False,
startupinfo=None, creationflags=0):
"""Create new Popen instance."""
_cleanup() # refer to Note 1
self._child_created = False
if not isinstance(bufsize, (int, long)):
raise TypeError("bufsize must be an integer")
# preexec_fn and close_fds are not supported by Windows
if mswindows:
if preexec_fn is not None:
raise ValueError("preexec_fn is not supported on Windows "
"platforms")
if close_fds and (stdin is not None or stdout is not None or
stderr is not None):
raise ValueError("close_fds is not supported on Windows "
"platforms if you redirect stdin/stdout/stderr")
# startupinfo and creationflags are not suppported by posix systems
else:
# POSIX
if startupinfo is not None:
raise ValueError("startupinfo is only supported on Windows "
"platforms")
if creationflags != 0:
raise ValueError("creationflags is only supported on Windows "
"platforms")
self.stdin = None
self.stdout = None
self.stderr = None
self.pid = None
self.returncode = None
self.universal_newlines = universal_newlines
# Input and output objects. The general principle is like
# this:
#
# Parent Child
# ------ -----
# p2cwrite ---stdin---> p2cread
# c2pread <--stdout--- c2pwrite
# errread <--stderr--- errwrite
#
# On POSIX, the child objects are file descriptors. On
# Windows, these are Windows file handles. The parent objects
# are file descriptors on both platforms. The parent objects
# are None when not using PIPEs. The child objects are None
# when not redirecting.
(p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite) = self._get_handles(stdin, stdout, stderr) # refer to Note 2
self._execute_child(args, executable, preexec_fn, close_fds,
cwd, env, universal_newlines,
startupinfo, creationflags, shell,
p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite) # launch the sub process. Refer to Note 3
if mswindows:
if p2cwrite is not None:
p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0)
if c2pread is not None:
c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0)
if errread is not None:
errread = msvcrt.open_osfhandle(errread.Detach(), 0)
# open fds of p2cwrite, c2pread, errread
if p2cwrite is not None:
self.stdin = os.fdopen(p2cwrite, 'wb', bufsize)
if c2pread is not None:
if universal_newlines:
# if universal_newlines is indicated
# tips about universal_newlines support:
# In addition to the standard fopen() values mode may be 'U' or 'rU'.
# Python is usually built with universal newline support;
# supplying 'U' opens the file as a text file, but lines may be terminated by any of the following:
# the Unix end-of-line convention '\n', the Macintosh convention '\r', or the Windows convention '\r\n'.
# All of these external representations are seen as '\n' by the Python program.
# If Python is built without universal newline support a mode with 'U' is the same as normal text mode.
# Note that file objects so opened also have an attribute called newlines which has a value of None (if no newlines have yet been seen), '\n', '\r', '\r\n', or a tuple containing all the newline types seen.
self.stdout = os.fdopen(c2pread, 'rU', bufsize)
else:
self.stdout = os.fdopen(c2pread, 'rb', bufsize)
if errread is not None:
if universal_newlines:
self.stderr = os.fdopen(errread, 'rU', bufsize)
else:
self.stderr = os.fdopen(errread, 'rb', bufsize)
Note 1:
Code of _cleanup() is like this:
def _cleanup():
'''
This function is used to remove the finished child process
'''
for inst in _active[:]:
if inst._internal_poll(_deadstate=sys.maxint) >= 0: # refer to Note 1-1
# return_value >= 0, means that the child process is finished.
try:
_active.remove(inst) # _active is initialized to [] when module of subprocess is first present (importted).
except ValueError:
# This can happen if two threads create a new Popen instance.
# It's harmless that it was already removed, so ignore.
pass
Note 1-1:
Code of self._internal_poll() is like this, which is defined within class Popen.
For finished sub processes, this function will return an integer GE 0, while None for the unfinished sub processes.
def _internal_poll(self, _deadstate=None):
"""Check if child process has terminated. Returns returncode
attribute."""
if self.returncode is None:
# means the child is not finished yet, then check it for another time.
try:
pid, sts = os.waitpid(self.pid, os.WNOHANG) # this will return immediately, if sub process finished, the return value is (pid, status)
# else, the return value is (0, 0)
if pid == self.pid:
# if child process finished, deal with return code
self._handle_exitstatus(sts) # refer to Note 1-1-1
else:
# else, do nothing, self.returncode would be None.
pass
except os.error:
if _deadstate is not None:
# if error occurs during watipid(), return the max int
self.returncode = _deadstate
# return the return code directly, if the process already finished
return self.returncode
Note 1-1-1:
Code of self._handle_exitstatus() is like the following code clips. And this function is implemented within class Popen.
If the child is terminated by a signal, then set the return code as the signal number;
if the child exits with system call exit(), then return the parameter of exit(). For example, if the exit statement is "exit(3)", then, 3 will be assigned to self.returncode.
def _handle_exitstatus(self, sts):
if os.WIFSIGNALED(sts):
# Return True if the process exited due to a signal, otherwise return False. Availability: Unix.
self.returncode = -os.WTERMSIG(sts) # Return the signal which caused the process to exit. Availability: Unix.
elif os.WIFEXITED(sts):
# Return True if the process exited using the exit(2) system call, otherwise return False. Availability: Unix.
self.returncode = os.WEXITSTATUS(sts) # If WIFEXITED(status) is true, return the integer parameter to the exit(2) system call. Otherwise, the return value is meaningless. Availability: Unix.
else:
# Should never happen
raise RuntimeError("Unknown child exit status!")
Note 2:
self._get_handles() is preparing the pipes for the sub process.
According to the official comments:
# Parent Child
# ------ -----
# p2cwrite ---stdin---> p2cread
# c2pread <--stdout--- c2pwrite
# errread <--stderr--- errwritewe call tell that,
p2cread is used for child to get the input from parent;
p2cwrite is used for parent to send input to child;
c2pread is used by parent to get output from chuild;
c2pwrite is used by child to send output to parent;
errread is used by parent to get error info from child;
errwirte is used by child to send error info to parent.
Code is like this:
def _get_handles(self, stdin, stdout, stderr):
"""Construct and return tupel with IO objects:
p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
"""
p2cread, p2cwrite = None, None
c2pread, c2pwrite = None, None
errread, errwrite = None, None
# stdin, stdout can be:
# None;
# PIPE;
# a file descirptor (a positive number);
# a file-object;
# With None, no redirection will occur; the child’s file handles will be inherited from the parent
# with PIPE, a new pipe will be craeted;
if stdin is None:
pass
elif stdin == PIPE:
p2cread, p2cwrite = os.pipe()
elif isinstance(stdin, int):
p2cread = stdin
else:
p2cread = stdin.fileno() # get the fd of the file-object
if stdout is None:
pass
elif stdout == PIPE:
c2pread, c2pwrite = os.pipe()
elif isinstance(stdout, int):
c2pwrite = stdout
else:
c2pwrite = stdout.fileno()
if stderr is None:
pass
elif stderr == PIPE:
errread, errwrite = os.pipe()
elif stderr == STDOUT:
# stedrr has another more option
errwrite = c2pwrite
elif isinstance(stderr, int):
errwrite = stderr
else:
# Assuming file-like object
errwrite = stderr.fileno()
return (p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite)
Note 3:
The child process is being launched by this, of which the code is:
def _execute_child(self, args, executable, preexec_fn, close_fds,
cwd, env, universal_newlines,
startupinfo, creationflags, shell,
p2cread, p2cwrite,
c2pread, c2pwrite,
errread, errwrite):
"""Execute program (POSIX version)"""
# convert the args to list
if isinstance(args, types.StringTypes): # module of types defines names for all type symbols known in the standard interpreter.
# types.StringTypes = (<type 'int'>, <type 'unicode'>)
# if args are passed in as string, code will arrive at here
args = [args]
else:
# if args are passed in as sequence, code will come here
args = list(args)
# handle shell
if shell:
args = ["/bin/sh", "-c"] + args
# handle executable
# consider this scenario, exectable is not indicated, means, it's None as default, and
# the args are passed in as a string, for example, 'ls -l'
# then what's will happen?
# this section of code is consider 'ls -l' as the executable, and absoultly, will get error
if executable is None:
executable = args[0]
# For transferring possible exec failure from child to parent
# The first char specifies the exception type: 0 means
# OSError, 1 means some other error.
# this pipe is used to get errors when try to luanch the executable.
# and after the executable is launched successfully, errread and errwrite will be used get the errors
errpipe_read, errpipe_write = os.pipe()
try:
try:
self._set_cloexec_flag(errpipe_write) # ?? don't konw what's this used for.
gc_was_enabled = gc.isenabled() # ?? don't konw why
# Disable gc to avoid bug where gc -> file_dealloc ->
# write to stderr -> hang. http://bugs.python.org/issue1336
# disable gobage collection before fork a clild process and then enable it.
gc.disable()
try:
self.pid = os.fork() # fork a sub process
except:
if gc_was_enabled:
gc.enable()
raise
self._child_created = True
if self.pid == 0:
# Child
try:
# Close parent's pipe ends
if p2cwrite is not None:
os.close(p2cwrite)
if c2pread is not None:
os.close(c2pread)
if errread is not None:
os.close(errread)
os.close(errpipe_read)
# Dup fds for child
# This is preparing for the call of self._close_fds() if close_fds is indicated
if p2cread is not None:
os.dup2(p2cread, 0)
if c2pwrite is not None:
os.dup2(c2pwrite, 1)
if errwrite is not None:
os.dup2(errwrite, 2)
# Close pipe fds. Make sure we don't close the same
# fd more than once, or standard fds.
if p2cread is not None and p2cread not in (0,):
os.close(p2cread)
if c2pwrite is not None and c2pwrite not in (p2cread, 1):
os.close(c2pwrite)
if errwrite is not None and errwrite not in (p2cread, c2pwrite, 2):
os.close(errwrite)
# Close all other fds, if asked for
# If close_fds is true, all file descriptors except 0, 1 and 2 will be closed before the child process is executed. (Unix only).
if close_fds:
self._close_fds(but=errpipe_write) # Refer to Note 3-1
# change cwd if cwd is indicated
if cwd is not None:
os.chdir(cwd)
# run preexec_fn if it's indicated
if preexec_fn:
preexec_fn()
# launch the executable
# On Unix, the new executable is loaded into the current process, and will have the same process id as the caller.
# So, the executable will have the same pid with the child process
if env is None:
os.execvp(executable, args)
else:
os.execvpe(executable, args, env)
except:
exc_type, exc_value, tb = sys.exc_info() # get exception info
exc_lines = traceback.format_exception(exc_type,
exc_value,
tb) # format exception info
exc_value.child_traceback = ''.join(exc_lines) # ?? Why do we need attribute of child_traceback here??
os.write(errpipe_write, pickle.dumps(exc_value)) # pickle Exception object to string and wirte the pipe.
# note, the errpipe_write is being used here, instead of errwite
# This exitcode won't be reported to applications, so it
# really doesn't matter what we return.
os._exit(255) # note the difference here between os.exit() and os._exit()
# Parent
if gc_was_enabled:
gc.enable()
finally:
# be sure the FD is closed no matter what, and it's closed by both parent and child
# Parent is closing this, becuase it's no use to parent;
# Child is closing this, becuase it has wirotten error info to it
os.close(errpipe_write)
if p2cread is not None and p2cwrite is not None:
os.close(p2cread) # close p2cread, which are intend for child, and leave p2cwrite to input something to child
if c2pwrite is not None and c2pread is not None:
os.close(c2pwrite) # close c2pwrite, which is intend for child, and leave c2pread to get output of child
if errwrite is not None and errread is not None:
os.close(errwrite) # close errwrite, which is intend for child, and leave errread to get the errors of child
# Wait for exec to fail or succeed; possibly raising exception
# Exception limited to 1M
# check the status of launching exectable.
data = _eintr_retry_call(os.read, errpipe_read, 1048576) # Refer to Note 3-2
finally:
# be sure the FD is closed no matter what
# errpipe_read is only closed in parent process, becuase it's no use now
# and the child has already closed it's own errpipe_read.
os.close(errpipe_read)
# data != "" means, there is somthing wrong when launching the executable
# if the executable is not launched successfully, should:
# wait sub process finishing,
# load the exception object,
# close the pipes,
# and then raise error
if data != "":
_eintr_retry_call(os.waitpid, self.pid, 0)
child_exception = pickle.loads(data)
for fd in (p2cwrite, c2pread, errread):
if fd is not None:
os.close(fd)
raise child_exception
Note 3-1:
Code of self._close_fds() is like this. And as you see, this function will close all the fds except 0, 1, 2 and the one indicated by "but"
def _close_fds(self, but):
os.closerange(3, but) # close all fds fo [3, but), but will not be closed.
os.closerange(but + 1, MAXFD)
Note 3-2:
Code of _eintr_retry_call() is like this. Not very sure about this function, but looks like its behavior is like this:
If a system call (if the func is a system call) is interrupted, then try again, till it finishes or other error occurs.
def _eintr_retry_call(func, *args):
while True:
try:
return func(*args)
except OSError, e:
if e.errno == errno.EINTR:
# errno.EINTR means interupted system call why this??
continue
raise
Other functions except the initialization function are very easy:
self.poll()
def poll(self):
return self._internal_poll() # Refer to Note 1-1self.wait()
def wait(self):
"""Wait for child process to terminate. Returns returncode
attribute."""
if self.returncode is None:
pid, sts = _eintr_retry_call(os.waitpid, self.pid, 0) # please note, the parent should be blocked here. Refer to Note 3-2
self._handle_exitstatus(sts) # Refer to Note 1-1-1
return self.returncodeself.send_signal, self.terminate and self.kill():
def send_signal(self, sig):
"""Send a signal to the process
"""
os.kill(self.pid, sig) # Send signal sig to the process pid.
# Constants for the specific signals available on the host platform are defined in the signal module.
# Availability: Unix.
def terminate(self):
"""Terminate the process with SIGTERM
"""
self.send_signal(signal.SIGTERM)
def kill(self):
"""Kill the process with SIGKILL
"""
self.send_signal(signal.SIGKILL)
To Be Continued.