Guess what folks?! Python's PTY Module is full on broken. It randomly (inexplicably) dies on some IO. I could probably start looking at the source and fix it, but since I have to target OTHER people's machines, I would rather just work around it. The solution? subprocess_helper v2!

Code below:

#!/usr/bin/python2.4
"""Process helper application"""
#Enomalism; Web based XEN Management console
#Copyright (C) 2007 Enomaly
#
#This program is free software; you can redistribute it and/or
#modify it under the terms of the GNU Lesser General Public
#License as published by the Free Software Foundation; either
#version 2.1 of the License, or (at your option) any later version.
#
#This program is distributed in the hope that it will be useful,
#but WITHOUT ANY WARRANTY; without even the implied warranty of
#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#Lesser General Public License for more details.
#
#You should have received a copy of the GNU Lesser General Public
#License along with this program; if not, write to the Free Software
#Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
#
#Contact Enomaly at:
#180 Senneville,
#Montreal, Quebec H9X 3X2 Canada

import subprocess,os,sys,signal,pty,time,errno,thread

efp=None


def smart_wait_for_subprocess(sp,timeout=30):
    """
    Will wait for Process given to expire, and then kill it if
    the time elapses (or never if timeout==0). 
    @param sp: Subprocess to watch
    @param timeout: timeout length in seconds
    """
    running=1
    try:
        sleeps=timeout*4
        #Just wait around for termination...
        
        while sp.poll()==None:
            #print>>efp,"WAITING:",sleeps
            sleeps-=1
            time.sleep(.25)
            os.kill(sp.pid,0)
            if sleeps<=0 and timeout>0:
                os.kill(sp.pid,signal.SIGTERM)
                break
            elif sleeps<=0:
                break
        #Did it REALLY exit? 
        try:
            os.kill(sp.pid,0)
            sleeps=20
            while sp.poll()==None:
                #print>>efp,"KILLING:",sleeps
                sleeps-=1
                time.sleep(.25)
                
                #No? KILL IT!
                if sleeps<=0 and timeout>0:
                    #print>>efp,"KILLING2:",sleeps
                    os.kill(sp.pid,signal.SIGKILL)
                    break
                elif sleeps<=0:
                    break
        except:
            running=0
    except:
        #Make REALLY effing sure it exited.
        os.kill(sp.pid,signal.SIGKILL)
        #print>>efp,"KILLING2:",sleeps
        
    
    
        
    
def call_subprocess(args,timeout=30,stdin=None):
    """
    Will take a set of args and run them as a subprocess
    call inside of a thread, which will always exit after
    timeout seconds (or never if timeout==0). Pushes stdin
    into the pipe on the communicate call if it is given.    
    Designed to be called by run_in_subprocess, as it 
    doesn't have any forking in it.
    Returns tuple of (returncode,stdout,stderr)
    @param args: Subprocess arguments
    @param timeout: max time to wait for completion
    @param stdin: Input to stuff into stdin
    """
    stdout=False
    stderr=False
    try:
        P=subprocess.Popen(args,stdout=subprocess.PIPE,\
                            stderr=subprocess.PIPE,\
                            stdin=subprocess.PIPE)
    except Exception,e:
        return (1,'',str(e))
    if stdin==None or not stdin:
        P.stdin.close()
    else:
        P.stdin.write(stdin)
        P.stdin.close()
    try:
        smart_wait_for_subprocess(P,timeout)        
    except Exception,e:
        return (1,'',str(e))
    try:
        try:
            returncode=P.returncode
            stdout=P.stdout.read()
            stderr=P.stderr.read()
            if not stderr:
                stderr=""
        except Exception,e:
            stdout="ERR:",str(e)
    finally:
        try:
            del P
        except:
            pass
    return (returncode,stdout,stderr)
        

def run_in_subprocess(args,timeout=30,stdin=None):
    """
    Will take a set of arguments and run them as a subprocess
    call, which will always exit after
    timeout seconds. Pushes stdin
    into the pipe on the communicate call if it is given.    
    Uses fork, and requires no other signals to work. Good for
    running inside of twisted.
    Returns tuple of (returncode,stdout,stderr)
    @param args: Subprocess arguments
    @param timeout: max time to wait for completion
    @param stdin: Input to stuff into stdin
    """
    returncode,stdout,stderr=(1,"",\
                        "Undefined error calling subprocess")
    pid=os.fork() 
    if pid: 
        #We are in the original process. Wait for output with waitpid...
        try:
            #print "waiting for pid:",pid
            returned=os.waitpid(pid,0)
            #print "Returned:",returned
            returncode=os.WEXITSTATUS(returned[1])
            #print "Returned from ",pid
        except Exception,e:
            returncode=1
            stderr=str(e)
        try:
            sef=open(os.path.join('/tmp',str(pid)+'.stderr'),'r')
            stderr=sef.read()
            sef.close()
            os.unlink(os.path.join('/tmp',str(pid)+'.stderr'))
        except Exception,e:
            stderr=str(e)
        try:
            sof=open(os.path.join('/tmp',str(pid)+'.stdout'),'r')
            stdout=sof.read()
            sof.close()
            os.unlink(os.path.join('/tmp',str(pid)+'.stdout'))
        except Exception,e:
            stdout=str(e)
        
    else: 
        #We are in the forked process
        pid=os.getpid()
        try:
            #os.chdir("/") 
            #os.setsid() 
            #os.umask(0)
            try:
                returncode,stdout,stderr=call_subprocess(args,timeout,stdin)
            except:
                os._exit(1)
            if returncode==None:
                os._exit(1)
            if stdout==None:
                stdout=""
            if stderr==None:
                stderr=""
            try:
                sef=open(os.path.join('/tmp',str(pid)+'.stderr'),'w')
                sef.write(stderr)
                sef.close()
            except Exception,e:
                stderr=str(e)
            try:
                sof=open(os.path.join('/tmp',str(pid)+'.stdout'),'w')
                sof.write(stdout)
                sof.close()
            except Exception,e:
                stdout=str(e)     
            os._exit(returncode)       
                
        except:
            os._exit(returncode)
        os._exit(254) #Are we REALLY sure?
    return returncode,stdout,stderr
    

if __name__=='__main__':
    print time.time()
    print run_in_subprocess(['/bin/cat',],10,"This is a test!")
    print run_in_subprocess(['/usr/bin/sudo','dumpe2fs', '/dev/sda2'],10)
    print run_in_subprocess(['/bin/sleep','3',],15)
    print run_in_subprocess(\
                    ['/usr/bin/sudo','/sbin/iptables','-L','FORWARD'],10)
    print time.time()

__all__=['run_in_subprocess',]