Commit 693df703 authored by Luke Campagnola's avatar Luke Campagnola
Browse files

Fixed multiprocess deadlock on windows

Added several utilities and debugging tools
Merge branch 'multiprocess-fix' into develop
parents 9cfc3a9f 79cfd360
......@@ -39,6 +39,12 @@ pyqtgraph-0.9.9 [unreleased]
- Added BarGraphItem.shape() to allow better mouse interaction
- Added MeshData.cylinder
- Added ViewBox.setBackgroundColor() and GLViewWidget.setBackgroundColor()
- Utilities / debugging tools
- Mutex used for tracing deadlocks
- Color output on terminal
- Multiprocess debugging colors messages by process
- Stdout filter that colors text by thread
- PeriodicTrace used to report deadlocks
Bugfixes:
- PlotCurveItem now has correct clicking behavior--clicks within a few px
......@@ -69,6 +75,7 @@ pyqtgraph-0.9.9 [unreleased]
- Fixed possible infinite loop from FiniteCache
- Allow images with NaN in ImageView
- MeshData can generate edges from face-indexed vertexes
- Fixed multiprocess deadlocks on windows
pyqtgraph-0.9.8 2013-11-24
......
......@@ -7,10 +7,12 @@ Distributed under MIT/X11 license. See license.txt for more infomation.
from __future__ import print_function
import sys, traceback, time, gc, re, types, weakref, inspect, os, cProfile
import sys, traceback, time, gc, re, types, weakref, inspect, os, cProfile, threading
from . import ptime
from numpy import ndarray
from .Qt import QtCore, QtGui
from .util.mutex import Mutex
from .util import cprint
__ftraceDepth = 0
def ftrace(func):
......@@ -991,3 +993,75 @@ class PrintDetector(object):
def flush(self):
self.stdout.flush()
class PeriodicTrace(object):
"""
Used to debug freezing by starting a new thread that reports on the
location of the main thread periodically.
"""
class ReportThread(QtCore.QThread):
def __init__(self):
self.frame = None
self.ind = 0
self.lastInd = None
self.lock = Mutex()
QtCore.QThread.__init__(self)
def notify(self, frame):
with self.lock:
self.frame = frame
self.ind += 1
def run(self):
while True:
time.sleep(1)
with self.lock:
if self.lastInd != self.ind:
print("== Trace %d: ==" % self.ind)
traceback.print_stack(self.frame)
self.lastInd = self.ind
def __init__(self):
self.mainThread = threading.current_thread()
self.thread = PeriodicTrace.ReportThread()
self.thread.start()
sys.settrace(self.trace)
def trace(self, frame, event, arg):
if threading.current_thread() is self.mainThread: # and 'threading' not in frame.f_code.co_filename:
self.thread.notify(frame)
# print("== Trace ==", event, arg)
# traceback.print_stack(frame)
return self.trace
class ThreadColor(object):
"""
Wrapper on stdout/stderr that colors text by the current thread ID.
*stream* must be 'stdout' or 'stderr'.
"""
colors = {}
lock = Mutex()
def __init__(self, stream):
self.stream = getattr(sys, stream)
self.err = stream == 'stderr'
setattr(sys, stream, self)
def write(self, msg):
with self.lock:
cprint.cprint(self.stream, self.color(), msg, -1, stderr=self.err)
def flush(self):
with self.lock:
self.stream.flush()
def color(self):
tid = threading.current_thread()
if tid not in self.colors:
c = (len(self.colors) % 15) + 1
self.colors[tid] = c
return self.colors[tid]
from .remoteproxy import RemoteEventHandler, ClosedError, NoResultError, LocalObjectProxy, ObjectProxy
import subprocess, atexit, os, sys, time, random, socket, signal
import multiprocessing.connection
from ..Qt import USE_PYSIDE
try:
import cPickle as pickle
except ImportError:
import pickle
from .remoteproxy import RemoteEventHandler, ClosedError, NoResultError, LocalObjectProxy, ObjectProxy
from ..Qt import USE_PYSIDE
from ..util import cprint # color printing for debugging
__all__ = ['Process', 'QtProcess', 'ForkedProcess', 'ClosedError', 'NoResultError']
class Process(RemoteEventHandler):
......@@ -35,7 +37,8 @@ class Process(RemoteEventHandler):
return objects either by proxy or by value (if they are picklable). See
ProxyObject for more information.
"""
_process_count = 1 # just used for assigning colors to each process for debugging
def __init__(self, name=None, target=None, executable=None, copySysPath=True, debug=False, timeout=20, wrapStdout=None):
"""
============== =============================================================
......@@ -64,7 +67,7 @@ class Process(RemoteEventHandler):
name = str(self)
if executable is None:
executable = sys.executable
self.debug = debug
self.debug = 7 if debug is True else False # 7 causes printing in white
## random authentication key
authkey = os.urandom(20)
......@@ -75,21 +78,20 @@ class Process(RemoteEventHandler):
#print "key:", ' '.join([str(ord(x)) for x in authkey])
## Listen for connection from remote process (and find free port number)
port = 10000
while True:
try:
l = multiprocessing.connection.Listener(('localhost', int(port)), authkey=authkey)
break
except socket.error as ex:
if ex.errno != 98 and ex.errno != 10048: # unix=98, win=10048
raise
port += 1
l = multiprocessing.connection.Listener(('localhost', 0), authkey=authkey)
port = l.address[1]
## start remote process, instruct it to run target function
sysPath = sys.path if copySysPath else None
bootstrap = os.path.abspath(os.path.join(os.path.dirname(__file__), 'bootstrap.py'))
self.debugMsg('Starting child process (%s %s)' % (executable, bootstrap))
# Decide on printing color for this process
if debug:
procDebug = (Process._process_count%6) + 1 # pick a color for this process to print in
Process._process_count += 1
else:
procDebug = False
if wrapStdout is None:
wrapStdout = sys.platform.startswith('win')
......@@ -102,8 +104,8 @@ class Process(RemoteEventHandler):
self.proc = subprocess.Popen((executable, bootstrap), stdin=subprocess.PIPE, stdout=stdout, stderr=stderr)
## to circumvent the bug and still make the output visible, we use
## background threads to pass data from pipes to stdout/stderr
self._stdoutForwarder = FileForwarder(self.proc.stdout, "stdout")
self._stderrForwarder = FileForwarder(self.proc.stderr, "stderr")
self._stdoutForwarder = FileForwarder(self.proc.stdout, "stdout", procDebug)
self._stderrForwarder = FileForwarder(self.proc.stderr, "stderr", procDebug)
else:
self.proc = subprocess.Popen((executable, bootstrap), stdin=subprocess.PIPE)
......@@ -120,7 +122,7 @@ class Process(RemoteEventHandler):
targetStr=targetStr,
path=sysPath,
pyside=USE_PYSIDE,
debug=debug
debug=procDebug
)
pickle.dump(data, self.proc.stdin)
self.proc.stdin.close()
......@@ -136,8 +138,8 @@ class Process(RemoteEventHandler):
continue
else:
raise
RemoteEventHandler.__init__(self, conn, name+'_parent', pid=self.proc.pid, debug=debug)
RemoteEventHandler.__init__(self, conn, name+'_parent', pid=self.proc.pid, debug=self.debug)
self.debugMsg('Connected to child process.')
atexit.register(self.join)
......@@ -167,10 +169,11 @@ class Process(RemoteEventHandler):
def startEventLoop(name, port, authkey, ppid, debug=False):
if debug:
import os
print('[%d] connecting to server at port localhost:%d, authkey=%s..' % (os.getpid(), port, repr(authkey)))
cprint.cout(debug, '[%d] connecting to server at port localhost:%d, authkey=%s..\n'
% (os.getpid(), port, repr(authkey)), -1)
conn = multiprocessing.connection.Client(('localhost', int(port)), authkey=authkey)
if debug:
print('[%d] connected; starting remote proxy.' % os.getpid())
cprint.cout(debug, '[%d] connected; starting remote proxy.\n' % os.getpid(), -1)
global HANDLER
#ppid = 0 if not hasattr(os, 'getppid') else os.getppid()
HANDLER = RemoteEventHandler(conn, name, ppid, debug=debug)
......@@ -380,17 +383,17 @@ class QtProcess(Process):
def __init__(self, **kwds):
if 'target' not in kwds:
kwds['target'] = startQtEventLoop
from ..Qt import QtGui ## avoid module-level import to keep bootstrap snappy.
self._processRequests = kwds.pop('processRequests', True)
if self._processRequests and QtGui.QApplication.instance() is None:
raise Exception("Must create QApplication before starting QtProcess, or use QtProcess(processRequests=False)")
Process.__init__(self, **kwds)
self.startEventTimer()
def startEventTimer(self):
from ..Qt import QtGui, QtCore ## avoid module-level import to keep bootstrap snappy.
from ..Qt import QtCore ## avoid module-level import to keep bootstrap snappy.
self.timer = QtCore.QTimer()
if self._processRequests:
app = QtGui.QApplication.instance()
if app is None:
raise Exception("Must create QApplication before starting QtProcess, or use QtProcess(processRequests=False)")
self.startRequestProcessing()
def startRequestProcessing(self, interval=0.01):
......@@ -412,10 +415,10 @@ class QtProcess(Process):
def startQtEventLoop(name, port, authkey, ppid, debug=False):
if debug:
import os
print('[%d] connecting to server at port localhost:%d, authkey=%s..' % (os.getpid(), port, repr(authkey)))
cprint.cout(debug, '[%d] connecting to server at port localhost:%d, authkey=%s..\n' % (os.getpid(), port, repr(authkey)), -1)
conn = multiprocessing.connection.Client(('localhost', int(port)), authkey=authkey)
if debug:
print('[%d] connected; starting remote proxy.' % os.getpid())
cprint.cout(debug, '[%d] connected; starting remote proxy.\n' % os.getpid(), -1)
from ..Qt import QtGui, QtCore
#from PyQt4 import QtGui, QtCore
app = QtGui.QApplication.instance()
......@@ -445,11 +448,13 @@ class FileForwarder(threading.Thread):
which ensures that the correct behavior is achieved even if
sys.stdout/stderr are replaced at runtime.
"""
def __init__(self, input, output):
def __init__(self, input, output, color):
threading.Thread.__init__(self)
self.input = input
self.output = output
self.lock = threading.Lock()
self.daemon = True
self.color = color
self.start()
def run(self):
......@@ -457,12 +462,12 @@ class FileForwarder(threading.Thread):
while True:
line = self.input.readline()
with self.lock:
sys.stdout.write(line)
cprint.cout(self.color, line, -1)
elif self.output == 'stderr':
while True:
line = self.input.readline()
with self.lock:
sys.stderr.write(line)
cprint.cerr(self.color, line, -1)
else:
while True:
line = self.input.readline()
......
......@@ -7,6 +7,9 @@ except ImportError:
import builtins
import pickle
# color printing for debugging
from ..util import cprint
class ClosedError(Exception):
"""Raised when an event handler receives a request to close the connection
or discovers that the connection has been closed."""
......@@ -80,7 +83,7 @@ class RemoteEventHandler(object):
def debugMsg(self, msg):
if not self.debug:
return
print("[%d] %s" % (os.getpid(), str(msg)))
cprint.cout(self.debug, "[%d] %s\n" % (os.getpid(), str(msg)), -1)
def getProxyOption(self, opt):
return self.proxyOptions[opt]
......
Copyright (c) 2010 Jonathan Hartley
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
* Neither the name of the copyright holders, nor those of its contributors
may be used to endorse or promote products derived from this software without
specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Download and docs:
http://pypi.python.org/pypi/colorama
Development:
http://code.google.com/p/colorama
Discussion group:
https://groups.google.com/forum/#!forum/python-colorama
Description
===========
Makes ANSI escape character sequences for producing colored terminal text and
cursor positioning work under MS Windows.
ANSI escape character sequences have long been used to produce colored terminal
text and cursor positioning on Unix and Macs. Colorama makes this work on
Windows, too, by wrapping stdout, stripping ANSI sequences it finds (which
otherwise show up as gobbledygook in your output), and converting them into the
appropriate win32 calls to modify the state of the terminal. On other platforms,
Colorama does nothing.
Colorama also provides some shortcuts to help generate ANSI sequences
but works fine in conjunction with any other ANSI sequence generation library,
such as Termcolor (http://pypi.python.org/pypi/termcolor.)
This has the upshot of providing a simple cross-platform API for printing
colored terminal text from Python, and has the happy side-effect that existing
applications or libraries which use ANSI sequences to produce colored output on
Linux or Macs can now also work on Windows, simply by calling
``colorama.init()``.
An alternative approach is to install 'ansi.sys' on Windows machines, which
provides the same behaviour for all applications running in terminals. Colorama
is intended for situations where that isn't easy (e.g. maybe your app doesn't
have an installer.)
Demo scripts in the source code repository prints some colored text using
ANSI sequences. Compare their output under Gnome-terminal's built in ANSI
handling, versus on Windows Command-Prompt using Colorama:
.. image:: http://colorama.googlecode.com/hg/screenshots/ubuntu-demo.png
:width: 661
:height: 357
:alt: ANSI sequences on Ubuntu under gnome-terminal.
.. image:: http://colorama.googlecode.com/hg/screenshots/windows-demo.png
:width: 668
:height: 325
:alt: Same ANSI sequences on Windows, using Colorama.
These screengrabs show that Colorama on Windows does not support ANSI 'dim
text': it looks the same as 'normal text'.
License
=======
Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file.
Dependencies
============
None, other than Python. Tested on Python 2.5.5, 2.6.5, 2.7, 3.1.2, and 3.2
Usage
=====
Initialisation
--------------
Applications should initialise Colorama using::
from colorama import init
init()
If you are on Windows, the call to ``init()`` will start filtering ANSI escape
sequences out of any text sent to stdout or stderr, and will replace them with
equivalent Win32 calls.
Calling ``init()`` has no effect on other platforms (unless you request other
optional functionality, see keyword args below.) The intention is that
applications can call ``init()`` unconditionally on all platforms, after which
ANSI output should just work.
To stop using colorama before your program exits, simply call ``deinit()``.
This will restore stdout and stderr to their original values, so that Colorama
is disabled. To start using Colorama again, call ``reinit()``, which wraps
stdout and stderr again, but is cheaper to call than doing ``init()`` all over
again.
Colored Output
--------------
Cross-platform printing of colored text can then be done using Colorama's
constant shorthand for ANSI escape sequences::
from colorama import Fore, Back, Style
print(Fore.RED + 'some red text')
print(Back.GREEN + 'and with a green background')
print(Style.DIM + 'and in dim text')
print(Fore.RESET + Back.RESET + Style.RESET_ALL)
print('back to normal now')
or simply by manually printing ANSI sequences from your own code::
print('/033[31m' + 'some red text')
print('/033[30m' # and reset to default color)
or Colorama can be used happily in conjunction with existing ANSI libraries
such as Termcolor::
from colorama import init
from termcolor import colored
# use Colorama to make Termcolor work on Windows too
init()
# then use Termcolor for all colored text output
print(colored('Hello, World!', 'green', 'on_red'))
Available formatting constants are::
Fore: BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE, RESET.
Back: BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE, RESET.
Style: DIM, NORMAL, BRIGHT, RESET_ALL
Style.RESET_ALL resets foreground, background and brightness. Colorama will
perform this reset automatically on program exit.
Cursor Positioning
------------------
ANSI codes to reposition the cursor are supported. See demos/demo06.py for
an example of how to generate them.
Init Keyword Args
-----------------
``init()`` accepts some kwargs to override default behaviour.
init(autoreset=False):
If you find yourself repeatedly sending reset sequences to turn off color
changes at the end of every print, then ``init(autoreset=True)`` will
automate that::
from colorama import init
init(autoreset=True)
print(Fore.RED + 'some red text')
print('automatically back to default color again')
init(strip=None):
Pass ``True`` or ``False`` to override whether ansi codes should be
stripped from the output. The default behaviour is to strip if on Windows.
init(convert=None):
Pass ``True`` or ``False`` to override whether to convert ansi codes in the
output into win32 calls. The default behaviour is to convert if on Windows
and output is to a tty (terminal).
init(wrap=True):
On Windows, colorama works by replacing ``sys.stdout`` and ``sys.stderr``
with proxy objects, which override the .write() method to do their work. If
this wrapping causes you problems, then this can be disabled by passing
``init(wrap=False)``. The default behaviour is to wrap if autoreset or
strip or convert are True.
When wrapping is disabled, colored printing on non-Windows platforms will
continue to work as normal. To do cross-platform colored output, you can
use Colorama's ``AnsiToWin32`` proxy directly::
import sys
from colorama import init, AnsiToWin32
init(wrap=False)
stream = AnsiToWin32(sys.stderr).stream
# Python 2
print >>stream, Fore.BLUE + 'blue text on stderr'
# Python 3
print(Fore.BLUE + 'blue text on stderr', file=stream)
Status & Known Problems
=======================
I've personally only tested it on WinXP (CMD, Console2), Ubuntu
(gnome-terminal, xterm), and OSX.
Some presumably valid ANSI sequences aren't recognised (see details below)
but to my knowledge nobody has yet complained about this. Puzzling.
See outstanding issues and wishlist at:
http://code.google.com/p/colorama/issues/list
If anything doesn't work for you, or doesn't do what you expected or hoped for,
I'd love to hear about it on that issues list, would be delighted by patches,
and would be happy to grant commit access to anyone who submits a working patch
or two.
Recognised ANSI Sequences
=========================
ANSI sequences generally take the form:
ESC [ <param> ; <param> ... <command>
Where <param> is an integer, and <command> is a single letter. Zero or more
params are passed to a <command>. If no params are passed, it is generally
synonymous with passing a single zero. No spaces exist in the sequence, they
have just been inserted here to make it easy to read.
The only ANSI sequences that colorama converts into win32 calls are::
ESC [ 0 m # reset all (colors and brightness)
ESC [ 1 m # bright
ESC [ 2 m # dim (looks same as normal brightness)
ESC [ 22 m # normal brightness
# FOREGROUND:
ESC [ 30 m # black
ESC [ 31 m # red
ESC [ 32 m # green
ESC [ 33 m # yellow
ESC [ 34 m # blue
ESC [ 35 m # magenta
ESC [ 36 m # cyan
ESC [ 37 m # white
ESC [ 39 m # reset
# BACKGROUND
ESC [ 40 m # black
ESC [ 41 m # red
ESC [ 42 m # green
ESC [ 43 m # yellow
ESC [ 44 m # blue
ESC [ 45 m # magenta
ESC [ 46 m # cyan
ESC [ 47 m # white
ESC [ 49 m # reset
# cursor positioning
ESC [ y;x H # position cursor at x across, y down
# clear the screen
ESC [ mode J # clear the screen. Only mode 2 (clear entire screen)
# is supported. It should be easy to add other modes,
# let me know if that would be useful.
Multiple numeric params to the 'm' command can be combined into a single
sequence, eg::
ESC [ 36 ; 45 ; 1 m # bright cyan text on magenta background
All other ANSI sequences of the form ``ESC [ <param> ; <param> ... <command>``
are silently stripped from the output on Windows.
Any other form of ANSI sequence, such as single-character codes or alternative
initial characters, are not recognised nor stripped. It would be cool to add
them though. Let me know if it would be useful for you, via the issues on
google code.
Development
===========
Help and fixes welcome! Ask Jonathan for commit rights, you'll get them.
Running tests requires:
- Michael Foord's 'mock' module to be installed.
- Tests are written using the 2010 era updates to 'unittest', and require to
be run either using Python2.7 or greater, or else to have Michael Foord's
'unittest2' module installed.
unittest2 test discovery doesn't work for colorama, so I use 'nose'::
nosetests -s
The -s is required because 'nosetests' otherwise applies a proxy of its own to
stdout, which confuses the unit tests.
Contact
=======