dc-@sdZddlZddlZddlZddlZddlZddlZejddkZejddkZ ejdd!dskZ ejdd!dtkZ e edZ d Zd Zer d Zd Zd ZefZefZefZeZeZnKdZdZdZefZeefZeejfZeZeZeridZ n dZ de _erddZ!ddZ"ddZ$n$ddZ!ddZ"ddZ$de!_erdZ%dZ&dZ'd Z(n0ddl)Z)e)j*Z%e)j+Z&e)j,Z'e)j-Z(e.d!Z/d"Z0d#Z1d$Z2d%Z3d&Z4d'Z5d(Z6d)Z7d*Z8d+Z9erd,Z:ddd-Z;e<d.Z=nd/Z:d0j>dUd1e=_e;Z?d2Z@erd3ZAn d4ZAd5ZBd6ZCeZDeZEd7ZFd8ZGd9ZHd:ZId;ZJer~ddlKZKeLeKd<ZMnddd=ZMd>ZNdd?ZOy ePj3Wn#eQk rd@ZRdAZSnXdBZRdCZSerdDZTn dEZTdFdGdHdIdJdKdLdMdNdOdPdQdRdSdTdUdVdWdXdYdZd[d\d]d^d_d`dadbdcdddedfdgdhdidjdkdldmdndodpdqdrg-ZUdS(us< A selection of cross-compatible functions for Python 2 and 3. This module exports useful functions for 2/3 compatible code: * bind_method: binds functions to classes * ``native_str_to_bytes`` and ``bytes_to_native_str`` * ``native_str``: always equal to the native platform string object (because this may be shadowed by imports from future.builtins) * lists: lrange(), lmap(), lzip(), lfilter() * iterable method compatibility: - iteritems, iterkeys, itervalues - viewitems, viewkeys, viewvalues These use the original method if available, otherwise they use items, keys, values. * types: * text_type: unicode in Python 2, str in Python 3 * binary_type: str in Python 2, bythes in Python 3 * string_types: basestring in Python 2, str in Python 3 * bchr(c): Take an integer and make a 1-character byte string * bord(c) Take the result of indexing on a byte string and make an integer * tobytes(s) Take a text string, a byte string, or a sequence of characters taken from a byte string, and make a byte string. * raise_from() * raise_with_traceback() This module also defines these decorators: * ``python_2_unicode_compatible`` * ``with_metaclass`` * ``implements_iterator`` Some of the functions in this module come from the following sources: * Jinja2 (BSD licensed: see https://github.com/mitsuhiko/jinja2/blob/master/LICENSE) * Pandas compatibility module pandas.compat * six.py by Benjamin Peterson * Django iNiiiiitpypy_translation_infocCs%ts!|j|_d|_n|S(s A decorator that defines __unicode__ and __str__ methods under Python 2. Under Python 3, this decorator is a no-op. To support Python 2 and 3 with a single code base, define a __str__ method returning unicode text and apply this decorator to the class, like this:: >>> from future.utils import python_2_unicode_compatible >>> @python_2_unicode_compatible ... class MyClass(object): ... def __str__(self): ... return u'Unicode string: \u5b54\u5b50' >>> a = MyClass() Then, after this import: >>> from future.builtins import str the following is ``True`` on both Python 3 and 2:: >>> str(a) == a.encode('utf-8').decode('utf-8') True and, on a Unicode-enabled terminal with the right fonts, these both print the Chinese characters for Confucius:: >>> print(a) >>> print(str(a)) The implementation comes from django.utils.encoding. cSs|jjdS(Nsutf-8(t __unicode__tencode(tself((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pytft(tPY3t__str__R(tcls((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pytpython_2_unicode_compatibleAs# cs/dffdY}|ddiS(s Function from jinja2/_compat.py. License: BSD. Use it like this:: class BaseForm(object): pass class FormType(type): pass class Form(with_metaclass(FormType, BaseForm)): pass This requires a bit of explanation: the basic idea is to make a dummy metaclass for one level of class instantiation that replaces itself with the actual metaclass. Because of internal type checks we also need to make sure that we downgrade the custom metaclass for one level to something closer to type (that's why __call__ and __init__ comes back from type etc.). This has the advantage over six.with_metaclass of not introducing dummy classes into the final MRO. t metaclasscs,eZejZejZfdZRS(cs2|dkr"tj||d|S||S(N((tNonettypet__new__(Rtnamet this_basestd(tbasestmeta(sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pyR s (t__name__t __module__R t__call__t__init__R ((RR(sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pyR s  ttemporary_classN(R (RRR ((RRsc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pytwith_metaclassjscCs t|gS(N(tbytes(ts((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pytbchrscCs*t|trt|dSt|SdS(Nslatin-1(t isinstancetstrR(R((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pytbstrs cCs|S(N((R((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pytbordscCs t|S(N(tchr(R((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pyRscCs t|S(N(R(R((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pyRscCs t|S(N(tord(R((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pyRscCs=t|tr|St|tr/|jdSt|SdS(Nslatin-1(RRRR(R((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pyttobytess  cCs-t|tr|jdSdj|SdS(Nslatin-1R(RtunicodeRtjoin(R((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pyR"s sS Encodes to latin-1 (where the first 256 chars are the same as ASCII.) sutf-8cCs |j|S(N(R(Rtencoding((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pytnative_str_to_bytesscCs |j|S(N(tdecode(tbR%((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pytbytes_to_native_strscCs|S(N((ttR%((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pyttext_to_native_strscCsddlm}||S(Ni(tnewbytes(t future.typesR,(RR%R,((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pyR&scCs t|S(N(tnative(R(R%((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pyR)stasciicCst|j|S(s} Use this to create a Py2 native string when "from __future__ import unicode_literals" is in effect. (R#R(R*R%((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pyR+ssu On Py3, returns an encoded string. On Py2, returns a newbytes type, ignoring the ``encoding`` argument. cOstt||S(N(tlisttrange(targstkwargs((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pytlrangescOstt||S(N(R0tzip(R2R3((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pytlzipscOstt||S(N(R0tmap(R2R3((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pytlmapscOstt||S(N(R0tfilter(R2R3((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pytlfilterscCse|r#td|jdDStr3|jSddl}|jd}t|j|SdS(sE A function equivalent to the str.isidentifier method on Py3 css|]}t|VqdS(N(t isidentifier(t.0ta((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pys st.iNs[a-zA-Z_][a-zA-Z0-9_]*$(talltsplitRR;tretcompiletbooltmatch(RtdottedRAt_name_re((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pyR;s  cKs.t|dd}|s$|j}n||S(s Function for iterating over dictionary items with the same set-like behaviour on Py2.7 as on Py3. Passes kwargs to method.t viewitemsN(tgetattrR titems(tobjR3tfunc((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pyRGs cKs.t|dd}|s$|j}n||S(s Function for iterating over dictionary keys with the same set-like behaviour on Py2.7 as on Py3. Passes kwargs to method.tviewkeysN(RHR tkeys(RJR3RK((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pyRLs cKs.t|dd}|s$|j}n||S(s Function for iterating over dictionary values with the same set-like behaviour on Py2.7 as on Py3. Passes kwargs to method.t viewvaluesN(RHR tvalues(RJR3RK((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pyRN s cKs.t|dd}|s$|j}n||S(ssUse this only if compatibility with Python versions before 2.7 is required. Otherwise, prefer viewitems(). t iteritemsN(RHR RI(RJR3RK((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pyRP,s cKs.t|dd}|s$|j}n||S(srUse this only if compatibility with Python versions before 2.7 is required. Otherwise, prefer viewkeys(). titerkeysN(RHR RM(RJR3RK((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pyRQ6s cKs.t|dd}|s$|j}n||S(stUse this only if compatibility with Python versions before 2.7 is required. Otherwise, prefer viewvalues(). t itervaluesN(RHR RO(RJR3RK((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pyRR@s cCs<ts(t||tj|d|nt|||dS(s/Bind a method to class, python 2 and python 3 compatible. Parameters ---------- cls : type class to receive bound method name : basestring name of method on class instance func : function function to be bound as method Returns ------- None N(Rtsetattrttypest MethodTypeR (RRRK((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pyt bind_methodJs"cCstjdS(Ni(tsystexc_info(((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pyt getexceptionbscCs4tjd}|dj}|dj}||fS(sr Returns the globals and locals of the calling frame. Is there an alternative to frame hacking here? ii(tinspecttstackt f_globalstf_locals(t caller_framet myglobalstmylocals((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pyt_get_caller_globals_and_localsfs  cCs=t|}|jdr5|jdr5|dd!S|SdS(sA Returns the string without any initial or final quotes. t'iiN(treprt startswithtendswith(tmystringtr((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pyt _repr_striprs  cBsCe\}}|j}||d<||d>> isinstance(obj, str) after this import: >>> from future.builtins import str u(RR (RJ((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pytistextscCst|tdS(s Deprecated. Use:: >>> isinstance(obj, bytes) after this import: >>> from future.builtins import bytes R(RR (RJ((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pytisbytesscCs ddlm}t||kS(s Equivalent to the result of ``isinstance(obj, newbytes)`` were ``__instancecheck__`` not overridden on the newbytes subclass. In other words, it is REALLY a newbytes instance, not a Py2 native str object? i(R,(tfuture.types.newbytesR,R (RJR,((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pyt isnewbytes's cCst|tjS(s_ Deprecated. Tests whether an object is a Py3 ``int`` or either a Py2 ``int`` or ``long``. Instead of using this function, you can use: >>> from future.builtins import int >>> isinstance(obj, int) The following idiom is equivalent: >>> from numbers import Integral >>> isinstance(obj, Integral) (RtnumberstIntegral(RJ((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pytisint4scCs!t|dr|jS|SdS(sO On Py3, this is a no-op: native(obj) -> obj On Py2, returns the corresponding native Py2 types that are superclasses for backported objects from Py3: >>> from builtins import str, bytes, int >>> native(str(u'ABC')) u'ABC' >>> type(native(str(u'ABC'))) unicode >>> native(bytes(b'ABC')) b'ABC' >>> type(native(bytes(b'ABC'))) bytes >>> native(int(10**20)) 100000000000000000000L >>> type(native(int(10**20))) long Existing native types on Py2 will be returned unchanged: >>> type(native(u'ABC')) unicode t __native__N(RR(RJ((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pyR.Gs texeccBsc|dkrBejd}|j}|dkr<|j}n~n|dkrW|}nddUdS(sExecute code in a namespace.isexec code in globs, locsN(R RWt _getframeR\R](tcodetglobstlocstframe((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pytexec_os      cCs8t|tjr,t|tjr,||S||SdS(s DEPRECATED: import ``old_div`` from ``past.utils`` instead. Equivalent to ``a / b`` on Python 2 without ``from __future__ import division``. TODO: generalize this to other objects (like arrays etc.) N(RRR(R=R(((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pytold_div}s $cs$tr dSfd}|SdS(s~ A decorator to turn a function or method call that returns text, i.e. unicode, into one that returns a native platform str. Use it as a decorator like this:: from __future__ import unicode_literals class MyClass(object): @as_native_str(encoding='ascii') def __repr__(self): return next(self._iter).upper() cSs|S(N((tf((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pyRRcs%tjfd}|S(Ncs||jdS(NR%(R(R2R3(R%R(sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pytwrappers(t functoolstwraps(RR(R%(Rsc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pytencoders!N(R(R%R((R%sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pyt as_native_strscCst|jS(N(R0RO(R((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pyt listvaluesscCst|jS(N(R0RI(R((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pyt listitemsscCs |jS(N(RO(R((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pyRscCs |jS(N(RI(R((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pyRscCs|S(N((RJ((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pytensure_new_typescCsddlm}ddlm}ddlm}ddlm}tt |}t |t|r|t kr}||S|t kr||S|t kr||S|tkr||S|tkr||S|Sn"t|||gkst|SdS(Ni(R,(tnewstr(tnewint(tnewdict(RR,tfuture.types.newstrRtfuture.types.newintRtfuture.types.newdictRR R.R|RR#tinttlongtdicttAssertionError(RJR,RRRt native_type((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pyRs&          tPY2tPY26RtPYPYRRVRRR)RRRtget_nextRYRRRR;RRRRPRQRRR:RRR8R4R6R.t native_bytest native_strR&RR RwR{treraiseR+R"RGRLRNR(ii(ii(Vt__doc__RTRWRRRkRZt version_infoRRRtPY27RRR RRRRRt string_typesRt integer_typesR t class_typest text_typeRt binary_typet basestringRt ClassTypeR#R"R&R)R R+R4R6R8R:t __builtin__R1R5R7R9R}R;RGRLRNRPRQRRRVRYRaRhRoRwRxR{tstripRRRRRRRRRRRR.tbuiltinsRHRRRRtAttributeErrorRRRt__all__(((sc/home/josie/.config/GIMP/2.10/plug-ins/export_layers/pygimplib/_lib/future/future/utils/__init__.pyt0s       ) $                                 *       $                   """ This is Victor Stinner's pure-Python implementation of PEP 383: the "surrogateescape" error handler of Python 3. Source: misc/python/surrogateescape.py in https://bitbucket.org/haypo/misc """ # This code is released under the Python license and the BSD 2-clause license import codecs import sys from future import utils FS_ERRORS = 'surrogateescape' # # -- Python 2/3 compatibility ------------------------------------- # FS_ERRORS = 'my_surrogateescape' def u(text): if utils.PY3: return text else: return text.decode('unicode_escape') def b(data): if utils.PY3: return data.encode('latin1') else: return data if utils.PY3: _unichr = chr bytes_chr = lambda code: bytes((code,)) else: _unichr = unichr bytes_chr = chr def surrogateescape_handler(exc): """ Pure Python implementation of the PEP 383: the "surrogateescape" error handler of Python 3. Undecodable bytes will be replaced by a Unicode character U+DCxx on decoding, and these are translated into the original bytes on encoding. """ mystring = exc.object[exc.start:exc.end] try: if isinstance(exc, UnicodeDecodeError): # mystring is a byte-string in this case decoded = replace_surrogate_decode(mystring) elif isinstance(exc, UnicodeEncodeError): # In the case of u'\udcc3'.encode('ascii', # 'this_surrogateescape_handler'), both Python 2.x and 3.x raise an # exception anyway after this function is called, even though I think # it's doing what it should. It seems that the strict encoder is called # to encode the unicode string that this function returns ... decoded = replace_surrogate_encode(mystring) else: raise exc except NotASurrogateError: raise exc return (decoded, exc.end) class NotASurrogateError(Exception): pass def replace_surrogate_encode(mystring): """ Returns a (unicode) string, not the more logical bytes, because the codecs register_error functionality expects this. """ decoded = [] for ch in mystring: # if utils.PY3: # code = ch # else: code = ord(ch) # The following magic comes from Py3.3's Python/codecs.c file: if not 0xD800 <= code <= 0xDCFF: # Not a surrogate. Fail with the original exception. raise exc # mybytes = [0xe0 | (code >> 12), # 0x80 | ((code >> 6) & 0x3f), # 0x80 | (code & 0x3f)] # Is this a good idea? if 0xDC00 <= code <= 0xDC7F: decoded.append(_unichr(code - 0xDC00)) elif code <= 0xDCFF: decoded.append(_unichr(code - 0xDC00)) else: raise NotASurrogateError return str().join(decoded) def replace_surrogate_decode(mybytes): """ Returns a (unicode) string """ decoded = [] for ch in mybytes: # We may be parsing newbytes (in which case ch is an int) or a native # str on Py2 if isinstance(ch, int): code = ch else: code = ord(ch) if 0x80 <= code <= 0xFF: decoded.append(_unichr(0xDC00 + code)) elif code <= 0x7F: decoded.append(_unichr(code)) else: # # It may be a bad byte # # Try swallowing it. # continue # print("RAISE!") raise NotASurrogateError return str().join(decoded) def encodefilename(fn): if FS_ENCODING == 'ascii': # ASCII encoder of Python 2 expects that the error handler returns a # Unicode string encodable to ASCII, whereas our surrogateescape error # handler has to return bytes in 0x80-0xFF range. encoded = [] for index, ch in enumerate(fn): code = ord(ch) if code < 128: ch = bytes_chr(code) elif 0xDC80 <= code <= 0xDCFF: ch = bytes_chr(code - 0xDC00) else: raise UnicodeEncodeError(FS_ENCODING, fn, index, index+1, 'ordinal not in range(128)') encoded.append(ch) return bytes().join(encoded) elif FS_ENCODING == 'utf-8': # UTF-8 encoder of Python 2 encodes surrogates, so U+DC80-U+DCFF # doesn't go through our error handler encoded = [] for index, ch in enumerate(fn): code = ord(ch) if 0xD800 <= code <= 0xDFFF: if 0xDC80 <= code <= 0xDCFF: ch = bytes_chr(code - 0xDC00) encoded.append(ch) else: raise UnicodeEncodeError( FS_ENCODING, fn, index, index+1, 'surrogates not allowed') else: ch_utf8 = ch.encode('utf-8') encoded.append(ch_utf8) return bytes().join(encoded) else: return fn.encode(FS_ENCODING, FS_ERRORS) def decodefilename(fn): return fn.decode(FS_ENCODING, FS_ERRORS) FS_ENCODING = 'ascii'; fn = b('[abc\xff]'); encoded = u('[abc\udcff]') # FS_ENCODING = 'cp932'; fn = b('[abc\x81\x00]'); encoded = u('[abc\udc81\x00]') # FS_ENCODING = 'UTF-8'; fn = b('[abc\xff]'); encoded = u('[abc\udcff]') # normalize the filesystem encoding name. # For example, we expect "utf-8", not "UTF8". FS_ENCODING = codecs.lookup(FS_ENCODING).name def register_surrogateescape(): """ Registers the surrogateescape error handler on Python 2 (only) """ if utils.PY3: return try: codecs.lookup_error(FS_ERRORS) except LookupError: codecs.register_error(FS_ERRORS, surrogateescape_handler) if __name__ == '__main__': pass # # Tests: # register_surrogateescape() # b = decodefilename(fn) # assert b == encoded, "%r != %r" % (b, encoded) # c = encodefilename(b) # assert c == fn, '%r != %r' % (c, fn) # # print("ok") from __future__ import absolute_import import sys if sys.version_info[0] < 3: from future.moves.html import * else: raise ImportError('This package should not be accessible on Python 3. ' 'Either you are trying to run from the python-future src folder ' 'or your installation of python-future is corrupted.') from __future__ import absolute_import from future.utils import PY3 if PY3: from html.entities import * else: from future.moves.html.entities import * from __future__ import absolute_import import sys __future_module__ = True if sys.version_info[0] == 3: raise ImportError('Cannot import module from python-future source folder') else: from future.moves.html.parser import * from __future__ import absolute_import import sys if sys.version_info[0] < 3: pass else: raise ImportError('This package should not be accessible on Python 3. ' 'Either you are trying to run from the python-future src folder ' 'or your installation of python-future is corrupted.') from __future__ import absolute_import import sys assert sys.version_info[0] < 3 from httplib import * from httplib import HTTPMessage # These constants aren't included in __all__ in httplib.py: from httplib import (HTTP_PORT, HTTPS_PORT, _CS_IDLE, _CS_REQ_STARTED, _CS_REQ_SENT, CONTINUE, SWITCHING_PROTOCOLS, PROCESSING, OK, CREATED, ACCEPTED, NON_AUTHORITATIVE_INFORMATION, NO_CONTENT, RESET_CONTENT, PARTIAL_CONTENT, MULTI_STATUS, IM_USED, MULTIPLE_CHOICES, MOVED_PERMANENTLY, FOUND, SEE_OTHER, NOT_MODIFIED, USE_PROXY, TEMPORARY_REDIRECT, BAD_REQUEST, UNAUTHORIZED, PAYMENT_REQUIRED, FORBIDDEN, NOT_FOUND, METHOD_NOT_ALLOWED, NOT_ACCEPTABLE, PROXY_AUTHENTICATION_REQUIRED, REQUEST_TIMEOUT, CONFLICT, GONE, LENGTH_REQUIRED, PRECONDITION_FAILED, REQUEST_ENTITY_TOO_LARGE, REQUEST_URI_TOO_LONG, UNSUPPORTED_MEDIA_TYPE, REQUESTED_RANGE_NOT_SATISFIABLE, EXPECTATION_FAILED, UNPROCESSABLE_ENTITY, LOCKED, FAILED_DEPENDENCY, UPGRADE_REQUIRED, INTERNAL_SERVER_ERROR, NOT_IMPLEMENTED, BAD_GATEWAY, SERVICE_UNAVAILABLE, GATEWAY_TIMEOUT, HTTP_VERSION_NOT_SUPPORTED, INSUFFICIENT_STORAGE, NOT_EXTENDED, MAXAMOUNT, ) # These are not available on Python 2.6.x: try: from httplib import LineTooLong, LineAndFileWrapper except ImportError: pass # These may not be available on all versions of Python 2.6.x or 2.7.x try: from httplib import ( _MAXLINE, _MAXHEADERS, _is_legal_header_name, _is_illegal_header_value, _METHODS_EXPECTING_BODY ) except ImportError: pass from __future__ import absolute_import import sys assert sys.version_info[0] < 3 from cookielib import * from __future__ import absolute_import import sys assert sys.version_info[0] < 3 from Cookie import * from Cookie import Morsel # left out of __all__ on Py2.7! from __future__ import absolute_import import sys assert sys.version_info[0] < 3 from BaseHTTPServer import * from CGIHTTPServer import * from SimpleHTTPServer import * try: from CGIHTTPServer import _url_collapse_path # needed for a test except ImportError: try: # Python 2.7.0 to 2.7.3 from CGIHTTPServer import ( _url_collapse_path_split as _url_collapse_path) except ImportError: # Doesn't exist on Python 2.6.x. Ignore it. pass # empty to make this a package """ Utility functions from 2to3, 3to2 and python-modernize (and some home-grown ones). Licences: 2to3: PSF License v2 3to2: Apache Software License (from 3to2/setup.py) python-modernize licence: BSD (from python-modernize/LICENSE) """ from lib2to3.fixer_util import (FromImport, Newline, is_import, find_root, does_tree_import, Comma) from lib2to3.pytree import Leaf, Node from lib2to3.pygram import python_symbols as syms, python_grammar from lib2to3.pygram import token from lib2to3.fixer_util import (Node, Call, Name, syms, Comma, Number) import re def canonical_fix_name(fix, avail_fixes): """ Examples: >>> canonical_fix_name('fix_wrap_text_literals') 'libfuturize.fixes.fix_wrap_text_literals' >>> canonical_fix_name('wrap_text_literals') 'libfuturize.fixes.fix_wrap_text_literals' >>> canonical_fix_name('wrap_te') ValueError("unknown fixer name") >>> canonical_fix_name('wrap') ValueError("ambiguous fixer name") """ if ".fix_" in fix: return fix else: if fix.startswith('fix_'): fix = fix[4:] # Infer the full module name for the fixer. # First ensure that no names clash (e.g. # lib2to3.fixes.fix_blah and libfuturize.fixes.fix_blah): found = [f for f in avail_fixes if f.endswith('fix_{0}'.format(fix))] if len(found) > 1: raise ValueError("Ambiguous fixer name. Choose a fully qualified " "module name instead from these:\n" + "\n".join(" " + myf for myf in found)) elif len(found) == 0: raise ValueError("Unknown fixer. Use --list-fixes or -l for a list.") return found[0] ## These functions are from 3to2 by Joe Amenta: def Star(prefix=None): return Leaf(token.STAR, u'*', prefix=prefix) def DoubleStar(prefix=None): return Leaf(token.DOUBLESTAR, u'**', prefix=prefix) def Minus(prefix=None): return Leaf(token.MINUS, u'-', prefix=prefix) def commatize(leafs): """ Accepts/turns: (Name, Name, ..., Name, Name) Returns/into: (Name, Comma, Name, Comma, ..., Name, Comma, Name) """ new_leafs = [] for leaf in leafs: new_leafs.append(leaf) new_leafs.append(Comma()) del new_leafs[-1] return new_leafs def indentation(node): """ Returns the indentation for this node Iff a node is in a suite, then it has indentation. """ while node.parent is not None and node.parent.type != syms.suite: node = node.parent if node.parent is None: return u"" # The first three children of a suite are NEWLINE, INDENT, (some other node) # INDENT.value contains the indentation for this suite # anything after (some other node) has the indentation as its prefix. if node.type == token.INDENT: return node.value elif node.prev_sibling is not None and node.prev_sibling.type == token.INDENT: return node.prev_sibling.value elif node.prev_sibling is None: return u"" else: return node.prefix def indentation_step(node): """ Dirty little trick to get the difference between each indentation level Implemented by finding the shortest indentation string (technically, the "least" of all of the indentation strings, but tabs and spaces mixed won't get this far, so those are synonymous.) """ r = find_root(node) # Collect all indentations into one set. all_indents = set(i.value for i in r.pre_order() if i.type == token.INDENT) if not all_indents: # nothing is indented anywhere, so we get to pick what we want return u" " # four spaces is a popular convention else: return min(all_indents) def suitify(parent): """ Turn the stuff after the first colon in parent's children into a suite, if it wasn't already """ for node in parent.children: if node.type == syms.suite: # already in the prefered format, do nothing return # One-liners have no suite node, we have to fake one up for i, node in enumerate(parent.children): if node.type == token.COLON: break else: raise ValueError(u"No class suite and no ':'!") # Move everything into a suite node suite = Node(syms.suite, [Newline(), Leaf(token.INDENT, indentation(node) + indentation_step(node))]) one_node = parent.children[i+1] one_node.remove() one_node.prefix = u'' suite.append_child(one_node) parent.append_child(suite) def NameImport(package, as_name=None, prefix=None): """ Accepts a package (Name node), name to import it as (string), and optional prefix and returns a node: import [as ] """ if prefix is None: prefix = u"" children = [Name(u"import", prefix=prefix), package] if as_name is not None: children.extend([Name(u"as", prefix=u" "), Name(as_name, prefix=u" ")]) return Node(syms.import_name, children) _compound_stmts = (syms.if_stmt, syms.while_stmt, syms.for_stmt, syms.try_stmt, syms.with_stmt) _import_stmts = (syms.import_name, syms.import_from) def import_binding_scope(node): """ Generator yields all nodes for which a node (an import_stmt) has scope The purpose of this is for a call to _find() on each of them """ # import_name / import_from are small_stmts assert node.type in _import_stmts test = node.next_sibling # A small_stmt can only be followed by a SEMI or a NEWLINE. while test.type == token.SEMI: nxt = test.next_sibling # A SEMI can only be followed by a small_stmt or a NEWLINE if nxt.type == token.NEWLINE: break else: yield nxt # A small_stmt can only be followed by either a SEMI or a NEWLINE test = nxt.next_sibling # Covered all subsequent small_stmts after the import_stmt # Now to cover all subsequent stmts after the parent simple_stmt parent = node.parent assert parent.type == syms.simple_stmt test = parent.next_sibling while test is not None: # Yes, this will yield NEWLINE and DEDENT. Deal with it. yield test test = test.next_sibling context = parent.parent # Recursively yield nodes following imports inside of a if/while/for/try/with statement if context.type in _compound_stmts: # import is in a one-liner c = context while c.next_sibling is not None: yield c.next_sibling c = c.next_sibling context = context.parent # Can't chain one-liners on one line, so that takes care of that. p = context.parent if p is None: return # in a multi-line suite while p.type in _compound_stmts: if context.type == syms.suite: yield context context = context.next_sibling if context is None: context = p.parent p = context.parent if p is None: break def ImportAsName(name, as_name, prefix=None): new_name = Name(name) new_as = Name(u"as", prefix=u" ") new_as_name = Name(as_name, prefix=u" ") new_node = Node(syms.import_as_name, [new_name, new_as, new_as_name]) if prefix is not None: new_node.prefix = prefix return new_node def is_docstring(node): """ Returns True if the node appears to be a docstring """ return (node.type == syms.simple_stmt and len(node.children) > 0 and node.children[0].type == token.STRING) def future_import(feature, node): """ This seems to work """ root = find_root(node) if does_tree_import(u"__future__", feature, node): return # Look for a shebang or encoding line shebang_encoding_idx = None for idx, node in enumerate(root.children): # Is it a shebang or encoding line? if is_shebang_comment(node) or is_encoding_comment(node): shebang_encoding_idx = idx if is_docstring(node): # skip over docstring continue names = check_future_import(node) if not names: # not a future statement; need to insert before this break if feature in names: # already imported return import_ = FromImport(u'__future__', [Leaf(token.NAME, feature, prefix=" ")]) if shebang_encoding_idx == 0 and idx == 0: # If this __future__ import would go on the first line, # detach the shebang / encoding prefix from the current first line. # and attach it to our new __future__ import node. import_.prefix = root.children[0].prefix root.children[0].prefix = u'' # End the __future__ import line with a newline and add a blank line # afterwards: children = [import_ , Newline()] root.insert_child(idx, Node(syms.simple_stmt, children)) def future_import2(feature, node): """ An alternative to future_import() which might not work ... """ root = find_root(node) if does_tree_import(u"__future__", feature, node): return insert_pos = 0 for idx, node in enumerate(root.children): if node.type == syms.simple_stmt and node.children and \ node.children[0].type == token.STRING: insert_pos = idx + 1 break for thing_after in root.children[insert_pos:]: if thing_after.type == token.NEWLINE: insert_pos += 1 continue prefix = thing_after.prefix thing_after.prefix = u"" break else: prefix = u"" import_ = FromImport(u"__future__", [Leaf(token.NAME, feature, prefix=u" ")]) children = [import_, Newline()] root.insert_child(insert_pos, Node(syms.simple_stmt, children, prefix=prefix)) def parse_args(arglist, scheme): u""" Parse a list of arguments into a dict """ arglist = [i for i in arglist if i.type != token.COMMA] ret_mapping = dict([(k, None) for k in scheme]) for i, arg in enumerate(arglist): if arg.type == syms.argument and arg.children[1].type == token.EQUAL: # argument < NAME '=' any > slot = arg.children[0].value ret_mapping[slot] = arg.children[2] else: slot = scheme[i] ret_mapping[slot] = arg return ret_mapping # def is_import_from(node): # """Returns true if the node is a statement "from ... import ..." # """ # return node.type == syms.import_from def is_import_stmt(node): return (node.type == syms.simple_stmt and node.children and is_import(node.children[0])) def touch_import_top(package, name_to_import, node): """Works like `does_tree_import` but adds an import statement at the top if it was not imported (but below any __future__ imports) and below any comments such as shebang lines). Based on lib2to3.fixer_util.touch_import() Calling this multiple times adds the imports in reverse order. Also adds "standard_library.install_aliases()" after "from future import standard_library". This should probably be factored into another function. """ root = find_root(node) if does_tree_import(package, name_to_import, root): return # Ideally, we would look for whether futurize --all-imports has been run, # as indicated by the presence of ``from builtins import (ascii, ..., # zip)`` -- and, if it has, we wouldn't import the name again. # Look for __future__ imports and insert below them found = False for name in ['absolute_import', 'division', 'print_function', 'unicode_literals']: if does_tree_import('__future__', name, root): found = True break if found: # At least one __future__ import. We want to loop until we've seen them # all. start, end = None, None for idx, node in enumerate(root.children): if check_future_import(node): start = idx # Start looping idx2 = start while node: node = node.next_sibling idx2 += 1 if not check_future_import(node): end = idx2 break break assert start is not None assert end is not None insert_pos = end else: # No __future__ imports. # We look for a docstring and insert the new node below that. If no docstring # exists, just insert the node at the top. for idx, node in enumerate(root.children): if node.type != syms.simple_stmt: break if not is_docstring(node): # This is the usual case. break insert_pos = idx if package is None: import_ = Node(syms.import_name, [ Leaf(token.NAME, u"import"), Leaf(token.NAME, name_to_import, prefix=u" ") ]) else: import_ = FromImport(package, [Leaf(token.NAME, name_to_import, prefix=u" ")]) if name_to_import == u'standard_library': # Add: # standard_library.install_aliases() # after: # from future import standard_library install_hooks = Node(syms.simple_stmt, [Node(syms.power, [Leaf(token.NAME, u'standard_library'), Node(syms.trailer, [Leaf(token.DOT, u'.'), Leaf(token.NAME, u'install_aliases')]), Node(syms.trailer, [Leaf(token.LPAR, u'('), Leaf(token.RPAR, u')')]) ]) ] ) children_hooks = [install_hooks, Newline()] else: children_hooks = [] # FromImport(package, [Leaf(token.NAME, name_to_import, prefix=u" ")]) children_import = [import_, Newline()] old_prefix = root.children[insert_pos].prefix root.children[insert_pos].prefix = u'' root.insert_child(insert_pos, Node(syms.simple_stmt, children_import, prefix=old_prefix)) if len(children_hooks) > 0: root.insert_child(insert_pos + 1, Node(syms.simple_stmt, children_hooks)) ## The following functions are from python-modernize by Armin Ronacher: # (a little edited). def check_future_import(node): """If this is a future import, return set of symbols that are imported, else return None.""" # node should be the import statement here savenode = node if not (node.type == syms.simple_stmt and node.children): return set() node = node.children[0] # now node is the import_from node if not (node.type == syms.import_from and # node.type == token.NAME and # seems to break it hasattr(node.children[1], 'value') and node.children[1].value == u'__future__'): return set() node = node.children[3] # now node is the import_as_name[s] # print(python_grammar.number2symbol[node.type]) # breaks sometimes if node.type == syms.import_as_names: result = set() for n in node.children: if n.type == token.NAME: result.add(n.value) elif n.type == syms.import_as_name: n = n.children[0] assert n.type == token.NAME result.add(n.value) return result elif node.type == syms.import_as_name: node = node.children[0] assert node.type == token.NAME return set([node.value]) elif node.type == token.NAME: return set([node.value]) else: # TODO: handle brackets like this: # from __future__ import (absolute_import, division) assert False, "strange import: %s" % savenode SHEBANG_REGEX = r'^#!.*python' ENCODING_REGEX = r"^#.*coding[:=]\s*([-\w.]+)" def is_shebang_comment(node): """ Comments are prefixes for Leaf nodes. Returns whether the given node has a prefix that looks like a shebang line or an encoding line: #!/usr/bin/env python #!/usr/bin/python3 """ return bool(re.match(SHEBANG_REGEX, node.prefix)) def is_encoding_comment(node): """ Comments are prefixes for Leaf nodes. Returns whether the given node has a prefix that looks like an encoding line: # coding: utf-8 # encoding: utf-8 # -*- coding: -*- # vim: set fileencoding= : """ return bool(re.match(ENCODING_REGEX, node.prefix)) def wrap_in_fn_call(fn_name, args, prefix=None): """ Example: >>> wrap_in_fn_call("oldstr", (arg,)) oldstr(arg) >>> wrap_in_fn_call("olddiv", (arg1, arg2)) olddiv(arg1, arg2) """ assert len(args) > 0 if len(args) == 1: newargs = args elif len(args) == 2: expr1, expr2 = args newargs = [expr1, Comma(), expr2] else: assert NotImplementedError('write me') return Call(Name(fn_name), newargs, prefix=prefix) """ futurize: automatic conversion to clean 2/3 code using ``python-future`` ====================================================================== Like Armin Ronacher's modernize.py, ``futurize`` attempts to produce clean standard Python 3 code that runs on both Py2 and Py3. One pass -------- Use it like this on Python 2 code: $ futurize --verbose mypython2script.py This will attempt to port the code to standard Py3 code that also provides Py2 compatibility with the help of the right imports from ``future``. To write changes to the files, use the -w flag. Two stages ---------- The ``futurize`` script can also be called in two separate stages. First: $ futurize --stage1 mypython2script.py This produces more modern Python 2 code that is not yet compatible with Python 3. The tests should still run and the diff should be uncontroversial to apply to most Python projects that are willing to drop support for Python 2.5 and lower. After this, the recommended approach is to explicitly mark all strings that must be byte-strings with a b'' prefix and all text (unicode) strings with a u'' prefix, and then invoke the second stage of Python 2 to 2/3 conversion with:: $ futurize --stage2 mypython2script.py Stage 2 adds a dependency on ``future``. It converts most remaining Python 2-specific code to Python 3 code and adds appropriate imports from ``future`` to restore Py2 support. The command above leaves all unadorned string literals as native strings (byte-strings on Py2, unicode strings on Py3). If instead you would like all unadorned string literals to be promoted to unicode, you can also pass this flag: $ futurize --stage2 --unicode-literals mypython2script.py This adds the declaration ``from __future__ import unicode_literals`` to the top of each file, which implicitly declares all unadorned string literals to be unicode strings (``unicode`` on Py2). All imports ----------- The --all-imports option forces adding all ``__future__`` imports, ``builtins`` imports, and standard library aliases, even if they don't seem necessary for the current state of each module. (This can simplify testing, and can reduce the need to think about Py2 compatibility when editing the code further.) """ from __future__ import (absolute_import, print_function, unicode_literals) import future.utils from future import __version__ import sys import logging import optparse import os from lib2to3.main import main, warn, StdoutRefactoringTool from lib2to3 import refactor from libfuturize.fixes import (lib2to3_fix_names_stage1, lib2to3_fix_names_stage2, libfuturize_fix_names_stage1, libfuturize_fix_names_stage2) fixer_pkg = 'libfuturize.fixes' def main(args=None): """Main program. Args: fixer_pkg: the name of a package where the fixers are located. args: optional; a list of command line arguments. If omitted, sys.argv[1:] is used. Returns a suggested exit status (0, 1, 2). """ # Set up option parser parser = optparse.OptionParser(usage="futurize [options] file|dir ...") parser.add_option("-V", "--version", action="store_true", help="Report the version number of futurize") parser.add_option("-a", "--all-imports", action="store_true", help="Add all __future__ and future imports to each module") parser.add_option("-1", "--stage1", action="store_true", help="Modernize Python 2 code only; no compatibility with Python 3 (or dependency on ``future``)") parser.add_option("-2", "--stage2", action="store_true", help="Take modernized (stage1) code and add a dependency on ``future`` to provide Py3 compatibility.") parser.add_option("-0", "--both-stages", action="store_true", help="Apply both stages 1 and 2") parser.add_option("-u", "--unicode-literals", action="store_true", help="Add ``from __future__ import unicode_literals`` to implicitly convert all unadorned string literals '' into unicode strings") parser.add_option("-f", "--fix", action="append", default=[], help="Each FIX specifies a transformation; default: all.\nEither use '-f division -f metaclass' etc. or use the fully-qualified module name: '-f lib2to3.fixes.fix_types -f libfuturize.fixes.fix_unicode_keep_u'") parser.add_option("-j", "--processes", action="store", default=1, type="int", help="Run 2to3 concurrently") parser.add_option("-x", "--nofix", action="append", default=[], help="Prevent a fixer from being run.") parser.add_option("-l", "--list-fixes", action="store_true", help="List available transformations") parser.add_option("-p", "--print-function", action="store_true", help="Modify the grammar so that print() is a function") parser.add_option("-v", "--verbose", action="store_true", help="More verbose logging") parser.add_option("--no-diffs", action="store_true", help="Don't show diffs of the refactoring") parser.add_option("-w", "--write", action="store_true", help="Write back modified files") parser.add_option("-n", "--nobackups", action="store_true", default=False, help="Don't write backups for modified files.") parser.add_option("-o", "--output-dir", action="store", type="str", default="", help="Put output files in this directory " "instead of overwriting the input files. Requires -n. " "For Python >= 2.7 only.") parser.add_option("-W", "--write-unchanged-files", action="store_true", help="Also write files even if no changes were required" " (useful with --output-dir); implies -w.") parser.add_option("--add-suffix", action="store", type="str", default="", help="Append this string to all output filenames." " Requires -n if non-empty. For Python >= 2.7 only." "ex: --add-suffix='3' will generate .py3 files.") # Parse command line arguments flags = {} refactor_stdin = False options, args = parser.parse_args(args) if options.write_unchanged_files: flags["write_unchanged_files"] = True if not options.write: warn("--write-unchanged-files/-W implies -w.") options.write = True # If we allowed these, the original files would be renamed to backup names # but not replaced. if options.output_dir and not options.nobackups: parser.error("Can't use --output-dir/-o without -n.") if options.add_suffix and not options.nobackups: parser.error("Can't use --add-suffix without -n.") if not options.write and options.no_diffs: warn("not writing files and not printing diffs; that's not very useful") if not options.write and options.nobackups: parser.error("Can't use -n without -w") if "-" in args: refactor_stdin = True if options.write: print("Can't write to stdin.", file=sys.stderr) return 2 # Is this ever necessary? if options.print_function: flags["print_function"] = True # Set up logging handler level = logging.DEBUG if options.verbose else logging.INFO logging.basicConfig(format='%(name)s: %(message)s', level=level) logger = logging.getLogger('libfuturize.main') if options.stage1 or options.stage2: assert options.both_stages is None options.both_stages = False else: options.both_stages = True avail_fixes = set() if options.stage1 or options.both_stages: avail_fixes.update(lib2to3_fix_names_stage1) avail_fixes.update(libfuturize_fix_names_stage1) if options.stage2 or options.both_stages: avail_fixes.update(lib2to3_fix_names_stage2) avail_fixes.update(libfuturize_fix_names_stage2) if options.unicode_literals: avail_fixes.add('libfuturize.fixes.fix_unicode_literals_import') if options.version: print(__version__) return 0 if options.list_fixes: print("Available transformations for the -f/--fix option:") # for fixname in sorted(refactor.get_all_fix_names(fixer_pkg)): for fixname in sorted(avail_fixes): print(fixname) if not args: return 0 if not args: print("At least one file or directory argument required.", file=sys.stderr) print("Use --help to show usage.", file=sys.stderr) return 2 unwanted_fixes = set(fixer_pkg + ".fix_" + fix for fix in options.nofix) extra_fixes = set() if options.all_imports: if options.stage1: prefix = 'libfuturize.fixes.' extra_fixes.add(prefix + 'fix_add__future__imports_except_unicode_literals') else: # In case the user hasn't run stage1 for some reason: prefix = 'libpasteurize.fixes.' extra_fixes.add(prefix + 'fix_add_all__future__imports') extra_fixes.add(prefix + 'fix_add_future_standard_library_import') extra_fixes.add(prefix + 'fix_add_all_future_builtins') explicit = set() if options.fix: all_present = False for fix in options.fix: if fix == 'all': all_present = True else: if ".fix_" in fix: explicit.add(fix) else: # Infer the full module name for the fixer. # First ensure that no names clash (e.g. # lib2to3.fixes.fix_blah and libfuturize.fixes.fix_blah): found = [f for f in avail_fixes if f.endswith('fix_{0}'.format(fix))] if len(found) > 1: print("Ambiguous fixer name. Choose a fully qualified " "module name instead from these:\n" + "\n".join(" " + myf for myf in found), file=sys.stderr) return 2 elif len(found) == 0: print("Unknown fixer. Use --list-fixes or -l for a list.", file=sys.stderr) return 2 explicit.add(found[0]) if len(explicit & unwanted_fixes) > 0: print("Conflicting usage: the following fixers have been " "simultaneously requested and disallowed:\n" + "\n".join(" " + myf for myf in (explicit & unwanted_fixes)), file=sys.stderr) return 2 requested = avail_fixes.union(explicit) if all_present else explicit else: requested = avail_fixes.union(explicit) fixer_names = (requested | extra_fixes) - unwanted_fixes input_base_dir = os.path.commonprefix(args) if (input_base_dir and not input_base_dir.endswith(os.sep) and not os.path.isdir(input_base_dir)): # One or more similar names were passed, their directory is the base. # os.path.commonprefix() is ignorant of path elements, this corrects # for that weird API. input_base_dir = os.path.dirname(input_base_dir) if options.output_dir: input_base_dir = input_base_dir.rstrip(os.sep) logger.info('Output in %r will mirror the input directory %r layout.', options.output_dir, input_base_dir) # Initialize the refactoring tool if future.utils.PY26: extra_kwargs = {} else: extra_kwargs = { 'append_suffix': options.add_suffix, 'output_dir': options.output_dir, 'input_base_dir': input_base_dir, } rt = StdoutRefactoringTool( sorted(fixer_names), flags, sorted(explicit), options.nobackups, not options.no_diffs, **extra_kwargs) # Refactor all files and directories passed as arguments if not rt.errors: if refactor_stdin: rt.refactor_stdin() else: try: rt.refactor(args, options.write, None, options.processes) except refactor.MultiprocessingUnsupported: assert options.processes > 1 print("Sorry, -j isn't " \ "supported on this platform.", file=sys.stderr) return 1 rt.summarize() # Return error status (0 if rt.errors is zero) return int(bool(rt.errors)) import sys from lib2to3 import refactor # The following fixers are "safe": they convert Python 2 code to more # modern Python 2 code. They should be uncontroversial to apply to most # projects that are happy to drop support for Py2.5 and below. Applying # them first will reduce the size of the patch set for the real porting. lib2to3_fix_names_stage1 = set([ 'lib2to3.fixes.fix_apply', 'lib2to3.fixes.fix_except', 'lib2to3.fixes.fix_exec', 'lib2to3.fixes.fix_exitfunc', 'lib2to3.fixes.fix_funcattrs', 'lib2to3.fixes.fix_has_key', 'lib2to3.fixes.fix_idioms', # 'lib2to3.fixes.fix_import', # makes any implicit relative imports explicit. (Use with ``from __future__ import absolute_import) 'lib2to3.fixes.fix_intern', 'lib2to3.fixes.fix_isinstance', 'lib2to3.fixes.fix_methodattrs', 'lib2to3.fixes.fix_ne', # 'lib2to3.fixes.fix_next', # would replace ``next`` method names # with ``__next__``. 'lib2to3.fixes.fix_numliterals', # turns 1L into 1, 0755 into 0o755 'lib2to3.fixes.fix_paren', # 'lib2to3.fixes.fix_print', # see the libfuturize fixer that also # adds ``from __future__ import print_function`` # 'lib2to3.fixes.fix_raise', # uses incompatible with_traceback() method on exceptions 'lib2to3.fixes.fix_reduce', # reduce is available in functools on Py2.6/Py2.7 'lib2to3.fixes.fix_renames', # sys.maxint -> sys.maxsize # 'lib2to3.fixes.fix_set_literal', # this is unnecessary and breaks Py2.6 support 'lib2to3.fixes.fix_repr', 'lib2to3.fixes.fix_standarderror', 'lib2to3.fixes.fix_sys_exc', 'lib2to3.fixes.fix_throw', 'lib2to3.fixes.fix_tuple_params', 'lib2to3.fixes.fix_types', 'lib2to3.fixes.fix_ws_comma', # can perhaps decrease readability: see issue #58 'lib2to3.fixes.fix_xreadlines', ]) # The following fixers add a dependency on the ``future`` package on order to # support Python 2: lib2to3_fix_names_stage2 = set([ # 'lib2to3.fixes.fix_buffer', # perhaps not safe. Test this. # 'lib2to3.fixes.fix_callable', # not needed in Py3.2+ 'lib2to3.fixes.fix_dict', # TODO: add support for utils.viewitems() etc. and move to stage2 # 'lib2to3.fixes.fix_execfile', # some problems: see issue #37. # We use a custom fixer instead (see below) # 'lib2to3.fixes.fix_future', # we don't want to remove __future__ imports 'lib2to3.fixes.fix_getcwdu', # 'lib2to3.fixes.fix_imports', # called by libfuturize.fixes.fix_future_standard_library # 'lib2to3.fixes.fix_imports2', # we don't handle this yet (dbm) 'lib2to3.fixes.fix_input', 'lib2to3.fixes.fix_itertools', 'lib2to3.fixes.fix_itertools_imports', 'lib2to3.fixes.fix_filter', 'lib2to3.fixes.fix_long', 'lib2to3.fixes.fix_map', # 'lib2to3.fixes.fix_metaclass', # causes SyntaxError in Py2! Use the one from ``six`` instead 'lib2to3.fixes.fix_next', 'lib2to3.fixes.fix_nonzero', # TODO: cause this to import ``object`` and/or add a decorator for mapping __bool__ to __nonzero__ 'lib2to3.fixes.fix_operator', # we will need support for this by e.g. extending the Py2 operator module to provide those functions in Py3 'lib2to3.fixes.fix_raw_input', # 'lib2to3.fixes.fix_unicode', # strips off the u'' prefix, which removes a potentially helpful source of information for disambiguating unicode/byte strings # 'lib2to3.fixes.fix_urllib', # included in libfuturize.fix_future_standard_library_urllib # 'lib2to3.fixes.fix_xrange', # custom one because of a bug with Py3.3's lib2to3 'lib2to3.fixes.fix_zip', ]) libfuturize_fix_names_stage1 = set([ 'libfuturize.fixes.fix_absolute_import', 'libfuturize.fixes.fix_next_call', # obj.next() -> next(obj). Unlike # lib2to3.fixes.fix_next, doesn't change # the ``next`` method to ``__next__``. 'libfuturize.fixes.fix_print_with_import', 'libfuturize.fixes.fix_raise', # 'libfuturize.fixes.fix_order___future__imports', # TODO: consolidate to a single line to simplify testing ]) libfuturize_fix_names_stage2 = set([ 'libfuturize.fixes.fix_basestring', # 'libfuturize.fixes.fix_add__future__imports_except_unicode_literals', # just in case 'libfuturize.fixes.fix_cmp', 'libfuturize.fixes.fix_division_safe', 'libfuturize.fixes.fix_execfile', 'libfuturize.fixes.fix_future_builtins', 'libfuturize.fixes.fix_future_standard_library', 'libfuturize.fixes.fix_future_standard_library_urllib', 'libfuturize.fixes.fix_metaclass', 'libpasteurize.fixes.fix_newstyle', 'libfuturize.fixes.fix_object', # 'libfuturize.fixes.fix_order___future__imports', # TODO: consolidate to a single line to simplify testing 'libfuturize.fixes.fix_unicode_keep_u', # 'libfuturize.fixes.fix_unicode_literals_import', 'libfuturize.fixes.fix_xrange_with_import', # custom one because of a bug with Py3.3's lib2to3 ]) """Fix UserDict. Incomplete! TODO: base this on fix_urllib perhaps? """ # Local imports from lib2to3 import fixer_base from lib2to3.fixer_util import Name, attr_chain from lib2to3.fixes.fix_imports import alternates, build_pattern, FixImports MAPPING = {'UserDict': 'collections', } # def alternates(members): # return "(" + "|".join(map(repr, members)) + ")" # # # def build_pattern(mapping=MAPPING): # mod_list = ' | '.join(["module_name='%s'" % key for key in mapping]) # bare_names = alternates(mapping.keys()) # # yield """name_import=import_name< 'import' ((%s) | # multiple_imports=dotted_as_names< any* (%s) any* >) > # """ % (mod_list, mod_list) # yield """import_from< 'from' (%s) 'import' ['('] # ( any | import_as_name< any 'as' any > | # import_as_names< any* >) [')'] > # """ % mod_list # yield """import_name< 'import' (dotted_as_name< (%s) 'as' any > | # multiple_imports=dotted_as_names< # any* dotted_as_name< (%s) 'as' any > any* >) > # """ % (mod_list, mod_list) # # # Find usages of module members in code e.g. thread.foo(bar) # yield "power< bare_with_attr=(%s) trailer<'.' any > any* >" % bare_names # class FixUserDict(fixer_base.BaseFix): class FixUserdict(FixImports): BM_compatible = True keep_line_order = True # This is overridden in fix_imports2. mapping = MAPPING # We want to run this fixer late, so fix_import doesn't try to make stdlib # renames into relative imports. run_order = 6 def build_pattern(self): return "|".join(build_pattern(self.mapping)) def compile_pattern(self): # We override this, so MAPPING can be pragmatically altered and the # changes will be reflected in PATTERN. self.PATTERN = self.build_pattern() super(FixImports, self).compile_pattern() # Don't match the node if it's within another match. def match(self, node): match = super(FixImports, self).match results = match(node) if results: # Module usage could be in the trailer of an attribute lookup, so we # might have nested matches when "bare_with_attr" is present. if "bare_with_attr" not in results and \ any(match(obj) for obj in attr_chain(node, "parent")): return False return results return False def start_tree(self, tree, filename): super(FixImports, self).start_tree(tree, filename) self.replace = {} def transform(self, node, results): import_mod = results.get("module_name") if import_mod: mod_name = import_mod.value new_name = unicode(self.mapping[mod_name]) import_mod.replace(Name(new_name, prefix=import_mod.prefix)) if "name_import" in results: # If it's not a "from x import x, y" or "import x as y" import, # marked its usage to be replaced. self.replace[mod_name] = new_name if "multiple_imports" in results: # This is a nasty hack to fix multiple imports on a line (e.g., # "import StringIO, urlparse"). The problem is that I can't # figure out an easy way to make a pattern recognize the keys of # MAPPING randomly sprinkled in an import statement. results = self.match(node) if results: self.transform(node, results) else: # Replace usage of the module. bare_name = results["bare_with_attr"][0] new_name = self.replace.get(bare_name.value) if new_name: bare_name.replace(Name(new_name, prefix=bare_name.prefix)) """ Fixer for import statements, with a __future__ import line. Based on lib2to3/fixes/fix_import.py, but extended slightly so it also supports Cython modules. If spam is being imported from the local directory, this import: from spam import eggs becomes: from __future__ import absolute_import from .spam import eggs and this import: import spam becomes: from __future__ import absolute_import from . import spam """ from os.path import dirname, join, exists, sep from lib2to3.fixes.fix_import import FixImport from lib2to3.fixer_util import FromImport, syms from lib2to3.fixes.fix_import import traverse_imports from libfuturize.fixer_util import future_import class FixAbsoluteImport(FixImport): run_order = 9 def transform(self, node, results): """ Copied from FixImport.transform(), but with this line added in any modules that had implicit relative imports changed: from __future__ import absolute_import" """ if self.skip: return imp = results['imp'] if node.type == syms.import_from: # Some imps are top-level (eg: 'import ham') # some are first level (eg: 'import ham.eggs') # some are third level (eg: 'import ham.eggs as spam') # Hence, the loop while not hasattr(imp, 'value'): imp = imp.children[0] if self.probably_a_local_import(imp.value): imp.value = u"." + imp.value imp.changed() future_import(u"absolute_import", node) else: have_local = False have_absolute = False for mod_name in traverse_imports(imp): if self.probably_a_local_import(mod_name): have_local = True else: have_absolute = True if have_absolute: if have_local: # We won't handle both sibling and absolute imports in the # same statement at the moment. self.warning(node, "absolute and local imports together") return new = FromImport(u".", [imp]) new.prefix = node.prefix future_import(u"absolute_import", node) return new def probably_a_local_import(self, imp_name): """ Like the corresponding method in the base class, but this also supports Cython modules. """ if imp_name.startswith(u"."): # Relative imports are certainly not local imports. return False imp_name = imp_name.split(u".", 1)[0] base_path = dirname(self.filename) base_path = join(base_path, imp_name) # If there is no __init__.py next to the file its not in a package # so can't be a relative import. if not exists(join(dirname(base_path), "__init__.py")): return False for ext in [".py", sep, ".pyc", ".so", ".sl", ".pyd", ".pyx"]: if exists(base_path + ext): return True return False """ Fixer for adding: from __future__ import absolute_import from __future__ import division from __future__ import print_function This is "stage 1": hopefully uncontroversial changes. Stage 2 adds ``unicode_literals``. """ from lib2to3 import fixer_base from libfuturize.fixer_util import future_import class FixAddFutureImportsExceptUnicodeLiterals(fixer_base.BaseFix): BM_compatible = True PATTERN = "file_input" run_order = 9 def transform(self, node, results): # Reverse order: future_import(u"print_function", node) future_import(u"division", node) future_import(u"absolute_import", node) """ Fixer that adds ``from past.builtins import basestring`` if there is a reference to ``basestring`` """ from lib2to3 import fixer_base from libfuturize.fixer_util import touch_import_top class FixBasestring(fixer_base.BaseFix): BM_compatible = True PATTERN = "'basestring'" def transform(self, node, results): touch_import_top(u'past.builtins', 'basestring', node) """Optional fixer that changes all unprefixed string literals "..." to b"...". br'abcd' is a SyntaxError on Python 2 but valid on Python 3. ur'abcd' is a SyntaxError on Python 3 but valid on Python 2. """ from __future__ import unicode_literals import re from lib2to3.pgen2 import token from lib2to3 import fixer_base _literal_re = re.compile(r"[^bBuUrR]?[\'\"]") class FixBytes(fixer_base.BaseFix): BM_compatible = True PATTERN = "STRING" def transform(self, node, results): if node.type == token.STRING: if _literal_re.match(node.value): new = node.clone() new.value = u'b' + new.value return new # coding: utf-8 """ Fixer for the cmp() function on Py2, which was removed in Py3. Adds this import line:: from past.builtins import cmp if cmp() is called in the code. """ from __future__ import unicode_literals from lib2to3 import fixer_base from libfuturize.fixer_util import touch_import_top expression = "name='cmp'" class FixCmp(fixer_base.BaseFix): BM_compatible = True run_order = 9 PATTERN = """ power< ({0}) trailer< '(' args=[any] ')' > rest=any* > """.format(expression) def transform(self, node, results): name = results["name"] touch_import_top(u'past.builtins', name.value, node) """ UNFINISHED For the ``future`` package. Adds this import line: from __future__ import division at the top so the code runs identically on Py3 and Py2.6/2.7 """ from libpasteurize.fixes.fix_division import FixDivision """ For the ``future`` package. Adds this import line: from __future__ import division at the top and changes any old-style divisions to be calls to past.utils.old_div so the code runs as before on Py2.6/2.7 and has the same behaviour on Py3. If "from __future__ import division" is already in effect, this fixer does nothing. """ from lib2to3 import fixer_base from lib2to3.fixer_util import syms, does_tree_import from libfuturize.fixer_util import (token, future_import, touch_import_top, wrap_in_fn_call) def match_division(node): u""" __future__.division redefines the meaning of a single slash for division, so we match that and only that. """ slash = token.SLASH return node.type == slash and not node.next_sibling.type == slash and \ not node.prev_sibling.type == slash class FixDivisionSafe(fixer_base.BaseFix): # BM_compatible = True run_order = 4 # this seems to be ignored? _accept_type = token.SLASH PATTERN = """ term<(not('/') any)+ '/' ((not('/') any))> """ def start_tree(self, tree, name): """ Skip this fixer if "__future__.division" is already imported. """ super(FixDivisionSafe, self).start_tree(tree, name) self.skip = "division" in tree.future_features def match(self, node): u""" Since the tree needs to be fixed once and only once if and only if it matches, we can start discarding matches after the first. """ if (node.type == self.syms.term and len(node.children) == 3 and match_division(node.children[1])): expr1, expr2 = node.children[0], node.children[2] return expr1, expr2 else: return False def transform(self, node, results): if self.skip: return future_import(u"division", node) touch_import_top(u'past.utils', u'old_div', node) expr1, expr2 = results[0].clone(), results[1].clone() # Strip any leading space for the first number: expr1.prefix = u'' return wrap_in_fn_call("old_div", (expr1, expr2), prefix=node.prefix) # coding: utf-8 """ Fixer for the execfile() function on Py2, which was removed in Py3. The Lib/lib2to3/fixes/fix_execfile.py module has some problems: see python-future issue #37. This fixer merely imports execfile() from past.builtins and leaves the code alone. Adds this import line:: from past.builtins import execfile for the function execfile() that was removed from Py3. """ from __future__ import unicode_literals from lib2to3 import fixer_base from libfuturize.fixer_util import touch_import_top expression = "name='execfile'" class FixExecfile(fixer_base.BaseFix): BM_compatible = True run_order = 9 PATTERN = """ power< ({0}) trailer< '(' args=[any] ')' > rest=any* > """.format(expression) def transform(self, node, results): name = results["name"] touch_import_top(u'past.builtins', name.value, node) """ For the ``future`` package. Adds this import line:: from builtins import XYZ for each of the functions XYZ that is used in the module. Adds these imports after any other imports (in an initial block of them). """ from __future__ import unicode_literals from lib2to3 import fixer_base from lib2to3.pygram import python_symbols as syms from lib2to3.fixer_util import Name, Call, in_special_context from libfuturize.fixer_util import touch_import_top # All builtins are: # from future.builtins.iterators import (filter, map, zip) # from future.builtins.misc import (ascii, chr, hex, input, isinstance, oct, open, round, super) # from future.types import (bytes, dict, int, range, str) # We don't need isinstance any more. replaced_builtin_fns = '''filter map zip ascii chr hex input next oct bytes range str raw_input'''.split() # This includes raw_input as a workaround for the # lib2to3 fixer for raw_input on Py3 (only), allowing # the correct import to be included. (Py3 seems to run # the fixers the wrong way around, perhaps ignoring the # run_order class attribute below ...) expression = '|'.join(["name='{0}'".format(name) for name in replaced_builtin_fns]) class FixFutureBuiltins(fixer_base.BaseFix): BM_compatible = True run_order = 7 # Currently we only match uses as a function. This doesn't match e.g.: # if isinstance(s, str): # ... PATTERN = """ power< ({0}) trailer< '(' [arglist=any] ')' > rest=any* > | power< 'map' trailer< '(' [arglist=any] ')' > > """.format(expression) def transform(self, node, results): name = results["name"] touch_import_top(u'builtins', name.value, node) # name.replace(Name(u"input", prefix=name.prefix)) """ For the ``future`` package. Changes any imports needed to reflect the standard library reorganization. Also Also adds these import lines: from future import standard_library standard_library.install_aliases() after any __future__ imports but before any other imports. """ from lib2to3.fixes.fix_imports import FixImports from libfuturize.fixer_util import touch_import_top class FixFutureStandardLibrary(FixImports): run_order = 8 def transform(self, node, results): result = super(FixFutureStandardLibrary, self).transform(node, results) # TODO: add a blank line between any __future__ imports and this? touch_import_top(u'future', u'standard_library', node) return result """ For the ``future`` package. A special fixer that ensures that these lines have been added:: from future import standard_library standard_library.install_hooks() even if the only module imported was ``urllib``, in which case the regular fixer wouldn't have added these lines. """ from lib2to3.fixes.fix_urllib import FixUrllib from libfuturize.fixer_util import touch_import_top, find_root class FixFutureStandardLibraryUrllib(FixUrllib): # not a subclass of FixImports run_order = 8 def transform(self, node, results): # transform_member() in lib2to3/fixes/fix_urllib.py breaks node so find_root(node) # no longer works after the super() call below. So we find the root first: root = find_root(node) result = super(FixFutureStandardLibraryUrllib, self).transform(node, results) # TODO: add a blank line between any __future__ imports and this? touch_import_top(u'future', u'standard_library', root) return result # coding: utf-8 """Fixer for __metaclass__ = X -> (future.utils.with_metaclass(X)) methods. The various forms of classef (inherits nothing, inherits once, inherints many) don't parse the same in the CST so we look at ALL classes for a __metaclass__ and if we find one normalize the inherits to all be an arglist. For one-liner classes ('class X: pass') there is no indent/dedent so we normalize those into having a suite. Moving the __metaclass__ into the classdef can also cause the class body to be empty so there is some special casing for that as well. This fixer also tries very hard to keep original indenting and spacing in all those corner cases. """ # This is a derived work of Lib/lib2to3/fixes/fix_metaclass.py under the # copyright of the Python Software Foundation, licensed under the Python # Software Foundation License 2. # # Copyright notice: # # Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, # 2011, 2012, 2013 Python Software Foundation. All rights reserved. # # Full license text: http://docs.python.org/3.4/license.html # Author: Jack Diederich, Daniel Neuhäuser # Local imports from lib2to3 import fixer_base from lib2to3.pygram import token from lib2to3.fixer_util import Name, syms, Node, Leaf, touch_import, Call, \ String, Comma, parenthesize def has_metaclass(parent): """ we have to check the cls_node without changing it. There are two possiblities: 1) clsdef => suite => simple_stmt => expr_stmt => Leaf('__meta') 2) clsdef => simple_stmt => expr_stmt => Leaf('__meta') """ for node in parent.children: if node.type == syms.suite: return has_metaclass(node) elif node.type == syms.simple_stmt and node.children: expr_node = node.children[0] if expr_node.type == syms.expr_stmt and expr_node.children: left_side = expr_node.children[0] if isinstance(left_side, Leaf) and \ left_side.value == '__metaclass__': return True return False def fixup_parse_tree(cls_node): """ one-line classes don't get a suite in the parse tree so we add one to normalize the tree """ for node in cls_node.children: if node.type == syms.suite: # already in the preferred format, do nothing return # !%@#! oneliners have no suite node, we have to fake one up for i, node in enumerate(cls_node.children): if node.type == token.COLON: break else: raise ValueError("No class suite and no ':'!") # move everything into a suite node suite = Node(syms.suite, []) while cls_node.children[i+1:]: move_node = cls_node.children[i+1] suite.append_child(move_node.clone()) move_node.remove() cls_node.append_child(suite) node = suite def fixup_simple_stmt(parent, i, stmt_node): """ if there is a semi-colon all the parts count as part of the same simple_stmt. We just want the __metaclass__ part so we move everything efter the semi-colon into its own simple_stmt node """ for semi_ind, node in enumerate(stmt_node.children): if node.type == token.SEMI: # *sigh* break else: return node.remove() # kill the semicolon new_expr = Node(syms.expr_stmt, []) new_stmt = Node(syms.simple_stmt, [new_expr]) while stmt_node.children[semi_ind:]: move_node = stmt_node.children[semi_ind] new_expr.append_child(move_node.clone()) move_node.remove() parent.insert_child(i, new_stmt) new_leaf1 = new_stmt.children[0].children[0] old_leaf1 = stmt_node.children[0].children[0] new_leaf1.prefix = old_leaf1.prefix def remove_trailing_newline(node): if node.children and node.children[-1].type == token.NEWLINE: node.children[-1].remove() def find_metas(cls_node): # find the suite node (Mmm, sweet nodes) for node in cls_node.children: if node.type == syms.suite: break else: raise ValueError("No class suite!") # look for simple_stmt[ expr_stmt[ Leaf('__metaclass__') ] ] for i, simple_node in list(enumerate(node.children)): if simple_node.type == syms.simple_stmt and simple_node.children: expr_node = simple_node.children[0] if expr_node.type == syms.expr_stmt and expr_node.children: # Check if the expr_node is a simple assignment. left_node = expr_node.children[0] if isinstance(left_node, Leaf) and \ left_node.value == u'__metaclass__': # We found a assignment to __metaclass__. fixup_simple_stmt(node, i, simple_node) remove_trailing_newline(simple_node) yield (node, i, simple_node) def fixup_indent(suite): """ If an INDENT is followed by a thing with a prefix then nuke the prefix Otherwise we get in trouble when removing __metaclass__ at suite start """ kids = suite.children[::-1] # find the first indent while kids: node = kids.pop() if node.type == token.INDENT: break # find the first Leaf while kids: node = kids.pop() if isinstance(node, Leaf) and node.type != token.DEDENT: if node.prefix: node.prefix = u'' return else: kids.extend(node.children[::-1]) class FixMetaclass(fixer_base.BaseFix): BM_compatible = True PATTERN = """ classdef """ def transform(self, node, results): if not has_metaclass(node): return fixup_parse_tree(node) # find metaclasses, keep the last one last_metaclass = None for suite, i, stmt in find_metas(node): last_metaclass = stmt stmt.remove() text_type = node.children[0].type # always Leaf(nnn, 'class') # figure out what kind of classdef we have if len(node.children) == 7: # Node(classdef, ['class', 'name', '(', arglist, ')', ':', suite]) # 0 1 2 3 4 5 6 if node.children[3].type == syms.arglist: arglist = node.children[3] # Node(classdef, ['class', 'name', '(', 'Parent', ')', ':', suite]) else: parent = node.children[3].clone() arglist = Node(syms.arglist, [parent]) node.set_child(3, arglist) elif len(node.children) == 6: # Node(classdef, ['class', 'name', '(', ')', ':', suite]) # 0 1 2 3 4 5 arglist = Node(syms.arglist, []) node.insert_child(3, arglist) elif len(node.children) == 4: # Node(classdef, ['class', 'name', ':', suite]) # 0 1 2 3 arglist = Node(syms.arglist, []) node.insert_child(2, Leaf(token.RPAR, u')')) node.insert_child(2, arglist) node.insert_child(2, Leaf(token.LPAR, u'(')) else: raise ValueError("Unexpected class definition") # now stick the metaclass in the arglist meta_txt = last_metaclass.children[0].children[0] meta_txt.value = 'metaclass' orig_meta_prefix = meta_txt.prefix # Was: touch_import(None, u'future.utils', node) touch_import(u'future.utils', u'with_metaclass', node) metaclass = last_metaclass.children[0].children[2].clone() metaclass.prefix = u'' arguments = [metaclass] if arglist.children: if len(arglist.children) == 1: base = arglist.children[0].clone() base.prefix = u' ' else: # Unfortunately six.with_metaclass() only allows one base # class, so we have to dynamically generate a base class if # there is more than one. bases = parenthesize(arglist.clone()) bases.prefix = u' ' base = Call(Name('type'), [ String("'NewBase'"), Comma(), bases, Comma(), Node( syms.atom, [Leaf(token.LBRACE, u'{'), Leaf(token.RBRACE, u'}')], prefix=u' ' ) ], prefix=u' ') arguments.extend([Comma(), base]) arglist.replace(Call( Name(u'with_metaclass', prefix=arglist.prefix), arguments )) fixup_indent(suite) # check for empty suite if not suite.children: # one-liner that was just __metaclass_ suite.remove() pass_leaf = Leaf(text_type, u'pass') pass_leaf.prefix = orig_meta_prefix node.append_child(pass_leaf) node.append_child(Leaf(token.NEWLINE, u'\n')) elif len(suite.children) > 1 and \ (suite.children[-2].type == token.INDENT and suite.children[-1].type == token.DEDENT): # there was only one line in the class body and it was __metaclass__ pass_leaf = Leaf(text_type, u'pass') suite.insert_child(-1, pass_leaf) suite.insert_child(-1, Leaf(token.NEWLINE, u'\n')) """ Based on fix_next.py by Collin Winter. Replaces it.next() -> next(it), per PEP 3114. Unlike fix_next.py, this fixer doesn't replace the name of a next method with __next__, which would break Python 2 compatibility without further help from fixers in stage 2. """ # Local imports from lib2to3.pgen2 import token from lib2to3.pygram import python_symbols as syms from lib2to3 import fixer_base from lib2to3.fixer_util import Name, Call, find_binding bind_warning = "Calls to builtin next() possibly shadowed by global binding" class FixNextCall(fixer_base.BaseFix): BM_compatible = True PATTERN = """ power< base=any+ trailer< '.' attr='next' > trailer< '(' ')' > > | power< head=any+ trailer< '.' attr='next' > not trailer< '(' ')' > > | global=global_stmt< 'global' any* 'next' any* > """ order = "pre" # Pre-order tree traversal def start_tree(self, tree, filename): super(FixNextCall, self).start_tree(tree, filename) n = find_binding('next', tree) if n: self.warning(n, bind_warning) self.shadowed_next = True else: self.shadowed_next = False def transform(self, node, results): assert results base = results.get("base") attr = results.get("attr") name = results.get("name") if base: if self.shadowed_next: # Omit this: # attr.replace(Name("__next__", prefix=attr.prefix)) pass else: base = [n.clone() for n in base] base[0].prefix = "" node.replace(Call(Name("next", prefix=node.prefix), base)) elif name: # Omit this: # n = Name("__next__", prefix=name.prefix) # name.replace(n) pass elif attr: # We don't do this transformation if we're assigning to "x.next". # Unfortunately, it doesn't seem possible to do this in PATTERN, # so it's being done here. if is_assign_target(node): head = results["head"] if "".join([str(n) for n in head]).strip() == '__builtin__': self.warning(node, bind_warning) return # Omit this: # attr.replace(Name("__next__")) elif "global" in results: self.warning(node, bind_warning) self.shadowed_next = True ### The following functions help test if node is part of an assignment ### target. def is_assign_target(node): assign = find_assign(node) if assign is None: return False for child in assign.children: if child.type == token.EQUAL: return False elif is_subtree(child, node): return True return False def find_assign(node): if node.type == syms.expr_stmt: return node if node.type == syms.simple_stmt or node.parent is None: return None return find_assign(node.parent) def is_subtree(root, node): if root == node: return True return any(is_subtree(c, node) for c in root.children) """ Fixer that adds ``from builtins import object`` if there is a line like this: class Foo(object): """ from lib2to3 import fixer_base from libfuturize.fixer_util import touch_import_top class FixObject(fixer_base.BaseFix): PATTERN = u"classdef< 'class' NAME '(' name='object' ')' colon=':' any >" def transform(self, node, results): touch_import_top(u'builtins', 'object', node) """ For the ``future`` package. Adds this import line: from past.builtins import str as oldstr at the top and wraps any unadorned string literals 'abc' or explicit byte-string literals b'abc' in oldstr() calls so the code has the same behaviour on Py3 as on Py2.6/2.7. """ from __future__ import unicode_literals import re from lib2to3 import fixer_base from lib2to3.pgen2 import token from lib2to3.fixer_util import syms from libfuturize.fixer_util import (future_import, touch_import_top, wrap_in_fn_call) _literal_re = re.compile(r"[^uUrR]?[\'\"]") class FixOldstrWrap(fixer_base.BaseFix): BM_compatible = True PATTERN = "STRING" def transform(self, node, results): if node.type == token.STRING: touch_import_top(u'past.types', u'oldstr', node) if _literal_re.match(node.value): new = node.clone() # Strip any leading space or comments: # TODO: check: do we really want to do this? new.prefix = u'' new.value = u'b' + new.value wrapped = wrap_in_fn_call("oldstr", [new], prefix=node.prefix) return wrapped """ UNFINISHED Fixer for turning multiple lines like these: from __future__ import division from __future__ import absolute_import from __future__ import print_function into a single line like this: from __future__ import (absolute_import, division, print_function) This helps with testing of ``futurize``. """ from lib2to3 import fixer_base from libfuturize.fixer_util import future_import class FixOrderFutureImports(fixer_base.BaseFix): BM_compatible = True PATTERN = "file_input" run_order = 10 # def match(self, node): # """ # Match only once per file # """ # if hasattr(node, 'type') and node.type == syms.file_input: # return True # return False def transform(self, node, results): # TODO # write me pass # Copyright 2006 Google, Inc. All Rights Reserved. # Licensed to PSF under a Contributor Agreement. """Fixer for print. Change: "print" into "print()" "print ..." into "print(...)" "print(...)" not changed "print ... ," into "print(..., end=' ')" "print >>x, ..." into "print(..., file=x)" No changes are applied if print_function is imported from __future__ """ # Local imports from lib2to3 import patcomp, pytree, fixer_base from lib2to3.pgen2 import token from lib2to3.fixer_util import Name, Call, Comma, String # from libmodernize import add_future parend_expr = patcomp.compile_pattern( """atom< '(' [arith_expr|atom|power|term|STRING|NAME] ')' >""" ) class FixPrint(fixer_base.BaseFix): BM_compatible = True PATTERN = """ simple_stmt< any* bare='print' any* > | print_stmt """ def transform(self, node, results): assert results bare_print = results.get("bare") if bare_print: # Special-case print all by itself. bare_print.replace(Call(Name(u"print"), [], prefix=bare_print.prefix)) # The "from __future__ import print_function"" declaration is added # by the fix_print_with_import fixer, so we skip it here. # add_future(node, u'print_function') return assert node.children[0] == Name(u"print") args = node.children[1:] if len(args) == 1 and parend_expr.match(args[0]): # We don't want to keep sticking parens around an # already-parenthesised expression. return sep = end = file = None if args and args[-1] == Comma(): args = args[:-1] end = " " if args and args[0] == pytree.Leaf(token.RIGHTSHIFT, u">>"): assert len(args) >= 2 file = args[1].clone() args = args[3:] # Strip a possible comma after the file expression # Now synthesize a print(args, sep=..., end=..., file=...) node. l_args = [arg.clone() for arg in args] if l_args: l_args[0].prefix = u"" if sep is not None or end is not None or file is not None: if sep is not None: self.add_kwarg(l_args, u"sep", String(repr(sep))) if end is not None: self.add_kwarg(l_args, u"end", String(repr(end))) if file is not None: self.add_kwarg(l_args, u"file", file) n_stmt = Call(Name(u"print"), l_args) n_stmt.prefix = node.prefix # Note that there are corner cases where adding this future-import is # incorrect, for example when the file also has a 'print ()' statement # that was intended to print "()". # add_future(node, u'print_function') return n_stmt def add_kwarg(self, l_nodes, s_kwd, n_expr): # XXX All this prefix-setting may lose comments (though rarely) n_expr.prefix = u"" n_argument = pytree.Node(self.syms.argument, (Name(s_kwd), pytree.Leaf(token.EQUAL, u"="), n_expr)) if l_nodes: l_nodes.append(Comma()) n_argument.prefix = u" " l_nodes.append(n_argument) """ For the ``future`` package. Turns any print statements into functions and adds this import line: from __future__ import print_function at the top to retain compatibility with Python 2.6+. """ from libfuturize.fixes.fix_print import FixPrint from libfuturize.fixer_util import future_import class FixPrintWithImport(FixPrint): run_order = 7 def transform(self, node, results): # Add the __future__ import first. (Otherwise any shebang or encoding # comment line attached as a prefix to the print statement will be # copied twice and appear twice.) future_import(u'print_function', node) n_stmt = super(FixPrintWithImport, self).transform(node, results) return n_stmt """Fixer for 'raise E, V' From Armin Ronacher's ``python-modernize``. raise -> raise raise E -> raise E raise E, V -> raise E(V) raise (((E, E'), E''), E'''), V -> raise E(V) CAVEATS: 1) "raise E, V" will be incorrectly translated if V is an exception instance. The correct Python 3 idiom is raise E from V but since we can't detect instance-hood by syntax alone and since any client code would have to be changed as well, we don't automate this. """ # Author: Collin Winter, Armin Ronacher # Local imports from lib2to3 import pytree, fixer_base from lib2to3.pgen2 import token from lib2to3.fixer_util import Name, Call, is_tuple class FixRaise(fixer_base.BaseFix): BM_compatible = True PATTERN = """ raise_stmt< 'raise' exc=any [',' val=any] > """ def transform(self, node, results): syms = self.syms exc = results["exc"].clone() if exc.type == token.STRING: msg = "Python 3 does not support string exceptions" self.cannot_convert(node, msg) return # Python 2 supports # raise ((((E1, E2), E3), E4), E5), V # as a synonym for # raise E1, V # Since Python 3 will not support this, we recurse down any tuple # literals, always taking the first element. if is_tuple(exc): while is_tuple(exc): # exc.children[1:-1] is the unparenthesized tuple # exc.children[1].children[0] is the first element of the tuple exc = exc.children[1].children[0].clone() exc.prefix = u" " if "val" not in results: # One-argument raise new = pytree.Node(syms.raise_stmt, [Name(u"raise"), exc]) new.prefix = node.prefix return new val = results["val"].clone() if is_tuple(val): args = [c.clone() for c in val.children[1:-1]] else: val.prefix = u"" args = [val] return pytree.Node(syms.raise_stmt, [Name(u"raise"), Call(exc, args)], prefix=node.prefix) """ Fixer for removing any of these lines: from __future__ import with_statement from __future__ import nested_scopes from __future__ import generators The reason is that __future__ imports like these are required to be the first line of code (after docstrings) on Python 2.6+, which can get in the way. These imports are always enabled in Python 2.6+, which is the minimum sane version to target for Py2/3 compatibility. """ from lib2to3 import fixer_base from libfuturize.fixer_util import remove_future_import class FixRemoveOldFutureImports(fixer_base.BaseFix): BM_compatible = True PATTERN = "file_input" run_order = 1 def transform(self, node, results): remove_future_import(u"with_statement", node) remove_future_import(u"nested_scopes", node) remove_future_import(u"generators", node) """Fixer that changes unicode to str and unichr to chr, but -- unlike the lib2to3 fix_unicode.py fixer, does not change u"..." into "...". The reason is that Py3.3+ supports the u"..." string prefix, and, if present, the prefix may provide useful information for disambiguating between byte strings and unicode strings, which is often the hardest part of the porting task. """ from lib2to3.pgen2 import token from lib2to3 import fixer_base _mapping = {u"unichr" : u"chr", u"unicode" : u"str"} class FixUnicodeKeepU(fixer_base.BaseFix): BM_compatible = True PATTERN = "'unicode' | 'unichr'" def transform(self, node, results): if node.type == token.NAME: new = node.clone() new.value = _mapping[node.value] return new """ Adds this import: from __future__ import unicode_literals """ from lib2to3 import fixer_base from libfuturize.fixer_util import future_import class FixUnicodeLiteralsImport(fixer_base.BaseFix): BM_compatible = True PATTERN = "file_input" run_order = 9 def transform(self, node, results): future_import(u"unicode_literals", node) """ For the ``future`` package. Turns any xrange calls into range calls and adds this import line: from builtins import range at the top. """ from lib2to3.fixes.fix_xrange import FixXrange from libfuturize.fixer_util import touch_import_top class FixXrangeWithImport(FixXrange): def transform(self, node, results): result = super(FixXrangeWithImport, self).transform(node, results) touch_import_top('builtins', 'range', node) return result # empty to make this a package """ pasteurize: automatic conversion of Python 3 code to clean 2/3 code =================================================================== ``pasteurize`` attempts to convert existing Python 3 code into source-compatible Python 2 and 3 code. Use it like this on Python 3 code: $ pasteurize --verbose mypython3script.py This removes any Py3-only syntax (e.g. new metaclasses) and adds these import lines: from __future__ import absolute_import from __future__ import division from __future__ import print_function from __future__ import unicode_literals from future import standard_library standard_library.install_hooks() from builtins import * To write changes to the files, use the -w flag. It also adds any other wrappers needed for Py2/3 compatibility. Note that separate stages are not available (or needed) when converting from Python 3 with ``pasteurize`` as they are when converting from Python 2 with ``futurize``. The --all-imports option forces adding all ``__future__`` imports, ``builtins`` imports, and standard library aliases, even if they don't seem necessary for the current state of each module. (This can simplify testing, and can reduce the need to think about Py2 compatibility when editing the code further.) """ from __future__ import (absolute_import, print_function, unicode_literals) import sys import logging import optparse from lib2to3.main import main, warn, StdoutRefactoringTool from lib2to3 import refactor from future import __version__ from libpasteurize.fixes import fix_names def main(args=None): """Main program. Returns a suggested exit status (0, 1, 2). """ # Set up option parser parser = optparse.OptionParser(usage="pasteurize [options] file|dir ...") parser.add_option("-V", "--version", action="store_true", help="Report the version number of pasteurize") parser.add_option("-a", "--all-imports", action="store_true", help="Adds all __future__ and future imports to each module") parser.add_option("-f", "--fix", action="append", default=[], help="Each FIX specifies a transformation; default: all") parser.add_option("-j", "--processes", action="store", default=1, type="int", help="Run 2to3 concurrently") parser.add_option("-x", "--nofix", action="append", default=[], help="Prevent a fixer from being run.") parser.add_option("-l", "--list-fixes", action="store_true", help="List available transformations") # parser.add_option("-p", "--print-function", action="store_true", # help="Modify the grammar so that print() is a function") parser.add_option("-v", "--verbose", action="store_true", help="More verbose logging") parser.add_option("--no-diffs", action="store_true", help="Don't show diffs of the refactoring") parser.add_option("-w", "--write", action="store_true", help="Write back modified files") parser.add_option("-n", "--nobackups", action="store_true", default=False, help="Don't write backups for modified files.") # Parse command line arguments refactor_stdin = False flags = {} options, args = parser.parse_args(args) fixer_pkg = 'libpasteurize.fixes' avail_fixes = fix_names flags["print_function"] = True if not options.write and options.no_diffs: warn("not writing files and not printing diffs; that's not very useful") if not options.write and options.nobackups: parser.error("Can't use -n without -w") if options.version: print(__version__) return 0 if options.list_fixes: print("Available transformations for the -f/--fix option:") for fixname in sorted(avail_fixes): print(fixname) if not args: return 0 if not args: print("At least one file or directory argument required.", file=sys.stderr) print("Use --help to show usage.", file=sys.stderr) return 2 if "-" in args: refactor_stdin = True if options.write: print("Can't write to stdin.", file=sys.stderr) return 2 # Set up logging handler level = logging.DEBUG if options.verbose else logging.INFO logging.basicConfig(format='%(name)s: %(message)s', level=level) # Initialize the refactoring tool unwanted_fixes = set(fixer_pkg + ".fix_" + fix for fix in options.nofix) extra_fixes = set() if options.all_imports: prefix = 'libpasteurize.fixes.' extra_fixes.add(prefix + 'fix_add_all__future__imports') extra_fixes.add(prefix + 'fix_add_future_standard_library_import') extra_fixes.add(prefix + 'fix_add_all_future_builtins') fixer_names = avail_fixes | extra_fixes - unwanted_fixes rt = StdoutRefactoringTool(sorted(fixer_names), flags, set(), options.nobackups, not options.no_diffs) # Refactor all files and directories passed as arguments if not rt.errors: if refactor_stdin: rt.refactor_stdin() else: try: rt.refactor(args, options.write, None, options.processes) except refactor.MultiprocessingUnsupported: assert options.processes > 1 print("Sorry, -j isn't " \ "supported on this platform.", file=sys.stderr) return 1 rt.summarize() # Return error status (0 if rt.errors is zero) return int(bool(rt.errors)) import sys from lib2to3 import refactor # The original set of these fixes comes from lib3to2 (https://bitbucket.org/amentajo/lib3to2): fix_names = set([ 'libpasteurize.fixes.fix_add_all__future__imports', # from __future__ import absolute_import etc. on separate lines 'libpasteurize.fixes.fix_add_future_standard_library_import', # we force adding this import for now, even if it doesn't seem necessary to the fix_future_standard_library fixer, for ease of testing # 'libfuturize.fixes.fix_order___future__imports', # consolidates to a single line to simplify testing -- UNFINISHED 'libpasteurize.fixes.fix_future_builtins', # adds "from future.builtins import *" 'libfuturize.fixes.fix_future_standard_library', # adds "from future import standard_library" 'libpasteurize.fixes.fix_annotations', # 'libpasteurize.fixes.fix_bitlength', # ints have this in Py2.7 # 'libpasteurize.fixes.fix_bool', # need a decorator or Mixin # 'libpasteurize.fixes.fix_bytes', # leave bytes as bytes # 'libpasteurize.fixes.fix_classdecorator', # available in # Py2.6+ # 'libpasteurize.fixes.fix_collections', hmmm ... # 'libpasteurize.fixes.fix_dctsetcomp', # avail in Py27 'libpasteurize.fixes.fix_division', # yes # 'libpasteurize.fixes.fix_except', # avail in Py2.6+ # 'libpasteurize.fixes.fix_features', # ? 'libpasteurize.fixes.fix_fullargspec', # 'libpasteurize.fixes.fix_funcattrs', 'libpasteurize.fixes.fix_getcwd', 'libpasteurize.fixes.fix_imports', # adds "from future import standard_library" 'libpasteurize.fixes.fix_imports2', # 'libpasteurize.fixes.fix_input', # 'libpasteurize.fixes.fix_int', # 'libpasteurize.fixes.fix_intern', # 'libpasteurize.fixes.fix_itertools', 'libpasteurize.fixes.fix_kwargs', # yes, we want this # 'libpasteurize.fixes.fix_memoryview', # 'libpasteurize.fixes.fix_metaclass', # write a custom handler for # this # 'libpasteurize.fixes.fix_methodattrs', # __func__ and __self__ seem to be defined on Py2.7 already 'libpasteurize.fixes.fix_newstyle', # yes, we want this: explicit inheritance from object. Without new-style classes in Py2, super() will break etc. # 'libpasteurize.fixes.fix_next', # use a decorator for this # 'libpasteurize.fixes.fix_numliterals', # prob not # 'libpasteurize.fixes.fix_open', # huh? # 'libpasteurize.fixes.fix_print', # no way 'libpasteurize.fixes.fix_printfunction', # adds __future__ import print_function # 'libpasteurize.fixes.fix_raise_', # TODO: get this working! # 'libpasteurize.fixes.fix_range', # nope # 'libpasteurize.fixes.fix_reduce', # 'libpasteurize.fixes.fix_setliteral', # 'libpasteurize.fixes.fix_str', # 'libpasteurize.fixes.fix_super', # maybe, if our magic super() isn't robust enough 'libpasteurize.fixes.fix_throw', # yes, if Py3 supports it # 'libpasteurize.fixes.fix_unittest', 'libpasteurize.fixes.fix_unpacking', # yes, this is useful # 'libpasteurize.fixes.fix_with' # way out of date ]) u""" Base classes for features that are backwards-incompatible. Usage: features = Features() features.add(Feature("py3k_feature", "power< 'py3k' any* >", "2.7")) PATTERN = features.PATTERN """ pattern_unformatted = u"%s=%s" # name=pattern, for dict lookups message_unformatted = u""" %s is only supported in Python %s and above.""" class Feature(object): u""" A feature has a name, a pattern, and a minimum version of Python 2.x required to use the feature (or 3.x if there is no backwards-compatible version of 2.x) """ def __init__(self, name, PATTERN, version): self.name = name self._pattern = PATTERN self.version = version def message_text(self): u""" Format the above text with the name and minimum version required. """ return message_unformatted % (self.name, self.version) class Features(set): u""" A set of features that generates a pattern for the features it contains. This set will act like a mapping in that we map names to patterns. """ mapping = {} def update_mapping(self): u""" Called every time we care about the mapping of names to features. """ self.mapping = dict([(f.name, f) for f in iter(self)]) @property def PATTERN(self): u""" Uses the mapping of names to features to return a PATTERN suitable for using the lib2to3 patcomp. """ self.update_mapping() return u" |\n".join([pattern_unformatted % (f.name, f._pattern) for f in iter(self)]) def __getitem__(self, key): u""" Implement a simple mapping to get patterns from names. """ return self.mapping[key] """ Fixer for adding: from __future__ import absolute_import from __future__ import division from __future__ import print_function from __future__ import unicode_literals This is done when converting from Py3 to both Py3/Py2. """ from lib2to3 import fixer_base from libfuturize.fixer_util import future_import class FixAddAllFutureImports(fixer_base.BaseFix): BM_compatible = True PATTERN = "file_input" run_order = 1 def transform(self, node, results): future_import(u"unicode_literals", node) future_import(u"print_function", node) future_import(u"division", node) future_import(u"absolute_import", node) """ For the ``future`` package. Adds this import line:: from builtins import (ascii, bytes, chr, dict, filter, hex, input, int, list, map, next, object, oct, open, pow, range, round, str, super, zip) to a module, irrespective of whether each definition is used. Adds these imports after any other imports (in an initial block of them). """ from __future__ import unicode_literals from lib2to3 import fixer_base from libfuturize.fixer_util import touch_import_top class FixAddAllFutureBuiltins(fixer_base.BaseFix): BM_compatible = True PATTERN = "file_input" run_order = 1 def transform(self, node, results): # import_str = """(ascii, bytes, chr, dict, filter, hex, input, # int, list, map, next, object, oct, open, pow, # range, round, str, super, zip)""" touch_import_top(u'builtins', '*', node) # builtins = """ascii bytes chr dict filter hex input # int list map next object oct open pow # range round str super zip""" # for builtin in sorted(builtins.split(), reverse=True): # touch_import_top(u'builtins', builtin, node) """ For the ``future`` package. Adds this import line: from future import standard_library after any __future__ imports but before any other imports. Doesn't actually change the imports to Py3 style. """ from lib2to3 import fixer_base from libfuturize.fixer_util import touch_import_top class FixAddFutureStandardLibraryImport(fixer_base.BaseFix): BM_compatible = True PATTERN = "file_input" run_order = 8 def transform(self, node, results): # TODO: add a blank line between any __future__ imports and this? touch_import_top(u'future', u'standard_library', node) # TODO: also add standard_library.install_hooks() u""" Fixer to remove function annotations """ from lib2to3 import fixer_base from lib2to3.pgen2 import token from lib2to3.fixer_util import syms warning_text = u"Removing function annotations completely." def param_without_annotations(node): return node.children[0] class FixAnnotations(fixer_base.BaseFix): warned = False def warn_once(self, node, reason): if not self.warned: self.warned = True self.warning(node, reason=reason) PATTERN = u""" funcdef< 'def' any parameters< '(' [params=any] ')' > ['->' ret=any] ':' any* > """ def transform(self, node, results): u""" This just strips annotations from the funcdef completely. """ params = results.get(u"params") ret = results.get(u"ret") if ret is not None: assert ret.prev_sibling.type == token.RARROW, u"Invalid return annotation" self.warn_once(node, reason=warning_text) ret.prev_sibling.remove() ret.remove() if params is None: return if params.type == syms.typedargslist: # more than one param in a typedargslist for param in params.children: if param.type == syms.tname: self.warn_once(node, reason=warning_text) param.replace(param_without_annotations(param)) elif params.type == syms.tname: # one param self.warn_once(node, reason=warning_text) params.replace(param_without_annotations(params)) u""" Fixer for division: from __future__ import division if needed """ from lib2to3 import fixer_base from libfuturize.fixer_util import token, future_import def match_division(node): u""" __future__.division redefines the meaning of a single slash for division, so we match that and only that. """ slash = token.SLASH return node.type == slash and not node.next_sibling.type == slash and \ not node.prev_sibling.type == slash class FixDivision(fixer_base.BaseFix): run_order = 4 # this seems to be ignored? def match(self, node): u""" Since the tree needs to be fixed once and only once if and only if it matches, then we can start discarding matches after we make the first. """ return match_division(node) def transform(self, node, results): future_import(u"division", node) u""" Warn about features that are not present in Python 2.5, giving a message that points to the earliest version of Python 2.x (or 3.x, if none) that supports it """ from .feature_base import Feature, Features from lib2to3 import fixer_base FEATURES = [ #(FeatureName, # FeaturePattern, # FeatureMinVersion, #), (u"memoryview", u"power < 'memoryview' trailer < '(' any* ')' > any* >", u"2.7", ), (u"numbers", u"""import_from< 'from' 'numbers' 'import' any* > | import_name< 'import' ('numbers' dotted_as_names< any* 'numbers' any* >) >""", u"2.6", ), (u"abc", u"""import_name< 'import' ('abc' dotted_as_names< any* 'abc' any* >) > | import_from< 'from' 'abc' 'import' any* >""", u"2.6", ), (u"io", u"""import_name< 'import' ('io' dotted_as_names< any* 'io' any* >) > | import_from< 'from' 'io' 'import' any* >""", u"2.6", ), (u"bin", u"power< 'bin' trailer< '(' any* ')' > any* >", u"2.6", ), (u"formatting", u"power< any trailer< '.' 'format' > trailer< '(' any* ')' > >", u"2.6", ), (u"nonlocal", u"global_stmt< 'nonlocal' any* >", u"3.0", ), (u"with_traceback", u"trailer< '.' 'with_traceback' >", u"3.0", ), ] class FixFeatures(fixer_base.BaseFix): run_order = 9 # Wait until all other fixers have run to check for these # To avoid spamming, we only want to warn for each feature once. features_warned = set() # Build features from the list above features = Features([Feature(name, pattern, version) for \ name, pattern, version in FEATURES]) PATTERN = features.PATTERN def match(self, node): to_ret = super(FixFeatures, self).match(node) # We want the mapping only to tell us the node's specific information. try: del to_ret[u'node'] except Exception: # We want it to delete the 'node' from the results # if it's there, so we don't care if it fails for normal reasons. pass return to_ret def transform(self, node, results): for feature_name in results: if feature_name in self.features_warned: continue else: curr_feature = self.features[feature_name] if curr_feature.version >= u"3": fail = self.cannot_convert else: fail = self.warning fail(node, reason=curr_feature.message_text()) self.features_warned.add(feature_name) u""" Fixer for getfullargspec -> getargspec """ from lib2to3 import fixer_base from lib2to3.fixer_util import Name warn_msg = u"some of the values returned by getfullargspec are not valid in Python 2 and have no equivalent." class FixFullargspec(fixer_base.BaseFix): PATTERN = u"'getfullargspec'" def transform(self, node, results): self.warning(node, warn_msg) return Name(u"getargspec", prefix=node.prefix) """ Adds this import line: from builtins import XYZ for each of the functions XYZ that is used in the module. """ from __future__ import unicode_literals from lib2to3 import fixer_base from lib2to3.pygram import python_symbols as syms from lib2to3.fixer_util import Name, Call, in_special_context from libfuturize.fixer_util import touch_import_top # All builtins are: # from future.builtins.iterators import (filter, map, zip) # from future.builtins.misc import (ascii, chr, hex, input, isinstance, oct, open, round, super) # from future.types import (bytes, dict, int, range, str) # We don't need isinstance any more. replaced_builtins = '''filter map zip ascii chr hex input next oct open round super bytes dict int range str'''.split() expression = '|'.join(["name='{0}'".format(name) for name in replaced_builtins]) class FixFutureBuiltins(fixer_base.BaseFix): BM_compatible = True run_order = 9 # Currently we only match uses as a function. This doesn't match e.g.: # if isinstance(s, str): # ... PATTERN = """ power< ({0}) trailer< '(' args=[any] ')' > rest=any* > """.format(expression) def transform(self, node, results): name = results["name"] touch_import_top(u'builtins', name.value, node) # name.replace(Name(u"input", prefix=name.prefix)) u""" Fixer for os.getcwd() -> os.getcwdu(). Also warns about "from os import getcwd", suggesting the above form. """ from lib2to3 import fixer_base from lib2to3.fixer_util import Name class FixGetcwd(fixer_base.BaseFix): PATTERN = u""" power< 'os' trailer< dot='.' name='getcwd' > any* > | import_from< 'from' 'os' 'import' bad='getcwd' > """ def transform(self, node, results): if u"name" in results: name = results[u"name"] name.replace(Name(u"getcwdu", prefix=name.prefix)) elif u"bad" in results: # Can't convert to getcwdu and then expect to catch every use. self.cannot_convert(node, u"import os, use os.getcwd() instead.") return else: raise ValueError(u"For some reason, the pattern matcher failed.") u""" Fixer for standard library imports renamed in Python 3 """ from lib2to3 import fixer_base from lib2to3.fixer_util import Name, is_probably_builtin, Newline, does_tree_import from lib2to3.pygram import python_symbols as syms from lib2to3.pgen2 import token from lib2to3.pytree import Node, Leaf from libfuturize.fixer_util import touch_import_top # from ..fixer_util import NameImport # used in simple_mapping_to_pattern() MAPPING = {u"reprlib": u"repr", u"winreg": u"_winreg", u"configparser": u"ConfigParser", u"copyreg": u"copy_reg", u"queue": u"Queue", u"socketserver": u"SocketServer", u"_markupbase": u"markupbase", u"test.support": u"test.test_support", u"dbm.bsd": u"dbhash", u"dbm.ndbm": u"dbm", u"dbm.dumb": u"dumbdbm", u"dbm.gnu": u"gdbm", u"html.parser": u"HTMLParser", u"html.entities": u"htmlentitydefs", u"http.client": u"httplib", u"http.cookies": u"Cookie", u"http.cookiejar": u"cookielib", # "tkinter": "Tkinter", u"tkinter.dialog": u"Dialog", u"tkinter._fix": u"FixTk", u"tkinter.scrolledtext": u"ScrolledText", u"tkinter.tix": u"Tix", u"tkinter.constants": u"Tkconstants", u"tkinter.dnd": u"Tkdnd", u"tkinter.__init__": u"Tkinter", u"tkinter.colorchooser": u"tkColorChooser", u"tkinter.commondialog": u"tkCommonDialog", u"tkinter.font": u"tkFont", u"tkinter.ttk": u"ttk", u"tkinter.messagebox": u"tkMessageBox", u"tkinter.turtle": u"turtle", u"urllib.robotparser": u"robotparser", u"xmlrpc.client": u"xmlrpclib", u"builtins": u"__builtin__", } # generic strings to help build patterns # these variables mean (with http.client.HTTPConnection as an example): # name = http # attr = client # used = HTTPConnection # fmt_name is a formatted subpattern (simple_name_match or dotted_name_match) # helps match 'queue', as in 'from queue import ...' simple_name_match = u"name='%s'" # helps match 'client', to be used if client has been imported from http subname_match = u"attr='%s'" # helps match 'http.client', as in 'import urllib.request' dotted_name_match = u"dotted_name=dotted_name< %s '.' %s >" # helps match 'queue', as in 'queue.Queue(...)' power_onename_match = u"%s" # helps match 'http.client', as in 'http.client.HTTPConnection(...)' power_twoname_match = u"power< %s trailer< '.' %s > any* >" # helps match 'client.HTTPConnection', if 'client' has been imported from http power_subname_match = u"power< %s any* >" # helps match 'from http.client import HTTPConnection' from_import_match = u"from_import=import_from< 'from' %s 'import' imported=any >" # helps match 'from http import client' from_import_submod_match = u"from_import_submod=import_from< 'from' %s 'import' (%s | import_as_name< %s 'as' renamed=any > | import_as_names< any* (%s | import_as_name< %s 'as' renamed=any >) any* > ) >" # helps match 'import urllib.request' name_import_match = u"name_import=import_name< 'import' %s > | name_import=import_name< 'import' dotted_as_name< %s 'as' renamed=any > >" # helps match 'import http.client, winreg' multiple_name_import_match = u"name_import=import_name< 'import' dotted_as_names< names=any* > >" def all_patterns(name): u""" Accepts a string and returns a pattern of possible patterns involving that name Called by simple_mapping_to_pattern for each name in the mapping it receives. """ # i_ denotes an import-like node # u_ denotes a node that appears to be a usage of the name if u'.' in name: name, attr = name.split(u'.', 1) simple_name = simple_name_match % (name) simple_attr = subname_match % (attr) dotted_name = dotted_name_match % (simple_name, simple_attr) i_from = from_import_match % (dotted_name) i_from_submod = from_import_submod_match % (simple_name, simple_attr, simple_attr, simple_attr, simple_attr) i_name = name_import_match % (dotted_name, dotted_name) u_name = power_twoname_match % (simple_name, simple_attr) u_subname = power_subname_match % (simple_attr) return u' | \n'.join((i_name, i_from, i_from_submod, u_name, u_subname)) else: simple_name = simple_name_match % (name) i_name = name_import_match % (simple_name, simple_name) i_from = from_import_match % (simple_name) u_name = power_onename_match % (simple_name) return u' | \n'.join((i_name, i_from, u_name)) class FixImports(fixer_base.BaseFix): PATTERN = u' | \n'.join([all_patterns(name) for name in MAPPING]) PATTERN = u' | \n'.join((PATTERN, multiple_name_import_match)) def transform(self, node, results): touch_import_top(u'future', u'standard_library', node) u""" Fixer for complicated imports """ from lib2to3 import fixer_base from lib2to3.fixer_util import Name, String, FromImport, Newline, Comma from libfuturize.fixer_util import touch_import_top TK_BASE_NAMES = (u'ACTIVE', u'ALL', u'ANCHOR', u'ARC',u'BASELINE', u'BEVEL', u'BOTH', u'BOTTOM', u'BROWSE', u'BUTT', u'CASCADE', u'CENTER', u'CHAR', u'CHECKBUTTON', u'CHORD', u'COMMAND', u'CURRENT', u'DISABLED', u'DOTBOX', u'E', u'END', u'EW', u'EXCEPTION', u'EXTENDED', u'FALSE', u'FIRST', u'FLAT', u'GROOVE', u'HIDDEN', u'HORIZONTAL', u'INSERT', u'INSIDE', u'LAST', u'LEFT', u'MITER', u'MOVETO', u'MULTIPLE', u'N', u'NE', u'NO', u'NONE', u'NORMAL', u'NS', u'NSEW', u'NUMERIC', u'NW', u'OFF', u'ON', u'OUTSIDE', u'PAGES', u'PIESLICE', u'PROJECTING', u'RADIOBUTTON', u'RAISED', u'READABLE', u'RIDGE', u'RIGHT', u'ROUND', u'S', u'SCROLL', u'SE', u'SEL', u'SEL_FIRST', u'SEL_LAST', u'SEPARATOR', u'SINGLE', u'SOLID', u'SUNKEN', u'SW', u'StringTypes', u'TOP', u'TRUE', u'TclVersion', u'TkVersion', u'UNDERLINE', u'UNITS', u'VERTICAL', u'W', u'WORD', u'WRITABLE', u'X', u'Y', u'YES', u'wantobjects') PY2MODULES = { u'urllib2' : ( u'AbstractBasicAuthHandler', u'AbstractDigestAuthHandler', u'AbstractHTTPHandler', u'BaseHandler', u'CacheFTPHandler', u'FTPHandler', u'FileHandler', u'HTTPBasicAuthHandler', u'HTTPCookieProcessor', u'HTTPDefaultErrorHandler', u'HTTPDigestAuthHandler', u'HTTPError', u'HTTPErrorProcessor', u'HTTPHandler', u'HTTPPasswordMgr', u'HTTPPasswordMgrWithDefaultRealm', u'HTTPRedirectHandler', u'HTTPSHandler', u'OpenerDirector', u'ProxyBasicAuthHandler', u'ProxyDigestAuthHandler', u'ProxyHandler', u'Request', u'StringIO', u'URLError', u'UnknownHandler', u'addinfourl', u'build_opener', u'install_opener', u'parse_http_list', u'parse_keqv_list', u'randombytes', u'request_host', u'urlopen'), u'urllib' : ( u'ContentTooShortError', u'FancyURLopener',u'URLopener', u'basejoin', u'ftperrors', u'getproxies', u'getproxies_environment', u'localhost', u'pathname2url', u'quote', u'quote_plus', u'splitattr', u'splithost', u'splitnport', u'splitpasswd', u'splitport', u'splitquery', u'splittag', u'splittype', u'splituser', u'splitvalue', u'thishost', u'unquote', u'unquote_plus', u'unwrap', u'url2pathname', u'urlcleanup', u'urlencode', u'urlopen', u'urlretrieve',), u'urlparse' : ( u'parse_qs', u'parse_qsl', u'urldefrag', u'urljoin', u'urlparse', u'urlsplit', u'urlunparse', u'urlunsplit'), u'dbm' : ( u'ndbm', u'gnu', u'dumb'), u'anydbm' : ( u'error', u'open'), u'whichdb' : ( u'whichdb',), u'BaseHTTPServer' : ( u'BaseHTTPRequestHandler', u'HTTPServer'), u'CGIHTTPServer' : ( u'CGIHTTPRequestHandler',), u'SimpleHTTPServer' : ( u'SimpleHTTPRequestHandler',), u'FileDialog' : TK_BASE_NAMES + ( u'FileDialog', u'LoadFileDialog', u'SaveFileDialog', u'dialogstates', u'test'), u'tkFileDialog' : ( u'Directory', u'Open', u'SaveAs', u'_Dialog', u'askdirectory', u'askopenfile', u'askopenfilename', u'askopenfilenames', u'askopenfiles', u'asksaveasfile', u'asksaveasfilename'), u'SimpleDialog' : TK_BASE_NAMES + ( u'SimpleDialog',), u'tkSimpleDialog' : TK_BASE_NAMES + ( u'askfloat', u'askinteger', u'askstring', u'Dialog'), u'SimpleXMLRPCServer' : ( u'CGIXMLRPCRequestHandler', u'SimpleXMLRPCDispatcher', u'SimpleXMLRPCRequestHandler', u'SimpleXMLRPCServer', u'list_public_methods', u'remove_duplicates', u'resolve_dotted_attribute'), u'DocXMLRPCServer' : ( u'DocCGIXMLRPCRequestHandler', u'DocXMLRPCRequestHandler', u'DocXMLRPCServer', u'ServerHTMLDoc',u'XMLRPCDocGenerator'), } MAPPING = { u'urllib.request' : (u'urllib2', u'urllib'), u'urllib.error' : (u'urllib2', u'urllib'), u'urllib.parse' : (u'urllib2', u'urllib', u'urlparse'), u'dbm.__init__' : (u'anydbm', u'whichdb'), u'http.server' : (u'CGIHTTPServer', u'SimpleHTTPServer', u'BaseHTTPServer'), u'tkinter.filedialog' : (u'tkFileDialog', u'FileDialog'), u'tkinter.simpledialog' : (u'tkSimpleDialog', u'SimpleDialog'), u'xmlrpc.server' : (u'DocXMLRPCServer', u'SimpleXMLRPCServer'), } # helps match 'http', as in 'from http.server import ...' simple_name = u"name='%s'" # helps match 'server', as in 'from http.server import ...' simple_attr = u"attr='%s'" # helps match 'HTTPServer', as in 'from http.server import HTTPServer' simple_using = u"using='%s'" # helps match 'urllib.request', as in 'import urllib.request' dotted_name = u"dotted_name=dotted_name< %s '.' %s >" # helps match 'http.server', as in 'http.server.HTTPServer(...)' power_twoname = u"pow=power< %s trailer< '.' %s > trailer< '.' using=any > any* >" # helps match 'dbm.whichdb', as in 'dbm.whichdb(...)' power_onename = u"pow=power< %s trailer< '.' using=any > any* >" # helps match 'from http.server import HTTPServer' # also helps match 'from http.server import HTTPServer, SimpleHTTPRequestHandler' # also helps match 'from http.server import *' from_import = u"from_import=import_from< 'from' %s 'import' (import_as_name< using=any 'as' renamed=any> | in_list=import_as_names< using=any* > | using='*' | using=NAME) >" # helps match 'import urllib.request' name_import = u"name_import=import_name< 'import' (%s | in_list=dotted_as_names< imp_list=any* >) >" ############# # WON'T FIX # ############# # helps match 'import urllib.request as name' name_import_rename = u"name_import_rename=dotted_as_name< %s 'as' renamed=any >" # helps match 'from http import server' from_import_rename = u"from_import_rename=import_from< 'from' %s 'import' (%s | import_as_name< %s 'as' renamed=any > | in_list=import_as_names< any* (%s | import_as_name< %s 'as' renamed=any >) any* >) >" def all_modules_subpattern(): u""" Builds a pattern for all toplevel names (urllib, http, etc) """ names_dot_attrs = [mod.split(u".") for mod in MAPPING] ret = u"( " + u" | ".join([dotted_name % (simple_name % (mod[0]), simple_attr % (mod[1])) for mod in names_dot_attrs]) ret += u" | " ret += u" | ".join([simple_name % (mod[0]) for mod in names_dot_attrs if mod[1] == u"__init__"]) + u" )" return ret def build_import_pattern(mapping1, mapping2): u""" mapping1: A dict mapping py3k modules to all possible py2k replacements mapping2: A dict mapping py2k modules to the things they do This builds a HUGE pattern to match all ways that things can be imported """ # py3k: urllib.request, py2k: ('urllib2', 'urllib') yield from_import % (all_modules_subpattern()) for py3k, py2k in mapping1.items(): name, attr = py3k.split(u'.') s_name = simple_name % (name) s_attr = simple_attr % (attr) d_name = dotted_name % (s_name, s_attr) yield name_import % (d_name) yield power_twoname % (s_name, s_attr) if attr == u'__init__': yield name_import % (s_name) yield power_onename % (s_name) yield name_import_rename % (d_name) yield from_import_rename % (s_name, s_attr, s_attr, s_attr, s_attr) class FixImports2(fixer_base.BaseFix): run_order = 4 PATTERN = u" | \n".join(build_import_pattern(MAPPING, PY2MODULES)) def transform(self, node, results): touch_import_top(u'future', u'standard_library', node) u""" Fixer for Python 3 function parameter syntax This fixer is rather sensitive to incorrect py3k syntax. """ # Note: "relevant" parameters are parameters following the first STAR in the list. from lib2to3 import fixer_base from lib2to3.fixer_util import token, String, Newline, Comma, Name from libfuturize.fixer_util import indentation, suitify, DoubleStar _assign_template = u"%(name)s = %(kwargs)s['%(name)s']; del %(kwargs)s['%(name)s']" _if_template = u"if '%(name)s' in %(kwargs)s: %(assign)s" _else_template = u"else: %(name)s = %(default)s" _kwargs_default_name = u"_3to2kwargs" def gen_params(raw_params): u""" Generator that yields tuples of (name, default_value) for each parameter in the list If no default is given, then it is default_value is None (not Leaf(token.NAME, 'None')) """ assert raw_params[0].type == token.STAR and len(raw_params) > 2 curr_idx = 2 # the first place a keyword-only parameter name can be is index 2 max_idx = len(raw_params) while curr_idx < max_idx: curr_item = raw_params[curr_idx] prev_item = curr_item.prev_sibling if curr_item.type != token.NAME: curr_idx += 1 continue if prev_item is not None and prev_item.type == token.DOUBLESTAR: break name = curr_item.value nxt = curr_item.next_sibling if nxt is not None and nxt.type == token.EQUAL: default_value = nxt.next_sibling curr_idx += 2 else: default_value = None yield (name, default_value) curr_idx += 1 def remove_params(raw_params, kwargs_default=_kwargs_default_name): u""" Removes all keyword-only args from the params list and a bare star, if any. Does not add the kwargs dict if needed. Returns True if more action is needed, False if not (more action is needed if no kwargs dict exists) """ assert raw_params[0].type == token.STAR if raw_params[1].type == token.COMMA: raw_params[0].remove() raw_params[1].remove() kw_params = raw_params[2:] else: kw_params = raw_params[3:] for param in kw_params: if param.type != token.DOUBLESTAR: param.remove() else: return False else: return True def needs_fixing(raw_params, kwargs_default=_kwargs_default_name): u""" Returns string with the name of the kwargs dict if the params after the first star need fixing Otherwise returns empty string """ found_kwargs = False needs_fix = False for t in raw_params[2:]: if t.type == token.COMMA: # Commas are irrelevant at this stage. continue elif t.type == token.NAME and not found_kwargs: # Keyword-only argument: definitely need to fix. needs_fix = True elif t.type == token.NAME and found_kwargs: # Return 'foobar' of **foobar, if needed. return t.value if needs_fix else u'' elif t.type == token.DOUBLESTAR: # Found either '*' from **foobar. found_kwargs = True else: # Never found **foobar. Return a synthetic name, if needed. return kwargs_default if needs_fix else u'' class FixKwargs(fixer_base.BaseFix): run_order = 7 # Run after function annotations are removed PATTERN = u"funcdef< 'def' NAME parameters< '(' arglist=typedargslist< params=any* > ')' > ':' suite=any >" def transform(self, node, results): params_rawlist = results[u"params"] for i, item in enumerate(params_rawlist): if item.type == token.STAR: params_rawlist = params_rawlist[i:] break else: return # params is guaranteed to be a list starting with *. # if fixing is needed, there will be at least 3 items in this list: # [STAR, COMMA, NAME] is the minimum that we need to worry about. new_kwargs = needs_fixing(params_rawlist) # new_kwargs is the name of the kwargs dictionary. if not new_kwargs: return suitify(node) # At this point, params_rawlist is guaranteed to be a list # beginning with a star that includes at least one keyword-only param # e.g., [STAR, NAME, COMMA, NAME, COMMA, DOUBLESTAR, NAME] or # [STAR, COMMA, NAME], or [STAR, COMMA, NAME, COMMA, DOUBLESTAR, NAME] # Anatomy of a funcdef: ['def', 'name', parameters, ':', suite] # Anatomy of that suite: [NEWLINE, INDENT, first_stmt, all_other_stmts] # We need to insert our new stuff before the first_stmt and change the # first_stmt's prefix. suite = node.children[4] first_stmt = suite.children[2] ident = indentation(first_stmt) for name, default_value in gen_params(params_rawlist): if default_value is None: suite.insert_child(2, Newline()) suite.insert_child(2, String(_assign_template %{u'name':name, u'kwargs':new_kwargs}, prefix=ident)) else: suite.insert_child(2, Newline()) suite.insert_child(2, String(_else_template %{u'name':name, u'default':default_value}, prefix=ident)) suite.insert_child(2, Newline()) suite.insert_child(2, String(_if_template %{u'assign':_assign_template %{u'name':name, u'kwargs':new_kwargs}, u'name':name, u'kwargs':new_kwargs}, prefix=ident)) first_stmt.prefix = ident suite.children[2].prefix = u"" # Now, we need to fix up the list of params. must_add_kwargs = remove_params(params_rawlist) if must_add_kwargs: arglist = results[u'arglist'] if len(arglist.children) > 0 and arglist.children[-1].type != token.COMMA: arglist.append_child(Comma()) arglist.append_child(DoubleStar(prefix=u" ")) arglist.append_child(Name(new_kwargs)) u""" Fixer for memoryview(s) -> buffer(s). Explicit because some memoryview methods are invalid on buffer objects. """ from lib2to3 import fixer_base from lib2to3.fixer_util import Name class FixMemoryview(fixer_base.BaseFix): explicit = True # User must specify that they want this. PATTERN = u""" power< name='memoryview' trailer< '(' [any] ')' > rest=any* > """ def transform(self, node, results): name = results[u"name"] name.replace(Name(u"buffer", prefix=name.prefix)) u""" Fixer for (metaclass=X) -> __metaclass__ = X Some semantics (see PEP 3115) may be altered in the translation.""" from lib2to3 import fixer_base from lib2to3.fixer_util import Name, syms, Node, Leaf, Newline, find_root from lib2to3.pygram import token from libfuturize.fixer_util import indentation, suitify # from ..fixer_util import Name, syms, Node, Leaf, Newline, find_root, indentation, suitify def has_metaclass(parent): results = None for node in parent.children: kids = node.children if node.type == syms.argument: if kids[0] == Leaf(token.NAME, u"metaclass") and \ kids[1] == Leaf(token.EQUAL, u"=") and \ kids[2]: #Hack to avoid "class X(=):" with this case. results = [node] + kids break elif node.type == syms.arglist: # Argument list... loop through it looking for: # Node(*, [*, Leaf(token.NAME, u"metaclass"), Leaf(token.EQUAL, u"="), Leaf(*, *)] for child in node.children: if results: break if child.type == token.COMMA: #Store the last comma, which precedes the metaclass comma = child elif type(child) == Node: meta = equal = name = None for arg in child.children: if arg == Leaf(token.NAME, u"metaclass"): #We have the (metaclass) part meta = arg elif meta and arg == Leaf(token.EQUAL, u"="): #We have the (metaclass=) part equal = arg elif meta and equal: #Here we go, we have (metaclass=X) name = arg results = (comma, meta, equal, name) break return results class FixMetaclass(fixer_base.BaseFix): PATTERN = u""" classdef """ def transform(self, node, results): meta_results = has_metaclass(node) if not meta_results: return for meta in meta_results: meta.remove() target = Leaf(token.NAME, u"__metaclass__") equal = Leaf(token.EQUAL, u"=", prefix=u" ") # meta is the last item in what was returned by has_metaclass(): name name = meta name.prefix = u" " stmt_node = Node(syms.atom, [target, equal, name]) suitify(node) for item in node.children: if item.type == syms.suite: for stmt in item.children: if stmt.type == token.INDENT: # Insert, in reverse order, the statement, a newline, # and an indent right after the first indented line loc = item.children.index(stmt) + 1 # Keep consistent indentation form ident = Leaf(token.INDENT, stmt.value) item.insert_child(loc, ident) item.insert_child(loc, Newline()) item.insert_child(loc, stmt_node) break u""" Fixer for "class Foo: ..." -> "class Foo(object): ..." """ from lib2to3 import fixer_base from lib2to3.fixer_util import LParen, RParen, Name from libfuturize.fixer_util import touch_import_top def insert_object(node, idx): node.insert_child(idx, RParen()) node.insert_child(idx, Name(u"object")) node.insert_child(idx, LParen()) class FixNewstyle(fixer_base.BaseFix): # Match: # class Blah: # and: # class Blah(): PATTERN = u"classdef< 'class' NAME ['(' ')'] colon=':' any >" def transform(self, node, results): colon = results[u"colon"] idx = node.children.index(colon) if (node.children[idx-2].value == '(' and node.children[idx-1].value == ')'): del node.children[idx-2:idx] idx -= 2 insert_object(node, idx) touch_import_top(u'builtins', 'object', node) u""" Fixer for: it.__next__() -> it.next(). next(it) -> it.next(). """ from lib2to3.pgen2 import token from lib2to3.pygram import python_symbols as syms from lib2to3 import fixer_base from lib2to3.fixer_util import Name, Call, find_binding, Attr bind_warning = u"Calls to builtin next() possibly shadowed by global binding" class FixNext(fixer_base.BaseFix): PATTERN = u""" power< base=any+ trailer< '.' attr='__next__' > any* > | power< head='next' trailer< '(' arg=any ')' > any* > | classdef< 'class' base=any+ ':' suite< any* funcdef< 'def' attr='__next__' parameters< '(' NAME ')' > any+ > any* > > """ def transform(self, node, results): assert results base = results.get(u"base") attr = results.get(u"attr") head = results.get(u"head") arg_ = results.get(u"arg") if arg_: arg = arg_.clone() head.replace(Attr(Name(unicode(arg),prefix=head.prefix), Name(u"next"))) arg_.remove() elif base: attr.replace(Name(u"next", prefix=attr.prefix)) u""" Fixer for print: from __future__ import print_function. """ from lib2to3 import fixer_base from libfuturize.fixer_util import future_import class FixPrintfunction(fixer_base.BaseFix): # explicit = True PATTERN = u""" power< 'print' trailer < '(' any* ')' > any* > """ def transform(self, node, results): future_import(u"print_function", node) u"""Fixer for 'raise E(V).with_traceback(T)' -> 'raise E, V, T'""" from lib2to3 import fixer_base from lib2to3.fixer_util import Comma, Node, Leaf, token, syms class FixRaise(fixer_base.BaseFix): PATTERN = u""" raise_stmt< 'raise' (power< name=any [trailer< '(' val=any* ')' >] [trailer< '.' 'with_traceback' > trailer< '(' trc=any ')' >] > | any) ['from' chain=any] >""" def transform(self, node, results): name, val, trc = (results.get(u"name"), results.get(u"val"), results.get(u"trc")) chain = results.get(u"chain") if chain is not None: self.warning(node, u"explicit exception chaining is not supported in Python 2") chain.prev_sibling.remove() chain.remove() if trc is not None: val = val[0] if val else Leaf(token.NAME, u"None") val.prefix = trc.prefix = u" " kids = [Leaf(token.NAME, u"raise"), name.clone(), Comma(), val.clone(), Comma(), trc.clone()] raise_stmt = Node(syms.raise_stmt, kids) node.replace(raise_stmt) u"""Fixer for raise E(V).with_traceback(T) to: from future.utils import raise_ ... raise_(E, V, T) TODO: FIXME!! """ from lib2to3 import fixer_base from lib2to3.fixer_util import Comma, Node, Leaf, token, syms class FixRaise(fixer_base.BaseFix): PATTERN = u""" raise_stmt< 'raise' (power< name=any [trailer< '(' val=any* ')' >] [trailer< '.' 'with_traceback' > trailer< '(' trc=any ')' >] > | any) ['from' chain=any] >""" def transform(self, node, results): FIXME name, val, trc = (results.get(u"name"), results.get(u"val"), results.get(u"trc")) chain = results.get(u"chain") if chain is not None: self.warning(node, u"explicit exception chaining is not supported in Python 2") chain.prev_sibling.remove() chain.remove() if trc is not None: val = val[0] if val else Leaf(token.NAME, u"None") val.prefix = trc.prefix = u" " kids = [Leaf(token.NAME, u"raise"), name.clone(), Comma(), val.clone(), Comma(), trc.clone()] raise_stmt = Node(syms.raise_stmt, kids) node.replace(raise_stmt) u"""Fixer for 'g.throw(E(V).with_traceback(T))' -> 'g.throw(E, V, T)'""" from lib2to3 import fixer_base from lib2to3.pytree import Node, Leaf from lib2to3.pgen2 import token from lib2to3.fixer_util import Comma class FixThrow(fixer_base.BaseFix): PATTERN = u""" power< any trailer< '.' 'throw' > trailer< '(' args=power< exc=any trailer< '(' val=any* ')' > trailer< '.' 'with_traceback' > trailer< '(' trc=any ')' > > ')' > > """ def transform(self, node, results): syms = self.syms exc, val, trc = (results[u"exc"], results[u"val"], results[u"trc"]) val = val[0] if val else Leaf(token.NAME, u"None") val.prefix = trc.prefix = u" " kids = [exc.clone(), Comma(), val.clone(), Comma(), trc.clone()] args = results[u"args"] args.children = kids u""" Fixer for: (a,)* *b (,c)* [,] = s for (a,)* *b (,c)* [,] in d: ... """ from lib2to3 import fixer_base from itertools import count from lib2to3.fixer_util import (Assign, Comma, Call, Newline, Name, Number, token, syms, Node, Leaf) from libfuturize.fixer_util import indentation, suitify, commatize # from libfuturize.fixer_util import Assign, Comma, Call, Newline, Name, Number, indentation, suitify, commatize, token, syms, Node, Leaf def assignment_source(num_pre, num_post, LISTNAME, ITERNAME): u""" Accepts num_pre and num_post, which are counts of values before and after the starg (not including the starg) Returns a source fit for Assign() from fixer_util """ children = [] pre = unicode(num_pre) post = unicode(num_post) # This code builds the assignment source from lib2to3 tree primitives. # It's not very readable, but it seems like the most correct way to do it. if num_pre > 0: pre_part = Node(syms.power, [Name(LISTNAME), Node(syms.trailer, [Leaf(token.LSQB, u"["), Node(syms.subscript, [Leaf(token.COLON, u":"), Number(pre)]), Leaf(token.RSQB, u"]")])]) children.append(pre_part) children.append(Leaf(token.PLUS, u"+", prefix=u" ")) main_part = Node(syms.power, [Leaf(token.LSQB, u"[", prefix=u" "), Name(LISTNAME), Node(syms.trailer, [Leaf(token.LSQB, u"["), Node(syms.subscript, [Number(pre) if num_pre > 0 else Leaf(1, u""), Leaf(token.COLON, u":"), Node(syms.factor, [Leaf(token.MINUS, u"-"), Number(post)]) if num_post > 0 else Leaf(1, u"")]), Leaf(token.RSQB, u"]"), Leaf(token.RSQB, u"]")])]) children.append(main_part) if num_post > 0: children.append(Leaf(token.PLUS, u"+", prefix=u" ")) post_part = Node(syms.power, [Name(LISTNAME, prefix=u" "), Node(syms.trailer, [Leaf(token.LSQB, u"["), Node(syms.subscript, [Node(syms.factor, [Leaf(token.MINUS, u"-"), Number(post)]), Leaf(token.COLON, u":")]), Leaf(token.RSQB, u"]")])]) children.append(post_part) source = Node(syms.arith_expr, children) return source class FixUnpacking(fixer_base.BaseFix): PATTERN = u""" expl=expr_stmt< testlist_star_expr< pre=(any ',')* star_expr< '*' name=NAME > post=(',' any)* [','] > '=' source=any > | impl=for_stmt< 'for' lst=exprlist< pre=(any ',')* star_expr< '*' name=NAME > post=(',' any)* [','] > 'in' it=any ':' suite=any>""" def fix_explicit_context(self, node, results): pre, name, post, source = (results.get(n) for n in (u"pre", u"name", u"post", u"source")) pre = [n.clone() for n in pre if n.type == token.NAME] name.prefix = u" " post = [n.clone() for n in post if n.type == token.NAME] target = [n.clone() for n in commatize(pre + [name.clone()] + post)] # to make the special-case fix for "*z, = ..." correct with the least # amount of modification, make the left-side into a guaranteed tuple target.append(Comma()) source.prefix = u"" setup_line = Assign(Name(self.LISTNAME), Call(Name(u"list"), [source.clone()])) power_line = Assign(target, assignment_source(len(pre), len(post), self.LISTNAME, self.ITERNAME)) return setup_line, power_line def fix_implicit_context(self, node, results): u""" Only example of the implicit context is a for loop, so only fix that. """ pre, name, post, it = (results.get(n) for n in (u"pre", u"name", u"post", u"it")) pre = [n.clone() for n in pre if n.type == token.NAME] name.prefix = u" " post = [n.clone() for n in post if n.type == token.NAME] target = [n.clone() for n in commatize(pre + [name.clone()] + post)] # to make the special-case fix for "*z, = ..." correct with the least # amount of modification, make the left-side into a guaranteed tuple target.append(Comma()) source = it.clone() source.prefix = u"" setup_line = Assign(Name(self.LISTNAME), Call(Name(u"list"), [Name(self.ITERNAME)])) power_line = Assign(target, assignment_source(len(pre), len(post), self.LISTNAME, self.ITERNAME)) return setup_line, power_line def transform(self, node, results): u""" a,b,c,d,e,f,*g,h,i = range(100) changes to _3to2list = list(range(100)) a,b,c,d,e,f,g,h,i, = _3to2list[:6] + [_3to2list[6:-2]] + _3to2list[-2:] and for a,b,*c,d,e in iter_of_iters: do_stuff changes to for _3to2iter in iter_of_iters: _3to2list = list(_3to2iter) a,b,c,d,e, = _3to2list[:2] + [_3to2list[2:-2]] + _3to2list[-2:] do_stuff """ self.LISTNAME = self.new_name(u"_3to2list") self.ITERNAME = self.new_name(u"_3to2iter") expl, impl = results.get(u"expl"), results.get(u"impl") if expl is not None: setup_line, power_line = self.fix_explicit_context(node, results) setup_line.prefix = expl.prefix power_line.prefix = indentation(expl.parent) setup_line.append_child(Newline()) parent = node.parent i = node.remove() parent.insert_child(i, power_line) parent.insert_child(i, setup_line) elif impl is not None: setup_line, power_line = self.fix_implicit_context(node, results) suitify(node) suite = [k for k in node.children if k.type == syms.suite][0] setup_line.prefix = u"" power_line.prefix = suite.children[1].value suite.children[2].prefix = indentation(suite.children[2]) suite.insert_child(2, Newline()) suite.insert_child(2, power_line) suite.insert_child(2, Newline()) suite.insert_child(2, setup_line) results.get(u"lst").replace(Name(self.ITERNAME, prefix=u" ")) # coding=utf-8 """ past: compatibility with Python 2 from Python 3 =============================================== ``past`` is a package to aid with Python 2/3 compatibility. Whereas ``future`` contains backports of Python 3 constructs to Python 2, ``past`` provides implementations of some Python 2 constructs in Python 3 and tools to import and run Python 2 code in Python 3. It is intended to be used sparingly, as a way of running old Python 2 code from Python 3 until the code is ported properly. Potential uses for libraries: - as a step in porting a Python 2 codebase to Python 3 (e.g. with the ``futurize`` script) - to provide Python 3 support for previously Python 2-only libraries with the same APIs as on Python 2 -- particularly with regard to 8-bit strings (the ``past.builtins.str`` type). - to aid in providing minimal-effort Python 3 support for applications using libraries that do not yet wish to upgrade their code properly to Python 3, or wish to upgrade it gradually to Python 3 style. Here are some code examples that run identically on Python 3 and 2:: >>> from past.builtins import str as oldstr >>> philosopher = oldstr(u'\u5b54\u5b50'.encode('utf-8')) >>> # This now behaves like a Py2 byte-string on both Py2 and Py3. >>> # For example, indexing returns a Python 2-like string object, not >>> # an integer: >>> philosopher[0] '\xe5' >>> type(philosopher[0]) >>> # List-producing versions of range, reduce, map, filter >>> from past.builtins import range, reduce >>> range(10) [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] >>> reduce(lambda x, y: x+y, [1, 2, 3, 4, 5]) 15 >>> # Other functions removed in Python 3 are resurrected ... >>> from past.builtins import execfile >>> execfile('myfile.py') >>> from past.builtins import raw_input >>> name = raw_input('What is your name? ') What is your name? [cursor] >>> from past.builtins import reload >>> reload(mymodule) # equivalent to imp.reload(mymodule) in Python 3 >>> from past.builtins import xrange >>> for i in xrange(10): ... pass It also provides import hooks so you can import and use Python 2 modules like this:: $ python3 >>> from past import autotranslate >>> authotranslate('mypy2module') >>> import mypy2module until the authors of the Python 2 modules have upgraded their code. Then, for example:: >>> mypy2module.func_taking_py2_string(oldstr(b'abcd')) Credits ------- :Author: Ed Schofield :Sponsor: Python Charmers Pty Ltd, Australia: http://pythoncharmers.com Licensing --------- Copyright 2013-2016 Python Charmers Pty Ltd, Australia. The software is distributed under an MIT licence. See LICENSE.txt. """ from past.translation import install_hooks as autotranslate from future import __version__, __copyright__, __license__ __title__ = 'past' __author__ = 'Ed Schofield' """ A resurrection of some old functions from Python 2 for use in Python 3. These should be used sparingly, to help with porting efforts, since code using them is no longer standard Python 3 code. This module provides the following: 1. Implementations of these builtin functions which have no equivalent on Py3: - apply - chr - cmp - execfile 2. Aliases: - intern <- sys.intern - raw_input <- input - reduce <- functools.reduce - reload <- imp.reload - unichr <- chr - unicode <- str - xrange <- range 3. List-producing versions of the corresponding Python 3 iterator-producing functions: - filter - map - range - zip 4. Forward-ported Py2 types: - basestring - dict - str - long - unicode """ from future.utils import PY3 from past.builtins.noniterators import (filter, map, range, reduce, zip) # from past.builtins.misc import (ascii, hex, input, oct, open) if PY3: from past.types import (basestring, olddict as dict, oldstr as str, long, unicode) else: from __builtin__ import (basestring, dict, str, long, unicode) from past.builtins.misc import (apply, chr, cmp, execfile, intern, oct, raw_input, reload, unichr, unicode, xrange) from past import utils if utils.PY3: # We only import names that shadow the builtins on Py3. No other namespace # pollution on Py3. # Only shadow builtins on Py3; no new names __all__ = ['filter', 'map', 'range', 'reduce', 'zip', 'basestring', 'dict', 'str', 'long', 'unicode', 'apply', 'chr', 'cmp', 'execfile', 'intern', 'raw_input', 'reload', 'unichr', 'xrange' ] else: # No namespace pollution on Py2 __all__ = [] from __future__ import unicode_literals import sys import inspect from collections import Mapping from future.utils import PY3, exec_ if PY3: import builtins def apply(f, *args, **kw): return f(*args, **kw) from past.builtins import str as oldstr def chr(i): """ Return a byte-string of one character with ordinal i; 0 <= i <= 256 """ return oldstr(bytes((i,))) def cmp(x, y): """ cmp(x, y) -> integer Return negative if xy. """ return (x > y) - (x < y) from sys import intern def oct(number): """oct(number) -> string Return the octal representation of an integer """ return '0' + builtins.oct(number)[2:] raw_input = input from imp import reload unicode = str unichr = chr xrange = range else: import __builtin__ apply = __builtin__.apply chr = __builtin__.chr cmp = __builtin__.cmp execfile = __builtin__.execfile intern = __builtin__.intern oct = __builtin__.oct raw_input = __builtin__.raw_input reload = __builtin__.reload unicode = __builtin__.unicode unichr = __builtin__.unichr xrange = __builtin__.xrange if PY3: def execfile(filename, myglobals=None, mylocals=None): """ Read and execute a Python script from a file in the given namespaces. The globals and locals are dictionaries, defaulting to the current globals and locals. If only globals is given, locals defaults to it. """ if myglobals is None: # There seems to be no alternative to frame hacking here. caller_frame = inspect.stack()[1] myglobals = caller_frame[0].f_globals mylocals = caller_frame[0].f_locals elif mylocals is None: # Only if myglobals is given do we set mylocals to it. mylocals = myglobals if not isinstance(myglobals, Mapping): raise TypeError('globals must be a mapping') if not isinstance(mylocals, Mapping): raise TypeError('locals must be a mapping') with open(filename, "rbU") as fin: source = fin.read() code = compile(source, filename, "exec") exec_(code, myglobals, mylocals) if PY3: __all__ = ['apply', 'chr', 'cmp', 'execfile', 'intern', 'raw_input', 'reload', 'unichr', 'unicode', 'xrange'] else: __all__ = [] """ This module is designed to be used as follows:: from past.builtins.noniterators import filter, map, range, reduce, zip And then, for example:: assert isinstance(range(5), list) The list-producing functions this brings in are:: - ``filter`` - ``map`` - ``range`` - ``reduce`` - ``zip`` """ from __future__ import division, absolute_import, print_function from itertools import chain, starmap import itertools # since zip_longest doesn't exist on Py2 from past.types import basestring from past.utils import PY3 def flatmap(f, items): return chain.from_iterable(map(f, items)) if PY3: import builtins # list-producing versions of the major Python iterating functions def oldfilter(*args): """ filter(function or None, sequence) -> list, tuple, or string Return those items of sequence for which function(item) is true. If function is None, return the items that are true. If sequence is a tuple or string, return the same type, else return a list. """ mytype = type(args[1]) if isinstance(args[1], basestring): return mytype().join(builtins.filter(*args)) elif isinstance(args[1], (tuple, list)): return mytype(builtins.filter(*args)) else: # Fall back to list. Is this the right thing to do? return list(builtins.filter(*args)) # This is surprisingly difficult to get right. For example, the # solutions here fail with the test cases in the docstring below: # http://stackoverflow.com/questions/8072755/ def oldmap(func, *iterables): """ map(function, sequence[, sequence, ...]) -> list Return a list of the results of applying the function to the items of the argument sequence(s). If more than one sequence is given, the function is called with an argument list consisting of the corresponding item of each sequence, substituting None for missing values when not all sequences have the same length. If the function is None, return a list of the items of the sequence (or a list of tuples if more than one sequence). Test cases: >>> oldmap(None, 'hello world') ['h', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd'] >>> oldmap(None, range(4)) [0, 1, 2, 3] More test cases are in past.tests.test_builtins. """ zipped = itertools.zip_longest(*iterables) l = list(zipped) if len(l) == 0: return [] if func is None: result = l else: result = list(starmap(func, l)) # Inspect to see whether it's a simple sequence of tuples try: if max([len(item) for item in result]) == 1: return list(chain.from_iterable(result)) # return list(flatmap(func, result)) except TypeError as e: # Simple objects like ints have no len() pass return result ############################ ### For reference, the source code for Py2.7 map function: # static PyObject * # builtin_map(PyObject *self, PyObject *args) # { # typedef struct { # PyObject *it; /* the iterator object */ # int saw_StopIteration; /* bool: did the iterator end? */ # } sequence; # # PyObject *func, *result; # sequence *seqs = NULL, *sqp; # Py_ssize_t n, len; # register int i, j; # # n = PyTuple_Size(args); # if (n < 2) { # PyErr_SetString(PyExc_TypeError, # "map() requires at least two args"); # return NULL; # } # # func = PyTuple_GetItem(args, 0); # n--; # # if (func == Py_None) { # if (PyErr_WarnPy3k("map(None, ...) not supported in 3.x; " # "use list(...)", 1) < 0) # return NULL; # if (n == 1) { # /* map(None, S) is the same as list(S). */ # return PySequence_List(PyTuple_GetItem(args, 1)); # } # } # # /* Get space for sequence descriptors. Must NULL out the iterator # * pointers so that jumping to Fail_2 later doesn't see trash. # */ # if ((seqs = PyMem_NEW(sequence, n)) == NULL) { # PyErr_NoMemory(); # return NULL; # } # for (i = 0; i < n; ++i) { # seqs[i].it = (PyObject*)NULL; # seqs[i].saw_StopIteration = 0; # } # # /* Do a first pass to obtain iterators for the arguments, and set len # * to the largest of their lengths. # */ # len = 0; # for (i = 0, sqp = seqs; i < n; ++i, ++sqp) { # PyObject *curseq; # Py_ssize_t curlen; # # /* Get iterator. */ # curseq = PyTuple_GetItem(args, i+1); # sqp->it = PyObject_GetIter(curseq); # if (sqp->it == NULL) { # static char errmsg[] = # "argument %d to map() must support iteration"; # char errbuf[sizeof(errmsg) + 25]; # PyOS_snprintf(errbuf, sizeof(errbuf), errmsg, i+2); # PyErr_SetString(PyExc_TypeError, errbuf); # goto Fail_2; # } # # /* Update len. */ # curlen = _PyObject_LengthHint(curseq, 8); # if (curlen > len) # len = curlen; # } # # /* Get space for the result list. */ # if ((result = (PyObject *) PyList_New(len)) == NULL) # goto Fail_2; # # /* Iterate over the sequences until all have stopped. */ # for (i = 0; ; ++i) { # PyObject *alist, *item=NULL, *value; # int numactive = 0; # # if (func == Py_None && n == 1) # alist = NULL; # else if ((alist = PyTuple_New(n)) == NULL) # goto Fail_1; # # for (j = 0, sqp = seqs; j < n; ++j, ++sqp) { # if (sqp->saw_StopIteration) { # Py_INCREF(Py_None); # item = Py_None; # } # else { # item = PyIter_Next(sqp->it); # if (item) # ++numactive; # else { # if (PyErr_Occurred()) { # Py_XDECREF(alist); # goto Fail_1; # } # Py_INCREF(Py_None); # item = Py_None; # sqp->saw_StopIteration = 1; # } # } # if (alist) # PyTuple_SET_ITEM(alist, j, item); # else # break; # } # # if (!alist) # alist = item; # # if (numactive == 0) { # Py_DECREF(alist); # break; # } # # if (func == Py_None) # value = alist; # else { # value = PyEval_CallObject(func, alist); # Py_DECREF(alist); # if (value == NULL) # goto Fail_1; # } # if (i >= len) { # int status = PyList_Append(result, value); # Py_DECREF(value); # if (status < 0) # goto Fail_1; # } # else if (PyList_SetItem(result, i, value) < 0) # goto Fail_1; # } # # if (i < len && PyList_SetSlice(result, i, len, NULL) < 0) # goto Fail_1; # # goto Succeed; # # Fail_1: # Py_DECREF(result); # Fail_2: # result = NULL; # Succeed: # assert(seqs); # for (i = 0; i < n; ++i) # Py_XDECREF(seqs[i].it); # PyMem_DEL(seqs); # return result; # } def oldrange(*args, **kwargs): return list(builtins.range(*args, **kwargs)) def oldzip(*args, **kwargs): return list(builtins.zip(*args, **kwargs)) filter = oldfilter map = oldmap range = oldrange from functools import reduce zip = oldzip __all__ = ['filter', 'map', 'range', 'reduce', 'zip'] else: import __builtin__ # Python 2-builtin ranges produce lists filter = __builtin__.filter map = __builtin__.map range = __builtin__.range reduce = __builtin__.reduce zip = __builtin__.zip __all__ = [] """ Forward-ports of types from Python 2 for use with Python 3: - ``basestring``: equivalent to ``(str, bytes)`` in ``isinstance`` checks - ``dict``: with list-producing .keys() etc. methods - ``str``: bytes-like, but iterating over them doesn't product integers - ``long``: alias of Py3 int with ``L`` suffix in the ``repr`` - ``unicode``: alias of Py3 str with ``u`` prefix in the ``repr`` """ from past import utils if utils.PY2: import __builtin__ basestring = __builtin__.basestring dict = __builtin__.dict str = __builtin__.str long = __builtin__.long unicode = __builtin__.unicode __all__ = [] else: from .basestring import basestring from .olddict import olddict from .oldstr import oldstr long = int unicode = str # from .unicode import unicode __all__ = ['basestring', 'olddict', 'oldstr', 'long', 'unicode'] """ An implementation of the basestring type for Python 3 Example use: >>> s = b'abc' >>> assert isinstance(s, basestring) >>> from past.types import str as oldstr >>> s2 = oldstr(b'abc') >>> assert isinstance(s2, basestring) """ import sys from past.utils import with_metaclass, PY2 if PY2: str = unicode ver = sys.version_info[:2] class BaseBaseString(type): def __instancecheck__(cls, instance): return isinstance(instance, (bytes, str)) def __subclasshook__(cls, thing): # TODO: What should go here? raise NotImplemented class basestring(with_metaclass(BaseBaseString)): """ A minimal backport of the Python 2 basestring type to Py3 """ __all__ = ['basestring'] # -*- coding: utf-8 -*- """ past.translation ================== The ``past.translation`` package provides an import hook for Python 3 which transparently runs ``futurize`` fixers over Python 2 code on import to convert print statements into functions, etc. It is intended to assist users in migrating to Python 3.x even if some dependencies still only support Python 2.x. Usage ----- Once your Py2 package is installed in the usual module search path, the import hook is invoked as follows: >>> from past import autotranslate >>> autotranslate('mypackagename') Or: >>> autotranslate(['mypackage1', 'mypackage2']) You can unregister the hook using:: >>> from past.translation import remove_hooks >>> remove_hooks() Author: Ed Schofield. Inspired by and based on ``uprefix`` by Vinay M. Sajip. """ import imp import logging import marshal import os import sys import copy from lib2to3.pgen2.parse import ParseError from lib2to3.refactor import RefactoringTool from libfuturize import fixes logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) myfixes = (list(fixes.libfuturize_fix_names_stage1) + list(fixes.lib2to3_fix_names_stage1) + list(fixes.libfuturize_fix_names_stage2) + list(fixes.lib2to3_fix_names_stage2)) # We detect whether the code is Py2 or Py3 by applying certain lib2to3 fixers # to it. If the diff is empty, it's Python 3 code. py2_detect_fixers = [ # From stage 1: 'lib2to3.fixes.fix_apply', # 'lib2to3.fixes.fix_dict', # TODO: add support for utils.viewitems() etc. and move to stage2 'lib2to3.fixes.fix_except', 'lib2to3.fixes.fix_execfile', 'lib2to3.fixes.fix_exitfunc', 'lib2to3.fixes.fix_funcattrs', 'lib2to3.fixes.fix_filter', 'lib2to3.fixes.fix_has_key', 'lib2to3.fixes.fix_idioms', 'lib2to3.fixes.fix_import', # makes any implicit relative imports explicit. (Use with ``from __future__ import absolute_import) 'lib2to3.fixes.fix_intern', 'lib2to3.fixes.fix_isinstance', 'lib2to3.fixes.fix_methodattrs', 'lib2to3.fixes.fix_ne', 'lib2to3.fixes.fix_numliterals', # turns 1L into 1, 0755 into 0o755 'lib2to3.fixes.fix_paren', 'lib2to3.fixes.fix_print', 'lib2to3.fixes.fix_raise', # uses incompatible with_traceback() method on exceptions 'lib2to3.fixes.fix_renames', 'lib2to3.fixes.fix_reduce', # 'lib2to3.fixes.fix_set_literal', # this is unnecessary and breaks Py2.6 support 'lib2to3.fixes.fix_repr', 'lib2to3.fixes.fix_standarderror', 'lib2to3.fixes.fix_sys_exc', 'lib2to3.fixes.fix_throw', 'lib2to3.fixes.fix_tuple_params', 'lib2to3.fixes.fix_types', 'lib2to3.fixes.fix_ws_comma', 'lib2to3.fixes.fix_xreadlines', # From stage 2: 'lib2to3.fixes.fix_basestring', # 'lib2to3.fixes.fix_buffer', # perhaps not safe. Test this. # 'lib2to3.fixes.fix_callable', # not needed in Py3.2+ # 'lib2to3.fixes.fix_dict', # TODO: add support for utils.viewitems() etc. 'lib2to3.fixes.fix_exec', # 'lib2to3.fixes.fix_future', # we don't want to remove __future__ imports 'lib2to3.fixes.fix_getcwdu', # 'lib2to3.fixes.fix_imports', # called by libfuturize.fixes.fix_future_standard_library # 'lib2to3.fixes.fix_imports2', # we don't handle this yet (dbm) # 'lib2to3.fixes.fix_input', # 'lib2to3.fixes.fix_itertools', # 'lib2to3.fixes.fix_itertools_imports', 'lib2to3.fixes.fix_long', # 'lib2to3.fixes.fix_map', # 'lib2to3.fixes.fix_metaclass', # causes SyntaxError in Py2! Use the one from ``six`` instead 'lib2to3.fixes.fix_next', 'lib2to3.fixes.fix_nonzero', # TODO: add a decorator for mapping __bool__ to __nonzero__ # 'lib2to3.fixes.fix_operator', # we will need support for this by e.g. extending the Py2 operator module to provide those functions in Py3 'lib2to3.fixes.fix_raw_input', # 'lib2to3.fixes.fix_unicode', # strips off the u'' prefix, which removes a potentially helpful source of information for disambiguating unicode/byte strings # 'lib2to3.fixes.fix_urllib', 'lib2to3.fixes.fix_xrange', # 'lib2to3.fixes.fix_zip', ] class RTs: """ A namespace for the refactoring tools. This avoids creating these at the module level, which slows down the module import. (See issue #117). There are two possible grammars: with or without the print statement. Hence we have two possible refactoring tool implementations. """ _rt = None _rtp = None _rt_py2_detect = None _rtp_py2_detect = None @staticmethod def setup(): """ Call this before using the refactoring tools to create them on demand if needed. """ if None in [RTs._rt, RTs._rtp]: RTs._rt = RefactoringTool(myfixes) RTs._rtp = RefactoringTool(myfixes, {'print_function': True}) @staticmethod def setup_detect_python2(): """ Call this before using the refactoring tools to create them on demand if needed. """ if None in [RTs._rt_py2_detect, RTs._rtp_py2_detect]: RTs._rt_py2_detect = RefactoringTool(py2_detect_fixers) RTs._rtp_py2_detect = RefactoringTool(py2_detect_fixers, {'print_function': True}) # We need to find a prefix for the standard library, as we don't want to # process any files there (they will already be Python 3). # # The following method is used by Sanjay Vinip in uprefix. This fails for # ``conda`` environments: # # In a non-pythonv virtualenv, sys.real_prefix points to the installed Python. # # In a pythonv venv, sys.base_prefix points to the installed Python. # # Outside a virtual environment, sys.prefix points to the installed Python. # if hasattr(sys, 'real_prefix'): # _syslibprefix = sys.real_prefix # else: # _syslibprefix = getattr(sys, 'base_prefix', sys.prefix) # Instead, we use the portion of the path common to both the stdlib modules # ``math`` and ``urllib``. def splitall(path): """ Split a path into all components. From Python Cookbook. """ allparts = [] while True: parts = os.path.split(path) if parts[0] == path: # sentinel for absolute paths allparts.insert(0, parts[0]) break elif parts[1] == path: # sentinel for relative paths allparts.insert(0, parts[1]) break else: path = parts[0] allparts.insert(0, parts[1]) return allparts def common_substring(s1, s2): """ Returns the longest common substring to the two strings, starting from the left. """ chunks = [] path1 = splitall(s1) path2 = splitall(s2) for (dir1, dir2) in zip(path1, path2): if dir1 != dir2: break chunks.append(dir1) return os.path.join(*chunks) # _stdlibprefix = common_substring(math.__file__, urllib.__file__) def detect_python2(source, pathname): """ Returns a bool indicating whether we think the code is Py2 """ RTs.setup_detect_python2() try: tree = RTs._rt_py2_detect.refactor_string(source, pathname) except ParseError as e: if e.msg != 'bad input' or e.value != '=': raise tree = RTs._rtp.refactor_string(source, pathname) if source != str(tree)[:-1]: # remove added newline # The above fixers made changes, so we conclude it's Python 2 code logger.debug('Detected Python 2 code: {0}'.format(pathname)) with open('/tmp/original_code.py', 'w') as f: f.write('### Original code (detected as py2): %s\n%s' % (pathname, source)) with open('/tmp/py2_detection_code.py', 'w') as f: f.write('### Code after running py3 detection (from %s)\n%s' % (pathname, str(tree)[:-1])) return True else: logger.debug('Detected Python 3 code: {0}'.format(pathname)) with open('/tmp/original_code.py', 'w') as f: f.write('### Original code (detected as py3): %s\n%s' % (pathname, source)) try: os.remove('/tmp/futurize_code.py') except OSError: pass return False class Py2Fixer(object): """ An import hook class that uses lib2to3 for source-to-source translation of Py2 code to Py3. """ # See the comments on :class:future.standard_library.RenameImport. # We add this attribute here so remove_hooks() and install_hooks() can # unambiguously detect whether the import hook is installed: PY2FIXER = True def __init__(self): self.found = None self.base_exclude_paths = ['future', 'past'] self.exclude_paths = copy.copy(self.base_exclude_paths) self.include_paths = [] def include(self, paths): """ Pass in a sequence of module names such as 'plotrique.plotting' that, if present at the leftmost side of the full package name, would specify the module to be transformed from Py2 to Py3. """ self.include_paths += paths def exclude(self, paths): """ Pass in a sequence of strings such as 'mymodule' that, if present at the leftmost side of the full package name, would cause the module not to undergo any source transformation. """ self.exclude_paths += paths def find_module(self, fullname, path=None): logger.debug('Running find_module: {0}...'.format(fullname)) if '.' in fullname: parent, child = fullname.rsplit('.', 1) if path is None: loader = self.find_module(parent, path) mod = loader.load_module(parent) path = mod.__path__ fullname = child # Perhaps we should try using the new importlib functionality in Python # 3.3: something like this? # thing = importlib.machinery.PathFinder.find_module(fullname, path) try: self.found = imp.find_module(fullname, path) except Exception as e: logger.debug('Py2Fixer could not find {0}') logger.debug('Exception was: {0})'.format(fullname, e)) return None self.kind = self.found[-1][-1] if self.kind == imp.PKG_DIRECTORY: self.pathname = os.path.join(self.found[1], '__init__.py') elif self.kind == imp.PY_SOURCE: self.pathname = self.found[1] return self def transform(self, source): # This implementation uses lib2to3, # you can override and use something else # if that's better for you # lib2to3 likes a newline at the end RTs.setup() source += '\n' try: tree = RTs._rt.refactor_string(source, self.pathname) except ParseError as e: if e.msg != 'bad input' or e.value != '=': raise tree = RTs._rtp.refactor_string(source, self.pathname) # could optimise a bit for only doing str(tree) if # getattr(tree, 'was_changed', False) returns True return str(tree)[:-1] # remove added newline def load_module(self, fullname): logger.debug('Running load_module for {0}...'.format(fullname)) if fullname in sys.modules: mod = sys.modules[fullname] else: if self.kind in (imp.PY_COMPILED, imp.C_EXTENSION, imp.C_BUILTIN, imp.PY_FROZEN): convert = False # elif (self.pathname.startswith(_stdlibprefix) # and 'site-packages' not in self.pathname): # # We assume it's a stdlib package in this case. Is this too brittle? # # Please file a bug report at https://github.com/PythonCharmers/python-future # # if so. # convert = False # in theory, other paths could be configured to be excluded here too elif any([fullname.startswith(path) for path in self.exclude_paths]): convert = False elif any([fullname.startswith(path) for path in self.include_paths]): convert = True else: convert = False if not convert: logger.debug('Excluded {0} from translation'.format(fullname)) mod = imp.load_module(fullname, *self.found) else: logger.debug('Autoconverting {0} ...'.format(fullname)) mod = imp.new_module(fullname) sys.modules[fullname] = mod # required by PEP 302 mod.__file__ = self.pathname mod.__name__ = fullname mod.__loader__ = self # This: # mod.__package__ = '.'.join(fullname.split('.')[:-1]) # seems to result in "SystemError: Parent module '' not loaded, # cannot perform relative import" for a package's __init__.py # file. We use the approach below. Another option to try is the # minimal load_module pattern from the PEP 302 text instead. # Is the test in the next line more or less robust than the # following one? Presumably less ... # ispkg = self.pathname.endswith('__init__.py') if self.kind == imp.PKG_DIRECTORY: mod.__path__ = [ os.path.dirname(self.pathname) ] mod.__package__ = fullname else: #else, regular module mod.__path__ = [] mod.__package__ = fullname.rpartition('.')[0] try: cachename = imp.cache_from_source(self.pathname) if not os.path.exists(cachename): update_cache = True else: sourcetime = os.stat(self.pathname).st_mtime cachetime = os.stat(cachename).st_mtime update_cache = cachetime < sourcetime # # Force update_cache to work around a problem with it being treated as Py3 code??? # update_cache = True if not update_cache: with open(cachename, 'rb') as f: data = f.read() try: code = marshal.loads(data) except Exception: # pyc could be corrupt. Regenerate it update_cache = True if update_cache: if self.found[0]: source = self.found[0].read() elif self.kind == imp.PKG_DIRECTORY: with open(self.pathname) as f: source = f.read() if detect_python2(source, self.pathname): source = self.transform(source) with open('/tmp/futurized_code.py', 'w') as f: f.write('### Futurized code (from %s)\n%s' % (self.pathname, source)) code = compile(source, self.pathname, 'exec') dirname = os.path.dirname(cachename) if not os.path.exists(dirname): os.makedirs(dirname) try: with open(cachename, 'wb') as f: data = marshal.dumps(code) f.write(data) except Exception: # could be write-protected pass exec(code, mod.__dict__) except Exception as e: # must remove module from sys.modules del sys.modules[fullname] raise # keep it simple if self.found[0]: self.found[0].close() return mod _hook = Py2Fixer() def install_hooks(include_paths=(), exclude_paths=()): if isinstance(include_paths, str): include_paths = (include_paths,) if isinstance(exclude_paths, str): exclude_paths = (exclude_paths,) assert len(include_paths) + len(exclude_paths) > 0, 'Pass at least one argument' _hook.include(include_paths) _hook.exclude(exclude_paths) # _hook.debug = debug enable = sys.version_info[0] >= 3 # enabled for all 3.x if enable and _hook not in sys.meta_path: sys.meta_path.insert(0, _hook) # insert at beginning. This could be made a parameter # We could return the hook when there are ways of configuring it #return _hook def remove_hooks(): if _hook in sys.meta_path: sys.meta_path.remove(_hook) def detect_hooks(): """ Returns True if the import hooks are installed, False if not. """ return _hook in sys.meta_path # present = any([hasattr(hook, 'PY2FIXER') for hook in sys.meta_path]) # return present class hooks(object): """ Acts as a context manager. Use like this: >>> from past import translation >>> with translation.hooks(): ... import mypy2module >>> import requests # py2/3 compatible anyway >>> # etc. """ def __enter__(self): self.hooks_were_installed = detect_hooks() install_hooks() return self def __exit__(self, *args): if not self.hooks_were_installed: remove_hooks() class suspend_hooks(object): """ Acts as a context manager. Use like this: >>> from past import translation >>> translation.install_hooks() >>> import http.client >>> # ... >>> with translation.suspend_hooks(): >>> import requests # or others that support Py2/3 If the hooks were disabled before the context, they are not installed when the context is left. """ def __enter__(self): self.hooks_were_installed = detect_hooks() remove_hooks() return self def __exit__(self, *args): if self.hooks_were_installed: install_hooks() """ A dict subclass for Python 3 that behaves like Python 2's dict Example use: >>> from past.builtins import dict >>> d1 = dict() # instead of {} for an empty dict >>> d2 = dict(key1='value1', key2='value2') The keys, values and items methods now return lists on Python 3.x and there are methods for iterkeys, itervalues, iteritems, and viewkeys etc. >>> for d in (d1, d2): ... assert isinstance(d.keys(), list) ... assert isinstance(d.values(), list) ... assert isinstance(d.items(), list) """ import sys from past.utils import with_metaclass _builtin_dict = dict ver = sys.version_info[:2] class BaseOldDict(type): def __instancecheck__(cls, instance): return isinstance(instance, _builtin_dict) class olddict(with_metaclass(BaseOldDict, _builtin_dict)): """ A backport of the Python 3 dict object to Py2 """ iterkeys = _builtin_dict.keys viewkeys = _builtin_dict.keys def keys(self): return list(super(olddict, self).keys()) itervalues = _builtin_dict.values viewvalues = _builtin_dict.values def values(self): return list(super(olddict, self).values()) iteritems = _builtin_dict.items viewitems = _builtin_dict.items def items(self): return list(super(olddict, self).items()) def has_key(self, k): """ D.has_key(k) -> True if D has a key k, else False """ return k in self # def __new__(cls, *args, **kwargs): # """ # dict() -> new empty dictionary # dict(mapping) -> new dictionary initialized from a mapping object's # (key, value) pairs # dict(iterable) -> new dictionary initialized as if via: # d = {} # for k, v in iterable: # d[k] = v # dict(**kwargs) -> new dictionary initialized with the name=value pairs # in the keyword argument list. For example: dict(one=1, two=2) # """ # # if len(args) == 0: # return super(olddict, cls).__new__(cls) # # Was: elif isinstance(args[0], newbytes): # # We use type() instead of the above because we're redefining # # this to be True for all unicode string subclasses. Warning: # # This may render newstr un-subclassable. # elif type(args[0]) == olddict: # return args[0] # # elif isinstance(args[0], _builtin_dict): # # value = args[0] # else: # value = args[0] # return super(olddict, cls).__new__(cls, value) def __native__(self): """ Hook for the past.utils.native() function """ return super(oldbytes, self) __all__ = ['olddict'] """ Pure-Python implementation of a Python 2-like str object for Python 3. """ from collections import Iterable from numbers import Integral from past.utils import PY2, with_metaclass _builtin_bytes = bytes class BaseOldStr(type): def __instancecheck__(cls, instance): return isinstance(instance, _builtin_bytes) def unescape(s): """ Interprets strings with escape sequences Example: >>> s = unescape(r'abc\\def') # i.e. 'abc\\\\def' >>> print(s) 'abc\def' >>> s2 = unescape('abc\\ndef') >>> len(s2) 8 >>> print(s2) abc def """ return s.encode().decode('unicode_escape') class oldstr(with_metaclass(BaseOldStr, _builtin_bytes)): """ A forward port of the Python 2 8-bit string object to Py3 """ # Python 2 strings have no __iter__ method: @property def __iter__(self): raise AttributeError def __dir__(self): return [thing for thing in dir(_builtin_bytes) if thing != '__iter__'] # def __new__(cls, *args, **kwargs): # """ # From the Py3 bytes docstring: # bytes(iterable_of_ints) -> bytes # bytes(string, encoding[, errors]) -> bytes # bytes(bytes_or_buffer) -> immutable copy of bytes_or_buffer # bytes(int) -> bytes object of size given by the parameter initialized with null bytes # bytes() -> empty bytes object # # Construct an immutable array of bytes from: # - an iterable yielding integers in range(256) # - a text string encoded using the specified encoding # - any object implementing the buffer API. # - an integer # """ # # if len(args) == 0: # return super(newbytes, cls).__new__(cls) # # Was: elif isinstance(args[0], newbytes): # # We use type() instead of the above because we're redefining # # this to be True for all unicode string subclasses. Warning: # # This may render newstr un-subclassable. # elif type(args[0]) == newbytes: # return args[0] # elif isinstance(args[0], _builtin_bytes): # value = args[0] # elif isinstance(args[0], unicode): # if 'encoding' not in kwargs: # raise TypeError('unicode string argument without an encoding') # ### # # Was: value = args[0].encode(**kwargs) # # Python 2.6 string encode() method doesn't take kwargs: # # Use this instead: # newargs = [kwargs['encoding']] # if 'errors' in kwargs: # newargs.append(kwargs['errors']) # value = args[0].encode(*newargs) # ### # elif isinstance(args[0], Iterable): # if len(args[0]) == 0: # # What is this? # raise ValueError('unknown argument type') # elif len(args[0]) > 0 and isinstance(args[0][0], Integral): # # It's a list of integers # value = b''.join([chr(x) for x in args[0]]) # else: # raise ValueError('item cannot be interpreted as an integer') # elif isinstance(args[0], Integral): # if args[0] < 0: # raise ValueError('negative count') # value = b'\x00' * args[0] # else: # value = args[0] # return super(newbytes, cls).__new__(cls, value) def __repr__(self): s = super(oldstr, self).__repr__() # e.g. b'abc' on Py3, b'abc' on Py3 return s[1:] def __str__(self): s = super(oldstr, self).__str__() # e.g. "b'abc'" or "b'abc\\ndef' # TODO: fix this: assert s[:2] == "b'" and s[-1] == "'" return unescape(s[2:-1]) # e.g. 'abc' or 'abc\ndef' def __getitem__(self, y): if isinstance(y, Integral): return super(oldstr, self).__getitem__(slice(y, y+1)) else: return super(oldstr, self).__getitem__(y) def __getslice__(self, *args): return self.__getitem__(slice(*args)) def __contains__(self, key): if isinstance(key, int): return False def __native__(self): return bytes(self) __all__ = ['oldstr'] """ Various non-built-in utility functions and definitions for Py2 compatibility in Py3. For example: >>> # The old_div() function behaves like Python 2's / operator >>> # without "from __future__ import division" >>> from past.utils import old_div >>> old_div(3, 2) # like 3/2 in Py2 0 >>> old_div(3, 2.0) # like 3/2.0 in Py2 1.5 """ import sys import numbers PY3 = sys.version_info[0] == 3 PY2 = sys.version_info[0] == 2 PYPY = hasattr(sys, 'pypy_translation_info') def with_metaclass(meta, *bases): """ Function from jinja2/_compat.py. License: BSD. Use it like this:: class BaseForm(object): pass class FormType(type): pass class Form(with_metaclass(FormType, BaseForm)): pass This requires a bit of explanation: the basic idea is to make a dummy metaclass for one level of class instantiation that replaces itself with the actual metaclass. Because of internal type checks we also need to make sure that we downgrade the custom metaclass for one level to something closer to type (that's why __call__ and __init__ comes back from type etc.). This has the advantage over six.with_metaclass of not introducing dummy classes into the final MRO. """ class metaclass(meta): __call__ = type.__call__ __init__ = type.__init__ def __new__(cls, name, this_bases, d): if this_bases is None: return type.__new__(cls, name, (), d) return meta(name, bases, d) return metaclass('temporary_class', None, {}) def native(obj): """ On Py2, this is a no-op: native(obj) -> obj On Py3, returns the corresponding native Py3 types that are superclasses for forward-ported objects from Py2: >>> from past.builtins import str, dict >>> native(str(b'ABC')) # Output on Py3 follows. On Py2, output is 'ABC' b'ABC' >>> type(native(str(b'ABC'))) bytes Existing native types on Py3 will be returned unchanged: >>> type(native(b'ABC')) bytes """ if hasattr(obj, '__native__'): return obj.__native__() else: return obj # An alias for future.utils.old_div(): def old_div(a, b): """ Equivalent to ``a / b`` on Python 2 without ``from __future__ import division``. TODO: generalize this to other objects (like arrays etc.) """ if isinstance(a, numbers.Integral) and isinstance(b, numbers.Integral): return a // b else: return a / b __all__ = ['PY3', 'PY2', 'PYPY', 'with_metaclass', 'native', 'old_div'] from __future__ import absolute_import import sys __future_module__ = True if sys.version_info[0] < 3: from Queue import * else: raise ImportError('This package should not be accessible on Python 3. ' 'Either you are trying to run from the python-future src folder ' 'or your installation of python-future is corrupted.') from __future__ import absolute_import import sys if sys.version_info[0] < 3: from repr import * else: raise ImportError('This package should not be accessible on Python 3. ' 'Either you are trying to run from the python-future src folder ' 'or your installation of python-future is corrupted.') from __future__ import absolute_import import sys if sys.version_info[0] < 3: from SocketServer import * else: raise ImportError('This package should not be accessible on Python 3. ' 'Either you are trying to run from the python-future src folder ' 'or your installation of python-future is corrupted.') from __future__ import absolute_import import sys if sys.version_info[0] < 3: from Tkinter import * from Tkinter import (_cnfmerge, _default_root, _flatten, _join, _setit, _splitdict, _stringify, _support_default_root, _test, _tkinter) else: raise ImportError('This package should not be accessible on Python 3. ' 'Either you are trying to run from the python-future src folder ' 'or your installation of python-future is corrupted.') from __future__ import absolute_import from future.utils import PY3 if PY3: from tkinter.colorchooser import * else: try: from tkColorChooser import * except ImportError: raise ImportError('The tkColorChooser module is missing. Does your Py2 ' 'installation include tkinter?') from __future__ import absolute_import from future.utils import PY3 if PY3: from tkinter.commondialog import * else: try: from tkCommonDialog import * except ImportError: raise ImportError('The tkCommonDialog module is missing. Does your Py2 ' 'installation include tkinter?') from __future__ import absolute_import from future.utils import PY3 if PY3: from tkinter.constants import * else: try: from Tkconstants import * except ImportError: raise ImportError('The Tkconstants module is missing. Does your Py2 ' 'installation include tkinter?') from __future__ import absolute_import from future.utils import PY3 if PY3: from tkinter.dialog import * else: try: from Dialog import * except ImportError: raise ImportError('The Dialog module is missing. Does your Py2 ' 'installation include tkinter?') from __future__ import absolute_import from future.utils import PY3 if PY3: from tkinter.dnd import * else: try: from Tkdnd import * except ImportError: raise ImportError('The Tkdnd module is missing. Does your Py2 ' 'installation include tkinter?') from __future__ import absolute_import from future.utils import PY3 if PY3: from tkinter.filedialog import * else: try: from FileDialog import * except ImportError: raise ImportError('The FileDialog module is missing. Does your Py2 ' 'installation include tkinter?') try: from tkFileDialog import * except ImportError: raise ImportError('The tkFileDialog module is missing. Does your Py2 ' 'installation include tkinter?') from __future__ import absolute_import from future.utils import PY3 if PY3: from tkinter.font import * else: try: from tkFont import * except ImportError: raise ImportError('The tkFont module is missing. Does your Py2 ' 'installation include tkinter?') from __future__ import absolute_import from future.utils import PY3 if PY3: from tkinter.messagebox import * else: try: from tkMessageBox import * except ImportError: raise ImportError('The tkMessageBox module is missing. Does your Py2 ' 'installation include tkinter?') from __future__ import absolute_import from future.utils import PY3 if PY3: from tkinter.scrolledtext import * else: try: from ScrolledText import * except ImportError: raise ImportError('The ScrolledText module is missing. Does your Py2 ' 'installation include tkinter?') from __future__ import absolute_import from future.utils import PY3 if PY3: from tkinter.simpledialog import * else: try: from SimpleDialog import * except ImportError: raise ImportError('The SimpleDialog module is missing. Does your Py2 ' 'installation include tkinter?') from __future__ import absolute_import from future.utils import PY3 if PY3: from tkinter.tix import * else: try: from Tix import * except ImportError: raise ImportError('The Tix module is missing. Does your Py2 ' 'installation include tkinter?') from __future__ import absolute_import from future.utils import PY3 if PY3: from tkinter.ttk import * else: try: from ttk import * except ImportError: raise ImportError('The ttk module is missing. Does your Py2 ' 'installation include tkinter?') from __future__ import absolute_import import sys __future_module__ = True if sys.version_info[0] < 3: from _winreg import * else: raise ImportError('This package should not be accessible on Python 3. ' 'Either you are trying to run from the python-future src folder ' 'or your installation of python-future is corrupted.') from __future__ import absolute_import import sys if sys.version_info[0] < 3: pass else: raise ImportError('This package should not be accessible on Python 3. ' 'Either you are trying to run from the python-future src folder ' 'or your installation of python-future is corrupted.') from __future__ import absolute_import import sys assert sys.version_info[0] < 3 from xmlrpclib import * from __future__ import absolute_import import sys assert sys.version_info[0] < 3 from xmlrpclib import * A. HISTORY OF THE SOFTWARE ========================== Python was created in the early 1990s by Guido van Rossum at Stichting Mathematisch Centrum (CWI, see http://www.cwi.nl) in the Netherlands as a successor of a language called ABC. Guido remains Python's principal author, although it includes many contributions from others. In 1995, Guido continued his work on Python at the Corporation for National Research Initiatives (CNRI, see http://www.cnri.reston.va.us) in Reston, Virginia where he released several versions of the software. In May 2000, Guido and the Python core development team moved to BeOpen.com to form the BeOpen PythonLabs team. In October of the same year, the PythonLabs team moved to Digital Creations, which became Zope Corporation. In 2001, the Python Software Foundation (PSF, see https://www.python.org/psf/) was formed, a non-profit organization created specifically to own Python-related Intellectual Property. Zope Corporation was a sponsoring member of the PSF. All Python releases are Open Source (see http://www.opensource.org for the Open Source Definition). Historically, most, but not all, Python releases have also been GPL-compatible; the table below summarizes the various releases. Release Derived Year Owner GPL- from compatible? (1) 0.9.0 thru 1.2 1991-1995 CWI yes 1.3 thru 1.5.2 1.2 1995-1999 CNRI yes 1.6 1.5.2 2000 CNRI no 2.0 1.6 2000 BeOpen.com no 1.6.1 1.6 2001 CNRI yes (2) 2.1 2.0+1.6.1 2001 PSF no 2.0.1 2.0+1.6.1 2001 PSF yes 2.1.1 2.1+2.0.1 2001 PSF yes 2.1.2 2.1.1 2002 PSF yes 2.1.3 2.1.2 2002 PSF yes 2.2 and above 2.1.1 2001-now PSF yes Footnotes: (1) GPL-compatible doesn't mean that we're distributing Python under the GPL. All Python licenses, unlike the GPL, let you distribute a modified version without making your changes open source. The GPL-compatible licenses make it possible to combine Python with other software that is released under the GPL; the others don't. (2) According to Richard Stallman, 1.6.1 is not GPL-compatible, because its license has a choice of law clause. According to CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1 is "not incompatible" with the GPL. Thanks to the many outside volunteers who have worked under Guido's direction to make these releases possible. B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON =============================================================== PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 -------------------------------------------- 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and the Individual or Organization ("Licensee") accessing and otherwise using this software ("Python") in source or binary form and its associated documentation. 2. Subject to the terms and conditions of this License Agreement, PSF hereby grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License Agreement and PSF's notice of copyright, i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018 Python Software Foundation; All Rights Reserved" are retained in Python alone or in any derivative version prepared by Licensee. 3. In the event Licensee prepares a derivative work that is based on or incorporates Python or any part thereof, and wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in any such work a brief summary of the changes made to Python. 4. PSF is making Python available to Licensee on an "AS IS" basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT INFRINGE ANY THIRD PARTY RIGHTS. 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. 6. This License Agreement will automatically terminate upon a material breach of its terms and conditions. 7. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint venture between PSF and Licensee. This License Agreement does not grant permission to use PSF trademarks or trade name in a trademark sense to endorse or promote products or services of Licensee, or any third party. 8. By copying, installing or otherwise using Python, Licensee agrees to be bound by the terms and conditions of this License Agreement. BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0 ------------------------------------------- BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1 1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the Individual or Organization ("Licensee") accessing and otherwise using this software in source or binary form and its associated documentation ("the Software"). 2. Subject to the terms and conditions of this BeOpen Python License Agreement, BeOpen hereby grants Licensee a non-exclusive, royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative works, distribute, and otherwise use the Software alone or in any derivative version, provided, however, that the BeOpen Python License is retained in the Software, alone or in any derivative version prepared by Licensee. 3. BeOpen is making the Software available to Licensee on an "AS IS" basis. BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT INFRINGE ANY THIRD PARTY RIGHTS. 4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. 5. This License Agreement will automatically terminate upon a material breach of its terms and conditions. 6. This License Agreement shall be governed by and interpreted in all respects by the law of the State of California, excluding conflict of law provisions. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint venture between BeOpen and Licensee. This License Agreement does not grant permission to use BeOpen trademarks or trade names in a trademark sense to endorse or promote products or services of Licensee, or any third party. As an exception, the "BeOpen Python" logos available at http://www.pythonlabs.com/logos.html may be used according to the permissions granted on that web page. 7. By copying, installing or otherwise using the software, Licensee agrees to be bound by the terms and conditions of this License Agreement. CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1 --------------------------------------- 1. This LICENSE AGREEMENT is between the Corporation for National Research Initiatives, having an office at 1895 Preston White Drive, Reston, VA 20191 ("CNRI"), and the Individual or Organization ("Licensee") accessing and otherwise using Python 1.6.1 software in source or binary form and its associated documentation. 2. Subject to the terms and conditions of this License Agreement, CNRI hereby grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, analyze, test, perform and/or display publicly, prepare derivative works, distribute, and otherwise use Python 1.6.1 alone or in any derivative version, provided, however, that CNRI's License Agreement and CNRI's notice of copyright, i.e., "Copyright (c) 1995-2001 Corporation for National Research Initiatives; All Rights Reserved" are retained in Python 1.6.1 alone or in any derivative version prepared by Licensee. Alternately, in lieu of CNRI's License Agreement, Licensee may substitute the following text (omitting the quotes): "Python 1.6.1 is made available subject to the terms and conditions in CNRI's License Agreement. This Agreement together with Python 1.6.1 may be located on the Internet using the following unique, persistent identifier (known as a handle): 1895.22/1013. This Agreement may also be obtained from a proxy server on the Internet using the following URL: http://hdl.handle.net/1895.22/1013". 3. In the event Licensee prepares a derivative work that is based on or incorporates Python 1.6.1 or any part thereof, and wants to make the derivative work available to others as provided herein, then Licensee hereby agrees to include in any such work a brief summary of the changes made to Python 1.6.1. 4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS" basis. CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT INFRINGE ANY THIRD PARTY RIGHTS. 5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON 1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1, OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. 6. This License Agreement will automatically terminate upon a material breach of its terms and conditions. 7. This License Agreement shall be governed by the federal intellectual property law of the United States, including without limitation the federal copyright law, and, to the extent such U.S. federal law does not apply, by the law of the Commonwealth of Virginia, excluding Virginia's conflict of law provisions. Notwithstanding the foregoing, with regard to derivative works based on Python 1.6.1 that incorporate non-separable material that was previously distributed under the GNU General Public License (GPL), the law of the Commonwealth of Virginia shall govern this License Agreement only as to issues arising under or with respect to Paragraphs 4, 5, and 7 of this License Agreement. Nothing in this License Agreement shall be deemed to create any relationship of agency, partnership, or joint venture between CNRI and Licensee. This License Agreement does not grant permission to use CNRI trademarks or trade name in a trademark sense to endorse or promote products or services of Licensee, or any third party. 8. By clicking on the "ACCEPT" button where indicated, or by copying, installing or otherwise using Python 1.6.1, Licensee agrees to be bound by the terms and conditions of this License Agreement. ACCEPT CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2 -------------------------------------------------- Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam, The Netherlands. All rights reserved. Permission to use, copy, modify, and distribute this software and its documentation for any purpose and without fee is hereby granted, provided that the above copyright notice appear in all copies and that both that copyright notice and this permission notice appear in supporting documentation, and that the name of Stichting Mathematisch Centrum or CWI not be used in advertising or publicity pertaining to distribution of the software without specific, written prior permission. STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. """Backport of importlib.import_module from 3.x.""" # While not critical (and in no way guaranteed!), it would be nice to keep this # code compatible with Python 2.3. import sys def _resolve_name(name, package, level): """Return the absolute name of the module to be imported.""" if not hasattr(package, 'rindex'): raise ValueError("'package' not set to a string") dot = len(package) for x in xrange(level, 1, -1): try: dot = package.rindex('.', 0, dot) except ValueError: raise ValueError("attempted relative import beyond top-level " "package") return "%s.%s" % (package[:dot], name) def import_module(name, package=None): """Import a module. The 'package' argument is required when performing a relative import. It specifies the package to use as the anchor point from which to resolve the relative import to an absolute import. """ if name.startswith('.'): if not package: raise TypeError("relative imports require the 'package' argument") level = 0 for character in name: if character != '.': break level += 1 name = _resolve_name(name[level:], package, level) __import__(name) return sys.modules[name] # Copyright 2001-2014 by Vinay Sajip. All Rights Reserved. # # Permission to use, copy, modify, and distribute this software and its # documentation for any purpose and without fee is hereby granted, # provided that the above copyright notice appear in all copies and that # both that copyright notice and this permission notice appear in # supporting documentation, and that the name of Vinay Sajip # not be used in advertising or publicity pertaining to distribution # of the software without specific, written prior permission. # VINAY SAJIP DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING # ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL # VINAY SAJIP BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR # ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER # IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT # OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. """ Logging package for Python. Based on PEP 282 and comments thereto in comp.lang.python. Copyright (C) 2001-2014 Vinay Sajip. All Rights Reserved. To use, simply 'import logging' and log away! """ import sys, os, time, cStringIO, traceback, warnings, weakref, collections __all__ = ['BASIC_FORMAT', 'BufferingFormatter', 'CRITICAL', 'DEBUG', 'ERROR', 'FATAL', 'FileHandler', 'Filter', 'Formatter', 'Handler', 'INFO', 'LogRecord', 'Logger', 'LoggerAdapter', 'NOTSET', 'NullHandler', 'StreamHandler', 'WARN', 'WARNING', 'addLevelName', 'basicConfig', 'captureWarnings', 'critical', 'debug', 'disable', 'error', 'exception', 'fatal', 'getLevelName', 'getLogger', 'getLoggerClass', 'info', 'log', 'makeLogRecord', 'setLoggerClass', 'warn', 'warning'] try: import codecs except ImportError: codecs = None try: import thread import threading except ImportError: thread = None __author__ = "Vinay Sajip " __status__ = "production" # Note: the attributes below are no longer maintained. __version__ = "0.5.1.2" __date__ = "07 February 2010" #--------------------------------------------------------------------------- # Miscellaneous module data #--------------------------------------------------------------------------- try: unicode _unicode = True except NameError: _unicode = False # next bit filched from 1.5.2's inspect.py def currentframe(): """Return the frame object for the caller's stack frame.""" try: raise Exception except: return sys.exc_info()[2].tb_frame.f_back if hasattr(sys, '_getframe'): currentframe = lambda: sys._getframe(3) # done filching # # _srcfile is used when walking the stack to check when we've got the first # caller stack frame. # _srcfile = os.path.normcase(currentframe.__code__.co_filename) # _srcfile is only used in conjunction with sys._getframe(). # To provide compatibility with older versions of Python, set _srcfile # to None if _getframe() is not available; this value will prevent # findCaller() from being called. #if not hasattr(sys, "_getframe"): # _srcfile = None # #_startTime is used as the base when calculating the relative time of events # _startTime = time.time() # #raiseExceptions is used to see if exceptions during handling should be #propagated # raiseExceptions = 1 # # If you don't want threading information in the log, set this to zero # logThreads = 1 # # If you don't want multiprocessing information in the log, set this to zero # logMultiprocessing = 1 # # If you don't want process information in the log, set this to zero # logProcesses = 1 #--------------------------------------------------------------------------- # Level related stuff #--------------------------------------------------------------------------- # # Default levels and level names, these can be replaced with any positive set # of values having corresponding names. There is a pseudo-level, NOTSET, which # is only really there as a lower limit for user-defined levels. Handlers and # loggers are initialized with NOTSET so that they will log all messages, even # at user-defined levels. # CRITICAL = 50 FATAL = CRITICAL ERROR = 40 WARNING = 30 WARN = WARNING INFO = 20 DEBUG = 10 NOTSET = 0 _levelNames = { CRITICAL : 'CRITICAL', ERROR : 'ERROR', WARNING : 'WARNING', INFO : 'INFO', DEBUG : 'DEBUG', NOTSET : 'NOTSET', 'CRITICAL' : CRITICAL, 'ERROR' : ERROR, 'WARN' : WARNING, 'WARNING' : WARNING, 'INFO' : INFO, 'DEBUG' : DEBUG, 'NOTSET' : NOTSET, } def getLevelName(level): """ Return the textual representation of logging level 'level'. If the level is one of the predefined levels (CRITICAL, ERROR, WARNING, INFO, DEBUG) then you get the corresponding string. If you have associated levels with names using addLevelName then the name you have associated with 'level' is returned. If a numeric value corresponding to one of the defined levels is passed in, the corresponding string representation is returned. Otherwise, the string "Level %s" % level is returned. """ return _levelNames.get(level, ("Level %s" % level)) def addLevelName(level, levelName): """ Associate 'levelName' with 'level'. This is used when converting levels to text during message formatting. """ _acquireLock() try: #unlikely to cause an exception, but you never know... _levelNames[level] = levelName _levelNames[levelName] = level finally: _releaseLock() def _checkLevel(level): if isinstance(level, (int, long)): rv = level elif str(level) == level: if level not in _levelNames: raise ValueError("Unknown level: %r" % level) rv = _levelNames[level] else: raise TypeError("Level not an integer or a valid string: %r" % level) return rv #--------------------------------------------------------------------------- # Thread-related stuff #--------------------------------------------------------------------------- # #_lock is used to serialize access to shared data structures in this module. #This needs to be an RLock because fileConfig() creates and configures #Handlers, and so might arbitrary user threads. Since Handler code updates the #shared dictionary _handlers, it needs to acquire the lock. But if configuring, #the lock would already have been acquired - so we need an RLock. #The same argument applies to Loggers and Manager.loggerDict. # if thread: _lock = threading.RLock() else: _lock = None def _acquireLock(): """ Acquire the module-level lock for serializing access to shared data. This should be released with _releaseLock(). """ if _lock: _lock.acquire() def _releaseLock(): """ Release the module-level lock acquired by calling _acquireLock(). """ if _lock: _lock.release() #--------------------------------------------------------------------------- # The logging record #--------------------------------------------------------------------------- class LogRecord(object): """ A LogRecord instance represents an event being logged. LogRecord instances are created every time something is logged. They contain all the information pertinent to the event being logged. The main information passed in is in msg and args, which are combined using str(msg) % args to create the message field of the record. The record also includes information such as when the record was created, the source line where the logging call was made, and any exception information to be logged. """ def __init__(self, name, level, pathname, lineno, msg, args, exc_info, func=None): """ Initialize a logging record with interesting information. """ ct = time.time() self.name = name self.msg = msg # # The following statement allows passing of a dictionary as a sole # argument, so that you can do something like # logging.debug("a %(a)d b %(b)s", {'a':1, 'b':2}) # Suggested by Stefan Behnel. # Note that without the test for args[0], we get a problem because # during formatting, we test to see if the arg is present using # 'if self.args:'. If the event being logged is e.g. 'Value is %d' # and if the passed arg fails 'if self.args:' then no formatting # is done. For example, logger.warn('Value is %d', 0) would log # 'Value is %d' instead of 'Value is 0'. # For the use case of passing a dictionary, this should not be a # problem. # Issue #21172: a request was made to relax the isinstance check # to hasattr(args[0], '__getitem__'). However, the docs on string # formatting still seem to suggest a mapping object is required. # Thus, while not removing the isinstance check, it does now look # for collections.Mapping rather than, as before, dict. if (args and len(args) == 1 and isinstance(args[0], collections.Mapping) and args[0]): args = args[0] self.args = args self.levelname = getLevelName(level) self.levelno = level self.pathname = pathname try: self.filename = os.path.basename(pathname) self.module = os.path.splitext(self.filename)[0] except (TypeError, ValueError, AttributeError): self.filename = pathname self.module = "Unknown module" self.exc_info = exc_info self.exc_text = None # used to cache the traceback text self.lineno = lineno self.funcName = func self.created = ct self.msecs = (ct - long(ct)) * 1000 self.relativeCreated = (self.created - _startTime) * 1000 if logThreads and thread: self.thread = thread.get_ident() self.threadName = threading.current_thread().name else: self.thread = None self.threadName = None if not logMultiprocessing: self.processName = None else: self.processName = 'MainProcess' mp = sys.modules.get('multiprocessing') if mp is not None: # Errors may occur if multiprocessing has not finished loading # yet - e.g. if a custom import hook causes third-party code # to run when multiprocessing calls import. See issue 8200 # for an example try: self.processName = mp.current_process().name except StandardError: pass if logProcesses and hasattr(os, 'getpid'): self.process = os.getpid() else: self.process = None def __str__(self): return ''%(self.name, self.levelno, self.pathname, self.lineno, self.msg) def getMessage(self): """ Return the message for this LogRecord. Return the message for this LogRecord after merging any user-supplied arguments with the message. """ if not _unicode: #if no unicode support... msg = str(self.msg) else: msg = self.msg if not isinstance(msg, basestring): try: msg = str(self.msg) except UnicodeError: msg = self.msg #Defer encoding till later if self.args: msg = msg % self.args return msg def makeLogRecord(dict): """ Make a LogRecord whose attributes are defined by the specified dictionary, This function is useful for converting a logging event received over a socket connection (which is sent as a dictionary) into a LogRecord instance. """ rv = LogRecord(None, None, "", 0, "", (), None, None) rv.__dict__.update(dict) return rv #--------------------------------------------------------------------------- # Formatter classes and functions #--------------------------------------------------------------------------- class Formatter(object): """ Formatter instances are used to convert a LogRecord to text. Formatters need to know how a LogRecord is constructed. They are responsible for converting a LogRecord to (usually) a string which can be interpreted by either a human or an external system. The base Formatter allows a formatting string to be specified. If none is supplied, the default value of "%s(message)\\n" is used. The Formatter can be initialized with a format string which makes use of knowledge of the LogRecord attributes - e.g. the default value mentioned above makes use of the fact that the user's message and arguments are pre- formatted into a LogRecord's message attribute. Currently, the useful attributes in a LogRecord are described by: %(name)s Name of the logger (logging channel) %(levelno)s Numeric logging level for the message (DEBUG, INFO, WARNING, ERROR, CRITICAL) %(levelname)s Text logging level for the message ("DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL") %(pathname)s Full pathname of the source file where the logging call was issued (if available) %(filename)s Filename portion of pathname %(module)s Module (name portion of filename) %(lineno)d Source line number where the logging call was issued (if available) %(funcName)s Function name %(created)f Time when the LogRecord was created (time.time() return value) %(asctime)s Textual time when the LogRecord was created %(msecs)d Millisecond portion of the creation time %(relativeCreated)d Time in milliseconds when the LogRecord was created, relative to the time the logging module was loaded (typically at application startup time) %(thread)d Thread ID (if available) %(threadName)s Thread name (if available) %(process)d Process ID (if available) %(message)s The result of record.getMessage(), computed just as the record is emitted """ converter = time.localtime def __init__(self, fmt=None, datefmt=None): """ Initialize the formatter with specified format strings. Initialize the formatter either with the specified format string, or a default as described above. Allow for specialized date formatting with the optional datefmt argument (if omitted, you get the ISO8601 format). """ if fmt: self._fmt = fmt else: self._fmt = "%(message)s" self.datefmt = datefmt def formatTime(self, record, datefmt=None): """ Return the creation time of the specified LogRecord as formatted text. This method should be called from format() by a formatter which wants to make use of a formatted time. This method can be overridden in formatters to provide for any specific requirement, but the basic behaviour is as follows: if datefmt (a string) is specified, it is used with time.strftime() to format the creation time of the record. Otherwise, the ISO8601 format is used. The resulting string is returned. This function uses a user-configurable function to convert the creation time to a tuple. By default, time.localtime() is used; to change this for a particular formatter instance, set the 'converter' attribute to a function with the same signature as time.localtime() or time.gmtime(). To change it for all formatters, for example if you want all logging times to be shown in GMT, set the 'converter' attribute in the Formatter class. """ ct = self.converter(record.created) if datefmt: s = time.strftime(datefmt, ct) else: t = time.strftime("%Y-%m-%d %H:%M:%S", ct) s = "%s,%03d" % (t, record.msecs) return s def formatException(self, ei): """ Format and return the specified exception information as a string. This default implementation just uses traceback.print_exception() """ sio = cStringIO.StringIO() traceback.print_exception(ei[0], ei[1], ei[2], None, sio) s = sio.getvalue() sio.close() if s[-1:] == "\n": s = s[:-1] return s def usesTime(self): """ Check if the format uses the creation time of the record. """ return self._fmt.find("%(asctime)") >= 0 def format(self, record): """ Format the specified record as text. The record's attribute dictionary is used as the operand to a string formatting operation which yields the returned string. Before formatting the dictionary, a couple of preparatory steps are carried out. The message attribute of the record is computed using LogRecord.getMessage(). If the formatting string uses the time (as determined by a call to usesTime(), formatTime() is called to format the event time. If there is exception information, it is formatted using formatException() and appended to the message. """ record.message = record.getMessage() if self.usesTime(): record.asctime = self.formatTime(record, self.datefmt) try: s = self._fmt % record.__dict__ except UnicodeDecodeError as e: # Issue 25664. The logger name may be Unicode. Try again ... try: record.name = record.name.decode('utf-8') s = self._fmt % record.__dict__ except UnicodeDecodeError: raise e if record.exc_info: # Cache the traceback text to avoid converting it multiple times # (it's constant anyway) if not record.exc_text: record.exc_text = self.formatException(record.exc_info) if record.exc_text: if s[-1:] != "\n": s = s + "\n" try: s = s + record.exc_text except UnicodeError: # Sometimes filenames have non-ASCII chars, which can lead # to errors when s is Unicode and record.exc_text is str # See issue 8924. # We also use replace for when there are multiple # encodings, e.g. UTF-8 for the filesystem and latin-1 # for a script. See issue 13232. s = s + record.exc_text.decode(sys.getfilesystemencoding(), 'replace') return s # # The default formatter to use when no other is specified # _defaultFormatter = Formatter() class BufferingFormatter(object): """ A formatter suitable for formatting a number of records. """ def __init__(self, linefmt=None): """ Optionally specify a formatter which will be used to format each individual record. """ if linefmt: self.linefmt = linefmt else: self.linefmt = _defaultFormatter def formatHeader(self, records): """ Return the header string for the specified records. """ return "" def formatFooter(self, records): """ Return the footer string for the specified records. """ return "" def format(self, records): """ Format the specified records and return the result as a string. """ rv = "" if len(records) > 0: rv = rv + self.formatHeader(records) for record in records: rv = rv + self.linefmt.format(record) rv = rv + self.formatFooter(records) return rv #--------------------------------------------------------------------------- # Filter classes and functions #--------------------------------------------------------------------------- class Filter(object): """ Filter instances are used to perform arbitrary filtering of LogRecords. Loggers and Handlers can optionally use Filter instances to filter records as desired. The base filter class only allows events which are below a certain point in the logger hierarchy. For example, a filter initialized with "A.B" will allow events logged by loggers "A.B", "A.B.C", "A.B.C.D", "A.B.D" etc. but not "A.BB", "B.A.B" etc. If initialized with the empty string, all events are passed. """ def __init__(self, name=''): """ Initialize a filter. Initialize with the name of the logger which, together with its children, will have its events allowed through the filter. If no name is specified, allow every event. """ self.name = name self.nlen = len(name) def filter(self, record): """ Determine if the specified record is to be logged. Is the specified record to be logged? Returns 0 for no, nonzero for yes. If deemed appropriate, the record may be modified in-place. """ if self.nlen == 0: return 1 elif self.name == record.name: return 1 elif record.name.find(self.name, 0, self.nlen) != 0: return 0 return (record.name[self.nlen] == ".") class Filterer(object): """ A base class for loggers and handlers which allows them to share common code. """ def __init__(self): """ Initialize the list of filters to be an empty list. """ self.filters = [] def addFilter(self, filter): """ Add the specified filter to this handler. """ if not (filter in self.filters): self.filters.append(filter) def removeFilter(self, filter): """ Remove the specified filter from this handler. """ if filter in self.filters: self.filters.remove(filter) def filter(self, record): """ Determine if a record is loggable by consulting all the filters. The default is to allow the record to be logged; any filter can veto this and the record is then dropped. Returns a zero value if a record is to be dropped, else non-zero. """ rv = 1 for f in self.filters: if not f.filter(record): rv = 0 break return rv #--------------------------------------------------------------------------- # Handler classes and functions #--------------------------------------------------------------------------- _handlers = weakref.WeakValueDictionary() #map of handler names to handlers _handlerList = [] # added to allow handlers to be removed in reverse of order initialized def _removeHandlerRef(wr): """ Remove a handler reference from the internal cleanup list. """ # This function can be called during module teardown, when globals are # set to None. It can also be called from another thread. So we need to # pre-emptively grab the necessary globals and check if they're None, # to prevent race conditions and failures during interpreter shutdown. acquire, release, handlers = _acquireLock, _releaseLock, _handlerList if acquire and release and handlers: acquire() try: if wr in handlers: handlers.remove(wr) finally: release() def _addHandlerRef(handler): """ Add a handler to the internal cleanup list using a weak reference. """ _acquireLock() try: _handlerList.append(weakref.ref(handler, _removeHandlerRef)) finally: _releaseLock() class Handler(Filterer): """ Handler instances dispatch logging events to specific destinations. The base handler class. Acts as a placeholder which defines the Handler interface. Handlers can optionally use Formatter instances to format records as desired. By default, no formatter is specified; in this case, the 'raw' message as determined by record.message is logged. """ def __init__(self, level=NOTSET): """ Initializes the instance - basically setting the formatter to None and the filter list to empty. """ Filterer.__init__(self) self._name = None self.level = _checkLevel(level) self.formatter = None # Add the handler to the global _handlerList (for cleanup on shutdown) _addHandlerRef(self) self.createLock() def get_name(self): return self._name def set_name(self, name): _acquireLock() try: if self._name in _handlers: del _handlers[self._name] self._name = name if name: _handlers[name] = self finally: _releaseLock() name = property(get_name, set_name) def createLock(self): """ Acquire a thread lock for serializing access to the underlying I/O. """ if thread: self.lock = threading.RLock() else: self.lock = None def acquire(self): """ Acquire the I/O thread lock. """ if self.lock: self.lock.acquire() def release(self): """ Release the I/O thread lock. """ if self.lock: self.lock.release() def setLevel(self, level): """ Set the logging level of this handler. """ self.level = _checkLevel(level) def format(self, record): """ Format the specified record. If a formatter is set, use it. Otherwise, use the default formatter for the module. """ if self.formatter: fmt = self.formatter else: fmt = _defaultFormatter return fmt.format(record) def emit(self, record): """ Do whatever it takes to actually log the specified logging record. This version is intended to be implemented by subclasses and so raises a NotImplementedError. """ raise NotImplementedError('emit must be implemented ' 'by Handler subclasses') def handle(self, record): """ Conditionally emit the specified logging record. Emission depends on filters which may have been added to the handler. Wrap the actual emission of the record with acquisition/release of the I/O thread lock. Returns whether the filter passed the record for emission. """ rv = self.filter(record) if rv: self.acquire() try: self.emit(record) finally: self.release() return rv def setFormatter(self, fmt): """ Set the formatter for this handler. """ self.formatter = fmt def flush(self): """ Ensure all logging output has been flushed. This version does nothing and is intended to be implemented by subclasses. """ pass def close(self): """ Tidy up any resources used by the handler. This version removes the handler from an internal map of handlers, _handlers, which is used for handler lookup by name. Subclasses should ensure that this gets called from overridden close() methods. """ #get the module data lock, as we're updating a shared structure. _acquireLock() try: #unlikely to raise an exception, but you never know... if self._name and self._name in _handlers: del _handlers[self._name] finally: _releaseLock() def handleError(self, record): """ Handle errors which occur during an emit() call. This method should be called from handlers when an exception is encountered during an emit() call. If raiseExceptions is false, exceptions get silently ignored. This is what is mostly wanted for a logging system - most users will not care about errors in the logging system, they are more interested in application errors. You could, however, replace this with a custom handler if you wish. The record which was being processed is passed in to this method. """ if raiseExceptions and sys.stderr: # see issue 13807 ei = sys.exc_info() try: traceback.print_exception(ei[0], ei[1], ei[2], None, sys.stderr) sys.stderr.write('Logged from file %s, line %s\n' % ( record.filename, record.lineno)) except IOError: pass # see issue 5971 finally: del ei class StreamHandler(Handler): """ A handler class which writes logging records, appropriately formatted, to a stream. Note that this class does not close the stream, as sys.stdout or sys.stderr may be used. """ def __init__(self, stream=None): """ Initialize the handler. If stream is not specified, sys.stderr is used. """ Handler.__init__(self) if stream is None: stream = sys.stderr self.stream = stream def flush(self): """ Flushes the stream. """ self.acquire() try: if self.stream and hasattr(self.stream, "flush"): self.stream.flush() finally: self.release() def emit(self, record): """ Emit a record. If a formatter is specified, it is used to format the record. The record is then written to the stream with a trailing newline. If exception information is present, it is formatted using traceback.print_exception and appended to the stream. If the stream has an 'encoding' attribute, it is used to determine how to do the output to the stream. """ try: msg = self.format(record) stream = self.stream fs = "%s\n" if not _unicode: #if no unicode support... stream.write(fs % msg) else: try: if (isinstance(msg, unicode) and getattr(stream, 'encoding', None)): ufs = u'%s\n' try: stream.write(ufs % msg) except UnicodeEncodeError: #Printing to terminals sometimes fails. For example, #with an encoding of 'cp1251', the above write will #work if written to a stream opened or wrapped by #the codecs module, but fail when writing to a #terminal even when the codepage is set to cp1251. #An extra encoding step seems to be needed. stream.write((ufs % msg).encode(stream.encoding)) else: stream.write(fs % msg) except UnicodeError: stream.write(fs % msg.encode("UTF-8")) self.flush() except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) class FileHandler(StreamHandler): """ A handler class which writes formatted logging records to disk files. """ def __init__(self, filename, mode='a', encoding=None, delay=0): """ Open the specified file and use it as the stream for logging. """ #keep the absolute path, otherwise derived classes which use this #may come a cropper when the current directory changes if codecs is None: encoding = None self.baseFilename = os.path.abspath(filename) self.mode = mode self.encoding = encoding self.delay = delay if delay: #We don't open the stream, but we still need to call the #Handler constructor to set level, formatter, lock etc. Handler.__init__(self) self.stream = None else: StreamHandler.__init__(self, self._open()) def close(self): """ Closes the stream. """ self.acquire() try: try: if self.stream: try: self.flush() finally: stream = self.stream self.stream = None if hasattr(stream, "close"): stream.close() finally: # Issue #19523: call unconditionally to # prevent a handler leak when delay is set StreamHandler.close(self) finally: self.release() def _open(self): """ Open the current base file with the (original) mode and encoding. Return the resulting stream. """ if self.encoding is None: stream = open(self.baseFilename, self.mode) else: stream = codecs.open(self.baseFilename, self.mode, self.encoding) return stream def emit(self, record): """ Emit a record. If the stream was not opened because 'delay' was specified in the constructor, open it before calling the superclass's emit. """ if self.stream is None: self.stream = self._open() StreamHandler.emit(self, record) #--------------------------------------------------------------------------- # Manager classes and functions #--------------------------------------------------------------------------- class PlaceHolder(object): """ PlaceHolder instances are used in the Manager logger hierarchy to take the place of nodes for which no loggers have been defined. This class is intended for internal use only and not as part of the public API. """ def __init__(self, alogger): """ Initialize with the specified logger being a child of this placeholder. """ #self.loggers = [alogger] self.loggerMap = { alogger : None } def append(self, alogger): """ Add the specified logger as a child of this placeholder. """ #if alogger not in self.loggers: if alogger not in self.loggerMap: #self.loggers.append(alogger) self.loggerMap[alogger] = None # # Determine which class to use when instantiating loggers. # _loggerClass = None def setLoggerClass(klass): """ Set the class to be used when instantiating a logger. The class should define __init__() such that only a name argument is required, and the __init__() should call Logger.__init__() """ if klass != Logger: if not issubclass(klass, Logger): raise TypeError("logger not derived from logging.Logger: " + klass.__name__) global _loggerClass _loggerClass = klass def getLoggerClass(): """ Return the class to be used when instantiating a logger. """ return _loggerClass class Manager(object): """ There is [under normal circumstances] just one Manager instance, which holds the hierarchy of loggers. """ def __init__(self, rootnode): """ Initialize the manager with the root node of the logger hierarchy. """ self.root = rootnode self.disable = 0 self.emittedNoHandlerWarning = 0 self.loggerDict = {} self.loggerClass = None def getLogger(self, name): """ Get a logger with the specified name (channel name), creating it if it doesn't yet exist. This name is a dot-separated hierarchical name, such as "a", "a.b", "a.b.c" or similar. If a PlaceHolder existed for the specified name [i.e. the logger didn't exist but a child of it did], replace it with the created logger and fix up the parent/child references which pointed to the placeholder to now point to the logger. """ rv = None if not isinstance(name, basestring): raise TypeError('A logger name must be string or Unicode') if isinstance(name, unicode): name = name.encode('utf-8') _acquireLock() try: if name in self.loggerDict: rv = self.loggerDict[name] if isinstance(rv, PlaceHolder): ph = rv rv = (self.loggerClass or _loggerClass)(name) rv.manager = self self.loggerDict[name] = rv self._fixupChildren(ph, rv) self._fixupParents(rv) else: rv = (self.loggerClass or _loggerClass)(name) rv.manager = self self.loggerDict[name] = rv self._fixupParents(rv) finally: _releaseLock() return rv def setLoggerClass(self, klass): """ Set the class to be used when instantiating a logger with this Manager. """ if klass != Logger: if not issubclass(klass, Logger): raise TypeError("logger not derived from logging.Logger: " + klass.__name__) self.loggerClass = klass def _fixupParents(self, alogger): """ Ensure that there are either loggers or placeholders all the way from the specified logger to the root of the logger hierarchy. """ name = alogger.name i = name.rfind(".") rv = None while (i > 0) and not rv: substr = name[:i] if substr not in self.loggerDict: self.loggerDict[substr] = PlaceHolder(alogger) else: obj = self.loggerDict[substr] if isinstance(obj, Logger): rv = obj else: assert isinstance(obj, PlaceHolder) obj.append(alogger) i = name.rfind(".", 0, i - 1) if not rv: rv = self.root alogger.parent = rv def _fixupChildren(self, ph, alogger): """ Ensure that children of the placeholder ph are connected to the specified logger. """ name = alogger.name namelen = len(name) for c in ph.loggerMap.keys(): #The if means ... if not c.parent.name.startswith(nm) if c.parent.name[:namelen] != name: alogger.parent = c.parent c.parent = alogger #--------------------------------------------------------------------------- # Logger classes and functions #--------------------------------------------------------------------------- class Logger(Filterer): """ Instances of the Logger class represent a single logging channel. A "logging channel" indicates an area of an application. Exactly how an "area" is defined is up to the application developer. Since an application can have any number of areas, logging channels are identified by a unique string. Application areas can be nested (e.g. an area of "input processing" might include sub-areas "read CSV files", "read XLS files" and "read Gnumeric files"). To cater for this natural nesting, channel names are organized into a namespace hierarchy where levels are separated by periods, much like the Java or Python package namespace. So in the instance given above, channel names might be "input" for the upper level, and "input.csv", "input.xls" and "input.gnu" for the sub-levels. There is no arbitrary limit to the depth of nesting. """ def __init__(self, name, level=NOTSET): """ Initialize the logger with a name and an optional level. """ Filterer.__init__(self) self.name = name self.level = _checkLevel(level) self.parent = None self.propagate = 1 self.handlers = [] self.disabled = 0 def setLevel(self, level): """ Set the logging level of this logger. """ self.level = _checkLevel(level) def debug(self, msg, *args, **kwargs): """ Log 'msg % args' with severity 'DEBUG'. To pass exception information, use the keyword argument exc_info with a true value, e.g. logger.debug("Houston, we have a %s", "thorny problem", exc_info=1) """ if self.isEnabledFor(DEBUG): self._log(DEBUG, msg, args, **kwargs) def info(self, msg, *args, **kwargs): """ Log 'msg % args' with severity 'INFO'. To pass exception information, use the keyword argument exc_info with a true value, e.g. logger.info("Houston, we have a %s", "interesting problem", exc_info=1) """ if self.isEnabledFor(INFO): self._log(INFO, msg, args, **kwargs) def warning(self, msg, *args, **kwargs): """ Log 'msg % args' with severity 'WARNING'. To pass exception information, use the keyword argument exc_info with a true value, e.g. logger.warning("Houston, we have a %s", "bit of a problem", exc_info=1) """ if self.isEnabledFor(WARNING): self._log(WARNING, msg, args, **kwargs) warn = warning def error(self, msg, *args, **kwargs): """ Log 'msg % args' with severity 'ERROR'. To pass exception information, use the keyword argument exc_info with a true value, e.g. logger.error("Houston, we have a %s", "major problem", exc_info=1) """ if self.isEnabledFor(ERROR): self._log(ERROR, msg, args, **kwargs) def exception(self, msg, *args, **kwargs): """ Convenience method for logging an ERROR with exception information. """ kwargs['exc_info'] = 1 self.error(msg, *args, **kwargs) def critical(self, msg, *args, **kwargs): """ Log 'msg % args' with severity 'CRITICAL'. To pass exception information, use the keyword argument exc_info with a true value, e.g. logger.critical("Houston, we have a %s", "major disaster", exc_info=1) """ if self.isEnabledFor(CRITICAL): self._log(CRITICAL, msg, args, **kwargs) fatal = critical def log(self, level, msg, *args, **kwargs): """ Log 'msg % args' with the integer severity 'level'. To pass exception information, use the keyword argument exc_info with a true value, e.g. logger.log(level, "We have a %s", "mysterious problem", exc_info=1) """ if not isinstance(level, (int, long)): if raiseExceptions: raise TypeError("level must be an integer") else: return if self.isEnabledFor(level): self._log(level, msg, args, **kwargs) def findCaller(self): """ Find the stack frame of the caller so that we can note the source file name, line number and function name. """ f = currentframe() #On some versions of IronPython, currentframe() returns None if #IronPython isn't run with -X:Frames. if f is not None: f = f.f_back rv = "(unknown file)", 0, "(unknown function)" while hasattr(f, "f_code"): co = f.f_code filename = os.path.normcase(co.co_filename) if filename == _srcfile: f = f.f_back continue rv = (co.co_filename, f.f_lineno, co.co_name) break return rv def makeRecord(self, name, level, fn, lno, msg, args, exc_info, func=None, extra=None): """ A factory method which can be overridden in subclasses to create specialized LogRecords. """ rv = LogRecord(name, level, fn, lno, msg, args, exc_info, func) if extra is not None: for key in extra: if (key in ["message", "asctime"]) or (key in rv.__dict__): raise KeyError("Attempt to overwrite %r in LogRecord" % key) rv.__dict__[key] = extra[key] return rv def _log(self, level, msg, args, exc_info=None, extra=None): """ Low-level logging routine which creates a LogRecord and then calls all the handlers of this logger to handle the record. """ if _srcfile: #IronPython doesn't track Python frames, so findCaller raises an #exception on some versions of IronPython. We trap it here so that #IronPython can use logging. try: fn, lno, func = self.findCaller() except ValueError: fn, lno, func = "(unknown file)", 0, "(unknown function)" else: fn, lno, func = "(unknown file)", 0, "(unknown function)" if exc_info: if not isinstance(exc_info, tuple): exc_info = sys.exc_info() record = self.makeRecord(self.name, level, fn, lno, msg, args, exc_info, func, extra) self.handle(record) def handle(self, record): """ Call the handlers for the specified record. This method is used for unpickled records received from a socket, as well as those created locally. Logger-level filtering is applied. """ if (not self.disabled) and self.filter(record): self.callHandlers(record) def addHandler(self, hdlr): """ Add the specified handler to this logger. """ _acquireLock() try: if not (hdlr in self.handlers): self.handlers.append(hdlr) finally: _releaseLock() def removeHandler(self, hdlr): """ Remove the specified handler from this logger. """ _acquireLock() try: if hdlr in self.handlers: self.handlers.remove(hdlr) finally: _releaseLock() def callHandlers(self, record): """ Pass a record to all relevant handlers. Loop through all handlers for this logger and its parents in the logger hierarchy. If no handler was found, output a one-off error message to sys.stderr. Stop searching up the hierarchy whenever a logger with the "propagate" attribute set to zero is found - that will be the last logger whose handlers are called. """ c = self found = 0 while c: for hdlr in c.handlers: found = found + 1 if record.levelno >= hdlr.level: hdlr.handle(record) if not c.propagate: c = None #break out else: c = c.parent if (found == 0) and raiseExceptions and not self.manager.emittedNoHandlerWarning: sys.stderr.write("No handlers could be found for logger" " \"%s\"\n" % self.name) self.manager.emittedNoHandlerWarning = 1 def getEffectiveLevel(self): """ Get the effective level for this logger. Loop through this logger and its parents in the logger hierarchy, looking for a non-zero logging level. Return the first one found. """ logger = self while logger: if logger.level: return logger.level logger = logger.parent return NOTSET def isEnabledFor(self, level): """ Is this logger enabled for level 'level'? """ if self.manager.disable >= level: return 0 return level >= self.getEffectiveLevel() def getChild(self, suffix): """ Get a logger which is a descendant to this one. This is a convenience method, such that logging.getLogger('abc').getChild('def.ghi') is the same as logging.getLogger('abc.def.ghi') It's useful, for example, when the parent logger is named using __name__ rather than a literal string. """ if self.root is not self: suffix = '.'.join((self.name, suffix)) return self.manager.getLogger(suffix) class RootLogger(Logger): """ A root logger is not that different to any other logger, except that it must have a logging level and there is only one instance of it in the hierarchy. """ def __init__(self, level): """ Initialize the logger with the name "root". """ Logger.__init__(self, "root", level) _loggerClass = Logger class LoggerAdapter(object): """ An adapter for loggers which makes it easier to specify contextual information in logging output. """ def __init__(self, logger, extra): """ Initialize the adapter with a logger and a dict-like object which provides contextual information. This constructor signature allows easy stacking of LoggerAdapters, if so desired. You can effectively pass keyword arguments as shown in the following example: adapter = LoggerAdapter(someLogger, dict(p1=v1, p2="v2")) """ self.logger = logger self.extra = extra def process(self, msg, kwargs): """ Process the logging message and keyword arguments passed in to a logging call to insert contextual information. You can either manipulate the message itself, the keyword args or both. Return the message and kwargs modified (or not) to suit your needs. Normally, you'll only need to override this one method in a LoggerAdapter subclass for your specific needs. """ kwargs["extra"] = self.extra return msg, kwargs def debug(self, msg, *args, **kwargs): """ Delegate a debug call to the underlying logger, after adding contextual information from this adapter instance. """ msg, kwargs = self.process(msg, kwargs) self.logger.debug(msg, *args, **kwargs) def info(self, msg, *args, **kwargs): """ Delegate an info call to the underlying logger, after adding contextual information from this adapter instance. """ msg, kwargs = self.process(msg, kwargs) self.logger.info(msg, *args, **kwargs) def warning(self, msg, *args, **kwargs): """ Delegate a warning call to the underlying logger, after adding contextual information from this adapter instance. """ msg, kwargs = self.process(msg, kwargs) self.logger.warning(msg, *args, **kwargs) def error(self, msg, *args, **kwargs): """ Delegate an error call to the underlying logger, after adding contextual information from this adapter instance. """ msg, kwargs = self.process(msg, kwargs) self.logger.error(msg, *args, **kwargs) def exception(self, msg, *args, **kwargs): """ Delegate an exception call to the underlying logger, after adding contextual information from this adapter instance. """ msg, kwargs = self.process(msg, kwargs) kwargs["exc_info"] = 1 self.logger.error(msg, *args, **kwargs) def critical(self, msg, *args, **kwargs): """ Delegate a critical call to the underlying logger, after adding contextual information from this adapter instance. """ msg, kwargs = self.process(msg, kwargs) self.logger.critical(msg, *args, **kwargs) def log(self, level, msg, *args, **kwargs): """ Delegate a log call to the underlying logger, after adding contextual information from this adapter instance. """ msg, kwargs = self.process(msg, kwargs) self.logger.log(level, msg, *args, **kwargs) def isEnabledFor(self, level): """ See if the underlying logger is enabled for the specified level. """ return self.logger.isEnabledFor(level) root = RootLogger(WARNING) Logger.root = root Logger.manager = Manager(Logger.root) #--------------------------------------------------------------------------- # Configuration classes and functions #--------------------------------------------------------------------------- BASIC_FORMAT = "%(levelname)s:%(name)s:%(message)s" def basicConfig(**kwargs): """ Do basic configuration for the logging system. This function does nothing if the root logger already has handlers configured. It is a convenience method intended for use by simple scripts to do one-shot configuration of the logging package. The default behaviour is to create a StreamHandler which writes to sys.stderr, set a formatter using the BASIC_FORMAT format string, and add the handler to the root logger. A number of optional keyword arguments may be specified, which can alter the default behaviour. filename Specifies that a FileHandler be created, using the specified filename, rather than a StreamHandler. filemode Specifies the mode to open the file, if filename is specified (if filemode is unspecified, it defaults to 'a'). format Use the specified format string for the handler. datefmt Use the specified date/time format. level Set the root logger level to the specified level. stream Use the specified stream to initialize the StreamHandler. Note that this argument is incompatible with 'filename' - if both are present, 'stream' is ignored. Note that you could specify a stream created using open(filename, mode) rather than passing the filename and mode in. However, it should be remembered that StreamHandler does not close its stream (since it may be using sys.stdout or sys.stderr), whereas FileHandler closes its stream when the handler is closed. """ # Add thread safety in case someone mistakenly calls # basicConfig() from multiple threads _acquireLock() try: if len(root.handlers) == 0: filename = kwargs.get("filename") if filename: mode = kwargs.get("filemode", 'a') hdlr = FileHandler(filename, mode) else: stream = kwargs.get("stream") hdlr = StreamHandler(stream) fs = kwargs.get("format", BASIC_FORMAT) dfs = kwargs.get("datefmt", None) fmt = Formatter(fs, dfs) hdlr.setFormatter(fmt) root.addHandler(hdlr) level = kwargs.get("level") if level is not None: root.setLevel(level) finally: _releaseLock() #--------------------------------------------------------------------------- # Utility functions at module level. # Basically delegate everything to the root logger. #--------------------------------------------------------------------------- def getLogger(name=None): """ Return a logger with the specified name, creating it if necessary. If no name is specified, return the root logger. """ if name: return Logger.manager.getLogger(name) else: return root #def getRootLogger(): # """ # Return the root logger. # # Note that getLogger('') now does the same thing, so this function is # deprecated and may disappear in the future. # """ # return root def critical(msg, *args, **kwargs): """ Log a message with severity 'CRITICAL' on the root logger. """ if len(root.handlers) == 0: basicConfig() root.critical(msg, *args, **kwargs) fatal = critical def error(msg, *args, **kwargs): """ Log a message with severity 'ERROR' on the root logger. """ if len(root.handlers) == 0: basicConfig() root.error(msg, *args, **kwargs) def exception(msg, *args, **kwargs): """ Log a message with severity 'ERROR' on the root logger, with exception information. """ kwargs['exc_info'] = 1 error(msg, *args, **kwargs) def warning(msg, *args, **kwargs): """ Log a message with severity 'WARNING' on the root logger. """ if len(root.handlers) == 0: basicConfig() root.warning(msg, *args, **kwargs) warn = warning def info(msg, *args, **kwargs): """ Log a message with severity 'INFO' on the root logger. """ if len(root.handlers) == 0: basicConfig() root.info(msg, *args, **kwargs) def debug(msg, *args, **kwargs): """ Log a message with severity 'DEBUG' on the root logger. """ if len(root.handlers) == 0: basicConfig() root.debug(msg, *args, **kwargs) def log(level, msg, *args, **kwargs): """ Log 'msg % args' with the integer severity 'level' on the root logger. """ if len(root.handlers) == 0: basicConfig() root.log(level, msg, *args, **kwargs) def disable(level): """ Disable all logging calls of severity 'level' and below. """ root.manager.disable = level def shutdown(handlerList=_handlerList): """ Perform any cleanup actions in the logging system (e.g. flushing buffers). Should be called at application exit. """ for wr in reversed(handlerList[:]): #errors might occur, for example, if files are locked #we just ignore them if raiseExceptions is not set try: h = wr() if h: try: h.acquire() h.flush() h.close() except (IOError, ValueError): # Ignore errors which might be caused # because handlers have been closed but # references to them are still around at # application exit. pass finally: h.release() except: if raiseExceptions: raise #else, swallow #Let's try and shutdown automatically on application exit... import atexit atexit.register(shutdown) # Null handler class NullHandler(Handler): """ This handler does nothing. It's intended to be used to avoid the "No handlers could be found for logger XXX" one-off warning. This is important for library code, which may contain code to log events. If a user of the library does not configure logging, the one-off warning might be produced; to avoid this, the library developer simply needs to instantiate a NullHandler and add it to the top-level logger of the library module or package. """ def handle(self, record): pass def emit(self, record): pass def createLock(self): self.lock = None # Warnings integration _warnings_showwarning = None def _showwarning(message, category, filename, lineno, file=None, line=None): """ Implementation of showwarnings which redirects to logging, which will first check to see if the file parameter is None. If a file is specified, it will delegate to the original warnings implementation of showwarning. Otherwise, it will call warnings.formatwarning and will log the resulting string to a warnings logger named "py.warnings" with level logging.WARNING. """ if file is not None: if _warnings_showwarning is not None: _warnings_showwarning(message, category, filename, lineno, file, line) else: s = warnings.formatwarning(message, category, filename, lineno, line) logger = getLogger("py.warnings") if not logger.handlers: logger.addHandler(NullHandler()) logger.warning("%s", s) def captureWarnings(capture): """ If capture is true, redirect all warnings to the logging package. If capture is False, ensure that warnings are not redirected to logging but to their original destinations. """ global _warnings_showwarning if capture: if _warnings_showwarning is None: _warnings_showwarning = warnings.showwarning warnings.showwarning = _showwarning else: if _warnings_showwarning is not None: warnings.showwarning = _warnings_showwarning _warnings_showwarning = None # Copyright 2001-2014 by Vinay Sajip. All Rights Reserved. # # Permission to use, copy, modify, and distribute this software and its # documentation for any purpose and without fee is hereby granted, # provided that the above copyright notice appear in all copies and that # both that copyright notice and this permission notice appear in # supporting documentation, and that the name of Vinay Sajip # not be used in advertising or publicity pertaining to distribution # of the software without specific, written prior permission. # VINAY SAJIP DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING # ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL # VINAY SAJIP BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR # ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER # IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT # OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. """ Configuration functions for the logging package for Python. The core package is based on PEP 282 and comments thereto in comp.lang.python, and influenced by Apache's log4j system. Copyright (C) 2001-2014 Vinay Sajip. All Rights Reserved. To use, simply 'import logging' and log away! """ import cStringIO import errno import io import logging import logging.handlers import os import re import socket import struct import sys import traceback import types try: import thread import threading except ImportError: thread = None from SocketServer import ThreadingTCPServer, StreamRequestHandler DEFAULT_LOGGING_CONFIG_PORT = 9030 RESET_ERROR = errno.ECONNRESET # # The following code implements a socket listener for on-the-fly # reconfiguration of logging. # # _listener holds the server object doing the listening _listener = None def fileConfig(fname, defaults=None, disable_existing_loggers=True): """ Read the logging configuration from a ConfigParser-format file. This can be called several times from an application, allowing an end user the ability to select from various pre-canned configurations (if the developer provides a mechanism to present the choices and load the chosen configuration). """ import ConfigParser cp = ConfigParser.ConfigParser(defaults) if hasattr(fname, 'readline'): cp.readfp(fname) else: cp.read(fname) formatters = _create_formatters(cp) # critical section logging._acquireLock() try: logging._handlers.clear() del logging._handlerList[:] # Handlers add themselves to logging._handlers handlers = _install_handlers(cp, formatters) _install_loggers(cp, handlers, disable_existing_loggers) finally: logging._releaseLock() def _resolve(name): """Resolve a dotted name to a global object.""" name = name.split('.') used = name.pop(0) found = __import__(used) for n in name: used = used + '.' + n try: found = getattr(found, n) except AttributeError: __import__(used) found = getattr(found, n) return found def _strip_spaces(alist): return map(lambda x: x.strip(), alist) def _encoded(s): return s if isinstance(s, str) else s.encode('utf-8') def _create_formatters(cp): """Create and return formatters""" flist = cp.get("formatters", "keys") if not len(flist): return {} flist = flist.split(",") flist = _strip_spaces(flist) formatters = {} for form in flist: sectname = "formatter_%s" % form opts = cp.options(sectname) if "format" in opts: fs = cp.get(sectname, "format", 1) else: fs = None if "datefmt" in opts: dfs = cp.get(sectname, "datefmt", 1) else: dfs = None c = logging.Formatter if "class" in opts: class_name = cp.get(sectname, "class") if class_name: c = _resolve(class_name) f = c(fs, dfs) formatters[form] = f return formatters def _install_handlers(cp, formatters): """Install and return handlers""" hlist = cp.get("handlers", "keys") if not len(hlist): return {} hlist = hlist.split(",") hlist = _strip_spaces(hlist) handlers = {} fixups = [] #for inter-handler references for hand in hlist: sectname = "handler_%s" % hand klass = cp.get(sectname, "class") opts = cp.options(sectname) if "formatter" in opts: fmt = cp.get(sectname, "formatter") else: fmt = "" try: klass = eval(klass, vars(logging)) except (AttributeError, NameError): klass = _resolve(klass) args = cp.get(sectname, "args") args = eval(args, vars(logging)) h = klass(*args) if "level" in opts: level = cp.get(sectname, "level") h.setLevel(logging._levelNames[level]) if len(fmt): h.setFormatter(formatters[fmt]) if issubclass(klass, logging.handlers.MemoryHandler): if "target" in opts: target = cp.get(sectname,"target") else: target = "" if len(target): #the target handler may not be loaded yet, so keep for later... fixups.append((h, target)) handlers[hand] = h #now all handlers are loaded, fixup inter-handler references... for h, t in fixups: h.setTarget(handlers[t]) return handlers def _install_loggers(cp, handlers, disable_existing_loggers): """Create and install loggers""" # configure the root first llist = cp.get("loggers", "keys") llist = llist.split(",") llist = list(map(lambda x: x.strip(), llist)) llist.remove("root") sectname = "logger_root" root = logging.root log = root opts = cp.options(sectname) if "level" in opts: level = cp.get(sectname, "level") log.setLevel(logging._levelNames[level]) for h in root.handlers[:]: root.removeHandler(h) hlist = cp.get(sectname, "handlers") if len(hlist): hlist = hlist.split(",") hlist = _strip_spaces(hlist) for hand in hlist: log.addHandler(handlers[hand]) #and now the others... #we don't want to lose the existing loggers, #since other threads may have pointers to them. #existing is set to contain all existing loggers, #and as we go through the new configuration we #remove any which are configured. At the end, #what's left in existing is the set of loggers #which were in the previous configuration but #which are not in the new configuration. existing = list(root.manager.loggerDict.keys()) #The list needs to be sorted so that we can #avoid disabling child loggers of explicitly #named loggers. With a sorted list it is easier #to find the child loggers. existing.sort() #We'll keep the list of existing loggers #which are children of named loggers here... child_loggers = [] #now set up the new ones... for log in llist: sectname = "logger_%s" % log qn = cp.get(sectname, "qualname") opts = cp.options(sectname) if "propagate" in opts: propagate = cp.getint(sectname, "propagate") else: propagate = 1 logger = logging.getLogger(qn) if qn in existing: i = existing.index(qn) + 1 # start with the entry after qn prefixed = qn + "." pflen = len(prefixed) num_existing = len(existing) while i < num_existing: if existing[i][:pflen] == prefixed: child_loggers.append(existing[i]) i += 1 existing.remove(qn) if "level" in opts: level = cp.get(sectname, "level") logger.setLevel(logging._levelNames[level]) for h in logger.handlers[:]: logger.removeHandler(h) logger.propagate = propagate logger.disabled = 0 hlist = cp.get(sectname, "handlers") if len(hlist): hlist = hlist.split(",") hlist = _strip_spaces(hlist) for hand in hlist: logger.addHandler(handlers[hand]) #Disable any old loggers. There's no point deleting #them as other threads may continue to hold references #and by disabling them, you stop them doing any logging. #However, don't disable children of named loggers, as that's #probably not what was intended by the user. for log in existing: logger = root.manager.loggerDict[log] if log in child_loggers: logger.level = logging.NOTSET logger.handlers = [] logger.propagate = 1 else: logger.disabled = disable_existing_loggers IDENTIFIER = re.compile('^[a-z_][a-z0-9_]*$', re.I) def valid_ident(s): m = IDENTIFIER.match(s) if not m: raise ValueError('Not a valid Python identifier: %r' % s) return True class ConvertingMixin(object): """For ConvertingXXX's, this mixin class provides common functions""" def convert_with_key(self, key, value, replace=True): result = self.configurator.convert(value) #If the converted value is different, save for next time if value is not result: if replace: self[key] = result if type(result) in (ConvertingDict, ConvertingList, ConvertingTuple): result.parent = self result.key = key return result def convert(self, value): result = self.configurator.convert(value) if value is not result: if type(result) in (ConvertingDict, ConvertingList, ConvertingTuple): result.parent = self return result # The ConvertingXXX classes are wrappers around standard Python containers, # and they serve to convert any suitable values in the container. The # conversion converts base dicts, lists and tuples to their wrapped # equivalents, whereas strings which match a conversion format are converted # appropriately. # # Each wrapper should have a configurator attribute holding the actual # configurator to use for conversion. class ConvertingDict(dict, ConvertingMixin): """A converting dictionary wrapper.""" def __getitem__(self, key): value = dict.__getitem__(self, key) return self.convert_with_key(key, value) def get(self, key, default=None): value = dict.get(self, key, default) return self.convert_with_key(key, value) def pop(self, key, default=None): value = dict.pop(self, key, default) return self.convert_with_key(key, value, replace=False) class ConvertingList(list, ConvertingMixin): """A converting list wrapper.""" def __getitem__(self, key): value = list.__getitem__(self, key) return self.convert_with_key(key, value) def pop(self, idx=-1): value = list.pop(self, idx) return self.convert(value) class ConvertingTuple(tuple, ConvertingMixin): """A converting tuple wrapper.""" def __getitem__(self, key): value = tuple.__getitem__(self, key) # Can't replace a tuple entry. return self.convert_with_key(key, value, replace=False) class BaseConfigurator(object): """ The configurator base class which defines some useful defaults. """ CONVERT_PATTERN = re.compile(r'^(?P[a-z]+)://(?P.*)$') WORD_PATTERN = re.compile(r'^\s*(\w+)\s*') DOT_PATTERN = re.compile(r'^\.\s*(\w+)\s*') INDEX_PATTERN = re.compile(r'^\[\s*(\w+)\s*\]\s*') DIGIT_PATTERN = re.compile(r'^\d+$') value_converters = { 'ext' : 'ext_convert', 'cfg' : 'cfg_convert', } # We might want to use a different one, e.g. importlib importer = __import__ def __init__(self, config): self.config = ConvertingDict(config) self.config.configurator = self # Issue 12718: winpdb replaces __import__ with a Python function, which # ends up being treated as a bound method. To avoid problems, we # set the importer on the instance, but leave it defined in the class # so existing code doesn't break if type(__import__) == types.FunctionType: self.importer = __import__ def resolve(self, s): """ Resolve strings to objects using standard import and attribute syntax. """ name = s.split('.') used = name.pop(0) try: found = self.importer(used) for frag in name: used += '.' + frag try: found = getattr(found, frag) except AttributeError: self.importer(used) found = getattr(found, frag) return found except ImportError: e, tb = sys.exc_info()[1:] v = ValueError('Cannot resolve %r: %s' % (s, e)) v.__cause__, v.__traceback__ = e, tb raise v def ext_convert(self, value): """Default converter for the ext:// protocol.""" return self.resolve(value) def cfg_convert(self, value): """Default converter for the cfg:// protocol.""" rest = value m = self.WORD_PATTERN.match(rest) if m is None: raise ValueError("Unable to convert %r" % value) else: rest = rest[m.end():] d = self.config[m.groups()[0]] #print d, rest while rest: m = self.DOT_PATTERN.match(rest) if m: d = d[m.groups()[0]] else: m = self.INDEX_PATTERN.match(rest) if m: idx = m.groups()[0] if not self.DIGIT_PATTERN.match(idx): d = d[idx] else: try: n = int(idx) # try as number first (most likely) d = d[n] except TypeError: d = d[idx] if m: rest = rest[m.end():] else: raise ValueError('Unable to convert ' '%r at %r' % (value, rest)) #rest should be empty return d def convert(self, value): """ Convert values to an appropriate type. dicts, lists and tuples are replaced by their converting alternatives. Strings are checked to see if they have a conversion format and are converted if they do. """ if not isinstance(value, ConvertingDict) and isinstance(value, dict): value = ConvertingDict(value) value.configurator = self elif not isinstance(value, ConvertingList) and isinstance(value, list): value = ConvertingList(value) value.configurator = self elif not isinstance(value, ConvertingTuple) and\ isinstance(value, tuple): value = ConvertingTuple(value) value.configurator = self elif isinstance(value, basestring): # str for py3k m = self.CONVERT_PATTERN.match(value) if m: d = m.groupdict() prefix = d['prefix'] converter = self.value_converters.get(prefix, None) if converter: suffix = d['suffix'] converter = getattr(self, converter) value = converter(suffix) return value def configure_custom(self, config): """Configure an object with a user-supplied factory.""" c = config.pop('()') if not hasattr(c, '__call__') and hasattr(types, 'ClassType') and type(c) != types.ClassType: c = self.resolve(c) props = config.pop('.', None) # Check for valid identifiers kwargs = dict([(k, config[k]) for k in config if valid_ident(k)]) result = c(**kwargs) if props: for name, value in props.items(): setattr(result, name, value) return result def as_tuple(self, value): """Utility function which converts lists to tuples.""" if isinstance(value, list): value = tuple(value) return value class DictConfigurator(BaseConfigurator): """ Configure logging using a dictionary-like object to describe the configuration. """ def configure(self): """Do the configuration.""" config = self.config if 'version' not in config: raise ValueError("dictionary doesn't specify a version") if config['version'] != 1: raise ValueError("Unsupported version: %s" % config['version']) incremental = config.pop('incremental', False) EMPTY_DICT = {} logging._acquireLock() try: if incremental: handlers = config.get('handlers', EMPTY_DICT) for name in handlers: if name not in logging._handlers: raise ValueError('No handler found with ' 'name %r' % name) else: try: handler = logging._handlers[name] handler_config = handlers[name] level = handler_config.get('level', None) if level: handler.setLevel(logging._checkLevel(level)) except StandardError as e: raise ValueError('Unable to configure handler ' '%r: %s' % (name, e)) loggers = config.get('loggers', EMPTY_DICT) for name in loggers: try: self.configure_logger(name, loggers[name], True) except StandardError as e: raise ValueError('Unable to configure logger ' '%r: %s' % (name, e)) root = config.get('root', None) if root: try: self.configure_root(root, True) except StandardError as e: raise ValueError('Unable to configure root ' 'logger: %s' % e) else: disable_existing = config.pop('disable_existing_loggers', True) logging._handlers.clear() del logging._handlerList[:] # Do formatters first - they don't refer to anything else formatters = config.get('formatters', EMPTY_DICT) for name in formatters: try: formatters[name] = self.configure_formatter( formatters[name]) except StandardError as e: raise ValueError('Unable to configure ' 'formatter %r: %s' % (name, e)) # Next, do filters - they don't refer to anything else, either filters = config.get('filters', EMPTY_DICT) for name in filters: try: filters[name] = self.configure_filter(filters[name]) except StandardError as e: raise ValueError('Unable to configure ' 'filter %r: %s' % (name, e)) # Next, do handlers - they refer to formatters and filters # As handlers can refer to other handlers, sort the keys # to allow a deterministic order of configuration handlers = config.get('handlers', EMPTY_DICT) deferred = [] for name in sorted(handlers): try: handler = self.configure_handler(handlers[name]) handler.name = name handlers[name] = handler except StandardError as e: if 'target not configured yet' in str(e): deferred.append(name) else: raise ValueError('Unable to configure handler ' '%r: %s' % (name, e)) # Now do any that were deferred for name in deferred: try: handler = self.configure_handler(handlers[name]) handler.name = name handlers[name] = handler except StandardError as e: raise ValueError('Unable to configure handler ' '%r: %s' % (name, e)) # Next, do loggers - they refer to handlers and filters #we don't want to lose the existing loggers, #since other threads may have pointers to them. #existing is set to contain all existing loggers, #and as we go through the new configuration we #remove any which are configured. At the end, #what's left in existing is the set of loggers #which were in the previous configuration but #which are not in the new configuration. root = logging.root existing = root.manager.loggerDict.keys() #The list needs to be sorted so that we can #avoid disabling child loggers of explicitly #named loggers. With a sorted list it is easier #to find the child loggers. existing.sort() #We'll keep the list of existing loggers #which are children of named loggers here... child_loggers = [] #now set up the new ones... loggers = config.get('loggers', EMPTY_DICT) for name in loggers: name = _encoded(name) if name in existing: i = existing.index(name) prefixed = name + "." pflen = len(prefixed) num_existing = len(existing) i = i + 1 # look at the entry after name while (i < num_existing) and\ (existing[i][:pflen] == prefixed): child_loggers.append(existing[i]) i = i + 1 existing.remove(name) try: self.configure_logger(name, loggers[name]) except StandardError as e: raise ValueError('Unable to configure logger ' '%r: %s' % (name, e)) #Disable any old loggers. There's no point deleting #them as other threads may continue to hold references #and by disabling them, you stop them doing any logging. #However, don't disable children of named loggers, as that's #probably not what was intended by the user. for log in existing: logger = root.manager.loggerDict[log] if log in child_loggers: logger.level = logging.NOTSET logger.handlers = [] logger.propagate = True elif disable_existing: logger.disabled = True # And finally, do the root logger root = config.get('root', None) if root: try: self.configure_root(root) except StandardError as e: raise ValueError('Unable to configure root ' 'logger: %s' % e) finally: logging._releaseLock() def configure_formatter(self, config): """Configure a formatter from a dictionary.""" if '()' in config: factory = config['()'] # for use in exception handler try: result = self.configure_custom(config) except TypeError as te: if "'format'" not in str(te): raise #Name of parameter changed from fmt to format. #Retry with old name. #This is so that code can be used with older Python versions #(e.g. by Django) config['fmt'] = config.pop('format') config['()'] = factory result = self.configure_custom(config) else: fmt = config.get('format', None) dfmt = config.get('datefmt', None) result = logging.Formatter(fmt, dfmt) return result def configure_filter(self, config): """Configure a filter from a dictionary.""" if '()' in config: result = self.configure_custom(config) else: name = config.get('name', '') result = logging.Filter(name) return result def add_filters(self, filterer, filters): """Add filters to a filterer from a list of names.""" for f in filters: try: filterer.addFilter(self.config['filters'][f]) except StandardError as e: raise ValueError('Unable to add filter %r: %s' % (f, e)) def configure_handler(self, config): """Configure a handler from a dictionary.""" formatter = config.pop('formatter', None) if formatter: try: formatter = self.config['formatters'][formatter] except StandardError as e: raise ValueError('Unable to set formatter ' '%r: %s' % (formatter, e)) level = config.pop('level', None) filters = config.pop('filters', None) if '()' in config: c = config.pop('()') if not hasattr(c, '__call__') and hasattr(types, 'ClassType') and type(c) != types.ClassType: c = self.resolve(c) factory = c else: cname = config.pop('class') klass = self.resolve(cname) #Special case for handler which refers to another handler if issubclass(klass, logging.handlers.MemoryHandler) and\ 'target' in config: try: th = self.config['handlers'][config['target']] if not isinstance(th, logging.Handler): config['class'] = cname # restore for deferred configuration raise StandardError('target not configured yet') config['target'] = th except StandardError as e: raise ValueError('Unable to set target handler ' '%r: %s' % (config['target'], e)) elif issubclass(klass, logging.handlers.SMTPHandler) and\ 'mailhost' in config: config['mailhost'] = self.as_tuple(config['mailhost']) elif issubclass(klass, logging.handlers.SysLogHandler) and\ 'address' in config: config['address'] = self.as_tuple(config['address']) factory = klass kwargs = dict([(k, config[k]) for k in config if valid_ident(k)]) try: result = factory(**kwargs) except TypeError as te: if "'stream'" not in str(te): raise #The argument name changed from strm to stream #Retry with old name. #This is so that code can be used with older Python versions #(e.g. by Django) kwargs['strm'] = kwargs.pop('stream') result = factory(**kwargs) if formatter: result.setFormatter(formatter) if level is not None: result.setLevel(logging._checkLevel(level)) if filters: self.add_filters(result, filters) return result def add_handlers(self, logger, handlers): """Add handlers to a logger from a list of names.""" for h in handlers: try: logger.addHandler(self.config['handlers'][h]) except StandardError as e: raise ValueError('Unable to add handler %r: %s' % (h, e)) def common_logger_config(self, logger, config, incremental=False): """ Perform configuration which is common to root and non-root loggers. """ level = config.get('level', None) if level is not None: logger.setLevel(logging._checkLevel(level)) if not incremental: #Remove any existing handlers for h in logger.handlers[:]: logger.removeHandler(h) handlers = config.get('handlers', None) if handlers: self.add_handlers(logger, handlers) filters = config.get('filters', None) if filters: self.add_filters(logger, filters) def configure_logger(self, name, config, incremental=False): """Configure a non-root logger from a dictionary.""" logger = logging.getLogger(name) self.common_logger_config(logger, config, incremental) propagate = config.get('propagate', None) if propagate is not None: logger.propagate = propagate def configure_root(self, config, incremental=False): """Configure a root logger from a dictionary.""" root = logging.getLogger() self.common_logger_config(root, config, incremental) dictConfigClass = DictConfigurator def dictConfig(config): """Configure logging using a dictionary.""" dictConfigClass(config).configure() def listen(port=DEFAULT_LOGGING_CONFIG_PORT): """ Start up a socket server on the specified port, and listen for new configurations. These will be sent as a file suitable for processing by fileConfig(). Returns a Thread object on which you can call start() to start the server, and which you can join() when appropriate. To stop the server, call stopListening(). """ if not thread: raise NotImplementedError("listen() needs threading to work") class ConfigStreamHandler(StreamRequestHandler): """ Handler for a logging configuration request. It expects a completely new logging configuration and uses fileConfig to install it. """ def handle(self): """ Handle a request. Each request is expected to be a 4-byte length, packed using struct.pack(">L", n), followed by the config file. Uses fileConfig() to do the grunt work. """ import tempfile try: conn = self.connection chunk = conn.recv(4) if len(chunk) == 4: slen = struct.unpack(">L", chunk)[0] chunk = self.connection.recv(slen) while len(chunk) < slen: chunk = chunk + conn.recv(slen - len(chunk)) try: import json d =json.loads(chunk) assert isinstance(d, dict) dictConfig(d) except: #Apply new configuration. file = cStringIO.StringIO(chunk) try: fileConfig(file) except (KeyboardInterrupt, SystemExit): raise except: traceback.print_exc() if self.server.ready: self.server.ready.set() except socket.error as e: if e.errno != RESET_ERROR: raise class ConfigSocketReceiver(ThreadingTCPServer): """ A simple TCP socket-based logging config receiver. """ allow_reuse_address = 1 def __init__(self, host='localhost', port=DEFAULT_LOGGING_CONFIG_PORT, handler=None, ready=None): ThreadingTCPServer.__init__(self, (host, port), handler) logging._acquireLock() self.abort = 0 logging._releaseLock() self.timeout = 1 self.ready = ready def serve_until_stopped(self): import select abort = 0 while not abort: rd, wr, ex = select.select([self.socket.fileno()], [], [], self.timeout) if rd: self.handle_request() logging._acquireLock() abort = self.abort logging._releaseLock() self.socket.close() class Server(threading.Thread): def __init__(self, rcvr, hdlr, port): super(Server, self).__init__() self.rcvr = rcvr self.hdlr = hdlr self.port = port self.ready = threading.Event() def run(self): server = self.rcvr(port=self.port, handler=self.hdlr, ready=self.ready) if self.port == 0: self.port = server.server_address[1] self.ready.set() global _listener logging._acquireLock() _listener = server logging._releaseLock() server.serve_until_stopped() return Server(ConfigSocketReceiver, ConfigStreamHandler, port) def stopListening(): """ Stop the listening server which was created with a call to listen(). """ global _listener logging._acquireLock() try: if _listener: _listener.abort = 1 _listener = None finally: logging._releaseLock() # Copyright 2001-2013 by Vinay Sajip. All Rights Reserved. # # Permission to use, copy, modify, and distribute this software and its # documentation for any purpose and without fee is hereby granted, # provided that the above copyright notice appear in all copies and that # both that copyright notice and this permission notice appear in # supporting documentation, and that the name of Vinay Sajip # not be used in advertising or publicity pertaining to distribution # of the software without specific, written prior permission. # VINAY SAJIP DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING # ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL # VINAY SAJIP BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR # ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER # IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT # OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. """ Additional handlers for the logging package for Python. The core package is based on PEP 282 and comments thereto in comp.lang.python. Copyright (C) 2001-2013 Vinay Sajip. All Rights Reserved. To use, simply 'import logging.handlers' and log away! """ import errno, logging, socket, os, cPickle, struct, time, re from stat import ST_DEV, ST_INO, ST_MTIME try: import codecs except ImportError: codecs = None try: unicode _unicode = True except NameError: _unicode = False # # Some constants... # DEFAULT_TCP_LOGGING_PORT = 9020 DEFAULT_UDP_LOGGING_PORT = 9021 DEFAULT_HTTP_LOGGING_PORT = 9022 DEFAULT_SOAP_LOGGING_PORT = 9023 SYSLOG_UDP_PORT = 514 SYSLOG_TCP_PORT = 514 _MIDNIGHT = 24 * 60 * 60 # number of seconds in a day class BaseRotatingHandler(logging.FileHandler): """ Base class for handlers that rotate log files at a certain point. Not meant to be instantiated directly. Instead, use RotatingFileHandler or TimedRotatingFileHandler. """ def __init__(self, filename, mode, encoding=None, delay=0): """ Use the specified filename for streamed logging """ if codecs is None: encoding = None logging.FileHandler.__init__(self, filename, mode, encoding, delay) self.mode = mode self.encoding = encoding def emit(self, record): """ Emit a record. Output the record to the file, catering for rollover as described in doRollover(). """ try: if self.shouldRollover(record): self.doRollover() logging.FileHandler.emit(self, record) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) class RotatingFileHandler(BaseRotatingHandler): """ Handler for logging to a set of files, which switches from one file to the next when the current file reaches a certain size. """ def __init__(self, filename, mode='a', maxBytes=0, backupCount=0, encoding=None, delay=0): """ Open the specified file and use it as the stream for logging. By default, the file grows indefinitely. You can specify particular values of maxBytes and backupCount to allow the file to rollover at a predetermined size. Rollover occurs whenever the current log file is nearly maxBytes in length. If backupCount is >= 1, the system will successively create new files with the same pathname as the base file, but with extensions ".1", ".2" etc. appended to it. For example, with a backupCount of 5 and a base file name of "app.log", you would get "app.log", "app.log.1", "app.log.2", ... through to "app.log.5". The file being written to is always "app.log" - when it gets filled up, it is closed and renamed to "app.log.1", and if files "app.log.1", "app.log.2" etc. exist, then they are renamed to "app.log.2", "app.log.3" etc. respectively. If maxBytes is zero, rollover never occurs. """ # If rotation/rollover is wanted, it doesn't make sense to use another # mode. If for example 'w' were specified, then if there were multiple # runs of the calling application, the logs from previous runs would be # lost if the 'w' is respected, because the log file would be truncated # on each run. if maxBytes > 0: mode = 'a' BaseRotatingHandler.__init__(self, filename, mode, encoding, delay) self.maxBytes = maxBytes self.backupCount = backupCount def doRollover(self): """ Do a rollover, as described in __init__(). """ if self.stream: self.stream.close() self.stream = None if self.backupCount > 0: for i in range(self.backupCount - 1, 0, -1): sfn = "%s.%d" % (self.baseFilename, i) dfn = "%s.%d" % (self.baseFilename, i + 1) if os.path.exists(sfn): #print "%s -> %s" % (sfn, dfn) if os.path.exists(dfn): os.remove(dfn) os.rename(sfn, dfn) dfn = self.baseFilename + ".1" if os.path.exists(dfn): os.remove(dfn) # Issue 18940: A file may not have been created if delay is True. if os.path.exists(self.baseFilename): os.rename(self.baseFilename, dfn) if not self.delay: self.stream = self._open() def shouldRollover(self, record): """ Determine if rollover should occur. Basically, see if the supplied record would cause the file to exceed the size limit we have. """ if self.stream is None: # delay was set... self.stream = self._open() if self.maxBytes > 0: # are we rolling over? msg = "%s\n" % self.format(record) self.stream.seek(0, 2) #due to non-posix-compliant Windows feature if self.stream.tell() + len(msg) >= self.maxBytes: return 1 return 0 class TimedRotatingFileHandler(BaseRotatingHandler): """ Handler for logging to a file, rotating the log file at certain timed intervals. If backupCount is > 0, when rollover is done, no more than backupCount files are kept - the oldest ones are deleted. """ def __init__(self, filename, when='h', interval=1, backupCount=0, encoding=None, delay=False, utc=False): BaseRotatingHandler.__init__(self, filename, 'a', encoding, delay) self.when = when.upper() self.backupCount = backupCount self.utc = utc # Calculate the real rollover interval, which is just the number of # seconds between rollovers. Also set the filename suffix used when # a rollover occurs. Current 'when' events supported: # S - Seconds # M - Minutes # H - Hours # D - Days # midnight - roll over at midnight # W{0-6} - roll over on a certain day; 0 - Monday # # Case of the 'when' specifier is not important; lower or upper case # will work. if self.when == 'S': self.interval = 1 # one second self.suffix = "%Y-%m-%d_%H-%M-%S" self.extMatch = r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}-\d{2}$" elif self.when == 'M': self.interval = 60 # one minute self.suffix = "%Y-%m-%d_%H-%M" self.extMatch = r"^\d{4}-\d{2}-\d{2}_\d{2}-\d{2}$" elif self.when == 'H': self.interval = 60 * 60 # one hour self.suffix = "%Y-%m-%d_%H" self.extMatch = r"^\d{4}-\d{2}-\d{2}_\d{2}$" elif self.when == 'D' or self.when == 'MIDNIGHT': self.interval = 60 * 60 * 24 # one day self.suffix = "%Y-%m-%d" self.extMatch = r"^\d{4}-\d{2}-\d{2}$" elif self.when.startswith('W'): self.interval = 60 * 60 * 24 * 7 # one week if len(self.when) != 2: raise ValueError("You must specify a day for weekly rollover from 0 to 6 (0 is Monday): %s" % self.when) if self.when[1] < '0' or self.when[1] > '6': raise ValueError("Invalid day specified for weekly rollover: %s" % self.when) self.dayOfWeek = int(self.when[1]) self.suffix = "%Y-%m-%d" self.extMatch = r"^\d{4}-\d{2}-\d{2}$" else: raise ValueError("Invalid rollover interval specified: %s" % self.when) self.extMatch = re.compile(self.extMatch) self.interval = self.interval * interval # multiply by units requested if os.path.exists(filename): t = os.stat(filename)[ST_MTIME] else: t = int(time.time()) self.rolloverAt = self.computeRollover(t) def computeRollover(self, currentTime): """ Work out the rollover time based on the specified time. """ result = currentTime + self.interval # If we are rolling over at midnight or weekly, then the interval is already known. # What we need to figure out is WHEN the next interval is. In other words, # if you are rolling over at midnight, then your base interval is 1 day, # but you want to start that one day clock at midnight, not now. So, we # have to fudge the rolloverAt value in order to trigger the first rollover # at the right time. After that, the regular interval will take care of # the rest. Note that this code doesn't care about leap seconds. :) if self.when == 'MIDNIGHT' or self.when.startswith('W'): # This could be done with less code, but I wanted it to be clear if self.utc: t = time.gmtime(currentTime) else: t = time.localtime(currentTime) currentHour = t[3] currentMinute = t[4] currentSecond = t[5] # r is the number of seconds left between now and midnight r = _MIDNIGHT - ((currentHour * 60 + currentMinute) * 60 + currentSecond) result = currentTime + r # If we are rolling over on a certain day, add in the number of days until # the next rollover, but offset by 1 since we just calculated the time # until the next day starts. There are three cases: # Case 1) The day to rollover is today; in this case, do nothing # Case 2) The day to rollover is further in the interval (i.e., today is # day 2 (Wednesday) and rollover is on day 6 (Sunday). Days to # next rollover is simply 6 - 2 - 1, or 3. # Case 3) The day to rollover is behind us in the interval (i.e., today # is day 5 (Saturday) and rollover is on day 3 (Thursday). # Days to rollover is 6 - 5 + 3, or 4. In this case, it's the # number of days left in the current week (1) plus the number # of days in the next week until the rollover day (3). # The calculations described in 2) and 3) above need to have a day added. # This is because the above time calculation takes us to midnight on this # day, i.e. the start of the next day. if self.when.startswith('W'): day = t[6] # 0 is Monday if day != self.dayOfWeek: if day < self.dayOfWeek: daysToWait = self.dayOfWeek - day else: daysToWait = 6 - day + self.dayOfWeek + 1 newRolloverAt = result + (daysToWait * (60 * 60 * 24)) if not self.utc: dstNow = t[-1] dstAtRollover = time.localtime(newRolloverAt)[-1] if dstNow != dstAtRollover: if not dstNow: # DST kicks in before next rollover, so we need to deduct an hour addend = -3600 else: # DST bows out before next rollover, so we need to add an hour addend = 3600 newRolloverAt += addend result = newRolloverAt return result def shouldRollover(self, record): """ Determine if rollover should occur. record is not used, as we are just comparing times, but it is needed so the method signatures are the same """ t = int(time.time()) if t >= self.rolloverAt: return 1 #print "No need to rollover: %d, %d" % (t, self.rolloverAt) return 0 def getFilesToDelete(self): """ Determine the files to delete when rolling over. More specific than the earlier method, which just used glob.glob(). """ dirName, baseName = os.path.split(self.baseFilename) fileNames = os.listdir(dirName) result = [] prefix = baseName + "." plen = len(prefix) for fileName in fileNames: if fileName[:plen] == prefix: suffix = fileName[plen:] if self.extMatch.match(suffix): result.append(os.path.join(dirName, fileName)) result.sort() if len(result) < self.backupCount: result = [] else: result = result[:len(result) - self.backupCount] return result def doRollover(self): """ do a rollover; in this case, a date/time stamp is appended to the filename when the rollover happens. However, you want the file to be named for the start of the interval, not the current time. If there is a backup count, then we have to get a list of matching filenames, sort them and remove the one with the oldest suffix. """ if self.stream: self.stream.close() self.stream = None # get the time that this sequence started at and make it a TimeTuple currentTime = int(time.time()) dstNow = time.localtime(currentTime)[-1] t = self.rolloverAt - self.interval if self.utc: timeTuple = time.gmtime(t) else: timeTuple = time.localtime(t) dstThen = timeTuple[-1] if dstNow != dstThen: if dstNow: addend = 3600 else: addend = -3600 timeTuple = time.localtime(t + addend) dfn = self.baseFilename + "." + time.strftime(self.suffix, timeTuple) if os.path.exists(dfn): os.remove(dfn) # Issue 18940: A file may not have been created if delay is True. if os.path.exists(self.baseFilename): os.rename(self.baseFilename, dfn) if self.backupCount > 0: for s in self.getFilesToDelete(): os.remove(s) if not self.delay: self.stream = self._open() newRolloverAt = self.computeRollover(currentTime) while newRolloverAt <= currentTime: newRolloverAt = newRolloverAt + self.interval #If DST changes and midnight or weekly rollover, adjust for this. if (self.when == 'MIDNIGHT' or self.when.startswith('W')) and not self.utc: dstAtRollover = time.localtime(newRolloverAt)[-1] if dstNow != dstAtRollover: if not dstNow: # DST kicks in before next rollover, so we need to deduct an hour addend = -3600 else: # DST bows out before next rollover, so we need to add an hour addend = 3600 newRolloverAt += addend self.rolloverAt = newRolloverAt class WatchedFileHandler(logging.FileHandler): """ A handler for logging to a file, which watches the file to see if it has changed while in use. This can happen because of usage of programs such as newsyslog and logrotate which perform log file rotation. This handler, intended for use under Unix, watches the file to see if it has changed since the last emit. (A file has changed if its device or inode have changed.) If it has changed, the old file stream is closed, and the file opened to get a new stream. This handler is not appropriate for use under Windows, because under Windows open files cannot be moved or renamed - logging opens the files with exclusive locks - and so there is no need for such a handler. Furthermore, ST_INO is not supported under Windows; stat always returns zero for this value. This handler is based on a suggestion and patch by Chad J. Schroeder. """ def __init__(self, filename, mode='a', encoding=None, delay=0): logging.FileHandler.__init__(self, filename, mode, encoding, delay) self.dev, self.ino = -1, -1 self._statstream() def _statstream(self): if self.stream: sres = os.fstat(self.stream.fileno()) self.dev, self.ino = sres[ST_DEV], sres[ST_INO] def emit(self, record): """ Emit a record. First check if the underlying file has changed, and if it has, close the old stream and reopen the file to get the current stream. """ # Reduce the chance of race conditions by stat'ing by path only # once and then fstat'ing our new fd if we opened a new log stream. # See issue #14632: Thanks to John Mulligan for the problem report # and patch. try: # stat the file by path, checking for existence sres = os.stat(self.baseFilename) except OSError as err: if err.errno == errno.ENOENT: sres = None else: raise # compare file system stat with that of our stream file handle if not sres or sres[ST_DEV] != self.dev or sres[ST_INO] != self.ino: if self.stream is not None: # we have an open file handle, clean it up self.stream.flush() self.stream.close() self.stream = None # See Issue #21742: _open () might fail. # open a new file handle and get new stat info from that fd self.stream = self._open() self._statstream() logging.FileHandler.emit(self, record) class SocketHandler(logging.Handler): """ A handler class which writes logging records, in pickle format, to a streaming socket. The socket is kept open across logging calls. If the peer resets it, an attempt is made to reconnect on the next call. The pickle which is sent is that of the LogRecord's attribute dictionary (__dict__), so that the receiver does not need to have the logging module installed in order to process the logging event. To unpickle the record at the receiving end into a LogRecord, use the makeLogRecord function. """ def __init__(self, host, port): """ Initializes the handler with a specific host address and port. The attribute 'closeOnError' is set to 1 - which means that if a socket error occurs, the socket is silently closed and then reopened on the next logging call. """ logging.Handler.__init__(self) self.host = host self.port = port self.sock = None self.closeOnError = 0 self.retryTime = None # # Exponential backoff parameters. # self.retryStart = 1.0 self.retryMax = 30.0 self.retryFactor = 2.0 def makeSocket(self, timeout=1): """ A factory method which allows subclasses to define the precise type of socket they want. """ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) if hasattr(s, 'settimeout'): s.settimeout(timeout) s.connect((self.host, self.port)) return s def createSocket(self): """ Try to create a socket, using an exponential backoff with a max retry time. Thanks to Robert Olson for the original patch (SF #815911) which has been slightly refactored. """ now = time.time() # Either retryTime is None, in which case this # is the first time back after a disconnect, or # we've waited long enough. if self.retryTime is None: attempt = 1 else: attempt = (now >= self.retryTime) if attempt: try: self.sock = self.makeSocket() self.retryTime = None # next time, no delay before trying except socket.error: #Creation failed, so set the retry time and return. if self.retryTime is None: self.retryPeriod = self.retryStart else: self.retryPeriod = self.retryPeriod * self.retryFactor if self.retryPeriod > self.retryMax: self.retryPeriod = self.retryMax self.retryTime = now + self.retryPeriod def send(self, s): """ Send a pickled string to the socket. This function allows for partial sends which can happen when the network is busy. """ if self.sock is None: self.createSocket() #self.sock can be None either because we haven't reached the retry #time yet, or because we have reached the retry time and retried, #but are still unable to connect. if self.sock: try: if hasattr(self.sock, "sendall"): self.sock.sendall(s) else: sentsofar = 0 left = len(s) while left > 0: sent = self.sock.send(s[sentsofar:]) sentsofar = sentsofar + sent left = left - sent except socket.error: self.sock.close() self.sock = None # so we can call createSocket next time def makePickle(self, record): """ Pickles the record in binary format with a length prefix, and returns it ready for transmission across the socket. """ ei = record.exc_info if ei: # just to get traceback text into record.exc_text ... dummy = self.format(record) record.exc_info = None # to avoid Unpickleable error # See issue #14436: If msg or args are objects, they may not be # available on the receiving end. So we convert the msg % args # to a string, save it as msg and zap the args. d = dict(record.__dict__) d['msg'] = record.getMessage() d['args'] = None s = cPickle.dumps(d, 1) if ei: record.exc_info = ei # for next handler slen = struct.pack(">L", len(s)) return slen + s def handleError(self, record): """ Handle an error during logging. An error has occurred during logging. Most likely cause - connection lost. Close the socket so that we can retry on the next event. """ if self.closeOnError and self.sock: self.sock.close() self.sock = None #try to reconnect next time else: logging.Handler.handleError(self, record) def emit(self, record): """ Emit a record. Pickles the record and writes it to the socket in binary format. If there is an error with the socket, silently drop the packet. If there was a problem with the socket, re-establishes the socket. """ try: s = self.makePickle(record) self.send(s) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) def close(self): """ Closes the socket. """ self.acquire() try: sock = self.sock if sock: self.sock = None sock.close() finally: self.release() logging.Handler.close(self) class DatagramHandler(SocketHandler): """ A handler class which writes logging records, in pickle format, to a datagram socket. The pickle which is sent is that of the LogRecord's attribute dictionary (__dict__), so that the receiver does not need to have the logging module installed in order to process the logging event. To unpickle the record at the receiving end into a LogRecord, use the makeLogRecord function. """ def __init__(self, host, port): """ Initializes the handler with a specific host address and port. """ SocketHandler.__init__(self, host, port) self.closeOnError = 0 def makeSocket(self): """ The factory method of SocketHandler is here overridden to create a UDP socket (SOCK_DGRAM). """ s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) return s def send(self, s): """ Send a pickled string to a socket. This function no longer allows for partial sends which can happen when the network is busy - UDP does not guarantee delivery and can deliver packets out of sequence. """ if self.sock is None: self.createSocket() self.sock.sendto(s, (self.host, self.port)) class SysLogHandler(logging.Handler): """ A handler class which sends formatted logging records to a syslog server. Based on Sam Rushing's syslog module: http://www.nightmare.com/squirl/python-ext/misc/syslog.py Contributed by Nicolas Untz (after which minor refactoring changes have been made). """ # from : # ====================================================================== # priorities/facilities are encoded into a single 32-bit quantity, where # the bottom 3 bits are the priority (0-7) and the top 28 bits are the # facility (0-big number). Both the priorities and the facilities map # roughly one-to-one to strings in the syslogd(8) source code. This # mapping is included in this file. # # priorities (these are ordered) LOG_EMERG = 0 # system is unusable LOG_ALERT = 1 # action must be taken immediately LOG_CRIT = 2 # critical conditions LOG_ERR = 3 # error conditions LOG_WARNING = 4 # warning conditions LOG_NOTICE = 5 # normal but significant condition LOG_INFO = 6 # informational LOG_DEBUG = 7 # debug-level messages # facility codes LOG_KERN = 0 # kernel messages LOG_USER = 1 # random user-level messages LOG_MAIL = 2 # mail system LOG_DAEMON = 3 # system daemons LOG_AUTH = 4 # security/authorization messages LOG_SYSLOG = 5 # messages generated internally by syslogd LOG_LPR = 6 # line printer subsystem LOG_NEWS = 7 # network news subsystem LOG_UUCP = 8 # UUCP subsystem LOG_CRON = 9 # clock daemon LOG_AUTHPRIV = 10 # security/authorization messages (private) LOG_FTP = 11 # FTP daemon # other codes through 15 reserved for system use LOG_LOCAL0 = 16 # reserved for local use LOG_LOCAL1 = 17 # reserved for local use LOG_LOCAL2 = 18 # reserved for local use LOG_LOCAL3 = 19 # reserved for local use LOG_LOCAL4 = 20 # reserved for local use LOG_LOCAL5 = 21 # reserved for local use LOG_LOCAL6 = 22 # reserved for local use LOG_LOCAL7 = 23 # reserved for local use priority_names = { "alert": LOG_ALERT, "crit": LOG_CRIT, "critical": LOG_CRIT, "debug": LOG_DEBUG, "emerg": LOG_EMERG, "err": LOG_ERR, "error": LOG_ERR, # DEPRECATED "info": LOG_INFO, "notice": LOG_NOTICE, "panic": LOG_EMERG, # DEPRECATED "warn": LOG_WARNING, # DEPRECATED "warning": LOG_WARNING, } facility_names = { "auth": LOG_AUTH, "authpriv": LOG_AUTHPRIV, "cron": LOG_CRON, "daemon": LOG_DAEMON, "ftp": LOG_FTP, "kern": LOG_KERN, "lpr": LOG_LPR, "mail": LOG_MAIL, "news": LOG_NEWS, "security": LOG_AUTH, # DEPRECATED "syslog": LOG_SYSLOG, "user": LOG_USER, "uucp": LOG_UUCP, "local0": LOG_LOCAL0, "local1": LOG_LOCAL1, "local2": LOG_LOCAL2, "local3": LOG_LOCAL3, "local4": LOG_LOCAL4, "local5": LOG_LOCAL5, "local6": LOG_LOCAL6, "local7": LOG_LOCAL7, } #The map below appears to be trivially lowercasing the key. However, #there's more to it than meets the eye - in some locales, lowercasing #gives unexpected results. See SF #1524081: in the Turkish locale, #"INFO".lower() != "info" priority_map = { "DEBUG" : "debug", "INFO" : "info", "WARNING" : "warning", "ERROR" : "error", "CRITICAL" : "critical" } def __init__(self, address=('localhost', SYSLOG_UDP_PORT), facility=LOG_USER, socktype=None): """ Initialize a handler. If address is specified as a string, a UNIX socket is used. To log to a local syslogd, "SysLogHandler(address="/dev/log")" can be used. If facility is not specified, LOG_USER is used. If socktype is specified as socket.SOCK_DGRAM or socket.SOCK_STREAM, that specific socket type will be used. For Unix sockets, you can also specify a socktype of None, in which case socket.SOCK_DGRAM will be used, falling back to socket.SOCK_STREAM. """ logging.Handler.__init__(self) self.address = address self.facility = facility self.socktype = socktype if isinstance(address, basestring): self.unixsocket = 1 self._connect_unixsocket(address) else: self.unixsocket = False if socktype is None: socktype = socket.SOCK_DGRAM host, port = address ress = socket.getaddrinfo(host, port, 0, socktype) if not ress: raise socket.error("getaddrinfo returns an empty list") for res in ress: af, socktype, proto, _, sa = res err = sock = None try: sock = socket.socket(af, socktype, proto) if socktype == socket.SOCK_STREAM: sock.connect(sa) break except socket.error as exc: err = exc if sock is not None: sock.close() if err is not None: raise err self.socket = sock self.socktype = socktype def _connect_unixsocket(self, address): use_socktype = self.socktype if use_socktype is None: use_socktype = socket.SOCK_DGRAM self.socket = socket.socket(socket.AF_UNIX, use_socktype) try: self.socket.connect(address) # it worked, so set self.socktype to the used type self.socktype = use_socktype except socket.error: self.socket.close() if self.socktype is not None: # user didn't specify falling back, so fail raise use_socktype = socket.SOCK_STREAM self.socket = socket.socket(socket.AF_UNIX, use_socktype) try: self.socket.connect(address) # it worked, so set self.socktype to the used type self.socktype = use_socktype except socket.error: self.socket.close() raise # curious: when talking to the unix-domain '/dev/log' socket, a # zero-terminator seems to be required. this string is placed # into a class variable so that it can be overridden if # necessary. log_format_string = '<%d>%s\000' def encodePriority(self, facility, priority): """ Encode the facility and priority. You can pass in strings or integers - if strings are passed, the facility_names and priority_names mapping dictionaries are used to convert them to integers. """ if isinstance(facility, basestring): facility = self.facility_names[facility] if isinstance(priority, basestring): priority = self.priority_names[priority] return (facility << 3) | priority def close(self): """ Closes the socket. """ self.acquire() try: if self.unixsocket: self.socket.close() finally: self.release() logging.Handler.close(self) def mapPriority(self, levelName): """ Map a logging level name to a key in the priority_names map. This is useful in two scenarios: when custom levels are being used, and in the case where you can't do a straightforward mapping by lowercasing the logging level name because of locale- specific issues (see SF #1524081). """ return self.priority_map.get(levelName, "warning") def emit(self, record): """ Emit a record. The record is formatted, and then sent to the syslog server. If exception information is present, it is NOT sent to the server. """ try: msg = self.format(record) + '\000' """ We need to convert record level to lowercase, maybe this will change in the future. """ prio = '<%d>' % self.encodePriority(self.facility, self.mapPriority(record.levelname)) # Message is a string. Convert to bytes as required by RFC 5424 if type(msg) is unicode: msg = msg.encode('utf-8') msg = prio + msg if self.unixsocket: try: self.socket.send(msg) except socket.error: self.socket.close() # See issue 17981 self._connect_unixsocket(self.address) self.socket.send(msg) elif self.socktype == socket.SOCK_DGRAM: self.socket.sendto(msg, self.address) else: self.socket.sendall(msg) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) class SMTPHandler(logging.Handler): """ A handler class which sends an SMTP email for each logging event. """ def __init__(self, mailhost, fromaddr, toaddrs, subject, credentials=None, secure=None): """ Initialize the handler. Initialize the instance with the from and to addresses and subject line of the email. To specify a non-standard SMTP port, use the (host, port) tuple format for the mailhost argument. To specify authentication credentials, supply a (username, password) tuple for the credentials argument. To specify the use of a secure protocol (TLS), pass in a tuple for the secure argument. This will only be used when authentication credentials are supplied. The tuple will be either an empty tuple, or a single-value tuple with the name of a keyfile, or a 2-value tuple with the names of the keyfile and certificate file. (This tuple is passed to the `starttls` method). """ logging.Handler.__init__(self) if isinstance(mailhost, (list, tuple)): self.mailhost, self.mailport = mailhost else: self.mailhost, self.mailport = mailhost, None if isinstance(credentials, (list, tuple)): self.username, self.password = credentials else: self.username = None self.fromaddr = fromaddr if isinstance(toaddrs, basestring): toaddrs = [toaddrs] self.toaddrs = toaddrs self.subject = subject self.secure = secure self._timeout = 5.0 def getSubject(self, record): """ Determine the subject for the email. If you want to specify a subject line which is record-dependent, override this method. """ return self.subject def emit(self, record): """ Emit a record. Format the record and send it to the specified addressees. """ try: import smtplib from email.utils import formatdate port = self.mailport if not port: port = smtplib.SMTP_PORT smtp = smtplib.SMTP(self.mailhost, port, timeout=self._timeout) msg = self.format(record) msg = "From: %s\r\nTo: %s\r\nSubject: %s\r\nDate: %s\r\n\r\n%s" % ( self.fromaddr, ",".join(self.toaddrs), self.getSubject(record), formatdate(), msg) if self.username: if self.secure is not None: smtp.ehlo() smtp.starttls(*self.secure) smtp.ehlo() smtp.login(self.username, self.password) smtp.sendmail(self.fromaddr, self.toaddrs, msg) smtp.quit() except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) class NTEventLogHandler(logging.Handler): """ A handler class which sends events to the NT Event Log. Adds a registry entry for the specified application name. If no dllname is provided, win32service.pyd (which contains some basic message placeholders) is used. Note that use of these placeholders will make your event logs big, as the entire message source is held in the log. If you want slimmer logs, you have to pass in the name of your own DLL which contains the message definitions you want to use in the event log. """ def __init__(self, appname, dllname=None, logtype="Application"): logging.Handler.__init__(self) try: import win32evtlogutil, win32evtlog self.appname = appname self._welu = win32evtlogutil if not dllname: dllname = os.path.split(self._welu.__file__) dllname = os.path.split(dllname[0]) dllname = os.path.join(dllname[0], r'win32service.pyd') self.dllname = dllname self.logtype = logtype self._welu.AddSourceToRegistry(appname, dllname, logtype) self.deftype = win32evtlog.EVENTLOG_ERROR_TYPE self.typemap = { logging.DEBUG : win32evtlog.EVENTLOG_INFORMATION_TYPE, logging.INFO : win32evtlog.EVENTLOG_INFORMATION_TYPE, logging.WARNING : win32evtlog.EVENTLOG_WARNING_TYPE, logging.ERROR : win32evtlog.EVENTLOG_ERROR_TYPE, logging.CRITICAL: win32evtlog.EVENTLOG_ERROR_TYPE, } except ImportError: print("The Python Win32 extensions for NT (service, event "\ "logging) appear not to be available.") self._welu = None def getMessageID(self, record): """ Return the message ID for the event record. If you are using your own messages, you could do this by having the msg passed to the logger being an ID rather than a formatting string. Then, in here, you could use a dictionary lookup to get the message ID. This version returns 1, which is the base message ID in win32service.pyd. """ return 1 def getEventCategory(self, record): """ Return the event category for the record. Override this if you want to specify your own categories. This version returns 0. """ return 0 def getEventType(self, record): """ Return the event type for the record. Override this if you want to specify your own types. This version does a mapping using the handler's typemap attribute, which is set up in __init__() to a dictionary which contains mappings for DEBUG, INFO, WARNING, ERROR and CRITICAL. If you are using your own levels you will either need to override this method or place a suitable dictionary in the handler's typemap attribute. """ return self.typemap.get(record.levelno, self.deftype) def emit(self, record): """ Emit a record. Determine the message ID, event category and event type. Then log the message in the NT event log. """ if self._welu: try: id = self.getMessageID(record) cat = self.getEventCategory(record) type = self.getEventType(record) msg = self.format(record) self._welu.ReportEvent(self.appname, id, cat, type, [msg]) except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) def close(self): """ Clean up this handler. You can remove the application name from the registry as a source of event log entries. However, if you do this, you will not be able to see the events as you intended in the Event Log Viewer - it needs to be able to access the registry to get the DLL name. """ #self._welu.RemoveSourceFromRegistry(self.appname, self.logtype) logging.Handler.close(self) class HTTPHandler(logging.Handler): """ A class which sends records to a Web server, using either GET or POST semantics. """ def __init__(self, host, url, method="GET"): """ Initialize the instance with the host, the request URL, and the method ("GET" or "POST") """ logging.Handler.__init__(self) method = method.upper() if method not in ["GET", "POST"]: raise ValueError("method must be GET or POST") self.host = host self.url = url self.method = method def mapLogRecord(self, record): """ Default implementation of mapping the log record into a dict that is sent as the CGI data. Overwrite in your class. Contributed by Franz Glasner. """ return record.__dict__ def emit(self, record): """ Emit a record. Send the record to the Web server as a percent-encoded dictionary """ try: import httplib, urllib host = self.host h = httplib.HTTP(host) url = self.url data = urllib.urlencode(self.mapLogRecord(record)) if self.method == "GET": if (url.find('?') >= 0): sep = '&' else: sep = '?' url = url + "%c%s" % (sep, data) h.putrequest(self.method, url) # support multiple hosts on one IP address... # need to strip optional :port from host, if present i = host.find(":") if i >= 0: host = host[:i] h.putheader("Host", host) if self.method == "POST": h.putheader("Content-type", "application/x-www-form-urlencoded") h.putheader("Content-length", str(len(data))) h.endheaders(data if self.method == "POST" else None) h.getreply() #can't do anything with the result except (KeyboardInterrupt, SystemExit): raise except: self.handleError(record) class BufferingHandler(logging.Handler): """ A handler class which buffers logging records in memory. Whenever each record is added to the buffer, a check is made to see if the buffer should be flushed. If it should, then flush() is expected to do what's needed. """ def __init__(self, capacity): """ Initialize the handler with the buffer size. """ logging.Handler.__init__(self) self.capacity = capacity self.buffer = [] def shouldFlush(self, record): """ Should the handler flush its buffer? Returns true if the buffer is up to capacity. This method can be overridden to implement custom flushing strategies. """ return (len(self.buffer) >= self.capacity) def emit(self, record): """ Emit a record. Append the record. If shouldFlush() tells us to, call flush() to process the buffer. """ self.buffer.append(record) if self.shouldFlush(record): self.flush() def flush(self): """ Override to implement custom flushing behaviour. This version just zaps the buffer to empty. """ self.acquire() try: self.buffer = [] finally: self.release() def close(self): """ Close the handler. This version just flushes and chains to the parent class' close(). """ try: self.flush() finally: logging.Handler.close(self) class MemoryHandler(BufferingHandler): """ A handler class which buffers logging records in memory, periodically flushing them to a target handler. Flushing occurs whenever the buffer is full, or when an event of a certain severity or greater is seen. """ def __init__(self, capacity, flushLevel=logging.ERROR, target=None): """ Initialize the handler with the buffer size, the level at which flushing should occur and an optional target. Note that without a target being set either here or via setTarget(), a MemoryHandler is no use to anyone! """ BufferingHandler.__init__(self, capacity) self.flushLevel = flushLevel self.target = target def shouldFlush(self, record): """ Check for buffer full or a record at the flushLevel or higher. """ return (len(self.buffer) >= self.capacity) or \ (record.levelno >= self.flushLevel) def setTarget(self, target): """ Set the target handler for this handler. """ self.target = target def flush(self): """ For a MemoryHandler, flushing means just sending the buffered records to the target, if there is one. Override if you want different behaviour. """ self.acquire() try: if self.target: for record in self.buffer: self.target.handle(record) self.buffer = [] finally: self.release() def close(self): """ Flush, set the target to None and lose the buffer. """ try: self.flush() finally: self.acquire() try: self.target = None BufferingHandler.close(self) finally: self.release() """ Python unit testing framework, based on Erich Gamma's JUnit and Kent Beck's Smalltalk testing framework. This module contains the core framework classes that form the basis of specific test cases and suites (TestCase, TestSuite etc.), and also a text-based utility class for running the tests and reporting the results (TextTestRunner). Simple usage: import unittest class IntegerArithmeticTestCase(unittest.TestCase): def testAdd(self): ## test method names begin 'test*' self.assertEqual((1 + 2), 3) self.assertEqual(0 + 1, 1) def testMultiply(self): self.assertEqual((0 * 10), 0) self.assertEqual((5 * 8), 40) if __name__ == '__main__': unittest.main() Further information is available in the bundled documentation, and from http://docs.python.org/library/unittest.html Copyright (c) 1999-2003 Steve Purcell Copyright (c) 2003-2010 Python Software Foundation This module is free software, and you may redistribute it and/or modify it under the same terms as Python itself, so long as this copyright message and disclaimer are retained in their original form. IN NO EVENT SHALL THE AUTHOR BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. THE AUTHOR SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. """ __all__ = ['TestResult', 'TestCase', 'TestSuite', 'TextTestRunner', 'TestLoader', 'FunctionTestCase', 'main', 'defaultTestLoader', 'SkipTest', 'skip', 'skipIf', 'skipUnless', 'expectedFailure', 'TextTestResult', 'installHandler', 'registerResult', 'removeResult', 'removeHandler'] # Expose obsolete functions for backwards compatibility __all__.extend(['getTestCaseNames', 'makeSuite', 'findTestCases']) __unittest = True from .result import TestResult from .case import (TestCase, FunctionTestCase, SkipTest, skip, skipIf, skipUnless, expectedFailure) from .suite import BaseTestSuite, TestSuite from .loader import (TestLoader, defaultTestLoader, makeSuite, getTestCaseNames, findTestCases) from .main import TestProgram, main from .runner import TextTestRunner, TextTestResult from .signals import installHandler, registerResult, removeResult, removeHandler # deprecated _TextTestResult = TextTestResult """Main entry point""" import sys if sys.argv[0].endswith("__main__.py"): sys.argv[0] = "python -m unittest" __unittest = True from .main import main, TestProgram, USAGE_AS_MAIN TestProgram.USAGE = USAGE_AS_MAIN main(module=None) """Test case implementation""" import collections import sys import functools import difflib import pprint import re import types import warnings from . import result from .util import ( strclass, safe_repr, unorderable_list_difference, _count_diff_all_purpose, _count_diff_hashable ) __unittest = True DIFF_OMITTED = ('\nDiff is %s characters long. ' 'Set self.maxDiff to None to see it.') class SkipTest(Exception): """ Raise this exception in a test to skip it. Usually you can use TestCase.skipTest() or one of the skipping decorators instead of raising this directly. """ pass class _ExpectedFailure(Exception): """ Raise this when a test is expected to fail. This is an implementation detail. """ def __init__(self, exc_info): super(_ExpectedFailure, self).__init__() self.exc_info = exc_info class _UnexpectedSuccess(Exception): """ The test was supposed to fail, but it didn't! """ pass def _id(obj): return obj def skip(reason): """ Unconditionally skip a test. """ def decorator(test_item): if not isinstance(test_item, (type, types.ClassType)): @functools.wraps(test_item) def skip_wrapper(*args, **kwargs): raise SkipTest(reason) test_item = skip_wrapper test_item.__unittest_skip__ = True test_item.__unittest_skip_why__ = reason return test_item return decorator def skipIf(condition, reason): """ Skip a test if the condition is true. """ if condition: return skip(reason) return _id def skipUnless(condition, reason): """ Skip a test unless the condition is true. """ if not condition: return skip(reason) return _id def expectedFailure(func): @functools.wraps(func) def wrapper(*args, **kwargs): try: func(*args, **kwargs) except Exception: raise _ExpectedFailure(sys.exc_info()) raise _UnexpectedSuccess return wrapper class _AssertRaisesContext(object): """A context manager used to implement TestCase.assertRaises* methods.""" def __init__(self, expected, test_case, expected_regexp=None): self.expected = expected self.failureException = test_case.failureException self.expected_regexp = expected_regexp def __enter__(self): return self def __exit__(self, exc_type, exc_value, tb): if exc_type is None: try: exc_name = self.expected.__name__ except AttributeError: exc_name = str(self.expected) raise self.failureException( "{0} not raised".format(exc_name)) if not issubclass(exc_type, self.expected): # let unexpected exceptions pass through return False self.exception = exc_value # store for later retrieval if self.expected_regexp is None: return True expected_regexp = self.expected_regexp if not expected_regexp.search(str(exc_value)): raise self.failureException('"%s" does not match "%s"' % (expected_regexp.pattern, str(exc_value))) return True class TestCase(object): """A class whose instances are single test cases. By default, the test code itself should be placed in a method named 'runTest'. If the fixture may be used for many test cases, create as many test methods as are needed. When instantiating such a TestCase subclass, specify in the constructor arguments the name of the test method that the instance is to execute. Test authors should subclass TestCase for their own tests. Construction and deconstruction of the test's environment ('fixture') can be implemented by overriding the 'setUp' and 'tearDown' methods respectively. If it is necessary to override the __init__ method, the base class __init__ method must always be called. It is important that subclasses should not change the signature of their __init__ method, since instances of the classes are instantiated automatically by parts of the framework in order to be run. When subclassing TestCase, you can set these attributes: * failureException: determines which exception will be raised when the instance's assertion methods fail; test methods raising this exception will be deemed to have 'failed' rather than 'errored'. * longMessage: determines whether long messages (including repr of objects used in assert methods) will be printed on failure in *addition* to any explicit message passed. * maxDiff: sets the maximum length of a diff in failure messages by assert methods using difflib. It is looked up as an instance attribute so can be configured by individual tests if required. """ failureException = AssertionError longMessage = False maxDiff = 80*8 # If a string is longer than _diffThreshold, use normal comparison instead # of difflib. See #11763. _diffThreshold = 2**16 # Attribute used by TestSuite for classSetUp _classSetupFailed = False def __init__(self, methodName='runTest'): """Create an instance of the class that will use the named test method when executed. Raises a ValueError if the instance does not have a method with the specified name. """ self._testMethodName = methodName self._resultForDoCleanups = None try: testMethod = getattr(self, methodName) except AttributeError: raise ValueError("no such test method in %s: %s" % (self.__class__, methodName)) self._testMethodDoc = testMethod.__doc__ self._cleanups = [] # Map types to custom assertEqual functions that will compare # instances of said type in more detail to generate a more useful # error message. self._type_equality_funcs = {} self.addTypeEqualityFunc(dict, 'assertDictEqual') self.addTypeEqualityFunc(list, 'assertListEqual') self.addTypeEqualityFunc(tuple, 'assertTupleEqual') self.addTypeEqualityFunc(set, 'assertSetEqual') self.addTypeEqualityFunc(frozenset, 'assertSetEqual') try: self.addTypeEqualityFunc(unicode, 'assertMultiLineEqual') except NameError: # No unicode support in this build pass def addTypeEqualityFunc(self, typeobj, function): """Add a type specific assertEqual style function to compare a type. This method is for use by TestCase subclasses that need to register their own type equality functions to provide nicer error messages. Args: typeobj: The data type to call this function on when both values are of the same type in assertEqual(). function: The callable taking two arguments and an optional msg= argument that raises self.failureException with a useful error message when the two arguments are not equal. """ self._type_equality_funcs[typeobj] = function def addCleanup(self, function, *args, **kwargs): """Add a function, with arguments, to be called when the test is completed. Functions added are called on a LIFO basis and are called after tearDown on test failure or success. Cleanup items are called even if setUp fails (unlike tearDown).""" self._cleanups.append((function, args, kwargs)) def setUp(self): "Hook method for setting up the test fixture before exercising it." pass def tearDown(self): "Hook method for deconstructing the test fixture after testing it." pass @classmethod def setUpClass(cls): "Hook method for setting up class fixture before running tests in the class." @classmethod def tearDownClass(cls): "Hook method for deconstructing the class fixture after running all tests in the class." def countTestCases(self): return 1 def defaultTestResult(self): return result.TestResult() def shortDescription(self): """Returns a one-line description of the test, or None if no description has been provided. The default implementation of this method returns the first line of the specified test method's docstring. """ doc = self._testMethodDoc return doc and doc.split("\n")[0].strip() or None def id(self): return "%s.%s" % (strclass(self.__class__), self._testMethodName) def __eq__(self, other): if type(self) is not type(other): return NotImplemented return self._testMethodName == other._testMethodName def __ne__(self, other): return not self == other def __hash__(self): return hash((type(self), self._testMethodName)) def __str__(self): return "%s (%s)" % (self._testMethodName, strclass(self.__class__)) def __repr__(self): return "<%s testMethod=%s>" % \ (strclass(self.__class__), self._testMethodName) def _addSkip(self, result, reason): addSkip = getattr(result, 'addSkip', None) if addSkip is not None: addSkip(self, reason) else: warnings.warn("TestResult has no addSkip method, skips not reported", RuntimeWarning, 2) result.addSuccess(self) def run(self, result=None): orig_result = result if result is None: result = self.defaultTestResult() startTestRun = getattr(result, 'startTestRun', None) if startTestRun is not None: startTestRun() self._resultForDoCleanups = result result.startTest(self) testMethod = getattr(self, self._testMethodName) if (getattr(self.__class__, "__unittest_skip__", False) or getattr(testMethod, "__unittest_skip__", False)): # If the class or method was skipped. try: skip_why = (getattr(self.__class__, '__unittest_skip_why__', '') or getattr(testMethod, '__unittest_skip_why__', '')) self._addSkip(result, skip_why) finally: result.stopTest(self) return try: success = False try: self.setUp() except SkipTest as e: self._addSkip(result, str(e)) except KeyboardInterrupt: raise except: result.addError(self, sys.exc_info()) else: try: testMethod() except KeyboardInterrupt: raise except self.failureException: result.addFailure(self, sys.exc_info()) except _ExpectedFailure as e: addExpectedFailure = getattr(result, 'addExpectedFailure', None) if addExpectedFailure is not None: addExpectedFailure(self, e.exc_info) else: warnings.warn("TestResult has no addExpectedFailure method, reporting as passes", RuntimeWarning) result.addSuccess(self) except _UnexpectedSuccess: addUnexpectedSuccess = getattr(result, 'addUnexpectedSuccess', None) if addUnexpectedSuccess is not None: addUnexpectedSuccess(self) else: warnings.warn("TestResult has no addUnexpectedSuccess method, reporting as failures", RuntimeWarning) result.addFailure(self, sys.exc_info()) except SkipTest as e: self._addSkip(result, str(e)) except: result.addError(self, sys.exc_info()) else: success = True try: self.tearDown() except KeyboardInterrupt: raise except: result.addError(self, sys.exc_info()) success = False cleanUpSuccess = self.doCleanups() success = success and cleanUpSuccess if success: result.addSuccess(self) finally: result.stopTest(self) if orig_result is None: stopTestRun = getattr(result, 'stopTestRun', None) if stopTestRun is not None: stopTestRun() def doCleanups(self): """Execute all cleanup functions. Normally called for you after tearDown.""" result = self._resultForDoCleanups ok = True while self._cleanups: function, args, kwargs = self._cleanups.pop(-1) try: function(*args, **kwargs) except KeyboardInterrupt: raise except: ok = False result.addError(self, sys.exc_info()) return ok def __call__(self, *args, **kwds): return self.run(*args, **kwds) def debug(self): """Run the test without collecting errors in a TestResult""" self.setUp() getattr(self, self._testMethodName)() self.tearDown() while self._cleanups: function, args, kwargs = self._cleanups.pop(-1) function(*args, **kwargs) def skipTest(self, reason): """Skip this test.""" raise SkipTest(reason) def fail(self, msg=None): """Fail immediately, with the given message.""" raise self.failureException(msg) def assertFalse(self, expr, msg=None): """Check that the expression is false.""" if expr: msg = self._formatMessage(msg, "%s is not false" % safe_repr(expr)) raise self.failureException(msg) def assertTrue(self, expr, msg=None): """Check that the expression is true.""" if not expr: msg = self._formatMessage(msg, "%s is not true" % safe_repr(expr)) raise self.failureException(msg) def _formatMessage(self, msg, standardMsg): """Honour the longMessage attribute when generating failure messages. If longMessage is False this means: * Use only an explicit message if it is provided * Otherwise use the standard message for the assert If longMessage is True: * Use the standard message * If an explicit message is provided, plus ' : ' and the explicit message """ if not self.longMessage: return msg or standardMsg if msg is None: return standardMsg try: # don't switch to '{}' formatting in Python 2.X # it changes the way unicode input is handled return '%s : %s' % (standardMsg, msg) except UnicodeDecodeError: return '%s : %s' % (safe_repr(standardMsg), safe_repr(msg)) def assertRaises(self, excClass, callableObj=None, *args, **kwargs): """Fail unless an exception of class excClass is raised by callableObj when invoked with arguments args and keyword arguments kwargs. If a different type of exception is raised, it will not be caught, and the test case will be deemed to have suffered an error, exactly as for an unexpected exception. If called with callableObj omitted or None, will return a context object used like this:: with self.assertRaises(SomeException): do_something() The context manager keeps a reference to the exception as the 'exception' attribute. This allows you to inspect the exception after the assertion:: with self.assertRaises(SomeException) as cm: do_something() the_exception = cm.exception self.assertEqual(the_exception.error_code, 3) """ context = _AssertRaisesContext(excClass, self) if callableObj is None: return context with context: callableObj(*args, **kwargs) def _getAssertEqualityFunc(self, first, second): """Get a detailed comparison function for the types of the two args. Returns: A callable accepting (first, second, msg=None) that will raise a failure exception if first != second with a useful human readable error message for those types. """ # # NOTE(gregory.p.smith): I considered isinstance(first, type(second)) # and vice versa. I opted for the conservative approach in case # subclasses are not intended to be compared in detail to their super # class instances using a type equality func. This means testing # subtypes won't automagically use the detailed comparison. Callers # should use their type specific assertSpamEqual method to compare # subclasses if the detailed comparison is desired and appropriate. # See the discussion in http://bugs.python.org/issue2578. # if type(first) is type(second): asserter = self._type_equality_funcs.get(type(first)) if asserter is not None: if isinstance(asserter, basestring): asserter = getattr(self, asserter) return asserter return self._baseAssertEqual def _baseAssertEqual(self, first, second, msg=None): """The default assertEqual implementation, not type specific.""" if not first == second: standardMsg = '%s != %s' % (safe_repr(first), safe_repr(second)) msg = self._formatMessage(msg, standardMsg) raise self.failureException(msg) def assertEqual(self, first, second, msg=None): """Fail if the two objects are unequal as determined by the '==' operator. """ assertion_func = self._getAssertEqualityFunc(first, second) assertion_func(first, second, msg=msg) def assertNotEqual(self, first, second, msg=None): """Fail if the two objects are equal as determined by the '!=' operator. """ if not first != second: msg = self._formatMessage(msg, '%s == %s' % (safe_repr(first), safe_repr(second))) raise self.failureException(msg) def assertAlmostEqual(self, first, second, places=None, msg=None, delta=None): """Fail if the two objects are unequal as determined by their difference rounded to the given number of decimal places (default 7) and comparing to zero, or by comparing that the between the two objects is more than the given delta. Note that decimal places (from zero) are usually not the same as significant digits (measured from the most significant digit). If the two objects compare equal then they will automatically compare almost equal. """ if first == second: # shortcut return if delta is not None and places is not None: raise TypeError("specify delta or places not both") if delta is not None: if abs(first - second) <= delta: return standardMsg = '%s != %s within %s delta' % (safe_repr(first), safe_repr(second), safe_repr(delta)) else: if places is None: places = 7 if round(abs(second-first), places) == 0: return standardMsg = '%s != %s within %r places' % (safe_repr(first), safe_repr(second), places) msg = self._formatMessage(msg, standardMsg) raise self.failureException(msg) def assertNotAlmostEqual(self, first, second, places=None, msg=None, delta=None): """Fail if the two objects are equal as determined by their difference rounded to the given number of decimal places (default 7) and comparing to zero, or by comparing that the between the two objects is less than the given delta. Note that decimal places (from zero) are usually not the same as significant digits (measured from the most significant digit). Objects that are equal automatically fail. """ if delta is not None and places is not None: raise TypeError("specify delta or places not both") if delta is not None: if not (first == second) and abs(first - second) > delta: return standardMsg = '%s == %s within %s delta' % (safe_repr(first), safe_repr(second), safe_repr(delta)) else: if places is None: places = 7 if not (first == second) and round(abs(second-first), places) != 0: return standardMsg = '%s == %s within %r places' % (safe_repr(first), safe_repr(second), places) msg = self._formatMessage(msg, standardMsg) raise self.failureException(msg) # Synonyms for assertion methods # The plurals are undocumented. Keep them that way to discourage use. # Do not add more. Do not remove. # Going through a deprecation cycle on these would annoy many people. assertEquals = assertEqual assertNotEquals = assertNotEqual assertAlmostEquals = assertAlmostEqual assertNotAlmostEquals = assertNotAlmostEqual assert_ = assertTrue # These fail* assertion method names are pending deprecation and will # be a DeprecationWarning in 3.2; http://bugs.python.org/issue2578 def _deprecate(original_func): def deprecated_func(*args, **kwargs): warnings.warn( 'Please use {0} instead.'.format(original_func.__name__), PendingDeprecationWarning, 2) return original_func(*args, **kwargs) return deprecated_func failUnlessEqual = _deprecate(assertEqual) failIfEqual = _deprecate(assertNotEqual) failUnlessAlmostEqual = _deprecate(assertAlmostEqual) failIfAlmostEqual = _deprecate(assertNotAlmostEqual) failUnless = _deprecate(assertTrue) failUnlessRaises = _deprecate(assertRaises) failIf = _deprecate(assertFalse) def assertSequenceEqual(self, seq1, seq2, msg=None, seq_type=None): """An equality assertion for ordered sequences (like lists and tuples). For the purposes of this function, a valid ordered sequence type is one which can be indexed, has a length, and has an equality operator. Args: seq1: The first sequence to compare. seq2: The second sequence to compare. seq_type: The expected datatype of the sequences, or None if no datatype should be enforced. msg: Optional message to use on failure instead of a list of differences. """ if seq_type is not None: seq_type_name = seq_type.__name__ if not isinstance(seq1, seq_type): raise self.failureException('First sequence is not a %s: %s' % (seq_type_name, safe_repr(seq1))) if not isinstance(seq2, seq_type): raise self.failureException('Second sequence is not a %s: %s' % (seq_type_name, safe_repr(seq2))) else: seq_type_name = "sequence" differing = None try: len1 = len(seq1) except (TypeError, NotImplementedError): differing = 'First %s has no length. Non-sequence?' % ( seq_type_name) if differing is None: try: len2 = len(seq2) except (TypeError, NotImplementedError): differing = 'Second %s has no length. Non-sequence?' % ( seq_type_name) if differing is None: if seq1 == seq2: return seq1_repr = safe_repr(seq1) seq2_repr = safe_repr(seq2) if len(seq1_repr) > 30: seq1_repr = seq1_repr[:30] + '...' if len(seq2_repr) > 30: seq2_repr = seq2_repr[:30] + '...' elements = (seq_type_name.capitalize(), seq1_repr, seq2_repr) differing = '%ss differ: %s != %s\n' % elements for i in xrange(min(len1, len2)): try: item1 = seq1[i] except (TypeError, IndexError, NotImplementedError): differing += ('\nUnable to index element %d of first %s\n' % (i, seq_type_name)) break try: item2 = seq2[i] except (TypeError, IndexError, NotImplementedError): differing += ('\nUnable to index element %d of second %s\n' % (i, seq_type_name)) break if item1 != item2: differing += ('\nFirst differing element %d:\n%s\n%s\n' % (i, safe_repr(item1), safe_repr(item2))) break else: if (len1 == len2 and seq_type is None and type(seq1) != type(seq2)): # The sequences are the same, but have differing types. return if len1 > len2: differing += ('\nFirst %s contains %d additional ' 'elements.\n' % (seq_type_name, len1 - len2)) try: differing += ('First extra element %d:\n%s\n' % (len2, safe_repr(seq1[len2]))) except (TypeError, IndexError, NotImplementedError): differing += ('Unable to index element %d ' 'of first %s\n' % (len2, seq_type_name)) elif len1 < len2: differing += ('\nSecond %s contains %d additional ' 'elements.\n' % (seq_type_name, len2 - len1)) try: differing += ('First extra element %d:\n%s\n' % (len1, safe_repr(seq2[len1]))) except (TypeError, IndexError, NotImplementedError): differing += ('Unable to index element %d ' 'of second %s\n' % (len1, seq_type_name)) standardMsg = differing diffMsg = '\n' + '\n'.join( difflib.ndiff(pprint.pformat(seq1).splitlines(), pprint.pformat(seq2).splitlines())) standardMsg = self._truncateMessage(standardMsg, diffMsg) msg = self._formatMessage(msg, standardMsg) self.fail(msg) def _truncateMessage(self, message, diff): max_diff = self.maxDiff if max_diff is None or len(diff) <= max_diff: return message + diff return message + (DIFF_OMITTED % len(diff)) def assertListEqual(self, list1, list2, msg=None): """A list-specific equality assertion. Args: list1: The first list to compare. list2: The second list to compare. msg: Optional message to use on failure instead of a list of differences. """ self.assertSequenceEqual(list1, list2, msg, seq_type=list) def assertTupleEqual(self, tuple1, tuple2, msg=None): """A tuple-specific equality assertion. Args: tuple1: The first tuple to compare. tuple2: The second tuple to compare. msg: Optional message to use on failure instead of a list of differences. """ self.assertSequenceEqual(tuple1, tuple2, msg, seq_type=tuple) def assertSetEqual(self, set1, set2, msg=None): """A set-specific equality assertion. Args: set1: The first set to compare. set2: The second set to compare. msg: Optional message to use on failure instead of a list of differences. assertSetEqual uses ducktyping to support different types of sets, and is optimized for sets specifically (parameters must support a difference method). """ try: difference1 = set1.difference(set2) except TypeError, e: self.fail('invalid type when attempting set difference: %s' % e) except AttributeError, e: self.fail('first argument does not support set difference: %s' % e) try: difference2 = set2.difference(set1) except TypeError, e: self.fail('invalid type when attempting set difference: %s' % e) except AttributeError, e: self.fail('second argument does not support set difference: %s' % e) if not (difference1 or difference2): return lines = [] if difference1: lines.append('Items in the first set but not the second:') for item in difference1: lines.append(repr(item)) if difference2: lines.append('Items in the second set but not the first:') for item in difference2: lines.append(repr(item)) standardMsg = '\n'.join(lines) self.fail(self._formatMessage(msg, standardMsg)) def assertIn(self, member, container, msg=None): """Just like self.assertTrue(a in b), but with a nicer default message.""" if member not in container: standardMsg = '%s not found in %s' % (safe_repr(member), safe_repr(container)) self.fail(self._formatMessage(msg, standardMsg)) def assertNotIn(self, member, container, msg=None): """Just like self.assertTrue(a not in b), but with a nicer default message.""" if member in container: standardMsg = '%s unexpectedly found in %s' % (safe_repr(member), safe_repr(container)) self.fail(self._formatMessage(msg, standardMsg)) def assertIs(self, expr1, expr2, msg=None): """Just like self.assertTrue(a is b), but with a nicer default message.""" if expr1 is not expr2: standardMsg = '%s is not %s' % (safe_repr(expr1), safe_repr(expr2)) self.fail(self._formatMessage(msg, standardMsg)) def assertIsNot(self, expr1, expr2, msg=None): """Just like self.assertTrue(a is not b), but with a nicer default message.""" if expr1 is expr2: standardMsg = 'unexpectedly identical: %s' % (safe_repr(expr1),) self.fail(self._formatMessage(msg, standardMsg)) def assertDictEqual(self, d1, d2, msg=None): self.assertIsInstance(d1, dict, 'First argument is not a dictionary') self.assertIsInstance(d2, dict, 'Second argument is not a dictionary') if d1 != d2: standardMsg = '%s != %s' % (safe_repr(d1, True), safe_repr(d2, True)) diff = ('\n' + '\n'.join(difflib.ndiff( pprint.pformat(d1).splitlines(), pprint.pformat(d2).splitlines()))) standardMsg = self._truncateMessage(standardMsg, diff) self.fail(self._formatMessage(msg, standardMsg)) def assertDictContainsSubset(self, expected, actual, msg=None): """Checks whether actual is a superset of expected.""" missing = [] mismatched = [] for key, value in expected.iteritems(): if key not in actual: missing.append(key) elif value != actual[key]: mismatched.append('%s, expected: %s, actual: %s' % (safe_repr(key), safe_repr(value), safe_repr(actual[key]))) if not (missing or mismatched): return standardMsg = '' if missing: standardMsg = 'Missing: %s' % ','.join(safe_repr(m) for m in missing) if mismatched: if standardMsg: standardMsg += '; ' standardMsg += 'Mismatched values: %s' % ','.join(mismatched) self.fail(self._formatMessage(msg, standardMsg)) def assertItemsEqual(self, expected_seq, actual_seq, msg=None): """An unordered sequence specific comparison. It asserts that actual_seq and expected_seq have the same element counts. Equivalent to:: self.assertEqual(Counter(iter(actual_seq)), Counter(iter(expected_seq))) Asserts that each element has the same count in both sequences. Example: - [0, 1, 1] and [1, 0, 1] compare equal. - [0, 0, 1] and [0, 1] compare unequal. """ first_seq, second_seq = list(expected_seq), list(actual_seq) with warnings.catch_warnings(): if sys.py3kwarning: # Silence Py3k warning raised during the sorting for _msg in ["(code|dict|type) inequality comparisons", "builtin_function_or_method order comparisons", "comparing unequal types"]: warnings.filterwarnings("ignore", _msg, DeprecationWarning) try: first = collections.Counter(first_seq) second = collections.Counter(second_seq) except TypeError: # Handle case with unhashable elements differences = _count_diff_all_purpose(first_seq, second_seq) else: if first == second: return differences = _count_diff_hashable(first_seq, second_seq) if differences: standardMsg = 'Element counts were not equal:\n' lines = ['First has %d, Second has %d: %r' % diff for diff in differences] diffMsg = '\n'.join(lines) standardMsg = self._truncateMessage(standardMsg, diffMsg) msg = self._formatMessage(msg, standardMsg) self.fail(msg) def assertMultiLineEqual(self, first, second, msg=None): """Assert that two multi-line strings are equal.""" self.assertIsInstance(first, basestring, 'First argument is not a string') self.assertIsInstance(second, basestring, 'Second argument is not a string') if first != second: # don't use difflib if the strings are too long if (len(first) > self._diffThreshold or len(second) > self._diffThreshold): self._baseAssertEqual(first, second, msg) firstlines = first.splitlines(True) secondlines = second.splitlines(True) if len(firstlines) == 1 and first.strip('\r\n') == first: firstlines = [first + '\n'] secondlines = [second + '\n'] standardMsg = '%s != %s' % (safe_repr(first, True), safe_repr(second, True)) diff = '\n' + ''.join(difflib.ndiff(firstlines, secondlines)) standardMsg = self._truncateMessage(standardMsg, diff) self.fail(self._formatMessage(msg, standardMsg)) def assertLess(self, a, b, msg=None): """Just like self.assertTrue(a < b), but with a nicer default message.""" if not a < b: standardMsg = '%s not less than %s' % (safe_repr(a), safe_repr(b)) self.fail(self._formatMessage(msg, standardMsg)) def assertLessEqual(self, a, b, msg=None): """Just like self.assertTrue(a <= b), but with a nicer default message.""" if not a <= b: standardMsg = '%s not less than or equal to %s' % (safe_repr(a), safe_repr(b)) self.fail(self._formatMessage(msg, standardMsg)) def assertGreater(self, a, b, msg=None): """Just like self.assertTrue(a > b), but with a nicer default message.""" if not a > b: standardMsg = '%s not greater than %s' % (safe_repr(a), safe_repr(b)) self.fail(self._formatMessage(msg, standardMsg)) def assertGreaterEqual(self, a, b, msg=None): """Just like self.assertTrue(a >= b), but with a nicer default message.""" if not a >= b: standardMsg = '%s not greater than or equal to %s' % (safe_repr(a), safe_repr(b)) self.fail(self._formatMessage(msg, standardMsg)) def assertIsNone(self, obj, msg=None): """Same as self.assertTrue(obj is None), with a nicer default message.""" if obj is not None: standardMsg = '%s is not None' % (safe_repr(obj),) self.fail(self._formatMessage(msg, standardMsg)) def assertIsNotNone(self, obj, msg=None): """Included for symmetry with assertIsNone.""" if obj is None: standardMsg = 'unexpectedly None' self.fail(self._formatMessage(msg, standardMsg)) def assertIsInstance(self, obj, cls, msg=None): """Same as self.assertTrue(isinstance(obj, cls)), with a nicer default message.""" if not isinstance(obj, cls): standardMsg = '%s is not an instance of %r' % (safe_repr(obj), cls) self.fail(self._formatMessage(msg, standardMsg)) def assertNotIsInstance(self, obj, cls, msg=None): """Included for symmetry with assertIsInstance.""" if isinstance(obj, cls): standardMsg = '%s is an instance of %r' % (safe_repr(obj), cls) self.fail(self._formatMessage(msg, standardMsg)) def assertRaisesRegexp(self, expected_exception, expected_regexp, callable_obj=None, *args, **kwargs): """Asserts that the message in a raised exception matches a regexp. Args: expected_exception: Exception class expected to be raised. expected_regexp: Regexp (re pattern object or string) expected to be found in error message. callable_obj: Function to be called. args: Extra args. kwargs: Extra kwargs. """ if expected_regexp is not None: expected_regexp = re.compile(expected_regexp) context = _AssertRaisesContext(expected_exception, self, expected_regexp) if callable_obj is None: return context with context: callable_obj(*args, **kwargs) def assertRegexpMatches(self, text, expected_regexp, msg=None): """Fail the test unless the text matches the regular expression.""" if isinstance(expected_regexp, basestring): expected_regexp = re.compile(expected_regexp) if not expected_regexp.search(text): msg = msg or "Regexp didn't match" msg = '%s: %r not found in %r' % (msg, expected_regexp.pattern, text) raise self.failureException(msg) def assertNotRegexpMatches(self, text, unexpected_regexp, msg=None): """Fail the test if the text matches the regular expression.""" if isinstance(unexpected_regexp, basestring): unexpected_regexp = re.compile(unexpected_regexp) match = unexpected_regexp.search(text) if match: msg = msg or "Regexp matched" msg = '%s: %r matches %r in %r' % (msg, text[match.start():match.end()], unexpected_regexp.pattern, text) raise self.failureException(msg) class FunctionTestCase(TestCase): """A test case that wraps a test function. This is useful for slipping pre-existing test functions into the unittest framework. Optionally, set-up and tidy-up functions can be supplied. As with TestCase, the tidy-up ('tearDown') function will always be called if the set-up ('setUp') function ran successfully. """ def __init__(self, testFunc, setUp=None, tearDown=None, description=None): super(FunctionTestCase, self).__init__() self._setUpFunc = setUp self._tearDownFunc = tearDown self._testFunc = testFunc self._description = description def setUp(self): if self._setUpFunc is not None: self._setUpFunc() def tearDown(self): if self._tearDownFunc is not None: self._tearDownFunc() def runTest(self): self._testFunc() def id(self): return self._testFunc.__name__ def __eq__(self, other): if not isinstance(other, self.__class__): return NotImplemented return self._setUpFunc == other._setUpFunc and \ self._tearDownFunc == other._tearDownFunc and \ self._testFunc == other._testFunc and \ self._description == other._description def __ne__(self, other): return not self == other def __hash__(self): return hash((type(self), self._setUpFunc, self._tearDownFunc, self._testFunc, self._description)) def __str__(self): return "%s (%s)" % (strclass(self.__class__), self._testFunc.__name__) def __repr__(self): return "<%s tec=%s>" % (strclass(self.__class__), self._testFunc) def shortDescription(self): if self._description is not None: return self._description doc = self._testFunc.__doc__ return doc and doc.split("\n")[0].strip() or None """Loading unittests.""" import os import re import sys import traceback import types from functools import cmp_to_key as _CmpToKey from fnmatch import fnmatch from . import case, suite __unittest = True # what about .pyc or .pyo (etc) # we would need to avoid loading the same tests multiple times # from '.py', '.pyc' *and* '.pyo' VALID_MODULE_NAME = re.compile(r'[_a-z]\w*\.py$', re.IGNORECASE) def _make_failed_import_test(name, suiteClass): message = 'Failed to import test module: %s\n%s' % (name, traceback.format_exc()) return _make_failed_test('ModuleImportFailure', name, ImportError(message), suiteClass) def _make_failed_load_tests(name, exception, suiteClass): return _make_failed_test('LoadTestsFailure', name, exception, suiteClass) def _make_failed_test(classname, methodname, exception, suiteClass): def testFailure(self): raise exception attrs = {methodname: testFailure} TestClass = type(classname, (case.TestCase,), attrs) return suiteClass((TestClass(methodname),)) class TestLoader(object): """ This class is responsible for loading tests according to various criteria and returning them wrapped in a TestSuite """ testMethodPrefix = 'test' sortTestMethodsUsing = cmp suiteClass = suite.TestSuite _top_level_dir = None def loadTestsFromTestCase(self, testCaseClass): """Return a suite of all test cases contained in testCaseClass""" if issubclass(testCaseClass, suite.TestSuite): raise TypeError("Test cases should not be derived from TestSuite." \ " Maybe you meant to derive from TestCase?") testCaseNames = self.getTestCaseNames(testCaseClass) if not testCaseNames and hasattr(testCaseClass, 'runTest'): testCaseNames = ['runTest'] loaded_suite = self.suiteClass(map(testCaseClass, testCaseNames)) return loaded_suite def loadTestsFromModule(self, module, use_load_tests=True): """Return a suite of all test cases contained in the given module""" tests = [] for name in dir(module): obj = getattr(module, name) if isinstance(obj, type) and issubclass(obj, case.TestCase): tests.append(self.loadTestsFromTestCase(obj)) load_tests = getattr(module, 'load_tests', None) tests = self.suiteClass(tests) if use_load_tests and load_tests is not None: try: return load_tests(self, tests, None) except Exception, e: return _make_failed_load_tests(module.__name__, e, self.suiteClass) return tests def loadTestsFromName(self, name, module=None): """Return a suite of all test cases given a string specifier. The name may resolve either to a module, a test case class, a test method within a test case class, or a callable object which returns a TestCase or TestSuite instance. The method optionally resolves the names relative to a given module. """ parts = name.split('.') if module is None: parts_copy = parts[:] while parts_copy: try: module = __import__('.'.join(parts_copy)) break except ImportError: del parts_copy[-1] if not parts_copy: raise parts = parts[1:] obj = module for part in parts: parent, obj = obj, getattr(obj, part) if isinstance(obj, types.ModuleType): return self.loadTestsFromModule(obj) elif isinstance(obj, type) and issubclass(obj, case.TestCase): return self.loadTestsFromTestCase(obj) elif (isinstance(obj, types.UnboundMethodType) and isinstance(parent, type) and issubclass(parent, case.TestCase)): name = parts[-1] inst = parent(name) return self.suiteClass([inst]) elif isinstance(obj, suite.TestSuite): return obj elif hasattr(obj, '__call__'): test = obj() if isinstance(test, suite.TestSuite): return test elif isinstance(test, case.TestCase): return self.suiteClass([test]) else: raise TypeError("calling %s returned %s, not a test" % (obj, test)) else: raise TypeError("don't know how to make test from: %s" % obj) def loadTestsFromNames(self, names, module=None): """Return a suite of all test cases found using the given sequence of string specifiers. See 'loadTestsFromName()'. """ suites = [self.loadTestsFromName(name, module) for name in names] return self.suiteClass(suites) def getTestCaseNames(self, testCaseClass): """Return a sorted sequence of method names found within testCaseClass """ def isTestMethod(attrname, testCaseClass=testCaseClass, prefix=self.testMethodPrefix): return attrname.startswith(prefix) and \ hasattr(getattr(testCaseClass, attrname), '__call__') testFnNames = filter(isTestMethod, dir(testCaseClass)) if self.sortTestMethodsUsing: testFnNames.sort(key=_CmpToKey(self.sortTestMethodsUsing)) return testFnNames def discover(self, start_dir, pattern='test*.py', top_level_dir=None): """Find and return all test modules from the specified start directory, recursing into subdirectories to find them. Only test files that match the pattern will be loaded. (Using shell style pattern matching.) All test modules must be importable from the top level of the project. If the start directory is not the top level directory then the top level directory must be specified separately. If a test package name (directory with '__init__.py') matches the pattern then the package will be checked for a 'load_tests' function. If this exists then it will be called with loader, tests, pattern. If load_tests exists then discovery does *not* recurse into the package, load_tests is responsible for loading all tests in the package. The pattern is deliberately not stored as a loader attribute so that packages can continue discovery themselves. top_level_dir is stored so load_tests does not need to pass this argument in to loader.discover(). """ set_implicit_top = False if top_level_dir is None and self._top_level_dir is not None: # make top_level_dir optional if called from load_tests in a package top_level_dir = self._top_level_dir elif top_level_dir is None: set_implicit_top = True top_level_dir = start_dir top_level_dir = os.path.abspath(top_level_dir) if not top_level_dir in sys.path: # all test modules must be importable from the top level directory # should we *unconditionally* put the start directory in first # in sys.path to minimise likelihood of conflicts between installed # modules and development versions? sys.path.insert(0, top_level_dir) self._top_level_dir = top_level_dir is_not_importable = False if os.path.isdir(os.path.abspath(start_dir)): start_dir = os.path.abspath(start_dir) if start_dir != top_level_dir: is_not_importable = not os.path.isfile(os.path.join(start_dir, '__init__.py')) else: # support for discovery from dotted module names try: __import__(start_dir) except ImportError: is_not_importable = True else: the_module = sys.modules[start_dir] top_part = start_dir.split('.')[0] start_dir = os.path.abspath(os.path.dirname((the_module.__file__))) if set_implicit_top: self._top_level_dir = self._get_directory_containing_module(top_part) sys.path.remove(top_level_dir) if is_not_importable: raise ImportError('Start directory is not importable: %r' % start_dir) tests = list(self._find_tests(start_dir, pattern)) return self.suiteClass(tests) def _get_directory_containing_module(self, module_name): module = sys.modules[module_name] full_path = os.path.abspath(module.__file__) if os.path.basename(full_path).lower().startswith('__init__.py'): return os.path.dirname(os.path.dirname(full_path)) else: # here we have been given a module rather than a package - so # all we can do is search the *same* directory the module is in # should an exception be raised instead return os.path.dirname(full_path) def _get_name_from_path(self, path): path = os.path.splitext(os.path.normpath(path))[0] _relpath = os.path.relpath(path, self._top_level_dir) assert not os.path.isabs(_relpath), "Path must be within the project" assert not _relpath.startswith('..'), "Path must be within the project" name = _relpath.replace(os.path.sep, '.') return name def _get_module_from_name(self, name): __import__(name) return sys.modules[name] def _match_path(self, path, full_path, pattern): # override this method to use alternative matching strategy return fnmatch(path, pattern) def _find_tests(self, start_dir, pattern): """Used by discovery. Yields test suites it loads.""" paths = os.listdir(start_dir) for path in paths: full_path = os.path.join(start_dir, path) if os.path.isfile(full_path): if not VALID_MODULE_NAME.match(path): # valid Python identifiers only continue if not self._match_path(path, full_path, pattern): continue # if the test file matches, load it name = self._get_name_from_path(full_path) try: module = self._get_module_from_name(name) except: yield _make_failed_import_test(name, self.suiteClass) else: mod_file = os.path.abspath(getattr(module, '__file__', full_path)) realpath = os.path.splitext(os.path.realpath(mod_file))[0] fullpath_noext = os.path.splitext(os.path.realpath(full_path))[0] if realpath.lower() != fullpath_noext.lower(): module_dir = os.path.dirname(realpath) mod_name = os.path.splitext(os.path.basename(full_path))[0] expected_dir = os.path.dirname(full_path) msg = ("%r module incorrectly imported from %r. Expected %r. " "Is this module globally installed?") raise ImportError(msg % (mod_name, module_dir, expected_dir)) yield self.loadTestsFromModule(module) elif os.path.isdir(full_path): if not os.path.isfile(os.path.join(full_path, '__init__.py')): continue load_tests = None tests = None if fnmatch(path, pattern): # only check load_tests if the package directory itself matches the filter name = self._get_name_from_path(full_path) package = self._get_module_from_name(name) load_tests = getattr(package, 'load_tests', None) tests = self.loadTestsFromModule(package, use_load_tests=False) if load_tests is None: if tests is not None: # tests loaded from package file yield tests # recurse into the package for test in self._find_tests(full_path, pattern): yield test else: try: yield load_tests(self, tests, pattern) except Exception, e: yield _make_failed_load_tests(package.__name__, e, self.suiteClass) defaultTestLoader = TestLoader() def _makeLoader(prefix, sortUsing, suiteClass=None): loader = TestLoader() loader.sortTestMethodsUsing = sortUsing loader.testMethodPrefix = prefix if suiteClass: loader.suiteClass = suiteClass return loader def getTestCaseNames(testCaseClass, prefix, sortUsing=cmp): return _makeLoader(prefix, sortUsing).getTestCaseNames(testCaseClass) def makeSuite(testCaseClass, prefix='test', sortUsing=cmp, suiteClass=suite.TestSuite): return _makeLoader(prefix, sortUsing, suiteClass).loadTestsFromTestCase(testCaseClass) def findTestCases(module, prefix='test', sortUsing=cmp, suiteClass=suite.TestSuite): return _makeLoader(prefix, sortUsing, suiteClass).loadTestsFromModule(module) """Unittest main program""" import sys import os import types from . import loader, runner from .signals import installHandler __unittest = True FAILFAST = " -f, --failfast Stop on first failure\n" CATCHBREAK = " -c, --catch Catch control-C and display results\n" BUFFEROUTPUT = " -b, --buffer Buffer stdout and stderr during test runs\n" USAGE_AS_MAIN = """\ Usage: %(progName)s [options] [tests] Options: -h, --help Show this message -v, --verbose Verbose output -q, --quiet Minimal output %(failfast)s%(catchbreak)s%(buffer)s Examples: %(progName)s test_module - run tests from test_module %(progName)s module.TestClass - run tests from module.TestClass %(progName)s module.Class.test_method - run specified test method [tests] can be a list of any number of test modules, classes and test methods. Alternative Usage: %(progName)s discover [options] Options: -v, --verbose Verbose output %(failfast)s%(catchbreak)s%(buffer)s -s directory Directory to start discovery ('.' default) -p pattern Pattern to match test files ('test*.py' default) -t directory Top level directory of project (default to start directory) For test discovery all test modules must be importable from the top level directory of the project. """ USAGE_FROM_MODULE = """\ Usage: %(progName)s [options] [test] [...] Options: -h, --help Show this message -v, --verbose Verbose output -q, --quiet Minimal output %(failfast)s%(catchbreak)s%(buffer)s Examples: %(progName)s - run default set of tests %(progName)s MyTestSuite - run suite 'MyTestSuite' %(progName)s MyTestCase.testSomething - run MyTestCase.testSomething %(progName)s MyTestCase - run all 'test*' test methods in MyTestCase """ class TestProgram(object): """A command-line program that runs a set of tests; this is primarily for making test modules conveniently executable. """ USAGE = USAGE_FROM_MODULE # defaults for testing failfast = catchbreak = buffer = progName = None def __init__(self, module='__main__', defaultTest=None, argv=None, testRunner=None, testLoader=loader.defaultTestLoader, exit=True, verbosity=1, failfast=None, catchbreak=None, buffer=None): if isinstance(module, basestring): self.module = __import__(module) for part in module.split('.')[1:]: self.module = getattr(self.module, part) else: self.module = module if argv is None: argv = sys.argv self.exit = exit self.failfast = failfast self.catchbreak = catchbreak self.verbosity = verbosity self.buffer = buffer self.defaultTest = defaultTest self.testRunner = testRunner self.testLoader = testLoader self.progName = os.path.basename(argv[0]) self.parseArgs(argv) self.runTests() def usageExit(self, msg=None): if msg: print msg usage = {'progName': self.progName, 'catchbreak': '', 'failfast': '', 'buffer': ''} if self.failfast != False: usage['failfast'] = FAILFAST if self.catchbreak != False: usage['catchbreak'] = CATCHBREAK if self.buffer != False: usage['buffer'] = BUFFEROUTPUT print self.USAGE % usage sys.exit(2) def parseArgs(self, argv): if len(argv) > 1 and argv[1].lower() == 'discover': self._do_discovery(argv[2:]) return import getopt long_opts = ['help', 'verbose', 'quiet', 'failfast', 'catch', 'buffer'] try: options, args = getopt.getopt(argv[1:], 'hHvqfcb', long_opts) for opt, value in options: if opt in ('-h','-H','--help'): self.usageExit() if opt in ('-q','--quiet'): self.verbosity = 0 if opt in ('-v','--verbose'): self.verbosity = 2 if opt in ('-f','--failfast'): if self.failfast is None: self.failfast = True # Should this raise an exception if -f is not valid? if opt in ('-c','--catch'): if self.catchbreak is None: self.catchbreak = True # Should this raise an exception if -c is not valid? if opt in ('-b','--buffer'): if self.buffer is None: self.buffer = True # Should this raise an exception if -b is not valid? if len(args) == 0 and self.defaultTest is None: # createTests will load tests from self.module self.testNames = None elif len(args) > 0: self.testNames = args if __name__ == '__main__': # to support python -m unittest ... self.module = None else: self.testNames = (self.defaultTest,) self.createTests() except getopt.error, msg: self.usageExit(msg) def createTests(self): if self.testNames is None: self.test = self.testLoader.loadTestsFromModule(self.module) else: self.test = self.testLoader.loadTestsFromNames(self.testNames, self.module) def _do_discovery(self, argv, Loader=None): if Loader is None: Loader = lambda: self.testLoader # handle command line args for test discovery self.progName = '%s discover' % self.progName import optparse parser = optparse.OptionParser() parser.prog = self.progName parser.add_option('-v', '--verbose', dest='verbose', default=False, help='Verbose output', action='store_true') if self.failfast != False: parser.add_option('-f', '--failfast', dest='failfast', default=False, help='Stop on first fail or error', action='store_true') if self.catchbreak != False: parser.add_option('-c', '--catch', dest='catchbreak', default=False, help='Catch Ctrl-C and display results so far', action='store_true') if self.buffer != False: parser.add_option('-b', '--buffer', dest='buffer', default=False, help='Buffer stdout and stderr during tests', action='store_true') parser.add_option('-s', '--start-directory', dest='start', default='.', help="Directory to start discovery ('.' default)") parser.add_option('-p', '--pattern', dest='pattern', default='test*.py', help="Pattern to match tests ('test*.py' default)") parser.add_option('-t', '--top-level-directory', dest='top', default=None, help='Top level directory of project (defaults to start directory)') options, args = parser.parse_args(argv) if len(args) > 3: self.usageExit() for name, value in zip(('start', 'pattern', 'top'), args): setattr(options, name, value) # only set options from the parsing here # if they weren't set explicitly in the constructor if self.failfast is None: self.failfast = options.failfast if self.catchbreak is None: self.catchbreak = options.catchbreak if self.buffer is None: self.buffer = options.buffer if options.verbose: self.verbosity = 2 start_dir = options.start pattern = options.pattern top_level_dir = options.top loader = Loader() self.test = loader.discover(start_dir, pattern, top_level_dir) def runTests(self): if self.catchbreak: installHandler() if self.testRunner is None: self.testRunner = runner.TextTestRunner if isinstance(self.testRunner, (type, types.ClassType)): try: testRunner = self.testRunner(verbosity=self.verbosity, failfast=self.failfast, buffer=self.buffer) except TypeError: # didn't accept the verbosity, buffer or failfast arguments testRunner = self.testRunner() else: # it is assumed to be a TestRunner instance testRunner = self.testRunner self.result = testRunner.run(self.test) if self.exit: sys.exit(not self.result.wasSuccessful()) main = TestProgram """Test result object""" import os import sys import traceback from StringIO import StringIO from . import util from functools import wraps __unittest = True def failfast(method): @wraps(method) def inner(self, *args, **kw): if getattr(self, 'failfast', False): self.stop() return method(self, *args, **kw) return inner STDOUT_LINE = '\nStdout:\n%s' STDERR_LINE = '\nStderr:\n%s' class TestResult(object): """Holder for test result information. Test results are automatically managed by the TestCase and TestSuite classes, and do not need to be explicitly manipulated by writers of tests. Each instance holds the total number of tests run, and collections of failures and errors that occurred among those test runs. The collections contain tuples of (testcase, exceptioninfo), where exceptioninfo is the formatted traceback of the error that occurred. """ _previousTestClass = None _testRunEntered = False _moduleSetUpFailed = False def __init__(self, stream=None, descriptions=None, verbosity=None): self.failfast = False self.failures = [] self.errors = [] self.testsRun = 0 self.skipped = [] self.expectedFailures = [] self.unexpectedSuccesses = [] self.shouldStop = False self.buffer = False self._stdout_buffer = None self._stderr_buffer = None self._original_stdout = sys.stdout self._original_stderr = sys.stderr self._mirrorOutput = False def printErrors(self): "Called by TestRunner after test run" def startTest(self, test): "Called when the given test is about to be run" self.testsRun += 1 self._mirrorOutput = False self._setupStdout() def _setupStdout(self): if self.buffer: if self._stderr_buffer is None: self._stderr_buffer = StringIO() self._stdout_buffer = StringIO() sys.stdout = self._stdout_buffer sys.stderr = self._stderr_buffer def startTestRun(self): """Called once before any tests are executed. See startTest for a method called before each test. """ def stopTest(self, test): """Called when the given test has been run""" self._restoreStdout() self._mirrorOutput = False def _restoreStdout(self): if self.buffer: if self._mirrorOutput: output = sys.stdout.getvalue() error = sys.stderr.getvalue() if output: if not output.endswith('\n'): output += '\n' self._original_stdout.write(STDOUT_LINE % output) if error: if not error.endswith('\n'): error += '\n' self._original_stderr.write(STDERR_LINE % error) sys.stdout = self._original_stdout sys.stderr = self._original_stderr self._stdout_buffer.seek(0) self._stdout_buffer.truncate() self._stderr_buffer.seek(0) self._stderr_buffer.truncate() def stopTestRun(self): """Called once after all tests are executed. See stopTest for a method called after each test. """ @failfast def addError(self, test, err): """Called when an error has occurred. 'err' is a tuple of values as returned by sys.exc_info(). """ self.errors.append((test, self._exc_info_to_string(err, test))) self._mirrorOutput = True @failfast def addFailure(self, test, err): """Called when an error has occurred. 'err' is a tuple of values as returned by sys.exc_info().""" self.failures.append((test, self._exc_info_to_string(err, test))) self._mirrorOutput = True def addSuccess(self, test): "Called when a test has completed successfully" pass def addSkip(self, test, reason): """Called when a test is skipped.""" self.skipped.append((test, reason)) def addExpectedFailure(self, test, err): """Called when an expected failure/error occurred.""" self.expectedFailures.append( (test, self._exc_info_to_string(err, test))) @failfast def addUnexpectedSuccess(self, test): """Called when a test was expected to fail, but succeed.""" self.unexpectedSuccesses.append(test) def wasSuccessful(self): "Tells whether or not this result was a success" return len(self.failures) == len(self.errors) == 0 def stop(self): "Indicates that the tests should be aborted" self.shouldStop = True def _exc_info_to_string(self, err, test): """Converts a sys.exc_info()-style tuple of values into a string.""" exctype, value, tb = err # Skip test runner traceback levels while tb and self._is_relevant_tb_level(tb): tb = tb.tb_next if exctype is test.failureException: # Skip assert*() traceback levels length = self._count_relevant_tb_levels(tb) msgLines = traceback.format_exception(exctype, value, tb, length) else: msgLines = traceback.format_exception(exctype, value, tb) if self.buffer: output = sys.stdout.getvalue() error = sys.stderr.getvalue() if output: if not output.endswith('\n'): output += '\n' msgLines.append(STDOUT_LINE % output) if error: if not error.endswith('\n'): error += '\n' msgLines.append(STDERR_LINE % error) return ''.join(msgLines) def _is_relevant_tb_level(self, tb): return '__unittest' in tb.tb_frame.f_globals def _count_relevant_tb_levels(self, tb): length = 0 while tb and not self._is_relevant_tb_level(tb): length += 1 tb = tb.tb_next return length def __repr__(self): return ("<%s run=%i errors=%i failures=%i>" % (util.strclass(self.__class__), self.testsRun, len(self.errors), len(self.failures))) """Running tests""" import sys import time from . import result from .signals import registerResult __unittest = True class _WritelnDecorator(object): """Used to decorate file-like objects with a handy 'writeln' method""" def __init__(self,stream): self.stream = stream def __getattr__(self, attr): if attr in ('stream', '__getstate__'): raise AttributeError(attr) return getattr(self.stream,attr) def writeln(self, arg=None): if arg: self.write(arg) self.write('\n') # text-mode streams translate to \r\n if needed class TextTestResult(result.TestResult): """A test result class that can print formatted text results to a stream. Used by TextTestRunner. """ separator1 = '=' * 70 separator2 = '-' * 70 def __init__(self, stream, descriptions, verbosity): super(TextTestResult, self).__init__(stream, descriptions, verbosity) self.stream = stream self.showAll = verbosity > 1 self.dots = verbosity == 1 self.descriptions = descriptions def getDescription(self, test): doc_first_line = test.shortDescription() if self.descriptions and doc_first_line: return '\n'.join((str(test), doc_first_line)) else: return str(test) def startTest(self, test): super(TextTestResult, self).startTest(test) if self.showAll: self.stream.write(self.getDescription(test)) self.stream.write(" ... ") self.stream.flush() def addSuccess(self, test): super(TextTestResult, self).addSuccess(test) if self.showAll: self.stream.writeln("ok") elif self.dots: self.stream.write('.') self.stream.flush() def addError(self, test, err): super(TextTestResult, self).addError(test, err) if self.showAll: self.stream.writeln("ERROR") elif self.dots: self.stream.write('E') self.stream.flush() def addFailure(self, test, err): super(TextTestResult, self).addFailure(test, err) if self.showAll: self.stream.writeln("FAIL") elif self.dots: self.stream.write('F') self.stream.flush() def addSkip(self, test, reason): super(TextTestResult, self).addSkip(test, reason) if self.showAll: self.stream.writeln("skipped {0!r}".format(reason)) elif self.dots: self.stream.write("s") self.stream.flush() def addExpectedFailure(self, test, err): super(TextTestResult, self).addExpectedFailure(test, err) if self.showAll: self.stream.writeln("expected failure") elif self.dots: self.stream.write("x") self.stream.flush() def addUnexpectedSuccess(self, test): super(TextTestResult, self).addUnexpectedSuccess(test) if self.showAll: self.stream.writeln("unexpected success") elif self.dots: self.stream.write("u") self.stream.flush() def printErrors(self): if self.dots or self.showAll: self.stream.writeln() self.printErrorList('ERROR', self.errors) self.printErrorList('FAIL', self.failures) def printErrorList(self, flavour, errors): for test, err in errors: self.stream.writeln(self.separator1) self.stream.writeln("%s: %s" % (flavour,self.getDescription(test))) self.stream.writeln(self.separator2) self.stream.writeln("%s" % err) class TextTestRunner(object): """A test runner class that displays results in textual form. It prints out the names of tests as they are run, errors as they occur, and a summary of the results at the end of the test run. """ resultclass = TextTestResult def __init__(self, stream=sys.stderr, descriptions=True, verbosity=1, failfast=False, buffer=False, resultclass=None): self.stream = _WritelnDecorator(stream) self.descriptions = descriptions self.verbosity = verbosity self.failfast = failfast self.buffer = buffer if resultclass is not None: self.resultclass = resultclass def _makeResult(self): return self.resultclass(self.stream, self.descriptions, self.verbosity) def run(self, test): "Run the given test case or test suite." result = self._makeResult() registerResult(result) result.failfast = self.failfast result.buffer = self.buffer startTime = time.time() startTestRun = getattr(result, 'startTestRun', None) if startTestRun is not None: startTestRun() try: test(result) finally: stopTestRun = getattr(result, 'stopTestRun', None) if stopTestRun is not None: stopTestRun() stopTime = time.time() timeTaken = stopTime - startTime result.printErrors() if hasattr(result, 'separator2'): self.stream.writeln(result.separator2) run = result.testsRun self.stream.writeln("Ran %d test%s in %.3fs" % (run, run != 1 and "s" or "", timeTaken)) self.stream.writeln() expectedFails = unexpectedSuccesses = skipped = 0 try: results = map(len, (result.expectedFailures, result.unexpectedSuccesses, result.skipped)) except AttributeError: pass else: expectedFails, unexpectedSuccesses, skipped = results infos = [] if not result.wasSuccessful(): self.stream.write("FAILED") failed, errored = map(len, (result.failures, result.errors)) if failed: infos.append("failures=%d" % failed) if errored: infos.append("errors=%d" % errored) else: self.stream.write("OK") if skipped: infos.append("skipped=%d" % skipped) if expectedFails: infos.append("expected failures=%d" % expectedFails) if unexpectedSuccesses: infos.append("unexpected successes=%d" % unexpectedSuccesses) if infos: self.stream.writeln(" (%s)" % (", ".join(infos),)) else: self.stream.write("\n") return result import signal import weakref from functools import wraps __unittest = True class _InterruptHandler(object): def __init__(self, default_handler): self.called = False self.original_handler = default_handler if isinstance(default_handler, (int, long)): if default_handler == signal.SIG_DFL: # Pretend it's signal.default_int_handler instead. default_handler = signal.default_int_handler elif default_handler == signal.SIG_IGN: # Not quite the same thing as SIG_IGN, but the closest we # can make it: do nothing. def default_handler(unused_signum, unused_frame): pass else: raise TypeError("expected SIGINT signal handler to be " "signal.SIG_IGN, signal.SIG_DFL, or a " "callable object") self.default_handler = default_handler def __call__(self, signum, frame): installed_handler = signal.getsignal(signal.SIGINT) if installed_handler is not self: # if we aren't the installed handler, then delegate immediately # to the default handler self.default_handler(signum, frame) if self.called: self.default_handler(signum, frame) self.called = True for result in _results.keys(): result.stop() _results = weakref.WeakKeyDictionary() def registerResult(result): _results[result] = 1 def removeResult(result): return bool(_results.pop(result, None)) _interrupt_handler = None def installHandler(): global _interrupt_handler if _interrupt_handler is None: default_handler = signal.getsignal(signal.SIGINT) _interrupt_handler = _InterruptHandler(default_handler) signal.signal(signal.SIGINT, _interrupt_handler) def removeHandler(method=None): if method is not None: @wraps(method) def inner(*args, **kwargs): initial = signal.getsignal(signal.SIGINT) removeHandler() try: return method(*args, **kwargs) finally: signal.signal(signal.SIGINT, initial) return inner global _interrupt_handler if _interrupt_handler is not None: signal.signal(signal.SIGINT, _interrupt_handler.original_handler) """TestSuite""" import sys from . import case from . import util __unittest = True def _call_if_exists(parent, attr): func = getattr(parent, attr, lambda: None) func() class BaseTestSuite(object): """A simple test suite that doesn't provide class or module shared fixtures. """ def __init__(self, tests=()): self._tests = [] self.addTests(tests) def __repr__(self): return "<%s tests=%s>" % (util.strclass(self.__class__), list(self)) def __eq__(self, other): if not isinstance(other, self.__class__): return NotImplemented return list(self) == list(other) def __ne__(self, other): return not self == other # Can't guarantee hash invariant, so flag as unhashable __hash__ = None def __iter__(self): return iter(self._tests) def countTestCases(self): cases = 0 for test in self: cases += test.countTestCases() return cases def addTest(self, test): # sanity checks if not hasattr(test, '__call__'): raise TypeError("{} is not callable".format(repr(test))) if isinstance(test, type) and issubclass(test, (case.TestCase, TestSuite)): raise TypeError("TestCases and TestSuites must be instantiated " "before passing them to addTest()") self._tests.append(test) def addTests(self, tests): if isinstance(tests, basestring): raise TypeError("tests must be an iterable of tests, not a string") for test in tests: self.addTest(test) def run(self, result): for test in self: if result.shouldStop: break test(result) return result def __call__(self, *args, **kwds): return self.run(*args, **kwds) def debug(self): """Run the tests without collecting errors in a TestResult""" for test in self: test.debug() class TestSuite(BaseTestSuite): """A test suite is a composite test consisting of a number of TestCases. For use, create an instance of TestSuite, then add test case instances. When all tests have been added, the suite can be passed to a test runner, such as TextTestRunner. It will run the individual test cases in the order in which they were added, aggregating the results. When subclassing, do not forget to call the base class constructor. """ def run(self, result, debug=False): topLevel = False if getattr(result, '_testRunEntered', False) is False: result._testRunEntered = topLevel = True for test in self: if result.shouldStop: break if _isnotsuite(test): self._tearDownPreviousClass(test, result) self._handleModuleFixture(test, result) self._handleClassSetUp(test, result) result._previousTestClass = test.__class__ if (getattr(test.__class__, '_classSetupFailed', False) or getattr(result, '_moduleSetUpFailed', False)): continue if not debug: test(result) else: test.debug() if topLevel: self._tearDownPreviousClass(None, result) self._handleModuleTearDown(result) result._testRunEntered = False return result def debug(self): """Run the tests without collecting errors in a TestResult""" debug = _DebugResult() self.run(debug, True) ################################ def _handleClassSetUp(self, test, result): previousClass = getattr(result, '_previousTestClass', None) currentClass = test.__class__ if currentClass == previousClass: return if result._moduleSetUpFailed: return if getattr(currentClass, "__unittest_skip__", False): return try: currentClass._classSetupFailed = False except TypeError: # test may actually be a function # so its class will be a builtin-type pass setUpClass = getattr(currentClass, 'setUpClass', None) if setUpClass is not None: _call_if_exists(result, '_setupStdout') try: setUpClass() except Exception as e: if isinstance(result, _DebugResult): raise currentClass._classSetupFailed = True className = util.strclass(currentClass) errorName = 'setUpClass (%s)' % className self._addClassOrModuleLevelException(result, e, errorName) finally: _call_if_exists(result, '_restoreStdout') def _get_previous_module(self, result): previousModule = None previousClass = getattr(result, '_previousTestClass', None) if previousClass is not None: previousModule = previousClass.__module__ return previousModule def _handleModuleFixture(self, test, result): previousModule = self._get_previous_module(result) currentModule = test.__class__.__module__ if currentModule == previousModule: return self._handleModuleTearDown(result) result._moduleSetUpFailed = False try: module = sys.modules[currentModule] except KeyError: return setUpModule = getattr(module, 'setUpModule', None) if setUpModule is not None: _call_if_exists(result, '_setupStdout') try: setUpModule() except Exception, e: if isinstance(result, _DebugResult): raise result._moduleSetUpFailed = True errorName = 'setUpModule (%s)' % currentModule self._addClassOrModuleLevelException(result, e, errorName) finally: _call_if_exists(result, '_restoreStdout') def _addClassOrModuleLevelException(self, result, exception, errorName): error = _ErrorHolder(errorName) addSkip = getattr(result, 'addSkip', None) if addSkip is not None and isinstance(exception, case.SkipTest): addSkip(error, str(exception)) else: result.addError(error, sys.exc_info()) def _handleModuleTearDown(self, result): previousModule = self._get_previous_module(result) if previousModule is None: return if result._moduleSetUpFailed: return try: module = sys.modules[previousModule] except KeyError: return tearDownModule = getattr(module, 'tearDownModule', None) if tearDownModule is not None: _call_if_exists(result, '_setupStdout') try: tearDownModule() except Exception as e: if isinstance(result, _DebugResult): raise errorName = 'tearDownModule (%s)' % previousModule self._addClassOrModuleLevelException(result, e, errorName) finally: _call_if_exists(result, '_restoreStdout') def _tearDownPreviousClass(self, test, result): previousClass = getattr(result, '_previousTestClass', None) currentClass = test.__class__ if currentClass == previousClass: return if getattr(previousClass, '_classSetupFailed', False): return if getattr(result, '_moduleSetUpFailed', False): return if getattr(previousClass, "__unittest_skip__", False): return tearDownClass = getattr(previousClass, 'tearDownClass', None) if tearDownClass is not None: _call_if_exists(result, '_setupStdout') try: tearDownClass() except Exception, e: if isinstance(result, _DebugResult): raise className = util.strclass(previousClass) errorName = 'tearDownClass (%s)' % className self._addClassOrModuleLevelException(result, e, errorName) finally: _call_if_exists(result, '_restoreStdout') class _ErrorHolder(object): """ Placeholder for a TestCase inside a result. As far as a TestResult is concerned, this looks exactly like a unit test. Used to insert arbitrary errors into a test suite run. """ # Inspired by the ErrorHolder from Twisted: # http://twistedmatrix.com/trac/browser/trunk/twisted/trial/runner.py # attribute used by TestResult._exc_info_to_string failureException = None def __init__(self, description): self.description = description def id(self): return self.description def shortDescription(self): return None def __repr__(self): return "" % (self.description,) def __str__(self): return self.id() def run(self, result): # could call result.addError(...) - but this test-like object # shouldn't be run anyway pass def __call__(self, result): return self.run(result) def countTestCases(self): return 0 def _isnotsuite(test): "A crude way to tell apart testcases and suites with duck-typing" try: iter(test) except TypeError: return True return False class _DebugResult(object): "Used by the TestSuite to hold previous class when running in debug." _previousTestClass = None _moduleSetUpFailed = False shouldStop = False """Various utility functions.""" from collections import namedtuple, OrderedDict __unittest = True _MAX_LENGTH = 80 def safe_repr(obj, short=False): try: result = repr(obj) except Exception: result = object.__repr__(obj) if not short or len(result) < _MAX_LENGTH: return result return result[:_MAX_LENGTH] + ' [truncated]...' def strclass(cls): return "%s.%s" % (cls.__module__, cls.__name__) def sorted_list_difference(expected, actual): """Finds elements in only one or the other of two, sorted input lists. Returns a two-element tuple of lists. The first list contains those elements in the "expected" list but not in the "actual" list, and the second contains those elements in the "actual" list but not in the "expected" list. Duplicate elements in either input list are ignored. """ i = j = 0 missing = [] unexpected = [] while True: try: e = expected[i] a = actual[j] if e < a: missing.append(e) i += 1 while expected[i] == e: i += 1 elif e > a: unexpected.append(a) j += 1 while actual[j] == a: j += 1 else: i += 1 try: while expected[i] == e: i += 1 finally: j += 1 while actual[j] == a: j += 1 except IndexError: missing.extend(expected[i:]) unexpected.extend(actual[j:]) break return missing, unexpected def unorderable_list_difference(expected, actual, ignore_duplicate=False): """Same behavior as sorted_list_difference but for lists of unorderable items (like dicts). As it does a linear search per item (remove) it has O(n*n) performance. """ missing = [] unexpected = [] while expected: item = expected.pop() try: actual.remove(item) except ValueError: missing.append(item) if ignore_duplicate: for lst in expected, actual: try: while True: lst.remove(item) except ValueError: pass if ignore_duplicate: while actual: item = actual.pop() unexpected.append(item) try: while True: actual.remove(item) except ValueError: pass return missing, unexpected # anything left in actual is unexpected return missing, actual _Mismatch = namedtuple('Mismatch', 'actual expected value') def _count_diff_all_purpose(actual, expected): 'Returns list of (cnt_act, cnt_exp, elem) triples where the counts differ' # elements need not be hashable s, t = list(actual), list(expected) m, n = len(s), len(t) NULL = object() result = [] for i, elem in enumerate(s): if elem is NULL: continue cnt_s = cnt_t = 0 for j in range(i, m): if s[j] == elem: cnt_s += 1 s[j] = NULL for j, other_elem in enumerate(t): if other_elem == elem: cnt_t += 1 t[j] = NULL if cnt_s != cnt_t: diff = _Mismatch(cnt_s, cnt_t, elem) result.append(diff) for i, elem in enumerate(t): if elem is NULL: continue cnt_t = 0 for j in range(i, n): if t[j] == elem: cnt_t += 1 t[j] = NULL diff = _Mismatch(0, cnt_t, elem) result.append(diff) return result def _ordered_count(iterable): 'Return dict of element counts, in the order they were first seen' c = OrderedDict() for elem in iterable: c[elem] = c.get(elem, 0) + 1 return c def _count_diff_hashable(actual, expected): 'Returns list of (cnt_act, cnt_exp, elem) triples where the counts differ' # elements must be hashable s, t = _ordered_count(actual), _ordered_count(expected) result = [] for elem, cnt_s in s.items(): cnt_t = t.get(elem, 0) if cnt_s != cnt_t: diff = _Mismatch(cnt_s, cnt_t, elem) result.append(diff) for elem, cnt_t in t.items(): if elem not in s: diff = _Mismatch(0, cnt_t, elem) result.append(diff) return result # -*- coding: utf-8 -*- """Helper functions and classes for managing GUI elements in plug-ins.""" from __future__ import absolute_import, division, print_function, unicode_literals from future.builtins import * from .cell_renderers import * from .drag_and_drop_context import * from .editable_label import * from .entries import * from .entry_expander import * from .entry_popup import * from .entry_undo_context import * from .gimp_item_combo_box import * from .item_box import * from .overwrite_chooser import * from .parasite_box import * from .popup_hide_context import * from .progress_updater import * from .utils import * from .._gui_messages import *