Python sys.getdefaultencoding() Examples

def _ustr(obj):
        """Drop-in replacement for str(obj) that tries to be Unicode friendly. It first tries
           str(obj). If that fails with a UnicodeEncodeError, then it tries unicode(obj). It
           then < returns the unicode object | encodes it with the default encoding | ... >.
        """
        if isinstance(obj,unicode):
            return obj

        try:
            # If this works, then _ustr(obj) has the same behaviour as str(obj), so
            # it won't break any existing code.
            return str(obj)

        except UnicodeEncodeError:
            # Else encode it
            ret = unicode(obj).encode(sys.getdefaultencoding(), 'xmlcharrefreplace')
            xmlcharref = Regex('&#\d+;')
            xmlcharref.setParseAction(lambda t: '\\u' + hex(int(t[0][2:-1]))[2:])
            return xmlcharref.transformString(ret)

    # build list of single arg builtins, tolerant of Python version, that can be used as parse actions 

def sysexec(self, *argv, **popen_opts):
        """ return stdout text from executing a system child process,
            where the 'self' path points to executable.
            The process is directly invoked and not through a system shell.
        """
        from subprocess import Popen, PIPE
        argv = map_as_list(str, argv)
        popen_opts['stdout'] = popen_opts['stderr'] = PIPE
        proc = Popen([str(self)] + argv, **popen_opts)
        stdout, stderr = proc.communicate()
        ret = proc.wait()
        if py.builtin._isbytes(stdout):
            stdout = py.builtin._totext(stdout, sys.getdefaultencoding())
        if ret != 0:
            if py.builtin._isbytes(stderr):
                stderr = py.builtin._totext(stderr, sys.getdefaultencoding())
            raise py.process.cmdexec.Error(ret, ret, str(self),
                                           stdout, stderr,)
        return stdout 

def cmdexec(cmd):
    """ return unicode output of executing 'cmd' in a separate process.

    raise cmdexec.Error exeception if the command failed.
    the exception will provide an 'err' attribute containing
    the error-output from the command.
    if the subprocess module does not provide a proper encoding/unicode strings
    sys.getdefaultencoding() will be used, if that does not exist, 'UTF-8'.
    """
    process = subprocess.Popen(cmd, shell=True,
            universal_newlines=True,
            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    out, err = process.communicate()
    if sys.version_info[0] < 3: # on py3 we get unicode strings, on py2 not
        try:
            default_encoding = sys.getdefaultencoding() # jython may not have it
        except AttributeError:
            default_encoding = sys.stdout.encoding or 'UTF-8'
        out = unicode(out, process.stdout.encoding or default_encoding)
        err = unicode(err, process.stderr.encoding or default_encoding)
    status = process.poll()
    if status:
        raise ExecutionFailed(status, status, cmd, out, err)
    return out 

def write(self, data, mode='w', ensure=False):
        """ write data into path.   If ensure is True create
        missing parent directories.
        """
        if ensure:
            self.dirpath().ensure(dir=1)
        if 'b' in mode:
            if not py.builtin._isbytes(data):
                raise ValueError("can only process bytes")
        else:
            if not py.builtin._istext(data):
                if not py.builtin._isbytes(data):
                    data = str(data)
                else:
                    data = py.builtin._totext(data, sys.getdefaultencoding())
        f = self.open(mode)
        try:
            f.write(data)
        finally:
            f.close() 

def formatyearpage(self, theyear, width=3, css='calendar.css', encoding=None):
        """
        Return a formatted year as a complete HTML page.
        """
        if encoding is None:
            encoding = sys.getdefaultencoding()
        v = []
        a = v.append
        a('<?xml version="1.0" encoding="%s"?>\n' % encoding)
        a('<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">\n')
        a('<html>\n')
        a('<head>\n')
        a('<meta http-equiv="Content-Type" content="text/html; charset=%s" />\n' % encoding)
        if css is not None:
            a('<link rel="stylesheet" type="text/css" href="%s" />\n' % css)
        a('<title>Calendar for %d</title>\n' % theyear)
        a('</head>\n')
        a('<body>\n')
        a(self.formatyear(theyear, width))
        a('</body>\n')
        a('</html>\n')
        return ''.join(v).encode(encoding, "xmlcharrefreplace") 

def _ustr(obj):
        """Drop-in replacement for str(obj) that tries to be Unicode friendly. It first tries
           str(obj). If that fails with a UnicodeEncodeError, then it tries unicode(obj). It
           then < returns the unicode object | encodes it with the default encoding | ... >.
        """
        if isinstance(obj,unicode):
            return obj

        try:
            # If this works, then _ustr(obj) has the same behaviour as str(obj), so
            # it won't break any existing code.
            return str(obj)

        except UnicodeEncodeError:
            # Else encode it
            ret = unicode(obj).encode(sys.getdefaultencoding(), 'xmlcharrefreplace')
            xmlcharref = Regex('&#\d+;')
            xmlcharref.setParseAction(lambda t: '\\u' + hex(int(t[0][2:-1]))[2:])
            return xmlcharref.transformString(ret)

    # build list of single arg builtins, tolerant of Python version, that can be used as parse actions 

def write(self, data, mode='w', ensure=False):
        """ write data into path.   If ensure is True create
        missing parent directories.
        """
        if ensure:
            self.dirpath().ensure(dir=1)
        if 'b' in mode:
            if not py.builtin._isbytes(data):
                raise ValueError("can only process bytes")
        else:
            if not py.builtin._istext(data):
                if not py.builtin._isbytes(data):
                    data = str(data)
                else:
                    data = py.builtin._totext(data, sys.getdefaultencoding())
        f = self.open(mode)
        try:
            f.write(data)
        finally:
            f.close() 

def sysexec(self, *argv, **popen_opts):
        """ return stdout text from executing a system child process,
            where the 'self' path points to executable.
            The process is directly invoked and not through a system shell.
        """
        from subprocess import Popen, PIPE
        argv = map_as_list(str, argv)
        popen_opts['stdout'] = popen_opts['stderr'] = PIPE
        proc = Popen([str(self)] + argv, **popen_opts)
        stdout, stderr = proc.communicate()
        ret = proc.wait()
        if py.builtin._isbytes(stdout):
            stdout = py.builtin._totext(stdout, sys.getdefaultencoding())
        if ret != 0:
            if py.builtin._isbytes(stderr):
                stderr = py.builtin._totext(stderr, sys.getdefaultencoding())
            raise py.process.cmdexec.Error(ret, ret, str(self),
                                           stdout, stderr,)
        return stdout 

def cmdexec(cmd):
    """ return unicode output of executing 'cmd' in a separate process.

    raise cmdexec.Error exeception if the command failed.
    the exception will provide an 'err' attribute containing
    the error-output from the command.
    if the subprocess module does not provide a proper encoding/unicode strings
    sys.getdefaultencoding() will be used, if that does not exist, 'UTF-8'.
    """
    process = subprocess.Popen(cmd, shell=True,
            universal_newlines=True,
            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    out, err = process.communicate()
    if sys.version_info[0] < 3: # on py3 we get unicode strings, on py2 not
        try:
            default_encoding = sys.getdefaultencoding() # jython may not have it
        except AttributeError:
            default_encoding = sys.stdout.encoding or 'UTF-8'
        out = unicode(out, process.stdout.encoding or default_encoding)
        err = unicode(err, process.stderr.encoding or default_encoding)
    status = process.poll()
    if status:
        raise ExecutionFailed(status, status, cmd, out, err)
    return out 

def write(self, data, mode='w', ensure=False):
        """ write data into path.   If ensure is True create
        missing parent directories.
        """
        if ensure:
            self.dirpath().ensure(dir=1)
        if 'b' in mode:
            if not py.builtin._isbytes(data):
                raise ValueError("can only process bytes")
        else:
            if not py.builtin._istext(data):
                if not py.builtin._isbytes(data):
                    data = str(data)
                else:
                    data = py.builtin._totext(data, sys.getdefaultencoding())
        f = self.open(mode)
        try:
            f.write(data)
        finally:
            f.close() 

def cmdexec(cmd):
    """ return unicode output of executing 'cmd' in a separate process.

    raise cmdexec.Error exeception if the command failed.
    the exception will provide an 'err' attribute containing
    the error-output from the command.
    if the subprocess module does not provide a proper encoding/unicode strings
    sys.getdefaultencoding() will be used, if that does not exist, 'UTF-8'.
    """
    process = subprocess.Popen(cmd, shell=True,
            universal_newlines=True,
            stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    out, err = process.communicate()
    if sys.version_info[0] < 3: # on py3 we get unicode strings, on py2 not
        try:
            default_encoding = sys.getdefaultencoding() # jython may not have it
        except AttributeError:
            default_encoding = sys.stdout.encoding or 'UTF-8'
        out = unicode(out, process.stdout.encoding or default_encoding)
        err = unicode(err, process.stderr.encoding or default_encoding)
    status = process.poll()
    if status:
        raise ExecutionFailed(status, status, cmd, out, err)
    return out 

def formatyearpage(self, theyear, width=3, css='calendar.css', encoding=None):
        """
        Return a formatted year as a complete HTML page.
        """
        if encoding is None:
            encoding = sys.getdefaultencoding()
        v = []
        a = v.append
        a('<?xml version="1.0" encoding="%s"?>\n' % encoding)
        a('<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">\n')
        a('<html>\n')
        a('<head>\n')
        a('<meta http-equiv="Content-Type" content="text/html; charset=%s" />\n' % encoding)
        if css is not None:
            a('<link rel="stylesheet" type="text/css" href="%s" />\n' % css)
        a('<title>Calendar for %d</title>\n' % theyear)
        a('</head>\n')
        a('<body>\n')
        a(self.formatyear(theyear, width))
        a('</body>\n')
        a('</html>\n')
        return ''.join(v).encode(encoding, "xmlcharrefreplace") 

def test_encoding(self):
        f = file(self.temp_file, 'w')
        # we throw on flush, CPython throws on write, so both write & close need to catch
        try:
            f.write(u'\u6211')
            f.close()
            self.fail('UnicodeEncodeError should have been thrown')
        except UnicodeEncodeError:
            pass
        
        if hasattr(sys, "setdefaultencoding"):
            #and verify UTF8 round trips correctly
            setenc = sys.setdefaultencoding
            saved = sys.getdefaultencoding()
            try:
                setenc('utf8')
                with file(self.temp_file, 'w') as f:
                    f.write(u'\u6211')

                with file(self.temp_file, 'r') as f:
                    txt = f.read()
                self.assertEqual(txt, u'\u6211')
            finally:
                setenc(saved) 

def _ustr(obj):
        """Drop-in replacement for str(obj) that tries to be Unicode friendly. It first tries
           str(obj). If that fails with a UnicodeEncodeError, then it tries unicode(obj). It
           then < returns the unicode object | encodes it with the default encoding | ... >.
        """
        if isinstance(obj,unicode):
            return obj

        try:
            # If this works, then _ustr(obj) has the same behaviour as str(obj), so
            # it won't break any existing code.
            return str(obj)

        except UnicodeEncodeError:
            # Else encode it
            ret = unicode(obj).encode(sys.getdefaultencoding(), 'xmlcharrefreplace')
            xmlcharref = Regex('&#\d+;')
            xmlcharref.setParseAction(lambda t: '\\u' + hex(int(t[0][2:-1]))[2:])
            return xmlcharref.transformString(ret)

    # build list of single arg builtins, tolerant of Python version, that can be used as parse actions 

def log(self, rev_start=None, rev_end=1, verbose=False):
        """ return a list of LogEntry instances for this path.
rev_start is the starting revision (defaulting to the first one).
rev_end is the last revision (defaulting to HEAD).
if verbose is True, then the LogEntry instances also know which files changed.
"""
        assert self.check()   # make it simpler for the pipe
        rev_start = rev_start is None and "HEAD" or rev_start
        rev_end = rev_end is None and "HEAD" or rev_end
        if rev_start == "HEAD" and rev_end == 1:
                rev_opt = ""
        else:
            rev_opt = "-r %s:%s" % (rev_start, rev_end)
        verbose_opt = verbose and "-v" or ""
        locale_env = fixlocale()
        # some blather on stderr
        auth_opt = self._makeauthoptions()
        #stdin, stdout, stderr  = os.popen3(locale_env +
        #                                   'svn log --xml %s %s %s "%s"' % (
        #                                    rev_opt, verbose_opt, auth_opt,
        #                                    self.strpath))
        cmd = locale_env + 'svn log --xml %s %s %s "%s"' % (
            rev_opt, verbose_opt, auth_opt, self.strpath)

        popen = subprocess.Popen(cmd,
                    stdout=subprocess.PIPE,
                    stderr=subprocess.PIPE,
                    shell=True,
        )
        stdout, stderr = popen.communicate()
        stdout = py.builtin._totext(stdout, sys.getdefaultencoding())
        minidom,ExpatError = importxml()
        try:
            tree = minidom.parseString(stdout)
        except ExpatError:
            raise ValueError('no such revision')
        result = []
        for logentry in filter(None, tree.firstChild.childNodes):
            if logentry.nodeType == logentry.ELEMENT_NODE:
                result.append(LogEntry(logentry))
        return result 

def _get_encoding(self, file):
        encoding = getattr(file, "encoding", None)
        if not encoding:
            encoding = sys.getdefaultencoding()
        return encoding 

def glob1(dirname, pattern):
    if not dirname:
        dirname = os.curdir
    if isinstance(pattern, unicode) and not isinstance(dirname, unicode):
        dirname = unicode(dirname, sys.getfilesystemencoding() or
                                   sys.getdefaultencoding())
    try:
        names = os.listdir(dirname)
    except os.error:
        return []
    if pattern[0] != '.':
        names = filter(lambda x: x[0] != '.', names)
    return fnmatch.filter(names, pattern) 

def to_native(x, charset=sys.getdefaultencoding(), errors='strict'):
        if x is None or isinstance(x, str):
            return x
        return x.decode(charset, errors) 

def to_native(x, charset=sys.getdefaultencoding(), errors='strict'):
        if x is None or isinstance(x, str):
            return x
        return x.encode(charset, errors) 

def auto_decode(data):
    """Check a bytes string for a BOM to correctly detect the encoding

    Fallback to locale.getpreferredencoding(False) like open() on Python3"""
    for bom, encoding in BOMS:
        if data.startswith(bom):
            return data[len(bom):].decode(encoding)
    # Lets check the first two lines as in PEP263
    for line in data.split(b'\n')[:2]:
        if line[0:1] == b'#' and ENCODING_RE.search(line):
            encoding = ENCODING_RE.search(line).groups()[0].decode('ascii')
            return data.decode(encoding)
    return data.decode(
        locale.getpreferredencoding(False) or sys.getdefaultencoding(),
    ) 

def log(self, rev_start=None, rev_end=1, verbose=False):
        """ return a list of LogEntry instances for this path.
rev_start is the starting revision (defaulting to the first one).
rev_end is the last revision (defaulting to HEAD).
if verbose is True, then the LogEntry instances also know which files changed.
"""
        assert self.check()   # make it simpler for the pipe
        rev_start = rev_start is None and "HEAD" or rev_start
        rev_end = rev_end is None and "HEAD" or rev_end
        if rev_start == "HEAD" and rev_end == 1:
                rev_opt = ""
        else:
            rev_opt = "-r %s:%s" % (rev_start, rev_end)
        verbose_opt = verbose and "-v" or ""
        locale_env = fixlocale()
        # some blather on stderr
        auth_opt = self._makeauthoptions()
        #stdin, stdout, stderr  = os.popen3(locale_env +
        #                                   'svn log --xml %s %s %s "%s"' % (
        #                                    rev_opt, verbose_opt, auth_opt,
        #                                    self.strpath))
        cmd = locale_env + 'svn log --xml %s %s %s "%s"' % (
            rev_opt, verbose_opt, auth_opt, self.strpath)

        popen = subprocess.Popen(cmd,
                    stdout=subprocess.PIPE,
                    stderr=subprocess.PIPE,
                    shell=True,
        )
        stdout, stderr = popen.communicate()
        stdout = py.builtin._totext(stdout, sys.getdefaultencoding())
        minidom,ExpatError = importxml()
        try:
            tree = minidom.parseString(stdout)
        except ExpatError:
            raise ValueError('no such revision')
        result = []
        for logentry in filter(None, tree.firstChild.childNodes):
            if logentry.nodeType == logentry.ELEMENT_NODE:
                result.append(LogEntry(logentry))
        return result 

def test_read_write(self, tmpdir):
        x = tmpdir.join("hello")
        part = py.builtin._totext("hällo", "utf8")
        x.write(part)
        assert x.read() == part
        x.write(part.encode(sys.getdefaultencoding()))
        assert x.read() == part.encode(sys.getdefaultencoding()) 

def apply_changes_to_python_environment():
        global python_environment_changes_applied
        if python_environment_changes_applied or sys.getdefaultencoding() == 'utf-8':
            python_environment_changes_applied = True
            return

        try:
            reload(sys)
            sys.stdout = stdout
            sys.stderr = stderr
            sys.setdefaultencoding('utf-8')
        except NameError:  # Python 3
            sys.exit('This should not happen!')

        python_environment_changes_applied = True 

def bug_1771_with_user_config(var):
    # sys.getdefaultencoding is considered as a never
    # returning function in the inconsistent_returns.rc file.
    if var == 1:
        sys.getdefaultencoding()
    else:
        return var * 2 

def decoding_stream(stream, encoding, errors="strict"):
    try:
        reader_cls = codecs.getreader(encoding or sys.getdefaultencoding())
    except LookupError:
        reader_cls = codecs.getreader(sys.getdefaultencoding())
    return reader_cls(stream, errors) 

def safe_decode(line, encoding, *args, **kwargs):
    """return decoded line from encoding or decode with default encoding"""
    try:
        return line.decode(encoding or sys.getdefaultencoding(), *args, **kwargs)
    except LookupError:
        return line.decode(sys.getdefaultencoding(), *args, **kwargs) 

def get_encoding():
    """Return preferred encoding."""
    return locale.getpreferredencoding() or sys.getdefaultencoding() 

def safe_decode(line, encoding, *args, **kwargs):
    '''return decoded line from encoding or decode with default encoding'''
    try:
        return line.decode(encoding or sys.getdefaultencoding(), *args, **kwargs)
    except LookupError:
        return line.decode(sys.getdefaultencoding(), *args, **kwargs) 

def log(self, rev_start=None, rev_end=1, verbose=False):
        """ return a list of LogEntry instances for this path.
rev_start is the starting revision (defaulting to the first one).
rev_end is the last revision (defaulting to HEAD).
if verbose is True, then the LogEntry instances also know which files changed.
"""
        assert self.check()   # make it simpler for the pipe
        rev_start = rev_start is None and "HEAD" or rev_start
        rev_end = rev_end is None and "HEAD" or rev_end
        if rev_start == "HEAD" and rev_end == 1:
                rev_opt = ""
        else:
            rev_opt = "-r %s:%s" % (rev_start, rev_end)
        verbose_opt = verbose and "-v" or ""
        locale_env = fixlocale()
        # some blather on stderr
        auth_opt = self._makeauthoptions()
        #stdin, stdout, stderr  = os.popen3(locale_env +
        #                                   'svn log --xml %s %s %s "%s"' % (
        #                                    rev_opt, verbose_opt, auth_opt,
        #                                    self.strpath))
        cmd = locale_env + 'svn log --xml %s %s %s "%s"' % (
            rev_opt, verbose_opt, auth_opt, self.strpath)

        popen = subprocess.Popen(cmd,
                    stdout=subprocess.PIPE,
                    stderr=subprocess.PIPE,
                    shell=True,
        )
        stdout, stderr = popen.communicate()
        stdout = py.builtin._totext(stdout, sys.getdefaultencoding())
        minidom,ExpatError = importxml()
        try:
            tree = minidom.parseString(stdout)
        except ExpatError:
            raise ValueError('no such revision')
        result = []
        for logentry in filter(None, tree.firstChild.childNodes):
            if logentry.nodeType == logentry.ELEMENT_NODE:
                result.append(LogEntry(logentry))
        return result 

def to_native(x, charset=sys.getdefaultencoding(), errors='strict'):
        if x is None or isinstance(x, str):
            return x
        return x.encode(charset, errors)