Python fileinput.filelineno() Examples

def test_version(self):
        message = "\nFound a different version at line %d or file %r: %r (may be %r)."

        files = [os.path.join(self.source_dir, '__init__.py')]
        if self.package_dir is not None:
            files.extend([
                os.path.join(self.package_dir, 'setup.py'),
                os.path.join(self.package_dir, 'doc/conf.py'),
            ])
        version = filename = None
        for line in fileinput.input(files):
            if fileinput.isfirstline():
                filename = fileinput.filename()
            lineno = fileinput.filelineno()

            match = self.get_version.search(line)
            if match is not None:
                if version is None:
                    version = match.group(1).strip('\'\"')
                else:
                    self.assertTrue(
                        version == match.group(1).strip('\'\"'),
                        message % (lineno, filename, match.group(1).strip('\'\"'), version)
                    ) 

def more(filenames, pagesize=10, clear=False, fmt='{line}'):
    '''Display content of filenames pagesize lines at a time (cleared if specified) with format fmt for each output line'''

    fileinput.close()  # in case still open
    try:
        pageno = 1
        if clear:
            clear_screen()
        for line in fileinput.input(filenames, openhook=fileinput.hook_encoded("utf-8")):
            lineno, filename, filelineno = fileinput.lineno(), fileinput.filename(), fileinput.filelineno()
            print(fmt.format(**locals()), end='')
            if pagesize and lineno % pagesize == 0:
                console.alert('Abort or continue', filename, 'Next page')  # TODO: use less intrusive mechanism than alert
                pageno += 1
                if clear:
                    clear_screen()
    finally:
        fileinput.close()


# --- main 

def main(args):
    parser = argparse.ArgumentParser(
        description=__doc__,
        epilog='This is inefficient for long input, as StaSh pipes do not multitask'
    )
    parser.add_argument('file', help='files to display ("-" is stdin is default)', action='store', nargs='*')
    parser.add_argument('-p', '--pageno', help='number screen pages cumulatively', action='store_true')
    parser.add_argument('-l', '--lineno', help='number lines cumulatively', action='store_true')
    parser.add_argument('-f', '--filename', help='label lines by filename', action='store_true')
    parser.add_argument('-n', '--filelineno', '-#', help='number lines per file', action='store_true')
    parser.add_argument(
        '-s',
        '--pagesize',
        help='number of lines per screen page (0 for no pagination)',
        action='store',
        type=int,
        default=40
    )  # TODO: use actual number of lines on screen for dynamic screen page size
    parser.add_argument('-c', '--clear', help='clear terminal screen before each screen page', action='store_true')
    ns = parser.parse_args(args)
    ns.line = True
    fmt = ' '.join('{' + var + '}' for var in 'pageno lineno filename filelineno line'.split() if getattr(ns, var))
    more(ns.file, ns.pagesize, ns.clear, fmt) 

def input_stream(files=()):
    """ Handles input files similar to fileinput.
    The advantage of this function is it recovers from errors if one
    file is invalid and proceed with the next file
    """
    fileinput.close()
    try:
        if not files:
            for line in fileinput.input(files):
                yield line, '', fileinput.filelineno()

        else:
            while files:
                thefile = files.pop(0)
                try:
                    for line in fileinput.input(thefile):
                        yield line, fileinput.filename(), fileinput.filelineno()
                except IOError as e:
                    yield None, fileinput.filename(), e
    finally:
        fileinput.close() 

def test_zero_byte_files(self):
        t1 = t2 = t3 = t4 = None
        try:
            t1 = writeTmp(1, [""])
            t2 = writeTmp(2, [""])
            t3 = writeTmp(3, ["The only line there is.\n"])
            t4 = writeTmp(4, [""])
            fi = FileInput(files=(t1, t2, t3, t4))

            line = fi.readline()
            self.assertEqual(line, 'The only line there is.\n')
            self.assertEqual(fi.lineno(), 1)
            self.assertEqual(fi.filelineno(), 1)
            self.assertEqual(fi.filename(), t3)

            line = fi.readline()
            self.assertFalse(line)
            self.assertEqual(fi.lineno(), 1)
            self.assertEqual(fi.filelineno(), 0)
            self.assertEqual(fi.filename(), t4)
            fi.close()
        finally:
            remove_tempfiles(t1, t2, t3, t4) 

def test_zero_byte_files(self):
        t1 = t2 = t3 = t4 = None
        try:
            t1 = writeTmp(1, [""])
            t2 = writeTmp(2, [""])
            t3 = writeTmp(3, ["The only line there is.\n"])
            t4 = writeTmp(4, [""])
            fi = FileInput(files=(t1, t2, t3, t4))

            line = fi.readline()
            self.assertEqual(line, 'The only line there is.\n')
            self.assertEqual(fi.lineno(), 1)
            self.assertEqual(fi.filelineno(), 1)
            self.assertEqual(fi.filename(), t3)

            line = fi.readline()
            self.assertFalse(line)
            self.assertEqual(fi.lineno(), 1)
            self.assertEqual(fi.filelineno(), 0)
            self.assertEqual(fi.filename(), t4)
            fi.close()
        finally:
            remove_tempfiles(t1, t2, t3, t4) 

def test_zero_byte_files(self):
        t1 = t2 = t3 = t4 = None
        try:
            t1 = writeTmp(1, [""])
            t2 = writeTmp(2, [""])
            t3 = writeTmp(3, ["The only line there is.\n"])
            t4 = writeTmp(4, [""])
            fi = FileInput(files=(t1, t2, t3, t4))

            line = fi.readline()
            self.assertEqual(line, 'The only line there is.\n')
            self.assertEqual(fi.lineno(), 1)
            self.assertEqual(fi.filelineno(), 1)
            self.assertEqual(fi.filename(), t3)

            line = fi.readline()
            self.assertFalse(line)
            self.assertEqual(fi.lineno(), 1)
            self.assertEqual(fi.filelineno(), 0)
            self.assertEqual(fi.filename(), t4)
            fi.close()
        finally:
            remove_tempfiles(t1, t2, t3, t4) 

def print_error(self, error, line='',
                    filename=None, filelineno=None):
        if self.should_ignore(error):
            return

        warn = self.should_warn(error)

        if not filename:
            filename = fileinput.filename()
        if not filelineno:
            filelineno = fileinput.filelineno()
        if warn:
            self.warning_count = self.warning_count + 1
        else:
            self.error_count = self.error_count + 1

        self.log_error(error, line, filename, filelineno, warn) 

def test_context_manager(self):
        try:
            t1 = writeTmp(1, ["A\nB\nC"])
            t2 = writeTmp(2, ["D\nE\nF"])
            with FileInput(files=(t1, t2)) as fi:
                lines = list(fi)
            self.assertEqual(lines, ["A\n", "B\n", "C", "D\n", "E\n", "F"])
            self.assertEqual(fi.filelineno(), 3)
            self.assertEqual(fi.lineno(), 6)
            self.assertEqual(fi._files, ())
        finally:
            remove_tempfiles(t1, t2) 

def test_missing_debug_statements(self):
        # Exclude explicit debug statements written in the code
        exclude = {
            'regex.py': [240, 241],
            'codepoints.py': [534],
            'cli.py': [117, 133, 137, 140],
        }

        message = "\nFound a debug missing statement at line %d or file %r: %r"
        filename = None
        file_excluded = []
        files = glob.glob(os.path.join(self.source_dir, '*.py')) + \
            glob.glob(os.path.join(self.source_dir, 'validators/*.py'))
        for line in fileinput.input(files):
            if fileinput.isfirstline():
                filename = fileinput.filename()
                file_excluded = exclude.get(os.path.basename(filename), [])
            lineno = fileinput.filelineno()

            if lineno in file_excluded:
                continue

            match = self.missing_debug.search(line)
            if match is None or filename.endswith('/cli.py') and match.group(0) == 'print(':
                continue
            self.assertIsNone(match, message % (lineno, filename, match.group(0))) 

def test_state_is_None(self):
        """Tests fileinput.filelineno() when fileinput._state is None.
           Ensure that it raises RuntimeError with a meaningful error message
           and does not modify fileinput._state"""
        fileinput._state = None
        with self.assertRaises(RuntimeError) as cm:
            fileinput.filelineno()
        self.assertEqual(("no active input()",), cm.exception.args)
        self.assertIsNone(fileinput._state) 

def filelineno(self):
        self.invocation_counts["filelineno"] += 1
        return self.return_values["filelineno"] 

def test_context_manager(self):
        try:
            t1 = writeTmp(1, ["A\nB\nC"])
            t2 = writeTmp(2, ["D\nE\nF"])
            with FileInput(files=(t1, t2)) as fi:
                lines = list(fi)
            self.assertEqual(lines, ["A\n", "B\n", "C", "D\n", "E\n", "F"])
            self.assertEqual(fi.filelineno(), 3)
            self.assertEqual(fi.lineno(), 6)
            self.assertEqual(fi._files, ())
        finally:
            remove_tempfiles(t1, t2) 

def test_files_that_dont_end_with_newline(self):
        t1 = t2 = None
        try:
            t1 = writeTmp(1, ["A\nB\nC"])
            t2 = writeTmp(2, ["D\nE\nF"])
            fi = FileInput(files=(t1, t2))
            lines = list(fi)
            self.assertEqual(lines, ["A\n", "B\n", "C", "D\n", "E\n", "F"])
            self.assertEqual(fi.filelineno(), 3)
            self.assertEqual(fi.lineno(), 6)
        finally:
            remove_tempfiles(t1, t2)

##     def test_unicode_filenames(self):
##         # XXX A unicode string is always returned by writeTmp.
##         #     So is this needed?
##         try:
##             t1 = writeTmp(1, ["A\nB"])
##             encoding = sys.getfilesystemencoding()
##             if encoding is None:
##                 encoding = 'ascii'
##             fi = FileInput(files=str(t1, encoding))
##             lines = list(fi)
##             self.assertEqual(lines, ["A\n", "B"])
##         finally:
##             remove_tempfiles(t1) 

def log_error(self, error, line, filename, filelineno, warn=False):
        # following pycodestyle/pep8 default output format
        # https://github.com/PyCQA/pycodestyle/blob/master/pycodestyle.py#L108
        print("%(filename)s:%(filelineno)s:1: %(error)s" %
              {'filename': filename,
               'filelineno': filelineno,
               'warn': "W" if warn else "E",
               'error': error.replace(":", "", 1),
               'line': line.rstrip('\n')}) 

def test_state_is_None(self):
        """Tests fileinput.filelineno() when fileinput._state is None.
           Ensure that it raises RuntimeError with a meaningful error message
           and does not modify fileinput._state"""
        fileinput._state = None
        with self.assertRaises(RuntimeError) as cm:
            fileinput.filelineno()
        self.assertEqual(("no active input()",), cm.exception.args)
        self.assertIsNone(fileinput._state) 

def filelineno(self):
        self.invocation_counts["filelineno"] += 1
        return self.return_values["filelineno"] 

def test_files_that_dont_end_with_newline(self):
        t1 = t2 = None
        try:
            t1 = writeTmp(1, ["A\nB\nC"])
            t2 = writeTmp(2, ["D\nE\nF"])
            fi = FileInput(files=(t1, t2))
            lines = list(fi)
            self.assertEqual(lines, ["A\n", "B\n", "C", "D\n", "E\n", "F"])
            self.assertEqual(fi.filelineno(), 3)
            self.assertEqual(fi.lineno(), 6)
        finally:
            remove_tempfiles(t1, t2)

##     def test_unicode_filenames(self):
##         # XXX A unicode string is always returned by writeTmp.
##         #     So is this needed?
##         try:
##             t1 = writeTmp(1, ["A\nB"])
##             encoding = sys.getfilesystemencoding()
##             if encoding is None:
##                 encoding = 'ascii'
##             fi = FileInput(files=str(t1, encoding))
##             lines = list(fi)
##             self.assertEqual(lines, ["A\n", "B"])
##         finally:
##             remove_tempfiles(t1) 

def backport(rootdir="."):
    for folder, subs, files in os.walk(rootdir):
        for filename in files:
            src_filename = os.path.join(folder, filename)
            # Skip non python files
            if not src_filename.endswith(".py"):
                continue
            if (__file__ and os.path.basename(src_filename) ==
                    os.path.basename(__file__)):
                continue
            print(src_filename)
            last_class = ""
            for line in fileinput.input(src_filename, inplace=True):
                if fileinput.filelineno() == 1:
                    if line.startswith("#!"):
                        print(line, end="")
                        print("from __future__ import unicode_literals")
                    else:
                        print("from __future__ import unicode_literals")
                        print(line, end="")
                    continue
                if line.strip().startswith("class"):
                    last_class = line.strip().split()[1]
                    last_class = re.match(r'([a-zA-Z0-9]+)',
                                          last_class).group(1)
                if "__str__(" in line:
                    line = line.replace("__str__(", "__unicode__(")
                if "super()" in line:
                    old_super = "super({}, self)".format(last_class)
                    line = line.replace("super()", old_super)
                print(line, end="") 

def test_state_is_None(self):
        """Tests fileinput.filelineno() when fileinput._state is None.
           Ensure that it raises RuntimeError with a meaningful error message
           and does not modify fileinput._state"""
        fileinput._state = None
        with self.assertRaises(RuntimeError) as cm:
            fileinput.filelineno()
        self.assertEqual(("no active input()",), cm.exception.args)
        self.assertIsNone(fileinput._state) 

def filelineno(self):
        self.invocation_counts["filelineno"] += 1
        return self.return_values["filelineno"] 

def test_context_manager(self):
        try:
            t1 = writeTmp(1, ["A\nB\nC"])
            t2 = writeTmp(2, ["D\nE\nF"])
            with FileInput(files=(t1, t2)) as fi:
                lines = list(fi)
            self.assertEqual(lines, ["A\n", "B\n", "C", "D\n", "E\n", "F"])
            self.assertEqual(fi.filelineno(), 3)
            self.assertEqual(fi.lineno(), 6)
            self.assertEqual(fi._files, ())
        finally:
            remove_tempfiles(t1, t2) 

def test_files_that_dont_end_with_newline(self):
        t1 = t2 = None
        try:
            t1 = writeTmp(1, ["A\nB\nC"])
            t2 = writeTmp(2, ["D\nE\nF"])
            fi = FileInput(files=(t1, t2))
            lines = list(fi)
            self.assertEqual(lines, ["A\n", "B\n", "C", "D\n", "E\n", "F"])
            self.assertEqual(fi.filelineno(), 3)
            self.assertEqual(fi.lineno(), 6)
        finally:
            remove_tempfiles(t1, t2)

##     def test_unicode_filenames(self):
##         # XXX A unicode string is always returned by writeTmp.
##         #     So is this needed?
##         try:
##             t1 = writeTmp(1, ["A\nB"])
##             encoding = sys.getfilesystemencoding()
##             if encoding is None:
##                 encoding = 'ascii'
##             fi = FileInput(files=str(t1, encoding))
##             lines = list(fi)
##             self.assertEqual(lines, ["A\n", "B"])
##         finally:
##             remove_tempfiles(t1) 

def check_syntax(filename, report):
    # run the file through "bash -n" to catch basic syntax errors and
    # other warnings
    matches = []

    # sample lines we want to match:
    # foo.sh: line 4: warning: \
    #    here-document at line 1 delimited by end-of-file (wanted `EOF')
    # foo.sh: line 9: syntax error: unexpected end of file
    # foo.sh: line 7: syntax error near unexpected token `}'
    #
    # i.e. consistency with ":"'s isn't constant, so just do our
    # best...
    r = re.compile(
        '^(?P<file>.*): line (?P<lineno>[0-9]+): (?P<error>.*)')
    # we are parsing the error message, so force it to ignore the
    # system locale so we don't get messages in another language
    bash_environment = os.environ
    bash_environment['LC_ALL'] = 'C'
    proc = subprocess.Popen(
        ['bash', '-n', filename], stdout=subprocess.PIPE,
        stderr=subprocess.PIPE, env=bash_environment,
        universal_newlines=True)
    outputs = proc.communicate()

    for line in outputs[1].split('\n'):
        m = r.match(line)
        if m:
            matches.append(m)

    for m in matches:
        if 'syntax error' in m.group('error'):
            msg = '%s: %s' % (MESSAGES['E040'].msg, m.group('error'))
            report.print_error(msg, filename=filename,
                               filelineno=int(m.group('lineno')))

        # Matching output from bash warning about here-documents not
        # ending.
        # FIXME: are there other warnings that might come out
        # with "bash -n"?  A quick scan of the source code suggests
        # no, but there might be other interesting things we could
        # catch.
        if 'warning:' in m.group('error'):
            if 'delimited by end-of-file' in m.group('error'):
                start = re.match('^.*line (?P<start>[0-9]+).*$',
                                 m.group('error'))
                report.print_error(
                    MESSAGES['E012'].msg % int(start.group('start')),
                    filename=filename,
                    filelineno=int(m.group('lineno'))) 

def main(args):
    global _stash
    ap = argparse.ArgumentParser()
    ap.add_argument('pattern', help='the pattern to match')
    ap.add_argument('files', nargs='*', help='files to be searched')
    ap.add_argument('-i', '--ignore-case', action='store_true', help='ignore case while searching')
    ap.add_argument('-v', '--invert', action='store_true', help='invert the search result')
    ap.add_argument('-c', '--count', action='store_true', help='count the search results instead of normal output')
    ns = ap.parse_args(args)

    flags = 0
    if ns.ignore_case:
        flags |= re.IGNORECASE

    pattern = re.compile(ns.pattern, flags=flags)

    # Do not try to grep directories
    files = [f for f in ns.files if not os.path.isdir(f)]

    fileinput.close()  # in case it is not closed
    try:
        counts = collections.defaultdict(int)
        for line in fileinput.input(files, openhook=fileinput.hook_encoded("utf-8")):
            if bool(pattern.search(line)) != ns.invert:
                if ns.count:
                    counts[fileinput.filename()] += 1
                else:
                    if ns.invert:  # optimize: if ns.invert, then no match, so no highlight color needed
                        newline = line
                    else:
                        newline = re.sub(pattern, lambda m: _stash.text_color(m.group(), 'red'), line)
                    if fileinput.isstdin():
                        fmt = u'{lineno}: {line}'
                    else:
                        fmt = u'{filename}: {lineno}: {line}'

                    print(fmt.format(filename=fileinput.filename(), lineno=fileinput.filelineno(), line=newline.rstrip()))

        if ns.count:
            for filename, count in counts.items():
                fmt = u'{count:6} {filename}'
                print(fmt.format(filename=filename, count=count))

    except Exception as err:
        print("grep: {}: {!s}".format(type(err).__name__, err), file=sys.stderr)
    finally:
        fileinput.close()