Python fileinput.hook_encoded() Examples
The following are 30
code examples of fileinput.hook_encoded().
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
You may also want to check out all available functions/classes of the module
fileinput
, or try the search function
.
.
Example #1
| Source File: test_fileinput.py From ironpython2 with Apache License 2.0 | 7 votes |
|
def test_file_opening_hook(self):
try:
# cannot use openhook and inplace mode
fi = FileInput(inplace=1, openhook=lambda f,m: None)
self.fail("FileInput should raise if both inplace "
"and openhook arguments are given")
except ValueError:
pass
try:
fi = FileInput(openhook=1)
self.fail("FileInput should check openhook for being callable")
except ValueError:
pass
try:
# UTF-7 is a convenient, seldom used encoding
t1 = writeTmp(1, ['+AEE-\n+AEI-'], mode="wb")
fi = FileInput(files=t1, openhook=hook_encoded("utf-7"))
lines = list(fi)
self.assertEqual(lines, [u'A\n', u'B'])
finally:
remove_tempfiles(t1)
Example #2
| Source File: test_fileinput.py From ironpython2 with Apache License 2.0 | 6 votes |
|
def test_readline(self):
with open(TESTFN, 'wb') as f:
f.write('A\nB\r\nC\r')
# Fill TextIOWrapper buffer.
f.write('123456789\n' * 1000)
# Issue #20501: readline() shouldn't read whole file.
f.write('\x80')
self.addCleanup(safe_unlink, TESTFN)
fi = FileInput(files=TESTFN, openhook=hook_encoded('ascii'))
# The most likely failure is a UnicodeDecodeError due to the entire
# file being read when it shouldn't have been.
self.assertEqual(fi.readline(), u'A\n')
self.assertEqual(fi.readline(), u'B\r\n')
self.assertEqual(fi.readline(), u'C\r')
with self.assertRaises(UnicodeDecodeError):
# Read to the end of file.
list(fi)
fi.close()
Example #3
| Source File: test_fileinput.py From CTFCrackTools with GNU General Public License v3.0 | 6 votes |
|
def test_file_opening_hook(self):
try:
# cannot use openhook and inplace mode
fi = FileInput(inplace=1, openhook=lambda f,m: None)
self.fail("FileInput should raise if both inplace "
"and openhook arguments are given")
except ValueError:
pass
try:
fi = FileInput(openhook=1)
self.fail("FileInput should check openhook for being callable")
except ValueError:
pass
try:
t1 = writeTmp(1, ["A\nB"], mode="wb")
fi = FileInput(files=t1, openhook=hook_encoded("rot13"))
lines = list(fi)
self.assertEqual(lines, ["N\n", "O"])
finally:
remove_tempfiles(t1)
Example #4
| Source File: changepasswords.py From maas with GNU Affero General Public License v3.0 | 6 votes |
|
def handle(self, *args, **options):
count = 0
UserModel = get_user_model()
for line in fileinput(args, openhook=hook_encoded("utf-8")):
try:
username, password = line.rstrip("\r\n").split(":", 1)
except ValueError:
raise CommandError(
"Invalid input provided. "
"Format is 'username:password', one per line."
)
try:
user = UserModel._default_manager.get(
**{UserModel.USERNAME_FIELD: username}
)
except UserModel.DoesNotExist:
raise CommandError("User '%s' does not exist." % username)
user.set_password(password)
user.save()
count += 1
return "%d password(s) successfully changed." % count
Example #5
| Source File: test_fileinput.py From CTFCrackTools-V2 with GNU General Public License v3.0 | 6 votes |
|
def test_file_opening_hook(self):
try:
# cannot use openhook and inplace mode
fi = FileInput(inplace=1, openhook=lambda f,m: None)
self.fail("FileInput should raise if both inplace "
"and openhook arguments are given")
except ValueError:
pass
try:
fi = FileInput(openhook=1)
self.fail("FileInput should check openhook for being callable")
except ValueError:
pass
try:
t1 = writeTmp(1, ["A\nB"], mode="wb")
fi = FileInput(files=t1, openhook=hook_encoded("rot13"))
lines = list(fi)
self.assertEqual(lines, ["N\n", "O"])
finally:
remove_tempfiles(t1)
Example #6
| Source File: test_fileinput.py From Project-New-Reign---Nemesis-Main with GNU General Public License v3.0 | 6 votes |
|
def test_modes(self):
with open(TESTFN, 'wb') as f:
# UTF-7 is a convenient, seldom used encoding
f.write(b'A\nB\r\nC\rD+IKw-')
self.addCleanup(safe_unlink, TESTFN)
def check(mode, expected_lines):
with FileInput(files=TESTFN, mode=mode,
openhook=hook_encoded('utf-7')) as fi:
lines = list(fi)
self.assertEqual(lines, expected_lines)
check('r', ['A\n', 'B\n', 'C\n', 'D\u20ac'])
with self.assertWarns(DeprecationWarning):
check('rU', ['A\n', 'B\n', 'C\n', 'D\u20ac'])
with self.assertWarns(DeprecationWarning):
check('U', ['A\n', 'B\n', 'C\n', 'D\u20ac'])
with self.assertRaises(ValueError):
check('rb', ['A\n', 'B\r\n', 'C\r', 'D\u20ac'])
Example #7
| Source File: test_fileinput.py From Project-New-Reign---Nemesis-Main with GNU General Public License v3.0 | 6 votes |
|
def test(self):
encoding = object()
result = fileinput.hook_encoded(encoding)
fake_open = InvocationRecorder()
original_open = builtins.open
builtins.open = fake_open
try:
filename = object()
mode = object()
open_result = result(filename, mode)
finally:
builtins.open = original_open
self.assertEqual(fake_open.invocation_count, 1)
args, kwargs = fake_open.last_invocation
self.assertIs(args[0], filename)
self.assertIs(args[1], mode)
self.assertIs(kwargs.pop('encoding'), encoding)
self.assertFalse(kwargs)
Example #8
| Source File: test_fileinput.py From Project-New-Reign---Nemesis-Main with GNU General Public License v3.0 | 6 votes |
|
def test_readline(self):
with open(TESTFN, 'wb') as f:
f.write(b'A\nB\r\nC\r')
# Fill TextIOWrapper buffer.
f.write(b'123456789\n' * 1000)
# Issue #20501: readline() shouldn't read whole file.
f.write(b'\x80')
self.addCleanup(safe_unlink, TESTFN)
with FileInput(files=TESTFN,
openhook=hook_encoded('ascii')) as fi:
try:
self.assertEqual(fi.readline(), 'A\n')
self.assertEqual(fi.readline(), 'B\n')
self.assertEqual(fi.readline(), 'C\n')
except UnicodeDecodeError:
self.fail('Read to end of file')
with self.assertRaises(UnicodeDecodeError):
# Read to the end of file.
list(fi)
self.assertEqual(fi.readline(), '')
self.assertEqual(fi.readline(), '')
Example #9
| Source File: test_fileinput.py From gcblue with BSD 3-Clause "New" or "Revised" License | 6 votes |
|
def test_modes(self):
with open(TESTFN, 'wb') as f:
# UTF-7 is a convenient, seldom used encoding
f.write('A\nB\r\nC\rD+IKw-')
self.addCleanup(safe_unlink, TESTFN)
def check(mode, expected_lines):
fi = FileInput(files=TESTFN, mode=mode,
openhook=hook_encoded('utf-7'))
lines = list(fi)
fi.close()
self.assertEqual(lines, expected_lines)
check('r', [u'A\n', u'B\r\n', u'C\r', u'D\u20ac'])
check('rU', [u'A\n', u'B\r\n', u'C\r', u'D\u20ac'])
check('U', [u'A\n', u'B\r\n', u'C\r', u'D\u20ac'])
check('rb', [u'A\n', u'B\r\n', u'C\r', u'D\u20ac'])
Example #10
| Source File: test_fileinput.py From gcblue with BSD 3-Clause "New" or "Revised" License | 6 votes |
|
def test_readline(self):
with open(TESTFN, 'wb') as f:
f.write('A\nB\r\nC\r')
# Fill TextIOWrapper buffer.
f.write('123456789\n' * 1000)
# Issue #20501: readline() shouldn't read whole file.
f.write('\x80')
self.addCleanup(safe_unlink, TESTFN)
fi = FileInput(files=TESTFN, openhook=hook_encoded('ascii'), bufsize=8)
# The most likely failure is a UnicodeDecodeError due to the entire
# file being read when it shouldn't have been.
self.assertEqual(fi.readline(), u'A\n')
self.assertEqual(fi.readline(), u'B\r\n')
self.assertEqual(fi.readline(), u'C\r')
with self.assertRaises(UnicodeDecodeError):
# Read to the end of file.
list(fi)
fi.close()
Example #11
| Source File: test_fileinput.py From gcblue with BSD 3-Clause "New" or "Revised" License | 6 votes |
|
def test_file_opening_hook(self):
try:
# cannot use openhook and inplace mode
fi = FileInput(inplace=1, openhook=lambda f,m: None)
self.fail("FileInput should raise if both inplace "
"and openhook arguments are given")
except ValueError:
pass
try:
fi = FileInput(openhook=1)
self.fail("FileInput should check openhook for being callable")
except ValueError:
pass
try:
t1 = writeTmp(1, ["A\nB"], mode="wb")
fi = FileInput(files=t1, openhook=hook_encoded("rot13"))
lines = list(fi)
self.assertEqual(lines, ["N\n", "O"])
finally:
remove_tempfiles(t1)
Example #12
| Source File: test_fileinput.py From ironpython3 with Apache License 2.0 | 6 votes |
|
def test_modes(self):
with open(TESTFN, 'wb') as f:
# UTF-7 is a convenient, seldom used encoding
f.write(b'A\nB\r\nC\rD+IKw-')
self.addCleanup(safe_unlink, TESTFN)
def check(mode, expected_lines):
with FileInput(files=TESTFN, mode=mode,
openhook=hook_encoded('utf-7')) as fi:
lines = list(fi)
self.assertEqual(lines, expected_lines)
check('r', ['A\n', 'B\n', 'C\n', 'D\u20ac'])
with self.assertWarns(DeprecationWarning):
check('rU', ['A\n', 'B\n', 'C\n', 'D\u20ac'])
with self.assertWarns(DeprecationWarning):
check('U', ['A\n', 'B\n', 'C\n', 'D\u20ac'])
with self.assertRaises(ValueError):
check('rb', ['A\n', 'B\r\n', 'C\r', 'D\u20ac'])
Example #13
| Source File: test_fileinput.py From ironpython3 with Apache License 2.0 | 6 votes |
|
def test(self):
encoding = object()
result = fileinput.hook_encoded(encoding)
fake_open = InvocationRecorder()
original_open = builtins.open
builtins.open = fake_open
try:
filename = object()
mode = object()
open_result = result(filename, mode)
finally:
builtins.open = original_open
self.assertEqual(fake_open.invocation_count, 1)
args, kwargs = fake_open.last_invocation
self.assertIs(args[0], filename)
self.assertIs(args[1], mode)
self.assertIs(kwargs.pop('encoding'), encoding)
self.assertFalse(kwargs)
Example #14
| Source File: test_fileinput.py From ironpython3 with Apache License 2.0 | 6 votes |
|
def test_readline(self):
with open(TESTFN, 'wb') as f:
f.write(b'A\nB\r\nC\r')
# Fill TextIOWrapper buffer.
f.write(b'123456789\n' * 1000)
# Issue #20501: readline() shouldn't read whole file.
f.write(b'\x80')
self.addCleanup(safe_unlink, TESTFN)
with FileInput(files=TESTFN,
openhook=hook_encoded('ascii'), bufsize=8) as fi:
try:
self.assertEqual(fi.readline(), 'A\n')
self.assertEqual(fi.readline(), 'B\n')
self.assertEqual(fi.readline(), 'C\n')
except UnicodeDecodeError:
self.fail('Read to end of file')
with self.assertRaises(UnicodeDecodeError):
# Read to the end of file.
list(fi)
self.assertEqual(fi.readline(), '')
self.assertEqual(fi.readline(), '')
Example #15
| Source File: more.py From stash with MIT License | 6 votes |
|
def more(filenames, pagesize=10, clear=False, fmt='{line}'):
'''Display content of filenames pagesize lines at a time (cleared if specified) with format fmt for each output line'''
fileinput.close() # in case still open
try:
pageno = 1
if clear:
clear_screen()
for line in fileinput.input(filenames, openhook=fileinput.hook_encoded("utf-8")):
lineno, filename, filelineno = fileinput.lineno(), fileinput.filename(), fileinput.filelineno()
print(fmt.format(**locals()), end='')
if pagesize and lineno % pagesize == 0:
console.alert('Abort or continue', filename, 'Next page') # TODO: use less intrusive mechanism than alert
pageno += 1
if clear:
clear_screen()
finally:
fileinput.close()
# --- main
Example #16
| Source File: cat.py From stash with MIT License | 6 votes |
|
def main(args):
p = argparse.ArgumentParser(description=__doc__)
p.add_argument("files", action="store", nargs="*", help="files to print")
ns = p.parse_args(args)
status = 0
fileinput.close() # in case it is not closed
try:
for line in fileinput.input(ns.files, openhook=fileinput.hook_encoded("utf-8")):
print(filter_non_printable(line), end='')
except Exception as e:
print('cat: %s' % str(e))
status = 1
finally:
fileinput.close()
sys.exit(status)
Example #17
| Source File: test_fileinput.py From ironpython2 with Apache License 2.0 | 6 votes |
|
def test_modes(self):
with open(TESTFN, 'wb') as f:
# UTF-7 is a convenient, seldom used encoding
f.write('A\nB\r\nC\rD+IKw-')
self.addCleanup(safe_unlink, TESTFN)
def check(mode, expected_lines):
fi = FileInput(files=TESTFN, mode=mode,
openhook=hook_encoded('utf-7'))
lines = list(fi)
fi.close()
self.assertEqual(lines, expected_lines)
check('r', [u'A\n', u'B\r\n', u'C\r', u'D\u20ac'])
check('rU', [u'A\n', u'B\r\n', u'C\r', u'D\u20ac'])
check('U', [u'A\n', u'B\r\n', u'C\r', u'D\u20ac'])
check('rb', [u'A\n', u'B\r\n', u'C\r', u'D\u20ac'])
Example #18
| Source File: test_fileinput.py From Fluid-Designer with GNU General Public License v3.0 | 6 votes |
|
def test_modes(self):
with open(TESTFN, 'wb') as f:
# UTF-7 is a convenient, seldom used encoding
f.write(b'A\nB\r\nC\rD+IKw-')
self.addCleanup(safe_unlink, TESTFN)
def check(mode, expected_lines):
with FileInput(files=TESTFN, mode=mode,
openhook=hook_encoded('utf-7')) as fi:
lines = list(fi)
self.assertEqual(lines, expected_lines)
check('r', ['A\n', 'B\n', 'C\n', 'D\u20ac'])
with self.assertWarns(DeprecationWarning):
check('rU', ['A\n', 'B\n', 'C\n', 'D\u20ac'])
with self.assertWarns(DeprecationWarning):
check('U', ['A\n', 'B\n', 'C\n', 'D\u20ac'])
with self.assertRaises(ValueError):
check('rb', ['A\n', 'B\r\n', 'C\r', 'D\u20ac'])
Example #19
| Source File: test_fileinput.py From Fluid-Designer with GNU General Public License v3.0 | 6 votes |
|
def test(self):
encoding = object()
result = fileinput.hook_encoded(encoding)
fake_open = InvocationRecorder()
original_open = builtins.open
builtins.open = fake_open
try:
filename = object()
mode = object()
open_result = result(filename, mode)
finally:
builtins.open = original_open
self.assertEqual(fake_open.invocation_count, 1)
args, kwargs = fake_open.last_invocation
self.assertIs(args[0], filename)
self.assertIs(args[1], mode)
self.assertIs(kwargs.pop('encoding'), encoding)
self.assertFalse(kwargs)
Example #20
| Source File: test_fileinput.py From Fluid-Designer with GNU General Public License v3.0 | 6 votes |
|
def test_readline(self):
with open(TESTFN, 'wb') as f:
f.write(b'A\nB\r\nC\r')
# Fill TextIOWrapper buffer.
f.write(b'123456789\n' * 1000)
# Issue #20501: readline() shouldn't read whole file.
f.write(b'\x80')
self.addCleanup(safe_unlink, TESTFN)
with FileInput(files=TESTFN,
openhook=hook_encoded('ascii'), bufsize=8) as fi:
try:
self.assertEqual(fi.readline(), 'A\n')
self.assertEqual(fi.readline(), 'B\n')
self.assertEqual(fi.readline(), 'C\n')
except UnicodeDecodeError:
self.fail('Read to end of file')
with self.assertRaises(UnicodeDecodeError):
# Read to the end of file.
list(fi)
self.assertEqual(fi.readline(), '')
self.assertEqual(fi.readline(), '')
Example #21
| Source File: YoutubeDL.py From youtube-dl-GUI with MIT License | 6 votes |
|
def download_with_info_file(self, info_filename):
with contextlib.closing(fileinput.FileInput(
[info_filename], mode='r',
openhook=fileinput.hook_encoded('utf-8'))) as f:
# FileInput doesn't have a read method, we can't call json.load
info = self.filter_requested_info(json.loads('\n'.join(f)))
try:
self.process_ie_result(info, download=True)
except DownloadError:
webpage_url = info.get('webpage_url')
if webpage_url is not None:
self.report_warning('The info failed to download, trying with "%s"' % webpage_url)
return self.download([webpage_url])
else:
raise
return self._download_retcode
Example #22
| Source File: test_fileinput.py From BinderFilter with MIT License | 6 votes |
|
def test_file_opening_hook(self):
try:
# cannot use openhook and inplace mode
fi = FileInput(inplace=1, openhook=lambda f,m: None)
self.fail("FileInput should raise if both inplace "
"and openhook arguments are given")
except ValueError:
pass
try:
fi = FileInput(openhook=1)
self.fail("FileInput should check openhook for being callable")
except ValueError:
pass
try:
t1 = writeTmp(1, ["A\nB"], mode="wb")
fi = FileInput(files=t1, openhook=hook_encoded("rot13"))
lines = list(fi)
self.assertEqual(lines, ["N\n", "O"])
finally:
remove_tempfiles(t1)
Example #23
| Source File: YoutubeDL.py From tvalacarta with GNU General Public License v3.0 | 6 votes |
|
def download_with_info_file(self, info_filename):
with contextlib.closing(fileinput.FileInput(
[info_filename], mode='r',
openhook=fileinput.hook_encoded('utf-8'))) as f:
# FileInput doesn't have a read method, we can't call json.load
info = self.filter_requested_info(json.loads('\n'.join(f)))
try:
self.process_ie_result(info, download=True)
except DownloadError:
webpage_url = info.get('webpage_url')
if webpage_url is not None:
self.report_warning('The info failed to download, trying with "%s"' % webpage_url)
return self.download([webpage_url])
else:
raise
return self._download_retcode
Example #24
| Source File: test_fileinput.py From oss-ftp with MIT License | 6 votes |
|
def test_file_opening_hook(self):
try:
# cannot use openhook and inplace mode
fi = FileInput(inplace=1, openhook=lambda f,m: None)
self.fail("FileInput should raise if both inplace "
"and openhook arguments are given")
except ValueError:
pass
try:
fi = FileInput(openhook=1)
self.fail("FileInput should check openhook for being callable")
except ValueError:
pass
try:
t1 = writeTmp(1, ["A\nB"], mode="wb")
fi = FileInput(files=t1, openhook=hook_encoded("rot13"))
lines = list(fi)
self.assertEqual(lines, ["N\n", "O"])
finally:
remove_tempfiles(t1)
Example #25
| Source File: test_fileinput.py From oss-ftp with MIT License | 6 votes |
|
def test_modes(self):
with open(TESTFN, 'wb') as f:
# UTF-7 is a convenient, seldom used encoding
f.write('A\nB\r\nC\rD+IKw-')
self.addCleanup(safe_unlink, TESTFN)
def check(mode, expected_lines):
fi = FileInput(files=TESTFN, mode=mode,
openhook=hook_encoded('utf-7'))
lines = list(fi)
fi.close()
self.assertEqual(lines, expected_lines)
check('r', [u'A\n', u'B\r\n', u'C\r', u'D\u20ac'])
check('rU', [u'A\n', u'B\r\n', u'C\r', u'D\u20ac'])
check('U', [u'A\n', u'B\r\n', u'C\r', u'D\u20ac'])
check('rb', [u'A\n', u'B\r\n', u'C\r', u'D\u20ac'])
Example #26
| Source File: test_fileinput.py From oss-ftp with MIT License | 6 votes |
|
def test_readline(self):
with open(TESTFN, 'wb') as f:
f.write('A\nB\r\nC\r')
# Fill TextIOWrapper buffer.
f.write('123456789\n' * 1000)
# Issue #20501: readline() shouldn't read whole file.
f.write('\x80')
self.addCleanup(safe_unlink, TESTFN)
fi = FileInput(files=TESTFN, openhook=hook_encoded('ascii'), bufsize=8)
# The most likely failure is a UnicodeDecodeError due to the entire
# file being read when it shouldn't have been.
self.assertEqual(fi.readline(), u'A\n')
self.assertEqual(fi.readline(), u'B\r\n')
self.assertEqual(fi.readline(), u'C\r')
with self.assertRaises(UnicodeDecodeError):
# Read to the end of file.
list(fi)
fi.close()
Example #27
| Source File: interactive.py From NLP_Toolkit with Apache License 2.0 | 5 votes |
|
def buffered_read(input, buffer_size):
buffer = []
with fileinput.input(files=[input], openhook=fileinput.hook_encoded("utf-8")) as h:
for src_str in h:
buffer.append(src_str.strip())
if len(buffer) >= buffer_size:
yield buffer
buffer = []
if len(buffer) > 0:
yield buffer
Example #28
| Source File: interactive.py From helo_word with Apache License 2.0 | 5 votes |
|
def buffered_read(input, buffer_size):
buffer = []
with fileinput.input(files=[input], openhook=fileinput.hook_encoded("utf-8")) as h:
for src_str in h:
buffer.append(src_str.strip())
if len(buffer) >= buffer_size:
yield buffer
buffer = []
if len(buffer) > 0:
yield buffer
Example #29
| Source File: interactive.py From helo_word with Apache License 2.0 | 5 votes |
|
def buffered_read(input, buffer_size):
buffer = []
with fileinput.input(files=[input], openhook=fileinput.hook_encoded("utf-8")) as h:
for src_str in h:
buffer.append(src_str.strip())
if len(buffer) >= buffer_size:
yield buffer
buffer = []
if len(buffer) > 0:
yield buffer
Example #30
| Source File: interactive.py From fairseq with MIT License | 5 votes |
|
def buffered_read(input, buffer_size):
buffer = []
with fileinput.input(files=[input], openhook=fileinput.hook_encoded("utf-8")) as h:
for src_str in h:
buffer.append(src_str.strip())
if len(buffer) >= buffer_size:
yield buffer
buffer = []
if len(buffer) > 0:
yield buffer
