Python itertools.zip_longest() Examples

def safeZip(listOfLists, enforceLength):
    """
    A safe version of python's zip()

    If two sublists are of different sizes, python's zip will truncate
    the output to be the smaller of the two.

    safeZip throws an exception if the size of the any sublist is different
    from the rest.
    """
    if enforceLength is True:
        length = len(listOfLists[0])
        assert(all([length == len(subList) for subList in listOfLists]))
    
    try:
        zipFunc = itertools.izip_longest # Python 2.x
    except AttributeError:
        zipFunc = itertools.zip_longest # Python 3.x
    
    return zipFunc(*listOfLists) 

def base_equal(self, ob1, ob2):
        if type(ob1) != type(ob2):
            return False

        def cmp(obj1, obj2):
            if isinstance(obj1, np.ndarray):
                return np.array_equal(obj1, obj2)
            elif isinstance(obj1, Iterable) and \
                    not isinstance(obj1, str) and \
                    isinstance(obj2, Iterable) and \
                    not isinstance(obj2, str):
                return all(cmp(it1, it2) for it1, it2 in itertools.zip_longest(obj1, obj2))
            elif hasattr(obj1, 'key') and hasattr(obj2, 'key'):
                return obj1.key == obj2.key
            elif isinstance(obj1, ReferenceType) and isinstance(obj2, ReferenceType):
                return cmp(obj1(), obj2())
            else:
                return obj1 == obj2

        for slot in ob1.__slots__:
            if not cmp(getattr(ob1, slot, None), getattr(ob2, slot, None)):
                return False

        return True 

def loose_version_compare(a, b):
    for i, j in zip_longest(a.version, b.version, fillvalue=''):
        if type(i) != type(j):
            i = str(i)
            j = str(j)
        if i == j:
            continue
        elif i < j:
            return -1
        else:  # i > j
            return 1

    #Longer version strings with equal prefixes are equal, but if one version string is longer than it is greater
    aLen = len(a.version)
    bLen = len(b.version)
    if aLen == bLen:
        return 0
    elif aLen < bLen:
        return -1
    else:
        return 1 

def __add__(self, other):
        if isinstance(other, type(self)):
            other_tiers = other._tiers
        elif isinstance(other, collections.abc.Iterable):
            other_tiers = other
        new_tiers = []
        for tier1,x in itertools.zip_longest(self._tiers, other_tiers):
            if tier1 is None:
                tier1 = []
            if isinstance(x, str) and len(x) > 1:
                new_tier = tier1 + [x]
            elif isinstance(x, collections.abc.Iterable):
                new_tier = tier1 + list(x)
            elif x is not None:
                return NotImplemented
            else:
                new_tier = tier1
            new_tiers.append(new_tier)
        return type(self)(new_tiers, callback=self._callback) 

def check_maker(maker: tp.Callable[[], tp.Iterator[experiments.Experiment]]) -> None:
    generators = [maker() for _ in range(2)]
    # check 1 sample
    sample = next(maker())
    assert isinstance(sample, experiments.Experiment)
    # check names, coherence and non-randomness
    for k, (elem1, elem2) in enumerate(itertools.zip_longest(*generators)):
        assert not elem1.is_incoherent, f"Incoherent settings should be filtered out from generator:\n{elem1}"
        try:
            assert elem1 == elem2  # much faster but lacks explicit message
        except AssertionError:
            testing.printed_assert_equal(
                elem1.get_description(),
                elem2.get_description(),
                err_msg=f"Two paths on the generator differed (see element #{k})\n"
                "Generators need to be deterministic in order to split the workload!",
            ) 

def _format_args(args, defaults=None, annotations=None):
    values = []
    if args is None:
        return ""
    if annotations is None:
        annotations = []
    if defaults is not None:
        default_offset = len(args) - len(defaults)
    packed = itertools.zip_longest(args, annotations)
    for i, (arg, annotation) in enumerate(packed):
        if isinstance(arg, Tuple):
            values.append("(%s)" % _format_args(arg.elts))
        else:
            argname = arg.name
            if annotation is not None:
                argname += ":" + annotation.as_string()
            values.append(argname)

            if defaults is not None and i >= default_offset:
                if defaults[i - default_offset] is not None:
                    values[-1] += "=" + defaults[i - default_offset].as_string()
    return ", ".join(values) 

def _format_args(args, defaults=None, annotations=None):
    values = []
    if args is None:
        return ""
    if annotations is None:
        annotations = []
    if defaults is not None:
        default_offset = len(args) - len(defaults)
    packed = itertools.zip_longest(args, annotations)
    for i, (arg, annotation) in enumerate(packed):
        if isinstance(arg, Tuple):
            values.append("(%s)" % _format_args(arg.elts))
        else:
            argname = arg.name
            if annotation is not None:
                argname += ":" + annotation.as_string()
            values.append(argname)

            if defaults is not None and i >= default_offset:
                if defaults[i - default_offset] is not None:
                    values[-1] += "=" + defaults[i - default_offset].as_string()
    return ", ".join(values) 

def __init__(self, iterable, num_shards, shard_id, fill_value=None):
        if shard_id < 0 or shard_id >= num_shards:
            raise ValueError('shard_id must be between 0 and num_shards')
        sharded_len = int(math.ceil(len(iterable) / float(num_shards)))
        itr = map(
            operator.itemgetter(1),
            itertools.zip_longest(
                range(sharded_len),
                itertools.islice(iterable, shard_id, len(iterable), num_shards),
                fillvalue=fill_value,
            ),
        )
        super().__init__(
            itr,
            start=int(math.ceil(getattr(iterable, 'n', 0) / float(num_shards))),
            total=sharded_len,
        ) 

def veval_ast_arguments(astc : 'AstContext', local_field : 'values.Field', graph : 'Graph', context : 'functions.VEvalContext' = None):
    assert(isinstance(astc.nast, gast.gast.arguments))
    lineprop = utils.LineProperty(astc.lineno, astc.filename)

    ret = functions.FunctionArgCollection()

    argspec = inspect.FullArgSpec(astc.nast.args, astc.nast.vararg, astc.nast.kwarg,
                                  astc.nast.defaults, astc.nast.kwonlyargs, astc.nast.kw_defaults, None)

    assert not argspec.kwonlyargs, "Keyword only args are not supported"
    assert not argspec.varargs, "Varaibale arguments *args is not supported"
    assert not argspec.varkw, "Variable keywords **kwargs is not supported"

    defaults = [veval_ast(astc.c(default), local_field, graph, context) for default in argspec.defaults]
    arg_list = []
    for k, v in itertools.zip_longest(reversed(argspec.args), defaults):
        arg_list.append((k.id, v))

    # reverse the list
    for k, v in reversed(arg_list):
        ret.add_arg(k, v)

    return ret 

def grouper(n, iterable, fillvalue=None):
    "Collect data into fixed-length chunks or blocks"
    # grouper(3, 'ABCDEFG', 'x') --> ABC DEF Gxx
    args = [iter(iterable)] * n
    return zip_longest(fillvalue=fillvalue, *args) 

def izip_records(seqarrays, fill_value=None, flatten=True):
    """
    Returns an iterator of concatenated items from a sequence of arrays.

    Parameters
    ----------
    seqarrays : sequence of arrays
        Sequence of arrays.
    fill_value : {None, integer}
        Value used to pad shorter iterables.
    flatten : {True, False},
        Whether to
    """

    # Should we flatten the items, or just use a nested approach
    if flatten:
        zipfunc = _izip_fields_flat
    else:
        zipfunc = _izip_fields

    if sys.version_info[0] >= 3:
        zip_longest = itertools.zip_longest
    else:
        zip_longest = itertools.izip_longest

    for tup in zip_longest(*seqarrays, fillvalue=fill_value):
        yield tuple(zipfunc(tup)) 

def __call__(self, *sequences):
        if not all(sequences):
            return 0
        sequences = [list(self._calc_mra(s)) for s in sequences]
        lengths = list(map(len, sequences))
        count = len(lengths)
        max_length = max(lengths)
        if abs(max_length - min(lengths)) > count:
            return 0

        for _ in range(count):
            new_sequences = []
            minlen = min(lengths)
            for chars in zip(*sequences):
                if not self._ident(*chars):
                    new_sequences.append(chars)
            new_sequences = map(list, zip(*new_sequences))
            # update sequences
            ss = zip_longest(new_sequences, sequences, fillvalue=list())
            sequences = [s1 + s2[minlen:] for s1, s2 in ss]
            # update lengths
            lengths = list(map(len, sequences))

        if not lengths:
            return max_length
        return max_length - max(lengths) 

def hexdump(filename, chunk_size=16):
    """Generate hexdump of a binary file.

    Args:
        filename (str): Path of file.
        chunk_size (int): Chunk size.

    """
    chunks = [iter(range(chunk_size))] * 4
    header = '   '.join(
        ' '.join(format(x, '0>2x') for x in chunk)
        for chunk in itertools.zip_longest(*chunks, fillvalue=0))
    print('ADDRESS        {:<53}       ASCII'.format(header))
    print('')
    template = '{:0>8x}       {:<53}       {}'

    with open(filename, 'rb') as stream:
        for chunk_count in itertools.count(1):
            chunk = stream.read(chunk_size)
            if not chunk:
                return
            chunks = [iter(chunk)] * 4
            print(template.format(
                chunk_count * chunk_size,
                '   '.join(
                    ' '.join(format(x, '0>2x') for x in chunk)
                    for chunk in itertools.zip_longest(*chunks, fillvalue=0)),
                ''.join(
                    chr(x) if 33 <= x <= 126 else '.'
                    for x in chunk))) 

def grouper(iterable, n):
        args = [iter(iterable)] * n
        return itertools.zip_longest(*args, fillvalue=None) 

def _create_md5_dict(self):
        """Create md5 dictionary per source
        """
        self.sbo_md5 = {}
        md5_lists = SBoGrep(self.prgnam).checksum()
        for src, md5 in itertools.zip_longest(self.sources, md5_lists):
            self.sbo_md5[src] = md5 

def reference(self, install, upgrade):
        """Reference list with packages installed
        and upgraded
        """
        self.template(78)
        print(f"| Total {len(install)} {self.pkg(len(install))} installed and "
              f"{len(upgrade)} {self.pkg(len(upgrade))} upgraded")
        self.template(78)
        for installed, upgraded in itertools.zip_longest(install, upgrade):
            if upgraded:
                print(f"| Package {upgraded} upgraded successfully")
            if installed:
                print(f"| Package {installed} installed successfully")
        self.template(78)
        print() 

def partify(self, wildcard_perm):
        return [set(a.strip() for a in y.split(self.SUBPART_DIVIDER_TOKEN))
                for y in [x[0] if x[0] else x[1]
                          for x in itertools.zip_longest(
                          wildcard_perm.split(self.PART_DIVIDER_TOKEN),
                          [self.WILDCARD_TOKEN] * 3)
                          ]
                ] 

def _assert_columns_equal(
    left: Sequence[tab_api.TableDefinition.Column],
    right: Sequence[tab_api.TableDefinition.Column],
) -> None:
    """
    Helper function to validate if sequences of columns are equal.

    The TableauHyperAPI as of 0.0.8953 does not implement equality operations
    for Column instances, hence the need for this.
    """

    class DummyColumn:
        """Dummy class to match items needed for str repr of columns."""

        @property
        def name(self):
            return None

        @property
        def type(self):
            return None

        @property
        def nullability(self):
            return None

    for c1, c2 in itertools.zip_longest(left, right, fillvalue=DummyColumn()):
        if c1.name != c2.name or c1.type != c2.type or c1.nullability != c2.nullability:
            break  # go to error handler
    else:
        return None  # everything matched up, so bail out

    c1_str = ", ".join(
        f"(Name={x.name}, Type={x.type}, Nullability={x.nullability})" for x in left
    )
    c2_str = ", ".join(
        f"(Name={x.name}, Type={x.type}, Nullability={x.nullability})" for x in right
    )

    raise TypeError(f"Mismatched column definitions: {c1_str} != {c2_str}") 

def izip_records(seqarrays, fill_value=None, flatten=True):
    """
    Returns an iterator of concatenated items from a sequence of arrays.

    Parameters
    ----------
    seqarrays : sequence of arrays
        Sequence of arrays.
    fill_value : {None, integer}
        Value used to pad shorter iterables.
    flatten : {True, False},
        Whether to
    """

    # Should we flatten the items, or just use a nested approach
    if flatten:
        zipfunc = _izip_fields_flat
    else:
        zipfunc = _izip_fields

    if sys.version_info[0] >= 3:
        zip_longest = itertools.zip_longest
    else:
        zip_longest = itertools.izip_longest

    for tup in zip_longest(*seqarrays, fillvalue=fill_value):
        yield tuple(zipfunc(tup)) 

def _pickle_object(x: typing.Any) -> str:
    pck: str = base64.b85encode(gzip.compress(pickle.dumps(x, protocol=4))).decode().strip()
    segment_gen = map(''.join, itertools.zip_longest(*([iter(pck)] * 100), fillvalue=''))
    return '\n'.join(repr(x) for x in segment_gen) 

def test_dfs_labeled_edges(index_dtype, example=False):
    dgl_g = dgl.DGLGraph()
    dgl_g.add_nodes(6)
    dgl_g.add_edges([0, 1, 0, 3, 3], [1, 2, 2, 4, 5])
    if index_dtype == 'int32':
        dgl_g = dgl.graph(dgl_g.edges()).int()
    else:
        dgl_g = dgl.graph(dgl_g.edges()).long()
    dgl_edges, dgl_labels = dgl.dfs_labeled_edges_generator(
            dgl_g, [0, 3], has_reverse_edge=True, has_nontree_edge=True)
    dgl_edges = [toset(t) for t in dgl_edges]
    dgl_labels = [toset(t) for t in dgl_labels]
    g1_solutions = [
            # edges           labels
            [[0, 1, 1, 0, 2], [0, 0, 1, 1, 2]],
            [[2, 2, 0, 1, 0], [0, 1, 0, 2, 1]],
    ]
    g2_solutions = [
            # edges        labels
            [[3, 3, 4, 4], [0, 1, 0, 1]],
            [[4, 4, 3, 3], [0, 1, 0, 1]],
    ]

    def combine_frontiers(sol):
        es, ls = zip(*sol)
        es = [set(i for i in t if i is not None)
              for t in itertools.zip_longest(*es)]
        ls = [set(i for i in t if i is not None)
              for t in itertools.zip_longest(*ls)]
        return es, ls

    for sol_set in itertools.product(g1_solutions, g2_solutions):
        es, ls = combine_frontiers(sol_set)
        if es == dgl_edges and ls == dgl_labels:
            break
    else:
        assert False 

def _has_same_layout_slots(slots, assigned_value):
    inferred = next(assigned_value.infer())
    if isinstance(inferred, astroid.ClassDef):
        other_slots = inferred.slots()
        if all(
            first_slot and second_slot and first_slot.value == second_slot.value
            for (first_slot, second_slot) in zip_longest(slots, other_slots)
        ):
            return True
    return False 

def _has_different_parameters(original, overridden, dummy_parameter_regex):
    zipped = zip_longest(original, overridden)
    for original_param, overridden_param in zipped:
        params = (original_param, overridden_param)
        if not all(params):
            return True

        names = [param.name for param in params]
        if any(map(dummy_parameter_regex.match, names)):
            continue
        if original_param.name != overridden_param.name:
            return True
    return False 

def blocked_iter(blocksize, iterable, fillvalue = None):
    # From the Python "itertools recipes" (grouper)
    args = [iter(iterable)] * blocksize
    return itertools.zip_longest(fillvalue=fillvalue, *args) 

def grouper(n, iterable, fillvalue=None):
    "Collect data into fixed-length chunks or blocks"
    # grouper(3, 'ABCDEFG', 'x') --> ABC DEF Gxx
    args = [iter(iterable)] * n
    return zip_longest(fillvalue=fillvalue, *args) 

def _has_same_layout_slots(slots, assigned_value):
    inferred = next(assigned_value.infer())
    if isinstance(inferred, astroid.ClassDef):
        other_slots = inferred.slots()
        if all(
            first_slot and second_slot and first_slot.value == second_slot.value
            for (first_slot, second_slot) in zip_longest(slots, other_slots)
        ):
            return True
    return False 

def _has_different_parameters(original, overridden, dummy_parameter_regex):
    zipped = zip_longest(original, overridden)
    for original_param, overridden_param in zipped:
        params = (original_param, overridden_param)
        if not all(params):
            return True

        names = [param.name for param in params]
        if any(map(dummy_parameter_regex.match, names)):
            continue
        if original_param.name != overridden_param.name:
            return True
    return False 

def tabulate(vals):
    # From pfmoore on GitHub:
    # https://github.com/pypa/pip/issues/3651#issuecomment-216932564
    assert len(vals) > 0

    sizes = [0] * max(len(x) for x in vals)
    for row in vals:
        sizes = [max(s, len(str(c))) for s, c in zip_longest(sizes, row)]

    result = []
    for row in vals:
        display = " ".join([str(c).ljust(s) if c is not None else ''
                            for s, c in zip_longest(sizes, row)])
        result.append(display)

    return result, sizes 

def compare_tokens(content, expected_types, expected_values=None,
                   expected_lineno=None, expected_lexpos=None):
    lexer = AtfLexer().lexer
    lexer.input(content)
    if expected_values is None:
        expected_values = repeat(None)
    if expected_lineno is None:
        expected_lineno = repeat(None)
    if expected_lexpos is None:
        expected_lexpos = repeat(None)
    for e_type, e_value, e_lineno, e_lexpos, token in zip_longest(
            expected_types,
            expected_values,
            expected_lineno,
            expected_lexpos,
            lexer):
        if token is None and e_type is None:
            break
        assert token.type == e_type
        if e_value:
            assert token.value == e_value
        if e_lineno:
            assert token.lineno == e_lineno
        if e_lexpos:
            assert token.lexpos == e_lexpos 

def check(func, expected, terminal_size=None):
    expected, lines = run_func(func, expected, terminal_size)
    if expected:
        maxlen = max(map(len, expected))
        if sys.version_info < (3,):
            # Ensure same type for comparison.
            expected = [
                x.decode("utf8") if isinstance(x, bytes) else x for x in expected
            ]
    else:
        maxlen = 0
    all_ok = True
    print()
    for pattern, string in zip_longest(expected, lines):
        if pattern is not None and string is not None:
            try:
                ok = re.match(pattern, string)
            except re.error as exc:
                raise ValueError("re.match failed for {!r}: {!r}".format(pattern, exc))
        else:
            ok = False
            if pattern is None:
                pattern = '<None>'
            if string is None:
                string = '<None>'
        # Use "$" to mark end of line with trailing space
        if re.search(r'\s+$', string):
            string += '$'
        if re.search(r'\s+$', pattern):
            pattern += '$'
        pattern = trans_trn(pattern)
        string = trans_trn(string)
        print(pattern.ljust(maxlen+1), '| ', string, end='')
        if ok:
            print()
        else:
            print(pdbpp.Color.set(pdbpp.Color.red, '    <<<<<'))
            all_ok = False
    assert all_ok