Python tables.openFile() Examples

def read_mat_file_into_bag(mat_fname):
    try:
        import scipy.io as sio
        x = sio.loadmat(mat_fname)
        return Bag(**x)
    except NotImplementedError:
        import tables
        from src.utils import tables_utils as tu
        x = tables.openFile(mat_fname)
        ret = Bag(**tu.read_tables_into_dict(x))
        x.close()
        return ret
    return None 

def dump_test_set(self, h5filepath, nframes, framesize):
        # set rng to a hardcoded state, so we always have the same test set!
        self.numpy_rng.seed(1)
        with tables.openFile(h5filepath, 'w') as h5file:

            h5file.createArray(h5file.root, 'test_targets',
                               self.partitions['test']['targets'])

            vids = h5file.createCArray(
                h5file.root,
                'test_images',
                tables.Float32Atom(),
                shape=(10000,
                       nframes, framesize, framesize),
                filters=tables.Filters(complevel=5, complib='zlib'))

            pos = h5file.createCArray(
                h5file.root,
                'test_pos',
                tables.UInt16Atom(),
                shape=(10000,
                       nframes, 2),
                filters=tables.Filters(complevel=5, complib='zlib'))
            for i in range(100):
                print i
                (vids[i*100:(i+1)*100],
                 pos[i*100:(i+1)*100], _) = self.get_batch(
                     'test', 100, nframes, framesize,
                     idx=np.arange(i*100,(i+1)*100))
                h5file.flush() 

def load_h5(self, filename):
        h5file = tables.openFile(filename, 'r')
        new_params = {}
        for p in h5file.listNodes(h5file.root):
            new_params[p.name] = p.read()
        self.updateparams_fromdict(new_params)
        h5file.close() 

def save_h5(self, filename):
        try:
            shutil.copyfile(filename, '{}_bak'.format(filename))
        except IOError:
            print 'could not make backup of model param file (which is normal if we haven\'t saved one until now)'

        with tables.openFile(filename, 'w') as h5file:
            for p in self.params:
                h5file.createArray(h5file.root, p.name, p.get_value())
                h5file.flush() 

def test_output():
    import tables as T
    h5 = T.openFile('/Users/ebruning/out/LYLOUT_040526_213000_0600.dat.gz.flash.h5')
    flashes = h5.root.flashes.LMA_040526_213000_600
    events  = h5.root.events.LMA_040526_213000_600
    # flashes.cols.n_points[0:100]
    big = [fl['flash_id'] for fl in flashes if fl['n_points'] > 100]
    a_flash = big[0]
    points = [ev['lat'] for ev in events if ev['flash_id'] == a_flash]
    print(flashes.cols.init_lon[0:10]) 

def test_output():
    import tables as T
    h5 = T.openFile('/Users/ebruning/out/LYLOUT_040526_213000_0600.dat.gz.flash.h5')
    flashes = h5.root.flashes.LMA_040526_213000_600
    events  = h5.root.events.LMA_040526_213000_600
    # flashes.cols.n_points[0:100]
    big = [fl['flash_id'] for fl in flashes if fl['n_points'] > 100]
    a_flash = big[0]
    points = [ev['lat'] for ev in events if ev['flash_id'] == a_flash]
    print(flashes.cols.init_lon[0:10]) 

def open_h5_file_read(h5filename):
    """
    Open an existing H5 in read mode.
    Same function as in hdf5_utils, here so we avoid one import
    """
    return tables.openFile(h5filename, mode='r') 

def synchronized_open_file(*args, **kwargs):
    if tables.__version__[0] == '2':
        tbf = tables.openFile(*args, **kwargs)
    else:
        tbf = tables.open_file(*args, **kwargs)
    return tbf 

def get_length(path):
    if tables.__version__[0] == '2':
        target_table = tables.openFile(path, 'r')
        target_index = target_table.getNode('/indices')
    else:
        target_table = tables.open_file(path, 'r')
        target_index = target_table.get_node('/indices')

    return target_index.shape[0] 

def synchronized_open_file(*args, **kwargs):
    if tables.__version__[0] == '2':
        tbf = tables.openFile(*args, **kwargs)
    else:
        tbf = tables.open_file(*args, **kwargs)
    return tbf 

def get_length(path):
    if tables.__version__[0] == '2':
        target_table = tables.openFile(path, 'r')
        target_index = target_table.getNode('/indices')
    else:
        target_table = tables.open_file(path, 'r')
        target_index = target_table.get_node('/indices')

    return target_index.shape[0] 

def open_hdf5_file(file_name, mode='a'):
    try:
        return tables.open_file(file_name, mode=mode)
    except:
        return tables.openFile(file_name, mode=mode)


# dtype = np.dtype('int16') / np.dtype('float64') 

def create_hdf5_file(file_name):
    try:
        return tables.open_file(file_name, mode='w')
    except:
        return tables.openFile(file_name, mode='w') 

def safe_hdf(array, name):
    if os.path.isfile(name + '.hdf') and not args.overwrite:
        logger.warning("Not saving %s, already exists." % (name + '.hdf'))
    else:
        if os.path.isfile(name + '.hdf'):
            logger.info("Overwriting %s." % (name + '.hdf'))
        else:
            logger.info("Saving to %s." % (name + '.hdf'))
        with tables.openFile(name + '.hdf', 'w') as f:
            atom = tables.Atom.from_dtype(array.dtype)
            filters = tables.Filters(complib='blosc', complevel=5)
            ds = f.createCArray(f.root, name.replace('.', ''), atom,
                                array.shape, filters=filters)
            ds[:] = array 

def open_h5_file_read(h5filename):
    """
    Open an existing H5 in read mode.
    Same function as in hdf5_utils, here so we avoid one import
    """
    return tables.openFile(h5filename, mode='r') 

def open_h5_file_read(h5filename):
    """
    Open an existing H5 in read mode.
    Same function as in hdf5_utils, here so we avoid one import
    """
    return tables.openFile(h5filename, mode='r') 

def safe_hdf(array, name):
    if os.path.isfile(name + '.hdf') and not args.overwrite:
        logger.warning("Not saving %s, already exists." % (name + '.hdf'))
    else:
        if os.path.isfile(name + '.hdf'):
            logger.info("Overwriting %s." % (name + '.hdf'))
        else:
            logger.info("Saving to %s." % (name + '.hdf'))
        with tables.openFile(name + '.hdf', 'w') as f:
            atom = tables.Atom.from_dtype(array.dtype)
            filters = tables.Filters(complib='blosc', complevel=5)
            ds = f.createCArray(f.root, name.replace('.', ''), atom,
                                array.shape, filters=filters)
            ds[:] = array 

def init_hdf5(self, path, shapes,
                  title="Pytables Dataset",
                  y_dtype='float'):
        """
        Initializes the hdf5 file into which the data will be stored. This must
        be called before calling fill_hdf5.

        Parameters
        ----------
        path : string
            The name of the hdf5 file.
        shapes : tuple
            The shapes of X and y.
        title : string, optional
            Name of the dataset. e.g. For SVHN, set this to "SVHN Dataset".
            "Pytables Dataset" is used as title, by default.
        y_dtype : string, optional
            Either 'float' or 'int'. Decides the type of pytables atom
            used to store the y data. By default 'float' type is used.
        """
        assert y_dtype in ['float', 'int'], (
            "y_dtype can be 'float' or 'int' only"
        )

        x_shape, y_shape = shapes
        # make pytables
        ensure_tables()
        h5file = tables.openFile(path, mode="w", title=title)
        gcolumns = h5file.createGroup(h5file.root, "Data", "Data")
        atom = (tables.Float32Atom() if config.floatX == 'float32'
                else tables.Float64Atom())
        h5file.createCArray(gcolumns, 'X', atom=atom, shape=x_shape,
                            title="Data values", filters=self.filters)
        if y_dtype != 'float':
            # For 1D ndarray of int labels, override the atom to integer
            atom = (tables.Int32Atom() if config.floatX == 'float32'
                    else tables.Int64Atom())
        h5file.createCArray(gcolumns, 'y', atom=atom, shape=y_shape,
                            title="Data targets", filters=self.filters)
        return h5file, gcolumns 

def load_h5(self, filename):
        h5file = tables.openFile(filename, 'r')
        new_params = {}
        for p in h5file.listNodes(h5file.root):
            new_params[p.name] = p.read()
        self.updateparams_fromdict(new_params)
        h5file.close() 

def save_h5(self, filename):
        try:
            shutil.copyfile(filename, '{}_bak'.format(filename))
        except IOError:
            print 'could not make backup of model param file (which is normal if we haven\'t saved one until now)'

        with tables.openFile(filename, 'w') as h5file:
            for p in self.params:
                h5file.createArray(h5file.root, p.name, p.get_value())
                h5file.flush() 

def load_h5(self, filename):
        h5file = tables.openFile(filename, 'r')
        new_params = {}
        for p in h5file.listNodes(h5file.root):
            new_params[p.name] = p.read()
        self.updateparams_fromdict(new_params)
        h5file.close() 

def save_h5(self, filename):
        try:
            shutil.copyfile(filename, '{}_bak'.format(filename))
        except IOError:
            print 'could not make backup of model param file (which is normal if we haven\'t saved one until now)'

        with tables.openFile(filename, 'w') as h5file:
            for p in self.params:
                h5file.createArray(h5file.root, p.name, p.get_value())
                h5file.flush() 

def merge_all_files_into_pytables(file_dir, file_out):
    """
    process each file into pytables
    """
    start = None
    start = datetime.datetime.now()
    out_h5 = tables.openFile(file_out,
                             mode="w",
                             title="bars",
                             filters=tables.Filters(complevel=9,
                                                    complib='zlib'))
    table = None
    for file_in in glob.glob(file_dir + "/*.gz"):
        gzip_file = gzip.open(file_in)
        expected_header = ["dt", "sid", "open", "high", "low", "close",
                           "volume"]
        csv_reader = csv.DictReader(gzip_file)
        header = csv_reader.fieldnames
        if header != expected_header:
            logging.warn("expected header %s\n" % (expected_header))
            logging.warn("header_found %s" % (header))
            return

        for current_date, rows in parse_csv(csv_reader):
            table = out_h5.createTable("/TD", "date_" + current_date,
                                       OHLCTableDescription,
                                       expectedrows=len(rows),
                                       createparents=True)
            table.append(rows)
            table.flush()
        if table is not None:
            table.flush()
    end = datetime.datetime.now()
    diff = (end - start).seconds
    logging.debug("finished  it took %d." % (diff)) 

def _f_open(self, args):
        if not self._opened:
            self._filename = args["filename"]
            self._title = args["title"]
            if self._title is None or not isinstance(self._title, basestring):
                self._title = strftime("PicoTape-%Y%m%d-%H%M%S")
            self._limit = args["limit"]
            self._overwrite = args["overwrite"]
            if self._filename is not None:
                self._fhandle = None
                error = "OK"
                try:
                    if not os.path.exists(os.path.dirname(self._filename)):
                        error = "Path to %s not found" % self._filename
                    elif not self._overwrite and os.path.exists(self._filename):
                        error = "File %s exists" % self._filename
                    else:
                        self._fhandle = tb.openFile(self._filename, title=self._title, mode="w")
                except Exception as ex:
                    self._fhandle = None
                    error = ex.message
                if self._fhandle is not None:
                    self._opened = True
                self._readq.put(error)
            else:
                self._memstore = True
                self._opened = True
                self._readq.put("OK")
            self._stats = args["stats"] and not self._memstore 

def open_h5_file_read(h5filename):
    """
    Open an existing H5 in read mode.
    Same function as in hdf5_utils, here so we avoid one import
    """
    return tables.openFile(h5filename, mode='r')