Python netCDF4.Dataset() Examples

def createUniqueIDTable(self, in_nc, out_table):
        """Create a table of unique stream IDs"""
        data_nc = NET.Dataset(in_nc)
        comid_arr = data_nc.variables[self.vars_oi[0]][:]
        comid_size = len(comid_arr)
        comid_arr = comid_arr.reshape(comid_size, 1)
        arcpy.AddMessage(comid_arr.transpose())
        arcpy.AddMessage(self.vars_oi[0])

        #convert to numpy structured array
        str_arr = NUM.core.records.fromarrays(comid_arr.transpose(), NUM.dtype([(self.vars_oi[0], NUM.int32)]))

        # numpy structured array to table
        arcpy.da.NumPyArrayToTable(str_arr, out_table)

        data_nc.close()

        return 

def __init__(self, filename):
        self.data = netCDF4.Dataset(filename)
        # data variables and dimensions
        variables = set(self.data.variables.keys())
        dimensions = set(self.data.dimensions.keys())
        self.keys = tuple(variables - dimensions)
        # size of lat/lon dimensions
        self.lat_size = self.data.dimensions['latitude'].size
        self.lon_size = self.data.dimensions['longitude'].size
        # spatial resolution in degrees
        self.delta_lat = -180.0 / (self.lat_size - 1)  # from north to south
        self.delta_lon = 360.0 / self.lon_size  # from west to east
        # time resolution in hours
        self.time_size = self.data.dimensions['time'].size
        self.start_time = self.data['time'][0]
        self.stop_time = self.data['time'][-1]
        self.time_range = self.stop_time - self.start_time
        self.delta_time = self.time_range / (self.time_size - 1) 

def dataValidation(self, in_nc, messages):
        """Check the necessary dimensions and variables in the input netcdf data"""
        data_nc = NET.Dataset(in_nc)
        dims = data_nc.dimensions.keys()
        globalattrs = data_nc.__dict__.keys()
        for each in self.dimensions:
            if each not in dims:
                messages.addErrorMessage(self.errorMessages[1].format(each))
                raise arcpy.ExecuteError
        for each in self.globalattributes:
            if each not in globalattrs:
                messages.addErrorMessage(self.errorMessages[2].format(each))
                raise arcpy.ExecuteError

        data_nc.close()

        return 

def write_dataset(dataset: xr.Dataset,
                  output_path: str,
                  format_name: str = None,
                  **kwargs) -> xr.Dataset:
    """
    Write dataset to *output_path*.
    If *format* is not provided it will be guessed from *output_path*.
    :param dataset: Dataset to be written.
    :param output_path: output path
    :param format_name: format, e.g. "zarr" or "netcdf4"
    :param kwargs: format-specific keyword arguments
    :return: the input dataset
    """
    format_name = format_name if format_name else guess_dataset_format(output_path)
    if format_name is None:
        raise ValueError("Unknown output format")
    dataset_io = find_dataset_io(format_name, modes=["w"])
    if dataset_io is None:
        raise ValueError(f"Unknown output format {format_name!r} for {output_path}")
    dataset_io.write(dataset, output_path, **kwargs)
    return dataset 

def open_dataset(input_path: str,
                 format_name: str = None,
                 is_cube: bool = False,
                 **kwargs) -> xr.Dataset:
    """
    Open a dataset from *input_path*.
    If *format* is not provided it will be guessed from *output_path*.
    :param input_path: input path
    :param format_name: format, e.g. "zarr" or "netcdf4"
    :param is_cube: Whether a ValueError will be raised, if the dataset read from *input_path* is not a xcube dataset.
    :param kwargs: format-specific keyword arguments
    :return: dataset object
    """
    format_name = format_name if format_name else guess_dataset_format(input_path)
    if format_name is None:
        raise ValueError("Unknown input format")
    dataset_io = find_dataset_io(format_name, modes=["r"])
    if dataset_io is None:
        raise ValueError(f"Unknown input format {format_name!r} for {input_path}")
    dataset = dataset_io.read(input_path, **kwargs)
    if is_cube:
        assert_cube(dataset)
    return dataset 

def write_cube(cube: xr.Dataset,
               output_path: str,
               format_name: str = None,
               cube_asserted: bool = False,
               **kwargs) -> xr.Dataset:
    """
    Write a xcube dataset to *output_path*.
    If *format* is not provided it will be guessed from *output_path*.
    :param cube: xcube dataset to be written.
    :param output_path: output path
    :param format_name: format, e.g. "zarr" or "netcdf4"
    :param kwargs: format-specific keyword arguments
    :param cube_asserted: If False, *cube* will be verified, otherwise it is expected to be a valid cube.
    :return: xcube dataset *cube*
    """
    if not cube_asserted:
        assert_cube(cube)
    return write_dataset(cube, output_path, format_name=format_name, **kwargs) 

def updateMessages(self, parameters):
        """Modify the messages created by internal validation for each tool
        parameter.  This method is called after internal validation."""
        if parameters[0].altered:
            in_nc = parameters[0].valueAsText
            try:
                data_nc = NET.Dataset(in_nc)
                data_nc.close()
            except Exception as e:
                parameters[0].setErrorMessage(e.message)

        if parameters[1].altered:
            (dirnm, basenm) = os.path.split(parameters[1].valueAsText)
            if not basenm.endswith(".csv"):
                parameters[1].setErrorMessage("The weight table must be in CSV format")

        return 

def get_dataset(ncfile, dataset=None):
    """
    Utility to create a netCDF4 Dataset from a filename, list of filenames,
    or just pass it through if it's already a netCDF4.Dataset

    if dataset is not None, it should be a valid netCDF4 Dataset object,
    and it will simply be returned
    """
    if dataset is not None:
        return dataset
    if isinstance(ncfile, nc4.Dataset):
        return ncfile
    elif isinstance(ncfile, Iterable) and len(ncfile) == 1:
        return nc4.Dataset(ncfile[0])
    elif isstring(ncfile):
        return nc4.Dataset(ncfile)
    else:
        return nc4.MFDataset(ncfile) 

def get_writable_dataset(ncfile, format="netcdf4"):
    """
    Utility to create a writable netCDF4 Dataset from a filename, list of filenames,
    or just pass it through if it's already a netCDF4.Dataset

    if dataset is not None, it should be a valid netCDF4 Dataset object,
    and it will simply be returned
    """
    if isinstance(ncfile, nc4.Dataset):
        # fixme: check for writable
        return ncfile
    elif isstring(ncfile):  # Fixme: should be pathlike...
        print("filename is:", ncfile)
        return nc4.Dataset(ncfile,
                           mode="w",
                           clobber=True,
                           format="NETCDF4")
    else:
        raise ValueError("Must be a string path or a netcdf4 Dataset") 

def load_grid(nc):
    """
    Get a SGrid object from a netCDF4.Dataset or file/URL.

    :param str or netCDF4.Dataset nc: a netCDF4 Dataset or URL/filepath
                                       to the netCDF file
    :return: SGrid object
    :rtype: sgrid.SGrid

    """
    if isinstance(nc, Dataset):
        pass
    else:
        nc = Dataset(nc, 'r')

    return SGrid.load_grid(nc) 

def _apply_limits_and_filters(self, cont, limits, simple_filters):
        if isinstance(cont, xarray.Dataset):
            if len(limits)>0:
                raise NotImplementedError( 
                    "limits not implemented on xarray datasets")
            oldsize = cont[self.time_field].size
            for f in simple_filters:
                cont = f(cont)
            logger.debug("Filters reduced number from "
                "{:d} to {:d}".format(oldsize, cont[self.time_field].size))
            return cont
        oldsize = cont.size
        cont = tpmath.array.limit_ndarray(cont, limits)
        for f in simple_filters:
            cont = f(cont)
        if cont.size < oldsize:
            logger.debug("Applying limitations, reducing "
                "{:d} to {:d}".format(oldsize, cont.size))
        return cont 

def dataValidation(self, in_nc, messages):
        """Check the necessary dimensions and variables in the input netcdf data"""
        data_nc = NET.Dataset(in_nc)
        vars = data_nc.variables.keys()
        for each in self.vars_oi:
            if each not in vars:
                messages.addErrorMessage(self.errorMessages[3].format(each))
                raise arcpy.ExecuteError
            else:
                dims = data_nc.variables[each].dimensions
                if self.dims_var != dims:
                    messages.addErrorMessage(self.errorMessages[4].format(each))
                    raise arcpy.ExecuteError

        data_nc.close()

        return 

def __init__(self, filename, mode='w'):
        """Create NetCDF storage layer, creating or appending to an existing file.

        Parameters
        ----------
        filename : str
           Name of storage file to bind to.
        mode : str, optional, default='w'
           File open mode, 'w' for (over)write, 'a' for append.

        """
        self._filename = filename
        self._ncfile = netcdf.Dataset(self._filename, mode=mode)
        self._envname = None
        self._modname = None

        # Create standard dimensions.
        if 'iterations' not in self._ncfile.dimensions:
            self._ncfile.createDimension('iterations', size=None)
        if 'spatial' not in self._ncfile.dimensions:
            self._ncfile.createDimension('spatial', size=3) 

def read(self, file_info, fields=None, **kwargs):
        """Read a CSV file and return an xarray.Dataset with its content

        Args:
            file_info: Path and name of the file as string or FileInfo object.
            fields: Field that you want to extract from the file. If not given,
                all fields are going to be extracted.
            **kwargs: Additional keyword arguments for the pandas function
                `pandas.read_csv`. See for more details:
                https://pandas.pydata.org/pandas-docs/stable/generated/pandas.read_csv.html

        Returns:
            A xarray.Dataset object.
        """

        data = pd.read_csv(file_info.path, **kwargs).to_xarray()

        if fields is None:
            return data
        else:
            return data[fields] 

def __init__(self, ds, window, init_time, *args, **kwargs):
        """Initialise a DatasetDeque

        Arguments:

            ds [Dataset]: Dataset that this belongs to, for example,
                HIRS().  This dataset must have an implementation of
                .as_xarray_dataset().

            window [timedelta]: Duration that should be kept in memory.
                For example, datetime.timedelta(hours=48).

            init_time [datetime]: Instant around which initial window
                shall be centred.

            Remaining arguments passed to read_period.
        """
        self.dsobj = ds
        self.window = window
        self.init_time = init_time
        self.center_time = init_time

        self.reset(newtime=self.init_time, *args, **kwargs) 

def nc_var_has_attr_vals(ds, var_name, att_dict):
    """
    Checks that the variable, var_name, as the attributes (and values)
    in the att_dict.

    """
    if not isinstance(ds, netCDF4.Dataset):
        ds = netCDF4.Dataset(ds)

    for key, val in att_dict.items():
        try:
            if val != getattr(ds.variables[var_name], key):
                return False
        except AttributeError:
            return False
    return True 

def get_additional_field(self, M, fld):
        """Get additional field.

        Get field from other dataset, original objects, or otherwise.
        To be implemented by subclass implementations.

        Exact fields depend on subclass.

        Arguments:

            M (ndarray): ndarray with existing data
                A (masked) array with a dtype such as returned from
                `self.read <Dataset.read>`.

            fld (str): Additional field to read from original data

        Returns:
            
            ndarray with fields of M + fld.
        """
        raise NotImplementedError("Must be implemented by child-class") 

def _add_cont_to_arr(self, arr, N, cont):
        """Changes arr in-situ, does not return"""
        if isinstance(cont, xarray.Dataset):
            # we should already know it's large enough
            # for arr[self.time_field] I start at N
            # for the other time coordinates at the relative "speed" they
            # are behind N
            # but this is not guaranteed to be regular so I would need to
            # keep trac of each individually, or inspect it on-the-fly
            # this approximation may be good enough for pre-allocation
            # (which is approximate anyway), when actually storing we need
            # to do a better job… for each time coordinate, check when it
            # “dies”
            raise NotImplementedError("This is not used for xarrays. "
                "But see comment in source-code for some thoughts.")
        else:
            arr[N:(N+cont.size)] = cont
            #arr = self._finalise_arr(arr, N) 

def __init__(self, **kwargs):
        """Initialise a Dataset object.

        All keyword arguments will be translated into attributes.
        Does not take positional arguments.

        Note that if you create a dataset with a name that already exists,
        the existing object is returned, but __init__ is still called
        (Python does this, see
        https://docs.python.org/3.7/reference/datamodel.html#object.__new__).
        """
        self.mandatory_fields = set()
        for (k, v) in kwargs.items():
            setattr(self, k, v)
        self.setlocal()
        if self.my_pseudo_fields is None:
            self.my_pseudo_fields = collections.OrderedDict() 

def append(self, dataset: xr.Dataset, output_path: str, **kwargs):
        """"Append *dataset* to existing *output_path* using format-specific write parameters *kwargs*."""
        raise NotImplementedError() 

def read_nc_data(self):
        '''
        file_name: nc data file path
        '''
        nc_data = nc.Dataset(self.input_filepath_name)     # read nc-format file
        nc_dimensions, self.nc_variables= nc_data.dimensions, nc_data.variables   # 获取文件中的维度和变量
        #days_num = self.nc_variables.shape[0]

        self.var_obs = [] # Observation var name list
        self.var_ruitu=[] # Ruitu var name list

        for v in self.nc_variables: 
            if v.find("_obs") != -1:
                self.var_obs.append(v)
            if v.find('_M') != -1:
                self.var_ruitu.append(v)

        assert len(self.var_obs) == 9, 'Error, length of var_obs should be 9'
        assert len(self.var_ruitu) == 29, 'Error, length of var_ruitu should be 29'

        print('obs_var:',self.var_obs)
        print('obs_var numbers:', len(self.var_obs))

        print('ruitu_var:',self.var_ruitu)
        print('ruitu_var numbers:', len(self.var_ruitu))

        self.sta_id = self.nc_variables['station'][:].data
        self.foretime_index = self.nc_variables['foretimes'][:].data
        self.day_index = self.nc_variables['date'][:].data
        # Build a dictionary map key: station_id, value: from 0-9
        # Value of station_dic: {90001: 0, 90002: 1, 90003: 2, 90004: 3, 90005: 4, 90006: 5, 90007: 6, 90008: 7, 90009: 8, 90010: 9}
        self.station_dic ={}
        for i,s in enumerate(self.sta_id):
            self.station_dic[s]=i 
        assert self.station_dic == {90001: 0, 90002: 1, 90003: 2, 90004: 3, 90005: 4, 90006: 5, 90007: 6, 90008: 7, 90009: 8, 90010: 9}, 'Error of station_dic!'

        # Transform observation data 

def test_read_from_nc_dataset():
    """
    Minimal test, but makes sure you can read from an already
    open netCDF4.Dataset.

    """
    with chdir(files):
        with netCDF4.Dataset('ElevenPoints_UGRIDv0.9.nc') as nc:
            grid = UGrid.from_nc_dataset(nc)
    assert grid.mesh_name == 'Mesh2'
    assert grid.nodes.shape == (11, 2)
    assert grid.faces.shape == (13, 3) 

def ncds():
    """
    provides a new netCDF4 Dataset
    """
    if not os.path.isdir(temp_files):
        os.mkdir(temp_files)
    fname = os.path.join(temp_files, 'temp_{}.nc'.format(file_counter[0]))
    file_counter[0] = file_counter[0] + 1    # remove file if it's already there
    # "clobber=True" should do this, but
    # it doesn't always clean up properly
    if os.path.isfile(fname):
        os.remove(fname)
    ncds = netCDF4.Dataset(fname,
                           mode="w",
                           clobber=True,
                           format="NETCDF4")
    yield (fname, ncds)

    if not ncds.isopen:
        ncds.close()

    if not KEEP_TEMP_FILES:
        try:
            os.remove(fname)
        except OSError:
            pass 

def test_get_mesh_names():
    """
    Check that it can find the mesh variable names.

    """
    with chdir(files):
        nc = netCDF4.Dataset('ElevenPoints_UGRIDv0.9.nc')
    names = read_netcdf.find_mesh_names(nc)
    assert names == [u'Mesh2'] 

def test_create_from_netcdf_dataset():
    ds = netCDF4.Dataset(sample_sgrid_file)

    var = Variable.from_netCDF(dataset=ds,
                               varname='u',
                               )
    print(var.info)

    assert var.data_file == 'arakawa_c_test_grid.nc'
    assert var.data.shape == (1,12,11) 

def test_gen_celltree_mask_from_center_mask():
    center_mask = np.array(([True, True, True, True, True],
                     [True, False, True, True, True],
                     [True, False, False, False, True],
                     [True, True, True, True, True]))
    center_sl = np.s_[1:-1,1:-1] #'both' padding

    m = utilities.gen_celltree_mask_from_center_mask(center_mask, center_sl)

    expected_mask = np.array(([False, True, True],
                              [False, False, False]))

    assert np.all(m == expected_mask)

    testds = nc.Dataset('foo', mode='w', diskless=True)
    testds.createDimension('x', 5)
    testds.createDimension('y', 4)
    testds.createVariable('mask', 'b', dimensions=('y', 'x'))
    testds['mask'][:] = center_mask

    m3 = utilities.gen_celltree_mask_from_center_mask(center_mask, center_sl)

    assert np.all(m3 == expected_mask)

    testds['mask'][:] = ~center_mask
    testds['mask'].flag_values = [0, 1]
    testds['mask'].flag_meanings = ['land', 'water']

    m4 = utilities.gen_celltree_mask_from_center_mask(center_mask, center_sl)

    assert np.all(m4 == expected_mask)

    testds['mask'][:] = ~center_mask
    testds['mask'].flag_values = [0, 1]
    # because sometimes it's a damn string
    testds['mask'].flag_meanings = 'land water'

    m5 = utilities.gen_celltree_mask_from_center_mask(center_mask, center_sl)

    assert np.all(m5 == expected_mask)
    testds.close() 

def test_get_dataset_attrs():
    filename = os.path.join(data_dir, 'UGRIDv0.9_eleven_points.nc')
    ds = nc.Dataset(filename)
    attrs = utilities.get_dataset_attrs(ds)

    assert len(attrs) == 4
    assert attrs['Conventions'] == "UGRID-0.9"
    assert attrs['Title'] == "UGRID for GNOME" 

def test_nc_variable():
    """
    test that it works with a netcdf variable object
    """
    import netCDF4

    # make a variable
    with chdir(test_files):
        fname = 'junk.nc'
        ds = netCDF4.Dataset(fname, mode='w')
        ds.createDimension('dim', (10))
        var = ds.createVariable('a_var', float, ('dim'))
        var[:] = np.arange(10)
        # give it some attributes
        var.attr_1 = 'some value'
        var.attr_2 = 'another value'

        # make a UVar from it
        uvar = UVar("a_var", 'node', data=var)

        assert uvar._data is var  # preserved the netcdf variable
        print(uvar.attributes)
        assert uvar.attributes == {'attr_1': 'some value',
                                   'attr_2': 'another value'}
        # access the data
        assert np.array_equal(uvar[3:5], [3.0, 4.0]) 

def sg_data():
    filename = os.path.join(get_test_file_dir(), 'arakawa_c_test_grid.nc')
    return filename, nc.Dataset(filename) 

def ug_data():
    filename = os.path.join(get_test_file_dir(), 'tri_ring.nc')
    return filename, nc.Dataset(filename)