Python pandas.datetime() Examples

def get_new(self):
        """List new files since last recorded file state.

        pysat stores filenames in the user_home/.pysat directory. Returns
        a list of all new fileanmes since the last known change to files.
        Filenames are stored if there is a change and either update_files
        is True at instrument object level or files.refresh() is called.

        Returns
        -------
        pandas.Series
            files are indexed by datetime

        """

        # refresh files
        self.refresh()
        # current files
        new_info = self._load()
        # previous set of files
        old_info = self._load(prev_version=True)
        new_files = new_info[-new_info.isin(old_info)]
        return new_files 

def test_deprecated_season_date_range():
    """Tests that deprecation of season_date_range is working"""

    import warnings

    start = pds.datetime(2012, 2, 28)
    stop = pds.datetime(2012, 3, 1)
    warnings.simplefilter("always")
    with warnings.catch_warnings(record=True) as war1:
        season1 = pytime.create_date_range(start, stop, freq='D')
    with warnings.catch_warnings(record=True) as war2:
        season2 = pytime.season_date_range(start, stop, freq='D')

    assert len(season1) == len(season2)
    assert (season1 == season2).all()
    assert len(war1) == 0
    assert len(war2) == 1
    assert war2[0].category == DeprecationWarning 

def test_iterate_over_bounds_set_by_fname_via_next(self):
        start = '2009-01-01.nofile'
        stop = '2009-01-15.nofile'
        start_d = pysat.datetime(2009, 1, 1)
        stop_d = pysat.datetime(2009, 1, 15)
        self.testInst.bounds = (start, stop)
        dates = []
        loop_next = True
        while loop_next:
            try:
                self.testInst.next()
                dates.append(self.testInst.date)
            except StopIteration:
                loop_next = False
        out = pds.date_range(start_d, stop_d).tolist()
        assert np.all(dates == out) 

def test_iterate_over_bounds_set_by_date_season_extra_time(self):
        start = [pysat.datetime(2009, 1, 1, 1, 10),
                 pysat.datetime(2009, 2, 1, 1, 10)]
        stop = [pysat.datetime(2009, 1, 15, 1, 10),
                pysat.datetime(2009, 2, 15, 1, 10)]
        self.testInst.bounds = (start, stop)
        # filter
        start = self.testInst._filter_datetime_input(start)
        stop = self.testInst._filter_datetime_input(stop)
        # iterate
        dates = []
        for inst in self.testInst:
            dates.append(inst.date)
        out = pds.date_range(start[0], stop[0]).tolist()
        out.extend(pds.date_range(start[1], stop[1]).tolist())
        assert np.all(dates == out) 

def test_prev_filename_load_default(self):
        """Test prev day is loaded when invoking .prev."""
        self.testInst.load(fname='2009-01-04.nofile')
        # print(self.testInst.date)
        self.testInst.prev()
        test_date = self.testInst.index[0]
        test_date = pysat.datetime(test_date.year, test_date.month,
                                   test_date.day)
        assert (test_date == pds.datetime(2009, 1, 3))
        assert (test_date == self.testInst.date)

    # --------------------------------------------------------------------------
    #
    # Test date helpers
    #
    # -------------------------------------------------------------------------- 

def filter_data(inst):
    """Remove data from instrument, simulating gaps"""

    times = [[pysat.datetime(2009, 1, 1, 10),
              pysat.datetime(2009, 1, 1, 12)],
             [pysat.datetime(2009, 1, 1, 4),
              pysat.datetime(2009, 1, 2, 5, 37)],
             [pysat.datetime(2009, 1, 1, 1, 37),
              pysat.datetime(2009, 1, 1, 3, 14)],
             [pysat.datetime(2009, 1, 1, 15),
              pysat.datetime(2009, 1, 1, 16)],
             [pysat.datetime(2009, 1, 1, 22),
              pysat.datetime(2009, 1, 2, 2)],
             [pysat.datetime(2009, 1, 13),
              pysat.datetime(2009, 1, 15)],
             [pysat.datetime(2009, 1, 20, 1),
              pysat.datetime(2009, 1, 25, 23)],
             [pysat.datetime(2009, 1, 25, 23, 30),
              pysat.datetime(2009, 1, 26, 3)]
             ]
    for time in times:
        idx, = np.where((inst.index > time[1]) | (inst.index < time[0]))
        inst.data = inst[idx] 

def setup(self):
        """Runs before every method to create a clean testing setup."""
        info = {'index': 'mlt'}
        self.testInst = pysat.Instrument('pysat', 'testing',
                                         clean_level='clean',
                                         orbit_info=info)
        times = [[pysat.datetime(2008, 12, 31, 4),
                  pysat.datetime(2008, 12, 31, 5, 37)],
                 [pysat.datetime(2009, 1, 1),
                  pysat.datetime(2009, 1, 1, 1, 37)]
                 ]
        for seconds in np.arange(38):
            day = pysat.datetime(2009, 1, 2) + \
                pds.DateOffset(days=int(seconds))
            times.append([day, day +
                          pds.DateOffset(hours=1, minutes=37,
                                         seconds=int(seconds)) -
                          pds.DateOffset(seconds=20)])

        self.testInst.custom.add(filter_data2, 'modify', times=times) 

def setup(self):
        """Runs before every method to create a clean testing setup."""
        info = {'index': 'mlt'}
        self.testInst = pysat.Instrument('pysat', 'testing_xarray',
                                         clean_level='clean',
                                         orbit_info=info)
        times = [[pysat.datetime(2008, 12, 31, 4),
                  pysat.datetime(2008, 12, 31, 5, 37)],
                 [pysat.datetime(2009, 1, 1),
                  pysat.datetime(2009, 1, 1, 1, 37)]
                 ]
        for seconds in np.arange(38):
            day = pysat.datetime(2009, 1, 2) + \
                pds.DateOffset(days=int(seconds))
            times.append([day, day +
                          pds.DateOffset(hours=1, minutes=37,
                                         seconds=int(seconds)) -
                          pds.DateOffset(seconds=20)])

        self.testInst.custom.add(filter_data2, 'modify', times=times) 

def test_iterate_over_bounds_set_by_fname_via_prev(self):
        start = '2009-01-01.nofile'
        stop = '2009-01-15.nofile'
        start_d = pysat.datetime(2009, 1, 1)
        stop_d = pysat.datetime(2009, 1, 15)
        self.testInst.bounds = (start, stop)
        dates = []
        loop = True
        while loop:
            try:
                self.testInst.prev()
                dates.append(self.testInst.date)
            except StopIteration:
                loop = False
        out = pds.date_range(start_d, stop_d).tolist()
        assert np.all(dates == out[::-1]) 

def test_setitem_mixed_datetime(self):
        # GH 9336
        expected = DataFrame({'a': [0, 0, 0, 0, 13, 14],
                              'b': [pd.datetime(2012, 1, 1),
                                    1,
                                    'x',
                                    'y',
                                    pd.datetime(2013, 1, 1),
                                    pd.datetime(2014, 1, 1)]})
        df = pd.DataFrame(0, columns=list('ab'), index=range(6))
        df['b'] = pd.NaT
        df.loc[0, 'b'] = pd.datetime(2012, 1, 1)
        df.loc[1, 'b'] = 1
        df.loc[[2, 3], 'b'] = 'x', 'y'
        A = np.array([[13, np.datetime64('2013-01-01T00:00:00')],
                      [14, np.datetime64('2014-01-01T00:00:00')]])
        df.loc[[4, 5], ['a', 'b']] = A
        assert_frame_equal(df, expected) 

def create_date_range(start, stop, freq='D'):
    """
    Return array of datetime objects using input frequency from start to stop

    Supports single datetime object or list, tuple, ndarray of start and
    stop dates.

    freq codes correspond to pandas date_range codes, D daily, M monthly,
    S secondly

    """

    if hasattr(start, '__iter__'):
        # missing check for datetime
        season = pds.date_range(start[0], stop[0], freq=freq)
        for (sta, stp) in zip(start[1:], stop[1:]):
            season = season.append(pds.date_range(sta, stp, freq=freq))
    else:
        season = pds.date_range(start, stop, freq=freq)
    return season 

def getyrdoy(date):
    """Return a tuple of year, day of year for a supplied datetime object.

    Parameters
    ----------
    date : datetime.datetime
        Datetime object

    Returns
    -------
    year : int
        Integer year
    doy : int
        Integer day of year

    """

    try:
        doy = date.toordinal() - datetime(date.year, 1, 1).toordinal() + 1
    except AttributeError:
        raise AttributeError("Must supply a pandas datetime object or " +
                             "equivalent")
    else:
        return date.year, doy 

def setup_method(self, datapath):
        self.dirpath = datapath("io", "sas", "data")
        self.data = []
        self.test_ix = [list(range(1, 16)), [16]]
        for j in 1, 2:
            fname = os.path.join(
                self.dirpath, "test_sas7bdat_{j}.csv".format(j=j))
            df = pd.read_csv(fname)
            epoch = pd.datetime(1960, 1, 1)
            t1 = pd.to_timedelta(df["Column4"], unit='d')
            df["Column4"] = epoch + t1
            t2 = pd.to_timedelta(df["Column12"], unit='d')
            df["Column12"] = epoch + t2
            for k in range(df.shape[1]):
                col = df.iloc[:, k]
                if col.dtype == np.int64:
                    df.iloc[:, k] = df.iloc[:, k].astype(np.float64)
                elif col.dtype == np.dtype('O'):
                    if PY2:
                        f = lambda x: (x.decode('utf-8') if
                                       isinstance(x, str) else x)
                        df.iloc[:, k] = df.iloc[:, k].apply(f)
            self.data.append(df) 

def _filter_datetime_input(self, date):
        """
        Returns datetime that only includes year, month, and day.

        Parameters
        ----------
        date : datetime (array_like or single input)

        Returns
        -------
        datetime (or list of datetimes)
            Only includes year, month, and day from original input

        """

        if date is None:
            return date
        else:
            if hasattr(date, '__iter__'):
                return [pds.datetime(da.year, da.month, da.day) for da in date]
            else:
                return pds.datetime(date.year, date.month, date.day) 

def test_is_datetime64_ns_dtype():
    assert not com.is_datetime64_ns_dtype(int)
    assert not com.is_datetime64_ns_dtype(str)
    assert not com.is_datetime64_ns_dtype(np.datetime64)
    assert not com.is_datetime64_ns_dtype(np.array([1, 2]))
    assert not com.is_datetime64_ns_dtype(np.array(['a', 'b']))
    assert not com.is_datetime64_ns_dtype(np.array([], dtype=np.datetime64))

    # This datetime array has the wrong unit (ps instead of ns)
    assert not com.is_datetime64_ns_dtype(np.array([], dtype="datetime64[ps]"))

    assert com.is_datetime64_ns_dtype(DatetimeTZDtype("ns", "US/Eastern"))
    assert com.is_datetime64_ns_dtype(
        pd.DatetimeIndex([1, 2, 3], dtype=np.dtype('datetime64[ns]'))) 

def test_single_orbit_call_by_0_index(self):
        self.testInst.load(2009, 1)
        self.testInst.orbits[0]
        ans = (self.testInst.index[0] == pds.datetime(2009, 1, 1))
        ans2 = (self.testInst.index[-1] == pds.datetime(2009, 1, 1, 1, 36, 59))
        assert ans & ans2 

def test_single_orbit_call_by_1_index(self):
        self.testInst.load(2009, 1)
        self.testInst.orbits[1]
        ans = (self.testInst.index[0] == pds.datetime(2009, 1, 1, 1, 37))
        ans2 = (self.testInst.index[-1] == pds.datetime(2009, 1, 1, 3, 13, 59))
        assert ans & ans2 

def test_single_orbit_call_by_negative_1_index(self):
        self.testInst.load(2008, 366)
        self.testInst.orbits[-1]
        ans = (self.testInst.index[0] ==
               (pds.datetime(2009, 1, 1)-relativedelta(hours=1, minutes=37)))
        ans2 = (self.testInst.index[-1] ==
                (pds.datetime(2009, 1, 1)-relativedelta(seconds=1)))
        assert ans & ans2 

def test_set_dataframe_column_ns_dtype(self):
        x = DataFrame([datetime.now(), datetime.now()])
        assert x[0].dtype == np.dtype('M8[ns]') 

def test_setitem_with_unaligned_tz_aware_datetime_column(self):
        # GH 12981
        # Assignment of unaligned offset-aware datetime series.
        # Make sure timezone isn't lost
        column = pd.Series(pd.date_range('2015-01-01', periods=3, tz='utc'),
                           name='dates')
        df = pd.DataFrame({'dates': column})
        df['dates'] = column[[1, 0, 2]]
        assert_series_equal(df['dates'], column)

        df = pd.DataFrame({'dates': column})
        df.loc[[0, 1, 2], 'dates'] = column[[1, 0, 2]]
        assert_series_equal(df['dates'], column) 

def test_single_orbit_call_orbit_starts_0_UT_using_prev(self):
        self.testInst.load(2009, 1)
        self.testInst.orbits.prev()
        assert (self.testInst.index[0] ==
                (pds.datetime(2009, 1, 1) +
                14 * relativedelta(hours=1, minutes=37)))
        assert (self.testInst.index[-1] ==
                (pds.datetime(2009, 1, 1) +
                15 * relativedelta(hours=1, minutes=37) -
                relativedelta(seconds=1))) 

def test_iterate_over_bounds_set_by_fname(self):
        start = '2009-01-01.nofile'
        stop = '2009-01-15.nofile'
        start_d = pysat.datetime(2009, 1, 1)
        stop_d = pysat.datetime(2009, 1, 15)
        self.testInst.bounds = (start, stop)
        dates = []
        for inst in self.testInst:
            dates.append(inst.date)
        out = pds.date_range(start_d, stop_d).tolist()
        assert np.all(dates == out) 

def test_iterate_over_bounds_set_by_date_season(self):
        start = [pysat.datetime(2009, 1, 1), pysat.datetime(2009, 2, 1)]
        stop = [pysat.datetime(2009, 1, 15), pysat.datetime(2009, 2, 15)]
        self.testInst.bounds = (start, stop)
        dates = []
        for inst in self.testInst:
            dates.append(inst.date)
        out = pds.date_range(start[0], stop[0]).tolist()
        out.extend(pds.date_range(start[1], stop[1]).tolist())
        assert np.all(dates == out) 

def test_set_bounds_by_date_season(self):
        start = [pysat.datetime(2009, 1, 1), pysat.datetime(2009, 2, 1)]
        stop = [pysat.datetime(2009, 1, 15), pysat.datetime(2009, 2, 15)]
        self.testInst.bounds = (start, stop)
        out = pds.date_range(start[0], stop[0]).tolist()
        out.extend(pds.date_range(start[1], stop[1]).tolist())
        assert np.all(self.testInst._iter_list == out) 

def test_iterate_over_bounds_set_by_date2(self):
        start = pysat.datetime(2008, 1, 1)
        stop = pysat.datetime(2010, 12, 31)
        self.testInst.bounds = (start, stop)
        dates = []
        for inst in self.testInst:
            dates.append(inst.date)
        out = pds.date_range(start, stop).tolist()
        assert np.all(dates == out) 

def test_iterate_over_bounds_set_by_date(self):
        start = pysat.datetime(2009, 1, 1)
        stop = pysat.datetime(2009, 1, 15)
        self.testInst.bounds = (start, stop)
        dates = []
        for inst in self.testInst:
            dates.append(inst.date)
        out = pds.date_range(start, stop).tolist()
        assert np.all(dates == out) 

def test_set_bounds_by_date_extra_time(self):
        start = pysat.datetime(2009, 1, 1, 1, 10)
        stop = pysat.datetime(2009, 1, 15, 1, 10)
        self.testInst.bounds = (start, stop)
        start = self.testInst._filter_datetime_input(start)
        stop = self.testInst._filter_datetime_input(stop)
        assert np.all(self.testInst._iter_list ==
                      pds.date_range(start, stop).tolist()) 

def test_set_bounds_by_date(self):
        start = pysat.datetime(2009, 1, 1)
        stop = pysat.datetime(2009, 1, 15)
        self.testInst.bounds = (start, stop)
        assert np.all(self.testInst._iter_list ==
                      pds.date_range(start, stop).tolist()) 

def test_set_bounds_mixed_iterabless(self):
        start = [pysat.datetime(2009, 1, 1)]*2
        self.testInst.bounds = [start, [pysat.datetime(2009, 1, 1), '2009-01-01.nofile']] 

def test_set_bounds_mixed_iterable(self):
        start = [pysat.datetime(2009, 1, 1)]*2
        self.testInst.bounds = [start, '2009-01-01.nofile']