Python ciso8601.parse_datetime() Examples

def test_invalid_tz_offsets_too_large(self):
        # The Python interpreter crashes if you give the datetime constructor a TZ offset with an absolute value >= 1440
        # TODO: Determine whether these are valid ISO 8601 values and therefore whether ciso8601 should support them.
        self.assertRaisesRegex(
            ValueError,
            # Error message differs whether or not we are using pytz or datetime.timezone
            r"^offset must be a timedelta strictly between" if sys.version_info.major >= 3 else r"\('absolute offset is too large', -5940\)",
            ciso8601.parse_datetime,
            '2018-01-01T00:00:00.00-99',
        )

        self.assertRaisesRegex(
            ValueError,
            r"tzminute must be in 0..59",
            ciso8601.parse_datetime,
            '2018-01-01T00:00:00.00-23:60',
        ) 

def _should_process_row(self, row, date_column, is_full_month, is_finalized=None):
        """Determine if we want to process this row.

        Args:
            row (dict): The line item entry from the AWS report file
            date_column (str): The name of date column to check
            is_full_month (boolean): If this is the first time we've processed this bill

        Kwargs:
            is_finalized (boolean): If this is a finalized bill

        Returns:
            (bool): Whether this row should be processed

        """
        if is_finalized or is_full_month:
            return True
        row_date = ciso8601.parse_datetime(row[date_column]).date()
        if row_date < self.data_cutoff_date:
            return False
        return True 

def convert_date(self, date_string):
        """Converts date from string format to date object, for use by DateField."""
        if date_string:
            try:
                # TODO: for now, return as a string.
                # When actually supporting DateField, then switch back to date.
                # ciso8601.parse_datetime(ts).astimezone(pytz.utc).date().isoformat()
                return self.date_field_for_converting.deserialize_from_string(date_string).isoformat()
            except ValueError:
                self.incr_counter(self.counter_category_name, 'Cannot convert to date', 1)
                # Don't bother to make sure we return a good value
                # within the interval, so we can find the output for
                # debugging.  Should not be necessary, as this is only
                # used for the column value, not the partitioning.
                return u"BAD: {}".format(date_string)
                # return self.lower_bound_date_string
        else:
            self.incr_counter(self.counter_category_name, 'Missing date', 1)
            return date_string 

def test_problem_check(self):
        template = self.event_templates['problem_check']
        event = self.create_event_log_line(template=template)
        expected_key = (self.DEFAULT_DATE, self.task.PROJECT_NAME)
        expected_dict = {
            'input_file': '',
            'source': self.task.PROJECT_NAME,
            'event_type': 'problem_check',
            'emitter_type': 'server',
            'timestamp': ciso8601.parse_datetime('2013-12-17T15:38:32.805444+00:00'),
            'received_at': ciso8601.parse_datetime('2013-12-17T15:38:32.805444+00:00'),
            'date': datetime.date(*[int(x) for x in self.DEFAULT_DATE.split('-')]),
            'username': 'test_user',
            'course_id': self.encoded_course_id,
            'org_id': self.encoded_org_id,
            'user_id': '10',
            'raw_event': self.get_raw_event(event),
        }
        expected_value = JsonEventRecord(**expected_dict).to_separated_values()
        self.assert_single_map_output(event, expected_key, expected_value) 

def test_valid_rfc3339_timestamps(self):
        """
        Validate that valid RFC 3339 datetimes are parseable by parse_rfc3339
        and produce the same result as parse_datetime.
        """
        for string in [
                '2018-01-02T03:04:05Z',
                '2018-01-02t03:04:05z',
                '2018-01-02 03:04:05z',
                '2018-01-02T03:04:05+00:00',
                '2018-01-02T03:04:05-00:00',
                '2018-01-02T03:04:05.12345Z',
                '2018-01-02T03:04:05+01:23',
                '2018-01-02T03:04:05-12:34',
                '2018-01-02T03:04:05-12:34',
        ]:
            self.assertEqual(ciso8601.parse_datetime(string),
                             ciso8601.parse_rfc3339(string)) 

def test_mixed_basic_and_extended_formats(self):
        """
        Both dates and times have "basic" and "extended" formats.
        But when you combine them into a datetime, the date and time components
        must have the same format.
        """
        self.assertRaisesRegex(
            ValueError,
            r"Cannot combine \"extended\" date format with \"basic\" time format",
            ciso8601.parse_datetime,
            '2014-01-02T010203',
        ),

        self.assertRaisesRegex(
            ValueError,
            r"Cannot combine \"basic\" date format with \"extended\" time format",
            ciso8601.parse_datetime,
            '20140102T01:02:03',
        ) 

def test_android_screen(self):
        template = self.event_templates['android_screen']
        event = self.create_event_log_line(template=template)
        expected_key = (self.DEFAULT_DATE, self.DEFAULT_PROJECT)
        expected_dict = {
            'input_file': '',
            'source': self.DEFAULT_PROJECT,
            'event_type': 'screen',
            'emitter_type': 'server',
            'timestamp': ciso8601.parse_datetime('2013-12-17T15:38:32.700000+00:00'),
            'received_at': ciso8601.parse_datetime('2013-12-17T15:38:32.796000+00:00'),
            'date': datetime.date(*[int(x) for x in self.DEFAULT_DATE.split('-')]),
            'agent_type': 'tablet',
            'agent_device_name': 'Samsung SM-N920A',
            'agent_os': 'Android',
            'agent_browser': 'Android',
            'agent_touch_capable': True,
            'anonymous_id': self.DEFAULT_ANONYMOUS_ID,
            'category': 'screen',
            'label': 'Launch\\0',
            'raw_event': self.get_raw_event(event),
        }
        expected_value = JsonEventRecord(**expected_dict).to_separated_values()
        self.assert_single_map_output(event, expected_key, expected_value) 

def get_date_parse_function():
    global parse_function
    if parse_function is None:
        try:
            import ciso8601
            parse_function = ciso8601.parse_datetime
        except ModuleNotFoundError:
            parse_function = parser.parse

    return parse_function 

def str_to_dt(s):
    dt = ciso8601.parse_datetime(s)
    if dt:
        return dt
    raise ValueError(f'Invalid datetime string: {dt!r}') 

def _serialize_timestamp(point):
    dt = point.get('time')
    if not dt:
        return ''
    elif isinstance(dt, int):
        return dt
    elif isinstance(dt, (str, bytes)):
        dt = ciso8601.parse_datetime(dt)
        if not dt:
            raise ValueError(f'Invalid datetime string: {dt!r}')

    if not dt.tzinfo:
        # Assume tz-naive input to be in UTC, not local time
        return int(dt.timestamp() - time.timezone) * 10 ** 9 + dt.microsecond * 1000
    return int(dt.timestamp()) * 10 ** 9 + dt.microsecond * 1000 

def parse_row(row: OrderedDict):
    """Parse row of CSV file into LineProtocol with structure:

    CSV format:
        date,symbol,open,close,low,high,volume
        2016-01-05,WLTW,123.43,125.839996,122.309998,126.25,2163600.0
        2016-01-06,WLTW,125.239998,119.980003,119.940002,125.540001,2386400.0
        2016-01-07,WLTW,116.379997,114.949997,114.93,119.739998,2489500.0
        2016-01-08,WLTW,115.480003,116.620003,113.5,117.440002,2006300.0
        2016-01-11,WLTW,117.010002,114.970001,114.089996,117.330002,1408600.0
        2016-01-12,WLTW,115.510002,115.550003,114.5,116.059998,1098000.0
        2016-01-13,WLTW,116.459999,112.849998,112.589996,117.07,949600.0
        ...

    :param row: the row of CSV file
    :return: Parsed csv row to LineProtocol
    """
    global _progress
    _progress += 1

    if _progress % 10000 == 0:
        print(_progress)

    time = (UTC.localize(ciso8601.parse_datetime(row["date"])) - EPOCH).total_seconds() * 1e9

    return f'financial-analysis,symbol={row["symbol"]} ' \
           f'close={row["close"]},high={row["high"]},low={row["low"]},open={row["open"]} ' \
           f'{int(time)}' 

def test_excessive_subsecond_precision(self):
        self.assertEqual(
            ciso8601.parse_datetime('20140203T103527.234567891234'),
            datetime.datetime(2014, 2, 3, 10, 35, 27, 234567)
        ) 

def str_to_datetime(str_repr):
    """Convert a string into a datetime."""
    # Allow the caller to be stupid.
    if type(str_repr) == datetime.datetime:
        return str_repr
    if not str_repr:
        return None
    return ciso8601.parse_datetime(str_repr) 

def dateToUNIX(date): #format: "YYYYMMDD hhmmss"
	ts = ciso8601.parse_datetime(date)
	return time.mktime(ts.timetuple()) 

def test_play_video(self):
        template = self.event_templates['play_video']
        event = self.create_event_log_line(template=template)
        expected_key = (self.DEFAULT_DATE, self.task.PROJECT_NAME)
        expected_dict = {
            'input_file': '',
            'source': self.task.PROJECT_NAME,
            'event_type': 'play_video',
            'emitter_type': 'browser',
            'timestamp': ciso8601.parse_datetime('2013-12-17T15:38:32.805444+00:00'),
            'received_at': ciso8601.parse_datetime('2013-12-17T15:38:32.805444+00:00'),
            'date': datetime.date(*[int(x) for x in self.DEFAULT_DATE.split('-')]),
            'username': 'test_user',
            'course_id': self.encoded_course_id,
            'org_id': self.encoded_org_id,
            'user_id': '10',
            'referrer': 'long meaningful url',
            'agent_type': 'desktop',
            'agent_device_name': 'Other',
            'agent_os': 'Mac OS X',
            'agent_browser': 'Safari',
            'agent_touch_capable': False,
            'raw_event': self.get_raw_event(event),
        }
        expected_value = JsonEventRecord(**expected_dict).to_separated_values()
        self.assert_single_map_output(event, expected_key, expected_value) 

def _localize_time(time_string):
    london_time = pytz.timezone("Europe/London")
    ret_time = time_string.replace(" ", "+")
    ret_time = ciso8601.parse_datetime(ret_time)
    ret_time = ret_time.astimezone(london_time)
    return ret_time.replace(tzinfo=None) 

def _get_last_modified_header(redis_key=None):
    # Default last modified is the UTC time now
    last_modified = format_datetime(
        datetime.datetime.utcnow().replace(tzinfo=timezone.utc),
        usegmt=True
    )

    # If we haven't been passed a Redis key, we just return the
    # current timeztamp as a last modified header.
    if redis_key is None:
        return last_modified

    # We have been given a Redis key, so attempt to pull it from Redis
    r = redis.Redis(host=REDIS_UCLAPI_HOST)
    redis_key = "http:headers:Last-Modified:" + redis_key
    value = r.get(redis_key)

    if value:
        # Convert the Redis bytes response to a string.
        value = value.decode('utf-8')

        # We need the UTC timezone so that we can convert to it.
        utc_tz = pytz.timezone("UTC")

        # Parse the ISO 8601 timestamp from Redis and represent it as UTC
        utc_timestamp = ciso8601.parse_datetime(value).astimezone(utc_tz)

        # Format the datetime object as per the HTTP Header RFC.
        # We replace the inner tzinfo in the timestamp to force it to be a UTC
        # timestamp as opposed to a naive one; this is a requirement for the
        # format_datetime function.
        last_modified = format_datetime(
            utc_timestamp.replace(tzinfo=timezone.utc),
            usegmt=True
        )

    return last_modified 

def normalize_time(self, event_time):
        """
        Convert time string to ISO-8601 format in UTC timezone.

        Returns None if string representation cannot be parsed.
        """
        datetime = ciso8601.parse_datetime(event_time)
        if datetime:
            return datetime.astimezone(pytz.utc).isoformat()
        else:
            return None 

def overloaded_parse_message(msg):
    """Parse a message string into a Message object."""

    # We are not using Decimals for parsing here.
    # We recognize that exposes data to potentially
    # lossy conversions.  However, this will affect
    # very few data points and we have chosen to
    # leave conversion as is for now.
    obj = simplejson.loads(msg, use_decimal=True)
    msg_type = _required_key(obj, 'type')

    if msg_type == 'RECORD':
        time_extracted = obj.get('time_extracted')
        if time_extracted:
            try:
                time_extracted = ciso8601.parse_datetime(time_extracted)
            except Exception:
                time_extracted = None
        return singer.RecordMessage(stream=_required_key(obj, 'stream'),
                                    record=_required_key(obj, 'record'),
                                    version=obj.get('version'),
                                    time_extracted=time_extracted)

    if msg_type == 'SCHEMA':
        return singer.SchemaMessage(stream=_required_key(obj, 'stream'),
                                    schema=_required_key(obj, 'schema'),
                                    key_properties=_required_key(obj, 'key_properties'),
                                    bookmark_properties=obj.get('bookmark_properties'))

    if msg_type == 'STATE':
        return singer.StateMessage(value=_required_key(obj, 'value'))

    if msg_type == 'ACTIVATE_VERSION':
        return singer.ActivateVersionMessage(stream=_required_key(obj, 'stream'),
                                             version=_required_key(obj, 'version'))
    return None 

def iso8601_to_days_since_epoch(date_str):
    dt = ciso8601.parse_datetime(date_str)
    return (dt - epoch).days 

def iso8601_to_nanos(date_str):
    """ Returns a nanoseconds since epoch for a given ISO-8601 date string

        Arguments:
        date: ISO-8601 UTC datetime string (e.g. "2016-01-01 12:00:00.000000")

        Return Values:
        int representing # of nanoseconds since "1970-01-01" for date
    """
    dt = ciso8601.parse_datetime(date_str)
    return unix_time_nanos(dt) 

def _date_converter(date_str):
    dt = ciso8601.parse_datetime(date_str)
    return (dt - epoch).days 

def _create_report(self, row, report_period_id, report_db_accessor):
        """Create a report object.

        Args:
            row (dict): A dictionary representation of a CSV file row
            report_period_id (str): report period object id

        Returns:
            (str): The DB id of the report object

        """
        table_name = OCPUsageReport
        start = ciso8601.parse_datetime(row.get("interval_start").replace(" +0000 UTC", "+0000"))
        end = ciso8601.parse_datetime(row.get("interval_end").replace(" +0000 UTC", "+0000"))

        key = (report_period_id, start)
        if key in self.processed_report.reports:
            return self.processed_report.reports[key]

        if key in self.existing_report_map:
            return self.existing_report_map[key]

        data = {"report_period_id": report_period_id, "interval_start": start, "interval_end": end}
        report_id = report_db_accessor.insert_on_conflict_do_nothing(
            table_name, data, conflict_columns=["report_period_id", "interval_start"]
        )

        self.processed_report.reports[key] = report_id

        return report_id 

def test_auto_generated_valid_formats(self):
        for (timestamp, expected_datetime) in generate_valid_timestamp_and_datetime():
            try:
                self.assertEqual(ciso8601.parse_datetime(timestamp), expected_datetime)
            except Exception:
                print("Had problems parsing: {timestamp}".format(timestamp=timestamp))
                raise 

def test_leap_year(self):
        # There is nothing unusual about leap years in ISO 8601.
        # We just want to make sure that they work in general.
        for leap_year in (1600, 2000, 2016):
            self.assertEqual(
                ciso8601.parse_datetime('{}-02-29'.format(leap_year)),
                datetime.datetime(leap_year, 2, 29, 0, 0, 0, 0)
            ) 

def test_special_midnight(self):
        self.assertEqual(
            ciso8601.parse_datetime('2014-02-03T24:00:00'),
            datetime.datetime(2014, 2, 4, 0, 0, 0)
        ) 

def test_parse_auto_generated_invalid_formats(self):
        for timestamp in generate_invalid_timestamp():
            try:
                with self.assertRaises(ValueError, msg="Timestamp '{0}' was supposed to be invalid, but parsing it didn't raise ValueError.".format(timestamp)):
                    ciso8601.parse_datetime(timestamp)
            except Exception as exc:
                print("Timestamp '{0}' was supposed to raise ValueError, but raised {1} instead".format(timestamp, type(exc).__name__))
                raise 

def test_invalid_calendar_separator(self):
        self.assertRaisesRegex(
            ValueError,
            r"Invalid character while parsing month",
            ciso8601.parse_datetime,
            '2018=01=01',
        )

        self.assertRaisesRegex(
            ValueError,
            r"Invalid character while parsing date separator \('-'\) \('=', Index: 7\)",
            ciso8601.parse_datetime,
            '2018-01=01',
        )

        self.assertRaisesRegex(
            ValueError,
            r"Invalid character while parsing date separator \('-'\) \('0', Index: 7\)",
            ciso8601.parse_datetime,
            '2018-0101',
        )

        self.assertRaisesRegex(
            ValueError,
            r"Invalid character while parsing day \('-', Index: 6\)",
            ciso8601.parse_datetime,
            '201801-01',
        ) 

def test_invalid_day_for_month(self):
        for non_leap_year in (1700, 1800, 1900, 2014):
            self.assertRaisesRegex(
                ValueError,
                r"day is out of range for month",
                ciso8601.parse_datetime,
                '{}-02-29'.format(non_leap_year)
            )

        self.assertRaisesRegex(
            ValueError,
            r"day is out of range for month",
            ciso8601.parse_datetime,
            '2014-01-32',
        )

        self.assertRaisesRegex(
            ValueError,
            r"day is out of range for month",
            ciso8601.parse_datetime,
            '2014-06-31',
        )

        self.assertRaisesRegex(
            ValueError,
            r"day is out of range for month",
            ciso8601.parse_datetime,
            '2014-06-00',
        ) 

def test_invalid_yyyymm_format(self):
        self.assertRaisesRegex(
            ValueError,
            r"Unexpected end of string while parsing day. Expected 2 more characters",
            ciso8601.parse_datetime,
            '201406',
        )