Python isodate.parse_datetime() Examples

def test_datetime_rfc(self):
        client = AutoRestRFC1123DateTimeTestService(base_url="http://localhost:3000")

        assert client.datetimerfc1123.get_null() is None

        with pytest.raises(DeserializationError):
            client.datetimerfc1123.get_invalid()

        with pytest.raises(DeserializationError):
            client.datetimerfc1123.get_underflow()

        with pytest.raises(DeserializationError):
            client.datetimerfc1123.get_overflow()

        client.datetimerfc1123.get_utc_lowercase_max_date_time()
        client.datetimerfc1123.get_utc_uppercase_max_date_time()
        client.datetimerfc1123.get_utc_min_date_time()

        max_date = isodate.parse_datetime("9999-12-31T23:59:59.999999Z")
        client.datetimerfc1123.put_utc_max_date_time(max_date)

        min_date = isodate.parse_datetime("0001-01-01T00:00:00Z")
        client.datetimerfc1123.put_utc_min_date_time(min_date) 

def _insert_post(url, user, data, session):
    try:
        date_time = parse_datetime(data["created_time"])
    except ISO8601Error:
        date_time = datetime.now()
    if not data.get("message") or len(data.get("message")) > 160:
        caption = ""
    else:
        caption = data.get("message").strip()
    facebook_id = data["id"]
    likes_count = data["reactions"]["summary"]["total_count"]
    permalink_url = data["permalink_url"]
    new_link = _insert_link(url, session)
    if not new_link:
        new_link = session.query(Link).filter(Link.url == url).first()
        new_post = UserPosts(
            user, new_link, date_time, caption, facebook_id, permalink_url
        )
        new_post.likes_count = likes_count
        session.add(new_post)
        return None
    new_post = UserPosts(user, new_link, date_time, caption, facebook_id, permalink_url)
    new_post.likes_count = likes_count
    session.add(new_post)
    return new_link 

def test_datetime_rfc(self):
        client = AutoRestRFC1123DateTimeTestService(base_url="http://localhost:3000")

        assert await client.datetimerfc1123.get_null() is None

        with pytest.raises(DeserializationError):
            await client.datetimerfc1123.get_invalid()

        with pytest.raises(DeserializationError):
            await client.datetimerfc1123.get_underflow()

        with pytest.raises(DeserializationError):
            await client.datetimerfc1123.get_overflow()

        await client.datetimerfc1123.get_utc_lowercase_max_date_time()
        await client.datetimerfc1123.get_utc_uppercase_max_date_time()
        await client.datetimerfc1123.get_utc_min_date_time()

        max_date = isodate.parse_datetime("9999-12-31T23:59:59.999999Z")
        await client.datetimerfc1123.put_utc_max_date_time(max_date)

        min_date = isodate.parse_datetime("0001-01-01T00:00:00Z")
        await client.datetimerfc1123.put_utc_min_date_time(min_date) 

def _parse_determine_date_datetime(s: str) -> Union[datetime.datetime,
                                                    datetime.date]:
    """Parse function parses a date, if time is included it parses as a
    datetime.
    """
    if sys.version_info >= (3, 7):
        # Check if the date includes time.
        if 'T' in s:
            return datetime.datetime.fromisoformat(s)
        else:
            return datetime.date.fromisoformat(s)
    else:
        # Check if the date includes time.
        if 'T' in s:
            return isodate.parse_datetime(s)
        else:
            return isodate.parse_date(s)


# Test if lists/tuples have contain matching items 

def marathon_log_line_passes_filter(
    line: str,
    levels: Sequence[str],
    service: str,
    components: Iterable[str],
    clusters: Sequence[str],
    instances: Iterable[str],
    start_time: datetime.datetime = None,
    end_time: datetime.datetime = None,
) -> bool:
    """Given a (JSON-formatted) log line where the message is a Marathon log line,
    return True if the line should be displayed given the provided service; return False
    otherwise."""
    try:
        parsed_line = json.loads(line)
    except ValueError:
        log.debug("Trouble parsing line as json. Skipping. Line: %r" % line)
        return False

    timestamp = isodate.parse_datetime(parsed_line.get("timestamp"))
    if not check_timestamp_in_range(timestamp, start_time, end_time):
        return False
    return format_job_id(service, "") in parsed_line.get("message", "") 

def extract_utc_timestamp_from_log_line(line: str) -> datetime.datetime:
    """
    Extracts the timestamp from a log line of the format "<timestamp> <other data>" and returns a UTC datetime object
    or None if it could not parse the line
    """
    # Extract ISO 8601 date per http://www.pelagodesign.com/blog/2009/05/20/iso-8601-date-validation-that-doesnt-suck/
    iso_re = (
        r"^([\+-]?\d{4}(?!\d{2}\b))((-?)((0[1-9]|1[0-2])(\3([12]\d|0[1-9]|3[01]))?|W([0-4]\d|5[0-2])(-?[1-7])?|"
        r"(00[1-9]|0[1-9]\d|[12]\d{2}|3([0-5]\d|6[1-6])))([T\s]((([01]\d|2[0-3])((:?)[0-5]\d)?|24\:?00)([\.,]\d+"
        r"(?!:))?)?(\17[0-5]\d([\.,]\d+)?)?([zZ]|([\+-])([01]\d|2[0-3]):?([0-5]\d)?)?)?)? "
    )

    tokens = re.match(iso_re, line)

    if not tokens:
        # Could not parse line
        return None
    timestamp = tokens.group(0).strip()
    dt = isodate.parse_datetime(timestamp)
    utc_timestamp = datetime_convert_timezone(dt, dt.tzinfo, tz.tzutc())
    return utc_timestamp 

def deserialize_iso(attr):
        """Deserialize ISO-8601 formatted string into Datetime object.

        :param str attr: response string to be deserialized.
        :rtype: Datetime
        :raises: DeserializationError if string format invalid.
        """
        if isinstance(attr, ET.Element):
            attr = attr.text
        try:
            attr = attr.upper()
            match = Deserializer.valid_date.match(attr)
            if not match:
                raise ValueError("Invalid datetime string: " + attr)

            check_decimal = attr.split('.')
            if len(check_decimal) > 1:
                decimal_str = ""
                for digit in check_decimal[1]:
                    if digit.isdigit():
                        decimal_str += digit
                    else:
                        break
                if len(decimal_str) > 6:
                    attr = attr.replace(decimal_str, decimal_str[0:6])

            date_obj = isodate.parse_datetime(attr)
            test_utc = date_obj.utctimetuple()
            if test_utc.tm_year > 9999 or test_utc.tm_year < 1:
                raise OverflowError("Hit max or min date")
        except(ValueError, OverflowError, AttributeError) as err:
            msg = "Cannot deserialize datetime object."
            raise_with_traceback(DeserializationError, msg, err)
        else:
            return date_obj 

def test_extract_utc_timestamp_from_log_line_ok():
    fake_timestamp = "2015-07-22T10:38:46-07:00"
    fake_utc_timestamp = isodate.parse_datetime("2015-07-22T17:38:46.000000")

    line = "%s this is a fake syslog test message" % fake_timestamp
    assert logs.extract_utc_timestamp_from_log_line(line) == fake_utc_timestamp 

def setUpClass(cls):
        cls.recipes = load('test_data/google-recipe-example.html',
                           python_objects=True)
        cls.recipe = cls.recipes[0]

        # input string (upload_date) is "2018-02-05T08:00:00+08:00"
        upload_date = cls.recipe['video'][0]['uploadDate']
        cls.datetime_test = isodate.parse_datetime(upload_date) 

def is_datetime(instance):
            if not isinstance(instance, str_types):
                return True
            return isodate.parse_datetime(instance) 

def is_datetime(instance):
            if not isinstance(instance, str_types):
                return True
            return isodate.parse_datetime(instance) 

def serialize_iso(attr, **kwargs):
        """Serialize Datetime object into ISO-8601 formatted string.

        :param Datetime attr: Object to be serialized.
        :rtype: str
        :raises: SerializationError if format invalid.
        """
        if isinstance(attr, str):
            attr = isodate.parse_datetime(attr)
        try:
            if not attr.tzinfo:
                _LOGGER.warning(
                    "Datetime with no tzinfo will be considered UTC.")
            utc = attr.utctimetuple()
            if utc.tm_year > 9999 or utc.tm_year < 1:
                raise OverflowError("Hit max or min date")

            microseconds = str(attr.microsecond).rjust(6,'0').rstrip('0').ljust(3, '0')
            if microseconds:
                microseconds = '.'+microseconds
            date = "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}".format(
                utc.tm_year, utc.tm_mon, utc.tm_mday,
                utc.tm_hour, utc.tm_min, utc.tm_sec)
            return date + microseconds + 'Z'
        except (ValueError, OverflowError) as err:
            msg = "Unable to serialize datetime object."
            raise_with_traceback(SerializationError, msg, err)
        except AttributeError as err:
            msg = "ISO-8601 object must be valid Datetime object."
            raise_with_traceback(TypeError, msg, err) 

def test_paasta_log_line_passes_filter_true_when_valid_time():
    service = "fake_service"
    levels = ["fake_level1", "fake_level2"]
    clusters = ["fake_cluster1", "fake_cluster2"]
    instance = "fake_instance"
    instances = [instance]
    components = ["build", "deploy"]
    line = "fake_line"
    formatted_line = format_log_line(
        levels[0],
        clusters[0],
        service,
        instance,
        components[0],
        line,
        timestamp="2016-06-07T23:46:03+00:00",
    )

    start_time = isodate.parse_datetime("2016-06-07T23:40:03+00:00")
    end_time = isodate.parse_datetime("2016-06-07T23:50:03+00:00")

    assert (
        logs.paasta_log_line_passes_filter(
            formatted_line,
            levels,
            service,
            components,
            clusters,
            instances,
            start_time=start_time,
            end_time=end_time,
        )
        is True
    ) 

def test_serialize_datetime(self):

        date_obj = isodate.parse_datetime('2015-01-01T00:00:00')
        date_str = Serializer.serialize_iso(date_obj)

        self.assertEqual(date_str, '2015-01-01T00:00:00.000Z')

        date_obj = isodate.parse_datetime('1999-12-31T23:59:59-12:00')
        date_str = Serializer.serialize_iso(date_obj)

        self.assertEqual(date_str, '2000-01-01T11:59:59.000Z')

        with self.assertRaises(SerializationError):
            date_obj = isodate.parse_datetime('9999-12-31T23:59:59-12:00')
            date_str = Serializer.serialize_iso(date_obj)

        with self.assertRaises(SerializationError):
            date_obj = isodate.parse_datetime('0001-01-01T00:00:00+23:59')
            date_str = Serializer.serialize_iso(date_obj)


        date_obj = isodate.parse_datetime("2015-06-01T16:10:08.0121-07:00")
        date_str = Serializer.serialize_iso(date_obj)

        self.assertEqual(date_str, '2015-06-01T23:10:08.0121Z')

        date_obj = datetime.min
        date_str = Serializer.serialize_iso(date_obj)
        self.assertEqual(date_str, '0001-01-01T00:00:00.000Z')

        date_obj = datetime.max
        date_str = Serializer.serialize_iso(date_obj)
        self.assertEqual(date_str, '9999-12-31T23:59:59.999999Z')

        date_obj = isodate.parse_datetime('2012-02-24T00:53:52.000001Z')
        date_str = Serializer.serialize_iso(date_obj)
        assert date_str == '2012-02-24T00:53:52.000001Z'

        date_obj = isodate.parse_datetime('2012-02-24T00:53:52.780Z')
        date_str = Serializer.serialize_iso(date_obj)
        assert date_str == '2012-02-24T00:53:52.780Z' 

def is_datetime(instance):
            if not isinstance(instance, str_types):
                return True
            return isodate.parse_datetime(instance) 

def is_date(instance):
            if not isinstance(instance, str_types):
                return True
            return isodate.parse_datetime(instance) 

def is_date(instance):
            if not isinstance(instance, str_types):
                return True
            return isodate.parse_datetime(instance) 

def is_datetime(instance):
            if not isinstance(instance, str_types):
                return True
            return isodate.parse_datetime(instance) 

def test_check_timestamp_in_range_true():
    timestamp = isodate.parse_datetime("2016-06-07T23:46:03+00:00")

    start_time = isodate.parse_datetime("2016-06-07T23:40:03+00:00")
    end_time = isodate.parse_datetime("2016-06-07T23:50:03+00:00")

    assert logs.check_timestamp_in_range(timestamp, start_time, end_time) is True 

def paasta_app_output_passes_filter(
    line: str,
    levels: Sequence[str],
    service: str,
    components: Iterable[str],
    clusters: Sequence[str],
    instances: Iterable[str],
    start_time: datetime.datetime = None,
    end_time: datetime.datetime = None,
) -> bool:
    try:
        parsed_line = json.loads(line)
    except ValueError:
        log.debug("Trouble parsing line as json. Skipping. Line: %r" % line)
        return False
    try:
        timestamp = isodate.parse_datetime(parsed_line.get("timestamp"))
    # https://github.com/gweis/isodate/issues/53
    except ValueError:
        return True
    if not check_timestamp_in_range(timestamp, start_time, end_time):
        return False
    return (
        parsed_line.get("component") in components
        and (
            parsed_line.get("cluster") in clusters
            or parsed_line.get("cluster") == ANY_CLUSTER
        )
        and (instances is None or parsed_line.get("instance") in instances)
    ) 

def paasta_log_line_passes_filter(
    line: str,
    levels: Sequence[str],
    service: str,
    components: Iterable[str],
    clusters: Sequence[str],
    instances: Iterable[str],
    start_time: datetime.datetime = None,
    end_time: datetime.datetime = None,
) -> bool:
    """Given a (JSON-formatted) log line, return True if the line should be
    displayed given the provided levels, components, and clusters; return False
    otherwise.
    """
    try:
        parsed_line = json.loads(line)
    except ValueError:
        log.debug("Trouble parsing line as json. Skipping. Line: %r" % line)
        return False

    timestamp = isodate.parse_datetime(parsed_line.get("timestamp"))
    if not check_timestamp_in_range(timestamp, start_time, end_time):
        return False
    return (
        parsed_line.get("level") in levels
        and parsed_line.get("component") in components
        and (
            parsed_line.get("cluster") in clusters
            or parsed_line.get("cluster") == ANY_CLUSTER
        )
        and (instances is None or parsed_line.get("instance") in instances)
    ) 

def marathon_app_status(
    app: MarathonApp,
    marathon_client: MarathonClient,
    dashboard_link: Optional[str],
    deploy_status: int,
    list_tasks: bool = False,
) -> MutableMapping[str, Any]:
    app_status = {
        "tasks_running": app.tasks_running,
        "tasks_healthy": app.tasks_healthy,
        "tasks_staged": app.tasks_staged,
        "tasks_total": app.instances,
        "create_timestamp": isodate.parse_datetime(app.version).timestamp(),
        "deploy_status": marathon_tools.MarathonDeployStatus.tostring(deploy_status),
    }

    app_queue = marathon_tools.get_app_queue(marathon_client, app.id)
    if deploy_status == marathon_tools.MarathonDeployStatus.Delayed:
        _, backoff_seconds = marathon_tools.get_app_queue_status_from_queue(app_queue)
        app_status["backoff_seconds"] = backoff_seconds

    unused_offers_summary = marathon_tools.summarize_unused_offers(app_queue)
    if unused_offers_summary is not None:
        app_status["unused_offers"] = unused_offers_summary

    if dashboard_link:
        app_status["dashboard_url"] = "{}/ui/#/apps/%2F{}".format(
            dashboard_link.rstrip("/"), app.id.lstrip("/")
        )

    if list_tasks is True:
        app_status["tasks"] = []
        for task in app.tasks:
            app_status["tasks"].append(build_marathon_task_dict(task))

    return app_status 

def iso8601_as_datetime(iso,
                        localize=False  # Default into local time zone
                        ):
    try:
        parsed = isodate.parse_datetime(iso)
        if localize and parsed.tzinfo is None:
            parsed = get_localzone().localize(parsed)
        return parsed
    except isodate.isoerror.ISO8601Error as ex:
        raise ValueError("Invalid ISO8601 date")

# TODO: This function exists in datetime as .isoformat() 

def is_datetime(instance):
            if not isinstance(instance, str_types):
                return True
            return isodate.parse_datetime(instance) 

def datetime(dt):
        return parse_datetime(dt).replace(tzinfo=utc) 

def is_date(instance):
            if not isinstance(instance, str_types):
                return True
            return isodate.parse_datetime(instance) 

def is_datetime(instance):
            if not isinstance(instance, str_types):
                return True
            return isodate.parse_datetime(instance) 

def generate_start_end_time(
    from_string: str = "30m", to_string: str = None
) -> Tuple[datetime.datetime, datetime.datetime]:
    """Parses the --from and --to command line arguments to create python
    datetime objects representing the start and end times for log retrieval

    :param from_string: The --from argument, defaults to 30 minutes
    :param to_string: The --to argument, defaults to the time right now
    :return: A tuple containing start_time, end_time, which specify the interval of log retrieval
    """
    if to_string is None:
        end_time = datetime.datetime.utcnow()
    else:
        # Try parsing as a a natural time duration first, if that fails move on to
        # parsing as an ISO-8601 timestamp
        to_duration = timeparse(to_string)

        if to_duration is not None:
            end_time = datetime.datetime.utcnow() - datetime.timedelta(
                seconds=to_duration
            )
        else:
            end_time = isodate.parse_datetime(to_string)
            if not end_time:
                raise ValueError(
                    "--to argument not in ISO8601 format and not a valid pytimeparse duration"
                )

    from_duration = timeparse(from_string)
    if from_duration is not None:
        start_time = datetime.datetime.utcnow() - datetime.timedelta(
            seconds=from_duration
        )
    else:
        start_time = isodate.parse_datetime(from_string)

        if not start_time:
            raise ValueError(
                "--from argument not in ISO8601 format and not a valid pytimeparse duration"
            )

    # Covert the timestamps to something timezone aware
    start_time = pytz.utc.localize(start_time)
    end_time = pytz.utc.localize(end_time)

    if start_time > end_time:
        raise ValueError("Start time bigger than end time")

    return start_time, end_time 

def filter_and_aggregate_scribe_logs(
        self,
        scribe_reader_ctx: ContextManager,
        scribe_env: str,
        stream_name: str,
        levels: Sequence[str],
        service: str,
        components: Iterable[str],
        clusters: Sequence[str],
        instances: Iterable[str],
        aggregated_logs: MutableSequence[Dict[str, Any]],
        parser_fn: Callable = None,
        filter_fn: Callable = None,
        start_time: datetime.datetime = None,
        end_time: datetime.datetime = None,
    ) -> None:
        with scribe_reader_ctx as scribe_reader:
            try:
                for line in scribe_reader:
                    # temporary until all log lines are strings not byte strings
                    if isinstance(line, bytes):
                        line = line.decode("utf-8")
                    if parser_fn:
                        line = parser_fn(line, clusters, service)
                    if filter_fn:
                        if filter_fn(
                            line,
                            levels,
                            service,
                            components,
                            clusters,
                            instances,
                            start_time=start_time,
                            end_time=end_time,
                        ):
                            try:
                                parsed_line = json.loads(line)
                                timestamp = isodate.parse_datetime(
                                    parsed_line.get("timestamp")
                                )
                                if not timestamp.tzinfo:
                                    timestamp = pytz.utc.localize(timestamp)
                            except ValueError:
                                timestamp = pytz.utc.localize(datetime.datetime.min)

                            line = {"raw_line": line, "sort_key": timestamp}
                            aggregated_logs.append(line)
            except StreamTailerSetupError as e:
                if "No data in stream" in str(e):
                    log.warning(f"Scribe stream {stream_name} is empty on {scribe_env}")
                    log.warning(
                        "Don't Panic! This may or may not be a problem depending on if you expect there to be"
                    )
                    log.warning("output within this stream.")
                elif "Failed to connect" in str(e):
                    log.warning(
                        f"Couldn't connect to Scribe to tail {stream_name} in {scribe_env}"
                    )
                    log.warning(f"Please be in {scribe_env} to tail this log.")
                else:
                    raise 

def get_video_description(video_id: str) -> str:
    parts = ["statistics", "contentDetails", "snippet"]
    request = get_video(video_id, parts)
    raise_api_errors(request)

    json = request.json()

    data = json["items"]
    if not data:
        raise NoResultsError()

    item = data[0]
    snippet = item["snippet"]
    statistics = item["statistics"]
    content_details = item["contentDetails"]

    out = "\x02{}\x02".format(snippet["title"])

    if not content_details.get("duration"):
        return out

    length = isodate.parse_duration(content_details["duration"])
    out += " - length \x02{}\x02".format(
        timeformat.format_time(int(length.total_seconds()), simple=True)
    )
    try:
        total_votes = float(statistics["likeCount"]) + float(statistics["dislikeCount"])
    except (LookupError, ValueError):
        total_votes = 0

    if total_votes != 0:
        # format
        likes = pluralize_suffix(int(statistics["likeCount"]), "like")
        dislikes = pluralize_suffix(int(statistics["dislikeCount"]), "dislike")

        percent = 100 * float(statistics["likeCount"]) / total_votes
        out += " - {}, {} (\x02{:.1f}\x02%)".format(likes, dislikes, percent)

    if "viewCount" in statistics:
        views = int(statistics["viewCount"])
        out += " - \x02{:,}\x02 view{}".format(views, "s"[views == 1 :])

    uploader = snippet["channelTitle"]

    upload_time = isodate.parse_datetime(snippet["publishedAt"])
    out += " - \x02{}\x02 on \x02{}\x02".format(
        uploader, upload_time.strftime("%Y.%m.%d")
    )

    try:
        yt_rating = content_details["contentRating"]["ytRating"]
    except KeyError:
        pass
    else:
        if yt_rating == "ytAgeRestricted":
            out += colors.parse(" - $(red)NSFW$(reset)")

    return out