Python dateutil.parser() Examples

def as_datetime(the_date, timezone=None):
    """Converts the_date to a DADateTime object with a timezone.  Uses the
    default timezone unless another timezone is explicitly provided."""
    ensure_definition(the_date, timezone)
    if timezone is None:
        timezone = get_default_timezone()
    if isinstance(the_date, datetime.date) and not isinstance(the_date, datetime.datetime):
        the_date = datetime.datetime.combine(the_date, datetime.datetime.min.time())
    if isinstance(the_date, datetime.datetime):
        new_datetime = the_date
    else:
        new_datetime = dateutil.parser.parse(the_date)
    if new_datetime.tzinfo:
        new_datetime = new_datetime.astimezone(pytz.timezone(timezone))
    else:
        new_datetime = pytz.timezone(timezone).localize(new_datetime)
    return dd(new_datetime) 

def perform_romeo_query(self, search_terms):
        search_terms = search_terms.copy()
        if self.api_key:
            search_terms['ak'] = self.api_key

        # Perform the query
        try:
            req = requests.get(self.base_url, params=search_terms, timeout=20)
        except requests.exceptions.RequestException as e:
            raise MetadataSourceException('Error while querying RoMEO.\n' +
                                          'URL was: '+self.base_url+'\n' +
                                          'Parameters were: '+str(search_terms)+'\n' +
                                          'Error is: '+str(e))

        # Parse it
        try:
            parser = ET.XMLParser(encoding='ISO-8859-1')
            root = ET.parse(BytesIO(req.content), parser)
        except ET.ParseError as e:
            raise MetadataSourceException('RoMEO returned an invalid XML response.\n' +
                                          'URL was: '+self.base_url+'\n' +
                                          'Parameters were: '+str(search_terms)+'\n' +
                                          'Error is: '+str(e))

        return root 

def test_fetch_updates(self):
        with requests_mock.mock() as http_mocker:
            http_mocker.get('http://www.sherpa.ac.uk/downloads/journal-title-issns.php?format=tsv',
                content=self.journals_dump_response)
            http_mocker.get('http://www.sherpa.ac.uk/downloads/download-dates.php?format=xml',
                content=self.latest_update_response)

            # Fetch all publishers initially
            self.api.fetch_updates()
            p = Publisher.objects.get(alias='GSA Today')
            self.assertEqual(p.last_updated, dateutil.parser.parse('2019-02-14T14:05:19Z'))
            p = Publisher.objects.get(romeo_id='2425')
            self.assertEqual(p.url, 'http://intranet.cvut.cz/')

            # Fetch updates again
            self.api.fetch_updates()

            # A publisher was updated
            p = Publisher.objects.get(romeo_id='2425')
            self.assertEqual(p.url, 'https://intranet.cvut.cz/') 

def _process_remove_objects_batch(self, bucket_name, objects_batch):
        """
        Requester and response parser for remove_objects
        """
        # assemble request content for objects_batch
        content = xml_marshal_delete_objects(objects_batch)

        # compute headers
        headers = {
            'Content-Md5': get_md5_base64digest(content),
            'Content-Length': len(content)
        }
        query = {'delete': ''}
        content_sha256_hex = get_sha256_hexdigest(content)

        # send multi-object delete request
        response = self._url_open(
            'POST', bucket_name=bucket_name,
            headers=headers, body=content,
            query=query, content_sha256=content_sha256_hex,
        )

        # parse response to find delete errors
        return parse_multi_delete_response(response.data) 

def parse_datetime_string(datetime_str):
    """
    :param datetime_str: A string representing date and time with timezone
                         information.
    :return: A datetime object, converted to UTC, with no timezone info.
    """
    # Parse the string to datetime object
    date_with_offset = dateutil.parser.parse(datetime_str)

    # Convert the date to UTC
    try:
        utc_date = date_with_offset.astimezone(pytz.utc)
    except ValueError:
        raise manager_exceptions.BadParametersError(
            'Date `{0}` missing timezone information, please provide'
            ' valid date. \nExpected format: YYYYMMDDHHMM+HHMM or'
            ' YYYYMMDDHHMM-HHMM i.e: 201801012230-0500'
            ' (Jan-01-18 10:30pm EST)'.format(datetime_str))

    # Date is in UTC, tzinfo is not necessary
    return utc_date.replace(tzinfo=None) 

def get_parsed_deployment(blueprint,
                          app_dir,
                          app_blueprint):
    file_server_root = config.instance.file_server_root
    blueprint_resource_dir = os.path.join(file_server_root,
                                          'blueprints',
                                          blueprint.tenant_name,
                                          blueprint.id)
    # The dsl parser expects a URL
    blueprint_resource_dir_url = 'file:{0}'.format(blueprint_resource_dir)
    app_path = os.path.join(file_server_root, app_dir, app_blueprint)

    try:
        return tasks.parse_dsl(
            app_path,
            resources_base_path=file_server_root,
            additional_resources=[blueprint_resource_dir_url],
            **app_context.get_parser_context()
        )
    except parser_exceptions.DSLParsingException as ex:
        raise manager_exceptions.InvalidBlueprintError(
            'Invalid blueprint - {0}'.format(ex)) 

def fetch_production(zone_key, session=None, target_datetime=None):    
    if zone_key != "MX":
        raise ValueError("MX parser cannot fetch production for zone {}".format(zone_key))
    
    if target_datetime is None:
        raise ValueError("Parser only supports fetching historical production data, please specify a terget_datetime in the past")
    
    # retrieve data for the month either from the cache or fetch it
    cache_key = target_datetime.strftime("%Y-%m")
    if cache_key in DATA_CACHE:
        df = DATA_CACHE[cache_key]
    else:
        df = fetch_csv_for_date(target_datetime, session=session)
        DATA_CACHE[cache_key] = df
    
    data = []
    for idx, series in df.iterrows():
        data.append({
            'zoneKey': zone_key,
            'datetime': series["instante"].to_pydatetime(),
            'production': convert_production(series),
            'source': 'cenace.gob.mx'
        })
    return data 

def fetch_MX_exchange(sorted_zone_keys, s):
    """
    Finds current flow between two Mexican control areas.
    Returns a float.
    """

    req = s.get(MX_EXCHANGE_URL)
    soup = BeautifulSoup(req.text, 'html.parser')
    exchange_div = soup.find("div", attrs={'id': EXCHANGES[sorted_zone_keys]})
    val = exchange_div.text

    # cenace html uses unicode hyphens instead of minus signs and , as thousand separator
    trantab = str.maketrans({chr(8208): chr(45), ",": ""})

    val = val.translate(trantab)
    flow = float(val)

    if sorted_zone_keys in ["BZ->MX-PN", "MX-CE->MX-OR", "MX-CE->MX-OC"]:
        # reversal needed for these zones due to EM ordering
        flow = -1*flow

    return flow 

def __json_date_parse(json_object):
        """
        Parse dates in certain known json fields, if possible.
        """
        known_date_fields = ["created_at", "week", "day", "expires_at", "scheduled_at", "updated_at", "last_status_at", "starts_at", "ends_at", "published_at"]
        for k, v in json_object.items():
            if k in known_date_fields:
                if v != None:
                    try:
                        if isinstance(v, int):
                            json_object[k] = datetime.datetime.fromtimestamp(v, pytz.utc)
                        else:
                            json_object[k] = dateutil.parser.parse(v)
                    except:
                        raise MastodonAPIError('Encountered invalid date.')
        return json_object 

def _normalize_gaf_date(date, report, taxon, line):
    if date is None or date == "":
        report.warning(line, Report.INVALID_DATE, date, "GORULE:0000001: empty",
            taxon=taxon, rule=1)
        return date

    # We check int(date)
    if len(date) == 8 and date.isdigit():
        d = datetime.datetime(int(date[0:4]), int(date[4:6]), int(date[6:8]), 0, 0, 0, 0)
    else:
        report.warning(line, Report.INVALID_DATE, date, "GORULE:0000001: Date field must be YYYYMMDD, got: {}".format(date),
            taxon=taxon, rule=1)
        try:
            d = dateutil.parser.parse(date)
        except:
            report.error(line, Report.INVALID_DATE, date, "GORULE:0000001: Could not parse date '{}' at all".format(date),
                taxon=taxon, rule=1)
            return None

    return d.strftime("%Y%m%d")

## we generate both qualifier and relation field
## Returns: (negated, relation, other_qualifiers) 

def load_datetime_tz(time_str):
    """
    Load datetime and ensure the result is timezone-aware.

    If the parsed timestamp is naive, transform it into a timezone-aware one
    using the local timezone.

    :param str time_str: string representing a timestamp
    :return datetime: the parsed timezone-aware datetime
    """
    # dateutil parser returns naive or tz-aware string depending on the format
    # of the input string
    timestamp = dateutil.parser.parse(time_str)
    # if the parsed timestamp is naive, forces it to local timezone
    if timestamp.tzinfo is None:
        timestamp = timestamp.replace(tzinfo=dateutil.tz.tzlocal())
    return timestamp 

def _defs(self, name, value):
        if name in ['cvss_vector', 'cvss_temporal_vector']:
            # Return a list of the Vectors instead of having everything in a
            # flat string.  This should allow for much easier parsing later.
            return value.split('/')

        elif name in ['cvss_base_score', 'cvss_temporal_score']:
            # CVSS scores are floats, so lets return them as such.
            return float(value)

        elif name in ['first_found', 'last_found', 'plugin_modification_date',
                      'plugin_publication_date', 'HOST_END', 'HOST_START']:
            # The first and last found attributes use a datetime timestamp
            # format that we should convert into a unix timestamp.
            return dateutil.parser.parse(value)

        elif name in ['port', 'pluginID', 'severity']:
            return int(value)

        else:
            return value 

def devseed_parser(self, img):
        """
        Args:
            img (dict): json metadata dict as shipped in devseed API response
        """
        p = img['properties']
        self.title = p['sentinel:product_id']
        self.utm_zone = int(p['sentinel:utm_zone'])
        self.lat_band = p['sentinel:latitude_band']
        self.sqid  = p['sentinel:grid_square']
        self.mgrs_id = '{}{}{}'.format(self.utm_zone, self.lat_band, self.sqid)

        self.date = dateutil.parser.parse(self.title.split('_')[2])
        #self.granule_date = dateutil.parser.parse(p['datetime'])
        self.satellite = p['eo:platform'].replace("sentinel-", "S").upper()  # sentinel-2b --> S2B
        self.relative_orbit = parse_safe_name_for_relative_orbit_number(self.title)

        self.thumbnail = img['assets']['thumbnail']['href'].replace('sentinel-s2-l1c.s3.amazonaws.com',
                                                                    'roda.sentinel-hub.com/sentinel-s2-l1c')
        self.cloud_cover = p['eo:cloud_cover']
        #self.id = img['id'] 

def scihub_parser(self, img):
        """
        Args:
            img (dict): json metadata dict for a single SAFE, as shipped in scihub
                opensearch API response
        """
        self.title = img['title']
        try:
            self.mgrs_id = img['tileid']
        except KeyError:
            self.mgrs_id = re.findall(r"_T([0-9]{2}[A-Z]{3})_", img['title'])[0]
        self.utm_zone, self.lat_band, self.sqid = split_mgrs_id(self.mgrs_id)
        self.date = dateutil.parser.parse(img['beginposition'], ignoretz=True)
        self.satellite = self.title[:3]  # S2A_MSIL1C_2018010... --> S2A
        self.absolute_orbit = img['orbitnumber']
        self.relative_orbit = img['relativeorbitnumber']
        self.datatake_id = img['s2datatakeid']
        self.processing_level = img['processinglevel'].split('-')[1]  # Level-1C --> L1C
        self.thumbnail = img['links']['icon'] 

def gcloud_parser(self, img):
        """
        Args:
            img (dict): json metadata dict for a single SAFE, as shipped in Gcloud
                API response
        """
        self.title = img['product_id']
        self.mgrs_id = img['mgrs_tile']
        self.utm_zone, self.lat_band, self.sqid = split_mgrs_id(self.mgrs_id)
        self.date = parse_safe_name_for_acquisition_date(self.title)  # 'sensing_time' contains the granule datetime
        self.satellite = img['product_id'][:3]
        self.relative_orbit = parse_safe_name_for_relative_orbit_number(self.title)

        self.absolute_orbit = int(img['granule_id'].split('_')[2][1:])
        self.granule_date = dateutil.parser.parse(img['sensing_time'], ignoretz=True)
        #self.granule_date = dateutil.parser.parse(img['granule_id'].split('_')[3])

        self.cloud_cover = img['cloud_cover']
        #self.is_old = True if '.' in img['granule_id'] else False 

def scihub_parser(self, img):
        """
        Args:
            img (dict): json metadata dict for a single SAFE, as shipped in scihub
                opensearch API response
        """
        self.title = img['title']
        self.date = dateutil.parser.parse(img['beginposition'], ignoretz=True)
        self.satellite = self.title[:3]  # S1A_IW_GRDH_1SDV_20191218... --> S1A
        self.absolute_orbit = img['orbitnumber']
        self.relative_orbit = img['relativeorbitnumber']
        self.orbit_direction = img['orbitdirection']
        self.operational_mode = img['sensoroperationalmode']
        self.polarisations = [p.lower() for p in img['polarisationmode'].split()]
        self.polarisation_string = parse_polarisation_string(img['polarisationmode'])
        self.product_type = img['producttype']
        self.footprint = img['footprint']
        self.thumbnail = img['links']['icon']
        self.id = img['id'] 

def planet_parser(self, img):  #TODO FIXME
        """
        Args:
            img (dict): json metadata dict for a single SAFE, as shipped in Planet
                API response
        """
        self.title = img['id']
        p = img['properties']
        self.mgrs_id = p['mgrs_grid_id']
        self.utm_zone, self.lat_band, self.sqid = split_mgrs_id(self.mgrs_id)
        self.date = parse_safe_name_for_acquisition_date(self.title)  # 'acquired' contains the granule datetime
        self.satellite = p['satellite_id'].replace("Sentinel-", "S")  # Sentinel-2A --> S2A
        self.relative_orbit = p['rel_orbit_number']
        self.absolute_orbit = p['abs_orbit_number']
        self.granule_date = dateutil.parser.parse(p['acquired'])
        #self.granule_date = dateutil.parser.parse(p['granule_id'].split('_')[3])
        self.thumbnail = img['_links']['thumbnail']

        self.cloud_cover = p['cloud_cover']
        self.sun_azimuth = p['sun_azimuth']
        self.sun_elevation = p['sun_elevation'] 

def month_of(the_date, as_word=False, language=None):
    """Interprets the_date as a date and returns the month.
    Set as_word to True if you want the month as a word."""
    ensure_definition(the_date, as_word, language)
    if language is None:
        language = get_language()
    try:
        if isinstance(the_date, datetime.datetime) or isinstance(the_date, datetime.date):
            date = the_date
        else:
            date = dateutil.parser.parse(the_date)
        if as_word:
            return(babel.dates.format_date(date, format='MMMM', locale=babel_language(language)))
        return(int(date.strftime('%m')))
    except:
        return word("Bad date") 

def dow_of(the_date, as_word=False, language=None):
    """Interprets the_date as a date and returns the day of week as a number from 1 to 7 for Sunday through Saturday.  Set as_word to True if you want the day of week as a word."""
    ensure_definition(the_date, as_word, language)
    if language is None:
        language = get_language()
    try:
        if isinstance(the_date, datetime.datetime) or isinstance(the_date, datetime.date):
            date = the_date
        else:
            date = dateutil.parser.parse(the_date)
        if as_word:
            return(babel.dates.format_date(date, format='EEEE', locale=babel_language(language)))
        else:
            return(int(date.strftime('%u')))
    except:
        return word("Bad date") 

def format_date(the_date, format=None, language=None):
    """Interprets the_date as a date and returns the date formatted for the current locale."""
    ensure_definition(the_date, format, language)
    if isinstance(the_date, DAEmpty):
        return ""
    if language is None:
        language = get_language()
    if format is None:
        format = interview_default('date format', 'long', language)
    try:
        if isinstance(the_date, datetime.datetime) or isinstance(the_date, datetime.date):
            date = the_date
        else:
            date = dateutil.parser.parse(the_date)
        return babel.dates.format_date(date, format=format, locale=babel_language(language))
    except:
        return word("Bad date") 

def get_s2_granule_id_of_scihub_item_from_sentinelhub(img):
    """
    Build the granule id of a given single tile SAFE.

    The hard part is to get the timestamp in the granule id. Unfortunately this
    timestamp is not part of the metadata returned by scihub. This function queries
    sentinelhub to retrieve it. It takes about 3 seconds.

    Args:
        img (Sentinel2Image instance): Sentinel-2 image metadata

    Return:
        str: granule id, e.g. L1C_T36RTV_A005095_20180226T084545
    """
    import sentinelhub
    t0 = (img.date - datetime.timedelta(hours=2)).isoformat()
    t1 = (img.date + datetime.timedelta(hours=2)).isoformat()
    r = sentinelhub.opensearch.get_tile_info('T{}'.format(img.mgrs_id), time=(t0, t1))
    assert(isinstance(r, dict))

    granule_date = dateutil.parser.parse(r['properties']['startDate']).strftime("%Y%m%dT%H%M%S")
    return "L1C_T{}_A{:06d}_{}".format(img.mgrs_id, img.relative_orbit, granule_date) 

def format_time(the_time, format=None, language=None):
    """Interprets the_time as a date/time and returns the time formatted for the current locale."""
    ensure_definition(the_time, format, language)
    if isinstance(the_time, DAEmpty):
        return ""
    if language is None:
        language = get_language()
    if format is None:
        format = interview_default('time format', 'short', language)
    try:
        if isinstance(the_time, datetime.datetime) or isinstance(the_time, datetime.date) or isinstance(the_time, datetime.time):
            time = the_time
        else:
            time = dateutil.parser.parse(the_time)
        return babel.dates.format_time(time, format=format, locale=babel_language(language))
    except Exception as errmess:
        return word("Bad date: " + str(errmess)) 

def testImportedModules(self):
        import dateutil.easter
        import dateutil.parser
        import dateutil.relativedelta
        import dateutil.rrule
        import dateutil.tz
        import dateutil.utils
        import dateutil.zoneinfo

        self.assertEquals(dateutil.easter, new_locals.pop("easter"))
        self.assertEquals(dateutil.parser, new_locals.pop("parser"))
        self.assertEquals(dateutil.relativedelta, new_locals.pop("relativedelta"))
        self.assertEquals(dateutil.rrule, new_locals.pop("rrule"))
        self.assertEquals(dateutil.tz, new_locals.pop("tz"))
        self.assertEquals(dateutil.utils, new_locals.pop("utils"))
        self.assertEquals(dateutil.zoneinfo, new_locals.pop("zoneinfo"))

        self.assertFalse(new_locals) 

def test_parsers_iso8601(self):
        # GH 12060
        # test only the iso parser - flexibility to different
        # separators and leadings 0s
        # Timestamp construction falls back to dateutil
        cases = {'2011-01-02': datetime(2011, 1, 2),
                 '2011-1-2': datetime(2011, 1, 2),
                 '2011-01': datetime(2011, 1, 1),
                 '2011-1': datetime(2011, 1, 1),
                 '2011 01 02': datetime(2011, 1, 2),
                 '2011.01.02': datetime(2011, 1, 2),
                 '2011/01/02': datetime(2011, 1, 2),
                 '2011\\01\\02': datetime(2011, 1, 2),
                 '2013-01-01 05:30:00': datetime(2013, 1, 1, 5, 30),
                 '2013-1-1 5:30:00': datetime(2013, 1, 1, 5, 30)}
        for date_str, exp in compat.iteritems(cases):
            actual = tslib._test_parse_iso8601(date_str)
            assert actual == exp

        # seperators must all match - YYYYMM not valid
        invalid_cases = ['2011-01/02', '2011^11^11',
                         '201401', '201111', '200101',
                         # mixed separated and unseparated
                         '2005-0101', '200501-01',
                         '20010101 12:3456', '20010101 1234:56',
                         # HHMMSS must have two digits in each component
                         # if unseparated
                         '20010101 1', '20010101 123', '20010101 12345',
                         '20010101 12345Z',
                         # wrong separator for HHMMSS
                         '2001-01-01 12-34-56']
        for date_str in invalid_cases:
            with pytest.raises(ValueError):
                tslib._test_parse_iso8601(date_str)
                # If no ValueError raised, let me know which case failed.
                raise Exception(date_str) 

def prep_search_value(self, term, lookup_type):
        if lookup_type in ('exact', 'in', 'range'):
            try:
                date_obj = dateutil.parser.parse(term)
            except ValueError:
                # This exception is theoretical, but it doesn't seem to raise.
                pass
            except TypeError:
                # Failed conversions can lead to the parser adding ints to None.
                pass
            else:
                return date_obj

        if lookup_type not in ('exact', 'in', 'range'):
            test_term = term
            if lookup_type == 'week_day':
                try:
                    test_term = int(test_term) - 1  # Django ORM uses 1-7, python strptime uses 0-6
                except:
                    return None
                else:
                    test_term = str(test_term)

            for test_format in STRPTIME_PLACEHOLDERS[lookup_type]:
                # Try to validate the term against the given date lookup type
                try:
                    date_obj = datetime.strptime(test_term, test_format)
                except ValueError:
                    pass
                else:
                    if lookup_type == 'week_day':
                        term = date_obj.weekday() + 1  # Django ORM uses 1-7, python strptime uses 0-6
                    else:
                        term = getattr(date_obj, lookup_type)
                    return str(term)

        return super(DateColumn, self).prep_search_value(term, lookup_type) 

def testParserAll(self):
        # All interface
        from dateutil.parser import parse
        from dateutil.parser import parserinfo

        # Other public classes
        from dateutil.parser import parser

        for var in (parse, parserinfo, parser):
            self.assertIsNot(var, None) 

def human_since(since, include_tz=False):
    tz = dateutil.tz.tzlocal() if include_tz else None
    return naturaltime(datetime.datetime.now(tz=tz) - dateutil.parser.parse(since)) 

def parse_date(self, line):
        try:
            r = dateutil.parser.parse(line, fuzzy=True)
            r = r.astimezone(tz.tzutc())
        # if the fuzzy parser failed to parse the line due to
        # incorrect timezone information issue 5 GitHub
        except ValueError:
            r = re.findall('^(.*?)\s*\(', line)
            if r:
                r = dateutil.parser.parse(r[0])
        return r

    # This code is originally from https://github.com/lnxg33k/MHA 

def send_alerts(self, observable, fail=True):
        communication = ObservableCommunication(observable, fail)
        for alert in self:
            seconds = observable.get_line_value('seconds', 0)
            since = self.sma.results.get_observable_result(observable).get('since')
            since = (dateutil.parser.parse(since) if since else datetime.datetime.now()).replace(tzinfo=tzlocal())
            dt = datetime.datetime.now().replace(tzinfo=tzlocal())
            if alert.section in self.sma.results.get_observable_result(observable)['alerted']:
                continue
            elif seconds and dt - since <= datetime.timedelta(seconds=seconds):
                continue
            success = alert.send(communication['subject'], communication['message'], **communication.alert_kwargs())
            if success:
                self.sma.results.add_alert_to_observable_result(observable, alert.section) 

def validate_timestamp(value):
    try:
        # Note: we must return our own exception message
        # as dateutil's parser returns different types / values on
        # different systems. OSX, for example, returns a tuple
        # containing a different error message than Linux
        dateutil.parser.parse(value)
    except Exception as e:
        original_err_msg = str(e)
        log.error(original_err_msg)
        ExceptionCollector.appendException(
            ValueError(_('"%(val)s" is not a valid timestamp. "%(msg)s"') %
                       {'val': value, 'msg': original_err_msg}))
    return