Python bson.json_util.default() Examples

def test_list_healings_empty_json_response(self, stdout, Request, urlopen):
        lines = []
        stdout.write.side_effect = lambda data, **kw: lines.append(data)
        request = mock.Mock()
        Request.return_value = request
        result = mock.Mock()
        result.getcode.return_value = 200
        urlopen.return_value = result
        healing_list = []
        result.read.return_value = json.dumps(healing_list, default=json_util.default)
        args = ['-s', self.service_name]
        admin_plugin.list_healings(args)
        Request.assert_called_with(self.target +
                                   "services/proxy/service/rpaas?" +
                                   "callback=/admin/healings?quantity=20")
        request.add_header.assert_any_call("Authorization", "bearer " + self.token)
        self.assertEqual("GET", request.get_method())
        expected_output = u"""
+----------+---------+------------+----------+--------+
| Instance | Machine | Start Time | Duration | Status |
+----------+---------+------------+----------+--------+
"""
        self.assertEqual(expected_output, "".join(lines)) 

def reply(self, e, reply):
    if type(reply) in [dict, list]:
      #reply = json.dumps(reply, sort_keys=True, indent=4, default=json_util.default, ensure_ascii=True)
      reply = json.dumps(reply, sort_keys=True, ensure_ascii=True, default=json_util.default)
    else:
      reply = str(reply)

    if e.target == self.connection.nickname:
      target=e.source.nick
    else:
      target=e.target
    _list = reply.split('\n')
    chunk_size = 512 - 12 - len(e.target) # 512 - len("PRIVMSG") - len(" :") - CR/LF - target

    _list = [[x[i:i+chunk_size] for i in range(0, len(x), chunk_size)] for x in _list]
    _list = [item for sublist in _list for item in sublist] # flatten list
    for r in _list[:4]:
      self.connection.privmsg(target, r) 

def test_list_healings(self):
        resp = self.api.get("/admin/healings")
        self.assertEqual(200, resp.status_code)
        self.assertEqual("[]", resp.data)
        loop_time = datetime.datetime(2016, 8, 2, 10, 53, 0)
        healing_list = []
        for x in range(1, 30):
            data = {"instance": "myinstance", "machine": "10.10.1.{}".format(x),
                    "start_time": loop_time, "end_time": loop_time, "status": "success"}
            healing_list.append(json.loads(json.dumps(data, default=json_util.default)))
            self.storage.db[self.storage.healing_collection].insert(data)
            loop_time = loop_time + datetime.timedelta(minutes=5)
        healing_list.reverse()
        resp = self.api.get("/admin/healings")
        self.assertEqual(200, resp.status_code)
        self.assertListEqual(healing_list[:20], json.loads(resp.data))
        resp = self.api.get("/admin/healings?quantity=10")
        self.assertEqual(200, resp.status_code)
        self.assertListEqual(healing_list[:10], json.loads(resp.data))
        resp = self.api.get("/admin/healings?quantity=aaaa")
        self.assertEqual(200, resp.status_code)
        self.assertListEqual(healing_list[:20], json.loads(resp.data)) 

def save(self, dist_dir='result'):
        '''
        save the search results to file
        '''
        if dist_dir not in os.listdir(os.curdir):
            os.mkdir(dist_dir)
        for w in self.results:
            file_name = ''.join([
                    '_'.join([k for k in w['keywords']]),
                    w['mid']
                    ])
            file_name += '.txt'
            f = codecs.open(os.path.join(dist_dir, file_name), 'w', 'utf-8')
            json.dump(w, f, ensure_ascii = False, default=json_util.default, indent = 2)
            # logging.info(w['text'])
            logging.info('writed to file {}'.format(file_name))
        return 

def verisSummary(verisRegex=None):
    try:
        # aggregate the veris tags from the incidents collection and return as json
        client = MongoClient(options.mongohost, options.mongoport)
        # use meteor db
        incidents = client.meteor['incidents']

        iveris = incidents.aggregate([
            {"$match": {"tags": {"$exists": True}}},
            {"$unwind": "$tags"},
            {"$match": {"tags": {"$regex": ''}}},
            {"$project": {
                "dateOpened": 1,
                "tags": 1,
                "phase": 1,
                "_id": 0
            }}
        ])
        if iveris:
            return json.dumps(list(iveris), default=json_util.default)
        else:
            return json.dumps(list())
    except Exception as e:
            logger.error('Exception while aggregating veris summary: {0}\n'.format(e)) 

def backup_mongo_collection(coll, filepath) :
	"""
	dumps all documents in collection in _backups_collections 
	"""

	app_log.warning('>>> backup_mongo_collection ... ')

	cursor 		= coll.find({})
	backup_file = open(filepath, "w")
	backup_file.write('[')
	for document in cursor:
		backup_file.write(json.dumps(document,indent=4, default=json_util.default))
		backup_file.write(',')
	backup_file.write(']')

### + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ###


### MAIN TORNADO APPLICATION WRAPPER 

def checknotif():
    if request.json["product"] == '':
        req = ':' + request.json['vendor'] + ':'
    else:
        req = request.json["vendor"] + ':' + request.json["product"] + ':' + request.json["version"]

    tab = []
    keytab = ['summary']
    for cves in mongo.db.cves.find({'vulnerable_configuration': {'$regex': req}}).sort("Modified", DESCENDING):
        dic = {}
        for key, value in cves.items():
            if key in keytab:
                dic[key] = cgi.escape(value, quote=True)
            else:
                if isinstance(value, datetime):
                    value = str(value)
                dic[key] = value
        tab.append(dic)
    return json.dumps(tab, sort_keys=True, default=json_util.default) 

def post(self, slug=None):

		### needs XSRF token to work
		### cf : 

		dic = tornado.escape.json_decode(self.request.body)
		app_log.info(dic)

		full_json = { "status" : "ok", "sent" : self.retrieve_results() }

		### write data as json
		# cf : https://stackoverflow.com/questions/35083374/how-to-decode-a-unicode-string-python
		results = json.dumps( full_json, ensure_ascii=False, default=json_util.default ).encode('utf8') 

		self.write( results )
		
		self.finish() 

def TRAC_OPTIONS(func):  # pylint: disable=invalid-name
    @click.option(
        '--trac-uri',
        default='http://localhost/xmlrpc',
        show_default=True,
        help='uri of the Trac instance XMLRpc endpoint',
    )
    @click.option(
        '--ssl-verify / --no-ssl-verify',
        default=True,
        show_default=True,
        help='Enable/disable SSL certificate verification'
    )
    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        return func(*args, **kwargs)

    return wrapper 

def get_default_columns(cls):
        """List default columns by category"""
        return {
            'schema': cls._default_columns.get('schema', _SchemaPreProcessing.default_columns),
            'mapping': cls._default_columns.get('mapping', _MappingPreProcessing.default_columns),
            'diff': cls._default_columns.get('diff', _DiffPreProcessing.default_columns)} 

def _format_types_count(types_count, array_types_count=None):
        """ Format types_count to a readable sting.

        >>> format_types_count({'integer': 10, 'boolean': 5, 'null': 3, })
        'integer : 10, boolean : 5, null : 3'

        >>> format_types_count({'ARRAY': 10, 'null': 3, }, {'float': 4})
        'ARRAY(float : 4) : 10, null : 3'

        :param types_count: dict
        :param array_types_count: dict, default None
        :return types_count_string : str
        """
        if types_count is None:
            return str(None)

        types_count = sorted(types_count.items(), key=lambda x: x[1], reverse=True)

        type_count_list = list()
        for type_name, count in types_count:
            if type_name == 'ARRAY':
                array_type_name = _SchemaPreProcessing._format_types_count(array_types_count)
                type_count_list.append('ARRAY(' + array_type_name + ') : ' + str(count))
            else:
                type_count_list.append(str(type_name) + ' : ' + str(count))

        types_count_string = ', '.join(type_count_list)
        return types_count_string 

def write_data(self, file_descr):
        """Use json module dump function to write into file_descr (opened with opener)."""
        json.dump(self.data, file_descr, indent=4, ensure_ascii=False,
                  default=json_util.default, sort_keys=True) 

def transform_data_to_file(data, formats, output=None, category='schema', **kwargs):
    """
    Transform data into each of output_formats and write result to output_filename or stdout.

    :param data: dict (schema, mapping or diff)
    :param formats: list of str - extensions of output desired among:
                            'json', 'yaml' (hierarchical formats)
                            'tsv', 'html', 'md' or 'xlsx' (list like formats)
    :param output: str full path to file where formatted output will be saved saved
                            (default is std out)
    :param category: string in 'schema', 'mapping', 'diff' - describe input data
    :param kwargs: may contain additional specific arguments
           columns: list of columns to display in the output for list like formats
           without_counts: bool to display count fields in output for hierarchical formats
    """
    wrong_formats = set(formats) - {'tsv', 'xlsx', 'json', 'yaml', 'html', 'md'}

    if wrong_formats:
        raise ValueError("Output format should be tsv, xlsx, html, md, json or yaml. "
                         "{} is/are not supported".format(wrong_formats))

    for output_format in formats:
        output_maker = rec_find_right_subclass(output_format)(
            data, category=category,
            columns_to_get=kwargs.get('columns'), without_counts=kwargs.get('without_counts'))
        with output_maker.open(output) as file_descr:
            output_maker.write_data(file_descr) 

def _dumps(obj, fmt=None):
    if fmt == 'toml':
        return toml.dumps(obj)
    elif fmt == 'json':
        return json.dumps(obj, sort_keys=True, indent=2, default=json_util.default)
    elif fmt == 'python':
        return pformat(obj, indent=2)
    elif fmt == 'pickle':
        return pickle.dumps(obj)
    else:
        return str(obj) 

def to_json(l):
    return json.dumps(list(l), indent=2, default=json_util.default) 

def to_json(self):
        """Returns a JSON string of the Data member"""
        return json.dumps(self.data, indent=4, sort_keys=True, default=json_util.default) 

def healings():
    manager = get_manager()
    quantity = request.args.get("quantity", type=int)
    if quantity is None or quantity <= 0:
        quantity = 20
    healing_list = manager.storage.list_healings(quantity)
    return json.dumps(healing_list, default=json_util.default) 

def test_list_healings(self, stdout, Request, urlopen):
        lines = []
        stdout.write.side_effect = lambda data, **kw: lines.append(data)
        request = mock.Mock()
        Request.return_value = request
        result = mock.Mock()
        result.getcode.return_value = 200
        urlopen.return_value = result
        start_time = datetime.datetime(2016, 8, 2, 10, 53, 0)
        healing_list = []
        status = ['success', None, 'Long and ugly error with a large testing description']
        for x in range(0, 3):
            end_time = start_time + datetime.timedelta(minutes=3)
            if x == 1:
                end_time = None
            data = {"instance": "myinstance", "machine": "10.10.1.{}".format(x),
                    "start_time": start_time, "end_time": end_time, "status": status[x]}
            healing_list.append(data)
            start_time = start_time + datetime.timedelta(minutes=5)
        result.read.return_value = json.dumps(healing_list, default=json_util.default)
        args = ['-s', self.service_name]
        admin_plugin.list_healings(args)
        Request.assert_called_with(self.target +
                                   "services/proxy/service/rpaas?" +
                                   "callback=/admin/healings?quantity=20")
        request.add_header.assert_any_call("Authorization", "bearer " + self.token)
        self.assertEqual("GET", request.get_method())
        expected_output = u"""
+------------+-----------+------------------+----------+--------------------------------+
| Instance   | Machine   | Start Time       | Duration | Status                         |
+------------+-----------+------------------+----------+--------------------------------+
| myinstance | 10.10.1.0 | Aug  02 05:53:00 | 00:03:00 | success                        |
+------------+-----------+------------------+----------+--------------------------------+
| myinstance | 10.10.1.1 | Aug  02 05:58:00 |          |                                |
+------------+-----------+------------------+----------+--------------------------------+
| myinstance | 10.10.1.2 | Aug  02 06:03:00 | 00:03:00 | Long and ugly error with a lar |
|            |           |                  |          | ge testing description         |
+------------+-----------+------------------+----------+--------------------------------+
"""
        self.assertEqual(expected_output, "".join(lines)) 

def handle(self, capture):
        if capture.file.size > 0:
            capture.file.open()
            pcapReader = dpkt.pcap.Reader(capture.file)

            for timestamp, packet in pcapReader:
                jsonPacket = generate_json(capture, packet, timestamp)

                # broadcast to socket
                jsonPacket["object_type"] = "packet:update"
                publishMessage("socket", message=json.dumps(jsonPacket, default=json_util.default)) 

def handle(self, capture):
        if capture.file.size > 0:
            capture.file.open()
            pcapReader = dpkt.pcap.Reader(capture.file)

            for timestamp, packet in pcapReader:
                jsonPacket = generate_json(capture, packet, timestamp)

                # broadcast to socket
                socketioemitter.io.Emit("packet:" + jsonPacket['mac_address_src'],
                                        json.dumps(jsonPacket, default=json_util.default))
                socketioemitter.io.Emit("packet:create", json.dumps(jsonPacket, default=json_util.default)) 

def token_required(funct):
    @wraps(funct)
    def api_token(*args, **kwargs):
      data = Advanced_API.authErrors()
      if data:
        return Response(json.dumps(data[0], indent=2, sort_keys=True, default=json_util.default), mimetype='application/json'), data[1]
      else: return API.api(funct)(*args, **kwargs)
    return api_token

  ##########
  # ROUTES #
  ##########
  # Overriding api_dbInfo to allow for logged-in users to get more info 

def api_search(self, vendor=None, product=None):
    if not (vendor and product): return {}
    search = vendor + ":" + product
    # Not using query.cvesForCPE, because that one gives too much info
    #return json.dumps(db.cvesForCPE(search), default=json_util.default)
    return db.cvesForCPE(search)

  # /api/search/<path:search> 

def format_message(self, message):
        if type(message) in [dict, list]:
            message = json.dumps(message, sort_keys=True, indent=4, default=json_util.default)
        else:
            message = str(message)
        return message 

def execute_analysis(self, indicator):

        # crits was the first intel platform supported by ace
        # so if there is no prefix to the value then it's assumed to be crits
        if ':' in indicator.value:
            return False

        analysis = self.create_analysis(indicator)

        # download the crits indicator JSOn directly from the crits mongo database
        mongodb_uri = saq.CONFIG['crits']['mongodb_uri']
        if not mongodb_uri:
            logging.warn("A Mongo DB URI is not configured for Crits Analysis.")
            return False
        client = MongoClient(saq.CONFIG['crits']['mongodb_uri'])
        db = client['crits']
        collection = db['indicators']

        mongo_object = analysis.details = collection.find_one({'_id' : ObjectId(indicator.value)})

        if analysis.details is None:
            logging.error("unable to find details of indicator {}".format(indicator.value))
            return False
        
        # this extra step is required to remove the ObjectId objects in the JSON result
        analysis.details = json.loads(json.dumps(analysis.details, default=json_util.default))

        # extract any tags (buckets) associated with the indicator
        if 'bucket_list' in mongo_object and isinstance(mongo_object['bucket_list'], list):
            for tag in mongo_object['bucket_list']:
                indicator.add_tag(tag)

        # add any associated campaigns as tags as well
        if 'campaign' in analysis.details:
            for actor in analysis.details['campaign']:
                indicator.add_tag('apt:{0}'.format(actor['name']))

        return True 

def init_session(request):
    pass
    # Put all the incident templates in the session
    request.session['incident_templates'] = list(IncidentTemplate.objects.exclude(name='default').values('name'))
    request.session['has_incident_templates'] = len(request.session['incident_templates']) > 0
    request.session['can_report_event'] = request.user.has_perm('incidents.handle_incidents', obj=Incident) or \
                                          request.user.has_perm('incidents.report_events', obj=Incident)


# audit trail ===================================================== 

def new_event(request):
    if request.method == 'POST':
        form = IncidentForm(request.POST, for_user=request.user)

        form.status = _('Open')

        if form.is_valid():
            i = form.save(commit=False)

            if not form.cleaned_data['is_major']:
                i.is_major = form.cleaned_data['category'].is_major

            if i.is_major:
                i.is_incident = True

            i.opened_by = request.user
            i.save()
            form.save_m2m()
            i.refresh_main_business_lines()
            i.done_creating()

            if i.is_incident:
                return redirect("incidents:details", incident_id=i.id)
            else:
                return redirect("events:details", incident_id=i.id)

    else:
        template = request.GET.get('template', 'default')
        try:
            template = IncidentTemplate.objects.get(name=template)
            data = model_to_dict(template)
            data['description'] = Template(data['description']).render(RequestContext(request))
        except ObjectDoesNotExist:
            data = {}
        form = IncidentForm(initial=data, for_user=request.user)

    return render(request, 'events/new.html', {'form': form, 'mode': 'new'}) 

def _stringify(self, iter, joinable='\n'):
        """
        Takes an interable (pymongo Cursor or Array) containing dictionaries and
        returns a stringified version using python join
        """
        return joinable.join([json.dumps(doc, default=json_util.default) for doc in iter]) 

def write_run_metadata(self, run_id, metadata):
        with open(self._run_meta_path(run_id), mode='w') as f:
            f.write(json.dumps(metadata, default=json_util.default)) 

def encode(value):
    from bson import json_util

    value = json_encode(value, ensure_ascii=False, default=json_util.default)
    if sys.version < '3':
        return unicode(value)
    return value 

def _stringify(iterable, joinable='\n'):
        """
        Takes an iterable (pymongo Cursor or Array) containing dictionaries and
        returns a stringified version using python join
        """
        return joinable.join(
            [json.dumps(doc, default=json_util.default) for doc in iterable]
        )