Java Code Examples for org.json.simple.JSONObject#putAll()
The following examples show how to use
org.json.simple.JSONObject#putAll() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: StatefulWindowingTest.java From incubator-heron with Apache License 2.0 | 6 votes |
|
@Override
public void execute(TupleWindow inputWindow) {
HashMap<String, Integer> wordCount = new HashMap<>();
for (Tuple tuple : inputWindow.get()) {
String key = tuple.getString(0);
if (wordCount.get(key) == null) {
wordCount.put(key, 1);
} else {
Integer val = wordCount.get(key);
wordCount.put(key, ++val);
}
}
myState.put(windowCount, wordCount);
windowCount++;
System.out.println("STATE: " + myState);
JSONObject jsonObject = new JSONObject();
jsonObject.putAll(myState);
collector.emit(new Values(jsonObject.toJSONString()));
}
Example 2
| Source File: AuthorizationResourceRequest.java From sqoop-on-spark with Apache License 2.0 | 6 votes |
|
public void grantRevokePrivilege(String serverUrl, List<MPrincipal> principals, List<MPrivilege> privileges, boolean isGrant) {
PrincipalsBean principalsBean = new PrincipalsBean(principals);
// Extract all config inputs including sensitive inputs
JSONObject jsonObject = new JSONObject();
jsonObject.putAll(principalsBean.extract(false));
if (privileges != null && privileges.size() != 0) {
PrivilegesBean privilegesBean = new PrivilegesBean(privileges);
jsonObject.putAll(privilegesBean.extract(false));
}
if (isGrant) {
super.put(serverUrl + RESOURCE + PRIVILEGES + GRANT, jsonObject.toJSONString());
} else {
super.put(serverUrl + RESOURCE + PRIVILEGES + REVOKE, jsonObject.toJSONString());
}
}
Example 3
| Source File: BoxResource.java From io with Apache License 2.0 | 6 votes |
|
/**
* boxインストールが実行されていないか、実行されたがキャッシュの有効期限が切れた場合のレスポンスを作成する.
* @return レスポンス用JSONオブジェクト
*/
@SuppressWarnings("unchecked")
private JSONObject createResponse(JSONObject values) {
JSONObject response = new JSONObject();
response.putAll(values);
response.remove("cell_id");
response.remove("box_id");
response.put("schema", this.getBox().getSchema());
ProgressInfo.STATUS status = ProgressInfo.STATUS.valueOf((String) values.get("status"));
if (status == ProgressInfo.STATUS.COMPLETED) {
response.remove("progress");
String startedAt = (String) response.remove("started_at");
response.put("installed_at", startedAt);
}
response.put("status", status.value());
return response;
}
Example 4
| Source File: CIFHbaseAdapter.java From metron with Apache License 2.0 | 5 votes |
|
@Override
public JSONObject enrich(CacheKey k) {
String metadata = k.coerceValue(String.class);
JSONObject output = new JSONObject();
LOGGER.debug("=======Looking Up For: {}", metadata);
output.putAll(getCIFObject(metadata));
return output;
}
Example 5
| Source File: MetronError.java From metron with Apache License 2.0 | 5 votes |
|
private void addMetadata(JSONObject errorMessage) {
if(metadata != null && metadata.keySet().size() > 0) {
// add each metadata element directly to the message. each metadata key already has
// a standard prefix, no need to add another prefix to avoid collisions. this mimics
// the behavior of merging metadata.
errorMessage.putAll(metadata);
}
}
Example 6
| Source File: ThreatHbaseAdapter.java From opensoc-streaming with Apache License 2.0 | 5 votes |
|
public JSONObject enrich(String metadata) {
JSONObject output = new JSONObject();
LOGGER.debug("=======Looking Up For:" + metadata);
output.putAll(getThreatObject(metadata));
return output;
}
Example 7
| Source File: APIMappingUtil.java From carbon-apimgt with Apache License 2.0 | 5 votes |
|
private static JSONObject handleEndpointSecurity(API api, JSONObject endpointSecurity) throws APIManagementException {
String tenantDomain = MultitenantUtils.getTenantDomain(APIUtil.replaceEmailDomainBack(api.getId()
.getProviderName()));
if (checkEndpointSecurityPasswordEnabled(tenantDomain)) {
return endpointSecurity;
}
JSONObject endpointSecurityElement = new JSONObject();
endpointSecurityElement.putAll(endpointSecurity);
if (endpointSecurityElement.get(APIConstants.ENDPOINT_SECURITY_SANDBOX)!= null){
JSONObject sandboxEndpointSecurity =
(JSONObject) endpointSecurityElement.get(APIConstants.ENDPOINT_SECURITY_SANDBOX);
if (sandboxEndpointSecurity.get(APIConstants.ENDPOINT_SECURITY_PASSWORD) != null){
sandboxEndpointSecurity.put(APIConstants.ENDPOINT_SECURITY_PASSWORD,"");
if (sandboxEndpointSecurity.get(APIConstants.ENDPOINT_SECURITY_TYPE)
.equals(APIConstants.ENDPOINT_SECURITY_TYPE_OAUTH)) {
sandboxEndpointSecurity.put(APIConstants.ENDPOINT_SECURITY_CLIENT_ID, "");
sandboxEndpointSecurity.put(APIConstants.ENDPOINT_SECURITY_CLIENT_SECRET, "");
}
}
}
if (endpointSecurityElement.get(APIConstants.ENDPOINT_SECURITY_PRODUCTION) != null) {
JSONObject productionEndpointSecurity =
(JSONObject) endpointSecurityElement.get(APIConstants.ENDPOINT_SECURITY_PRODUCTION);
if (productionEndpointSecurity.get(APIConstants.ENDPOINT_SECURITY_PASSWORD) != null) {
productionEndpointSecurity.put(APIConstants.ENDPOINT_SECURITY_PASSWORD, "");
if (productionEndpointSecurity.get(APIConstants.ENDPOINT_SECURITY_TYPE)
.equals(APIConstants.ENDPOINT_SECURITY_TYPE_OAUTH)) {
productionEndpointSecurity.put(APIConstants.ENDPOINT_SECURITY_CLIENT_ID, "");
productionEndpointSecurity.put(APIConstants.ENDPOINT_SECURITY_CLIENT_SECRET, "");
}
}
}
return endpointSecurityElement;
}
Example 8
| Source File: AuthorizationResourceRequest.java From sqoop-on-spark with Apache License 2.0 | 5 votes |
|
public void grantRevokeRole(String serverUrl, List<MRole> roles, List<MPrincipal> principals, boolean isGrant) {
RolesBean rolesBean = new RolesBean(roles);
PrincipalsBean principalsBean = new PrincipalsBean(principals);
// Extract all config inputs including sensitive inputs
JSONObject jsonObject = new JSONObject();
jsonObject.putAll(rolesBean.extract(false));
jsonObject.putAll(principalsBean.extract(false));
if (isGrant) {
super.put(serverUrl + RESOURCE + ROLES + GRANT, jsonObject.toJSONString());
} else {
super.put(serverUrl + RESOURCE + ROLES + REVOKE, jsonObject.toJSONString());
}
}
Example 9
| Source File: TriggerTest.java From SB_Elsinore_Server with MIT License | 5 votes |
|
public void addTrigger(String m, String s, String n)
{
Map<String, String> jMap = new HashMap<>();
jMap.put(WaitTrigger.WAITTIMEMINS, m);
jMap.put(WaitTrigger.WAITTIMESECS, s);
jMap.put(WaitTrigger.NOTES, n);
JSONObject j = new JSONObject();
j.putAll(jMap);
mTriggerControl.addTrigger(0, "Wait", j);
}
Example 10
| Source File: RegularExpressionsParser.java From metron with Apache License 2.0 | 5 votes |
|
/**
* Parses an unstructured text message into a json object based upon the regular expression
* configuration supplied.
*
* @param rawMessage incoming unstructured raw text.
* @return List of json parsed json objects. In this case list will have a single element only.
*/
@Override
public List<JSONObject> parse(byte[] rawMessage) {
String originalMessage = null;
try {
originalMessage = new String(rawMessage, getReadCharset()).trim();
LOG.debug(" raw message. {}", originalMessage);
if (originalMessage.isEmpty()) {
LOG.warn("Message is empty.");
return Arrays.asList(new JSONObject());
}
} catch (Exception e) {
LOG.error("[Metron] Could not read raw message. {} " + originalMessage, e);
throw new RuntimeException(e.getMessage(), e);
}
JSONObject parsedJson = new JSONObject();
if (messageHeaderPatternsMap.size() > 0) {
parsedJson.putAll(extractHeaderFields(originalMessage));
}
parsedJson.putAll(parse(originalMessage));
parsedJson.put(Constants.Fields.ORIGINAL.getName(), originalMessage);
/**
* Populate the output json with default timestamp.
*/
parsedJson.put(Constants.Fields.TIMESTAMP.getName(), System.currentTimeMillis());
applyFieldTransformations(parsedJson);
return Arrays.asList(parsedJson);
}
Example 11
| Source File: ProjectionJDBCDriverTests.java From jdbc-cb with Apache License 2.0 | 4 votes |
|
@SuppressWarnings("unchecked")
@Test
public void testQueryAggregateWithAlias() throws Exception
{
JDBCTestUtils.setConnection(null);
JSONObject obj = new JSONObject();
HashMap<String, Object> map = new HashMap<String, Object>();
map.put("name", "test_name");
map.put("specialchars", "()*&^%$!@{}{}:\"\\\';::");
map.put("id", 12345);
map.put("double", 12345.333);
map.put("data_time", "2001-01-01 01:01:01.00");
obj.putAll(map);
JSONArray expectedArray = new JSONArray();
HashMap<String,JSONObject> objMap = new HashMap<String,JSONObject>();
objMap.put("1_testQueryAggregateWithAlias", obj);
JSONObject expectedObject = new JSONObject();
expectedObject.put("name", "test_value");
expectedArray.add(expectedObject);
JDBCTestUtils.insertData(objMap, "default");
Thread.sleep(5000);
String query = "select count(*) as alias_count from default";
JDBCTestUtils.resetConnection();
try ( Connection con = JDBCTestUtils.con)
{
try (Statement stmt = con.createStatement())
{
try (ResultSet rs = stmt.executeQuery(query))
{
while(rs.next()){
CBResultSet cbrs = (CBResultSet) rs;
java.sql.ResultSetMetaData meta = cbrs.getMetaData();
SQLJSON jsonVal1 = cbrs.getSQLJSON("alias_count");
int actual = jsonVal1.getInt();
assertEquals(1, actual);
}
}
finally{
stmt.executeUpdate("delete from default");
Thread.sleep(10000);
}
}
}
}
Example 12
| Source File: JSONWriter.java From netbeans with Apache License 2.0 | 4 votes |
|
private static JSONObject store(Map<String, Object> map) {
JSONObject obj = newJSONObject();
obj.putAll(map);
return obj;
}
Example 13
| Source File: DataProcessAndPublishingAgent.java From carbon-apimgt with Apache License 2.0 | 4 votes |
|
public void run() {
JSONObject jsonObMap = new JSONObject();
org.apache.axis2.context.MessageContext axis2MessageContext = ((Axis2MessageContext) messageContext)
.getAxis2MessageContext();
//Set transport headers of the message
TreeMap<String, String> transportHeaderMap = (TreeMap<String, String>) axis2MessageContext
.getProperty(org.apache.axis2.context.MessageContext.TRANSPORT_HEADERS);
String remoteIP = GatewayUtils.getIp(axis2MessageContext);
if (log.isDebugEnabled()) {
log.debug("Remote IP address : " + remoteIP);
}
if (remoteIP != null && remoteIP.length() > 0) {
try {
InetAddress address = APIUtil.getAddress(remoteIP);
if (address instanceof Inet4Address) {
jsonObMap.put(APIThrottleConstants.IP, APIUtil.ipToLong(remoteIP));
jsonObMap.put(APIThrottleConstants.IPv6, 0);
} else if (address instanceof Inet6Address) {
jsonObMap.put(APIThrottleConstants.IPv6, APIUtil.ipToBigInteger(remoteIP));
jsonObMap.put(APIThrottleConstants.IP, 0);
}
} catch (UnknownHostException e) {
//send empty value as ip
log.error("Error while parsing host IP " + remoteIP, e);
jsonObMap.put(APIThrottleConstants.IPv6, 0);
jsonObMap.put(APIThrottleConstants.IP, 0);
}
}
//HeaderMap will only be set if the Header Publishing has been enabled.
if (this.headersMap != null) {
jsonObMap.putAll(this.headersMap);
}
//Setting query parameters
if (getThrottleProperties().isEnableQueryParamConditions()) {
Map<String, String> queryParams = GatewayUtils.getQueryParams(axis2MessageContext);
if (queryParams != null) {
jsonObMap.putAll(queryParams);
}
}
//Publish jwt claims
if (getThrottleProperties().isEnableJwtConditions()) {
if (authenticationContext.getCallerToken() != null) {
Map assertions = GatewayUtils.getJWTClaims(authenticationContext);
if (assertions != null) {
jsonObMap.putAll(assertions);
}
}
}
//this parameter will be used to capture message size and pass it to calculation logic
ArrayList<VerbInfoDTO> list = (ArrayList<VerbInfoDTO>) messageContext.getProperty(APIConstants.VERB_INFO_DTO);
boolean isVerbInfoContentAware = false;
if (list != null && !list.isEmpty()) {
VerbInfoDTO verbInfoDTO = list.get(0);
isVerbInfoContentAware = verbInfoDTO.isContentAware();
}
if (authenticationContext.isContentAwareTierPresent() || isVerbInfoContentAware) {
if (log.isDebugEnabled()) {
log.debug("Message size: " + messageSizeInBytes + "B");
}
jsonObMap.put(APIThrottleConstants.MESSAGE_SIZE, messageSizeInBytes);
if (!StringUtils.isEmpty(authenticationContext.getApplicationName())) {
jsonObMap.put(APIThrottleConstants.APPLICATION_NAME, authenticationContext.getApplicationName());
}
if (!StringUtils.isEmpty(authenticationContext.getProductName()) && !StringUtils
.isEmpty(authenticationContext.getProductProvider())) {
jsonObMap.put(APIThrottleConstants.SUBSCRIPTION_TYPE, APIConstants.API_PRODUCT_SUBSCRIPTION_TYPE);
} else {
jsonObMap.put(APIThrottleConstants.SUBSCRIPTION_TYPE, APIConstants.API_SUBSCRIPTION_TYPE);
}
}
Object[] objects = new Object[]{messageContext.getMessageID(),
this.applicationLevelThrottleKey, this.applicationLevelTier,
this.apiLevelThrottleKey, this.apiLevelTier,
this.subscriptionLevelThrottleKey, this.subscriptionLevelTier,
this.resourceLevelThrottleKey, this.resourceLevelTier,
this.authorizedUser, this.apiContext, this.apiVersion,
this.appTenant, this.apiTenant, this.appId, this.apiName, jsonObMap.toString()};
org.wso2.carbon.databridge.commons.Event event = new org.wso2.carbon.databridge.commons.Event(streamID,
System.currentTimeMillis(), null, null, objects);
dataPublisher.tryPublish(event);
}
Example 14
| Source File: ProjectionJDBCDriverTests.java From jdbc-cb with Apache License 2.0 | 4 votes |
|
@SuppressWarnings("unchecked")
@Test
public void testArrayAndNestedSelectAllData() throws Exception
{
JDBCTestUtils.setConnection(null);
JSONObject obj = new JSONObject();
JSONObject jsonObjNew = new JSONObject();
HashMap<String, Object> map = new HashMap<String, Object>();
JSONObject nestedObject = new JSONObject();
JSONArray jsonarrayForNesting = new JSONArray();
for(int i=0;i<100;++i){
jsonarrayForNesting.add(i);
}
nestedObject.put("nested_array", jsonarrayForNesting);
JSONArray expectedArray = new JSONArray();
obj.put("nested_data", nestedObject);
obj.putAll(map);
HashMap<String,JSONObject> objMap = new HashMap<String,JSONObject>();
objMap.put("1_testArrayAndNestedSelectAllData", obj);
expectedArray.add(obj);
JDBCTestUtils.insertData(objMap, "default");
Thread.sleep(30000);
String query = "select nested_data.* from default";
JDBCTestUtils.resetConnection();
try ( Connection con = JDBCTestUtils.con)
{
try (Statement stmt = con.createStatement())
{
try (ResultSet rs = stmt.executeQuery(query))
{
while(rs.next()){
CBResultSet cbrs = (CBResultSet) rs;
java.sql.ResultSetMetaData meta = cbrs.getMetaData();
SQLJSON jsonVal1 = cbrs.getSQLJSON("nested_array");
Object obj1 = jsonVal1.getObject();
assertEquals(jsonarrayForNesting.toString().replace(" ",""), obj1.toString().replace(" ",""));
}
}
finally{
stmt.executeUpdate("delete from default");
Thread.sleep(10000);
}
}
}
}
Example 15
| Source File: ProjectionJDBCDriverTests.java From jdbc-cb with Apache License 2.0 | 4 votes |
|
@SuppressWarnings("unchecked")
@Test
public void testNestedOperator() throws Exception
{
JDBCTestUtils.setConnection(null);
JSONObject obj = new JSONObject();
JSONObject nestedobj = new JSONObject();
JSONObject jsonObjNew = new JSONObject();
HashMap<String, Object> map = new HashMap<String, Object>();
nestedobj.put("nested.nested", "1");
map.put("name", "test_name");
map.put("specialchars", "()*&^%$!@{}{}:\"\\\';::");
map.put("id", 12345);
map.put("double", 12345.333);
map.put("data_time", "2001-01-01 01:01:01.00");
obj.put("nested", nestedobj);
obj.putAll(map);
JSONArray expectedArray = new JSONArray();
HashMap<String,JSONObject> objMap = new HashMap<String,JSONObject>();
objMap.put("1_testNestedOperator", obj);
expectedArray.add(nestedobj);
JDBCTestUtils.insertData(objMap, "default");
Thread.sleep(10000);
String query = "select default.nested.* from default";
JDBCTestUtils.resetConnection();
try ( Connection con = JDBCTestUtils.con)
{
try (Statement stmt = con.createStatement())
{
try (ResultSet rs = stmt.executeQuery(query))
{
while(rs.next()){
assertEquals("1", rs.getString("nested.nested"));
}
}
finally{
stmt.executeUpdate("delete from default");
Thread.sleep(10000);
}
}
}
}
Example 16
| Source File: ProjectionJDBCDriverTests.java From jdbc-cb with Apache License 2.0 | 4 votes |
|
@SuppressWarnings({ "unchecked", "deprecation", "resource" })
@Test
public void testSimpleDataDifferentDataTypesAsFields() throws Exception
{
JDBCTestUtils.setConnection(null);
JSONObject obj = new JSONObject();
HashMap<String, Object> map = new HashMap<String, Object>();
map.put("name", "test_name");
map.put("specialchars", "()*&^%$!@{}{}:\"\\\';::");
map.put("int", 12345);
map.put("double", 12345.333);
map.put("data_time", "2001-01-01 01:01:01.00");
obj.putAll(map);
JSONArray expectedArray = new JSONArray();
HashMap<String,JSONObject> objMap = new HashMap<String,JSONObject>();
objMap.put("1_testSimpleDataDifferentDataTypesAsFields", obj);
expectedArray.add(obj);
JDBCTestUtils.insertData(objMap, "default");
Thread.sleep(20000);
String query = "select name,specialchars,int,double,data_time from default";
JDBCTestUtils.setConnection(null);
try ( Connection con = JDBCTestUtils.con)
{
try (Statement stmt = con.createStatement())
{
try (ResultSet rs = stmt.executeQuery(query))
{
while(rs.next()){
CBResultSet cbrs = (CBResultSet) rs;
java.sql.ResultSetMetaData meta = cbrs.getMetaData();
SQLJSON jsonVal1 = cbrs.getSQLJSON("name");
assertEquals(12, jsonVal1.getJDBCType());
assertEquals("test_name", jsonVal1.getString());
jsonVal1 = cbrs.getSQLJSON("specialchars");
assertEquals(12, jsonVal1.getJDBCType());
assertEquals("()*&^%$!@{}{}:\"\\\';::", jsonVal1.getString());
jsonVal1 = cbrs.getSQLJSON("int");
assertEquals(2, jsonVal1.getJDBCType());
assertEquals(12345, jsonVal1.getInt());
jsonVal1 = cbrs.getSQLJSON("double");
assertEquals(2, jsonVal1.getJDBCType());
//assertEquals(12345.333, jsonVal1.getDouble());
}
}
finally{
stmt.executeUpdate("delete from default");
Thread.sleep(10000);
}
}
}
}
Example 17
| Source File: ProjectionJDBCDriverTests.java From jdbc-cb with Apache License 2.0 | 4 votes |
|
@SuppressWarnings("unchecked")
@Test
public void testSimpleDataWithNull() throws Exception
{
JDBCTestUtils.setConnection(null);
JDBCTestUtils.deleteDataFromBucket("default");
JSONObject obj = new JSONObject();
JSONObject jsonObjNew = new JSONObject();
HashMap<String, Object> map = new HashMap<String, Object>();
map.put("name", null);
obj.putAll(map);
JSONArray expectedArray = new JSONArray();
HashMap<String,JSONObject> objMap = new HashMap<String,JSONObject>();
objMap.put("1_testSimpleDataWithNull", obj);
expectedArray.add(obj);
JDBCTestUtils.insertData(objMap, "default");
Thread.sleep(5000);
String query = "select * from default";
JDBCTestUtils.resetConnection();
try ( Connection con = JDBCTestUtils.con)
{
try (Statement stmt = con.createStatement())
{
try (ResultSet rs = stmt.executeQuery(query))
{
while(rs.next()){
CBResultSet cbrs = (CBResultSet) rs;
java.sql.ResultSetMetaData meta = cbrs.getMetaData();
SQLJSON jsonVal1 = cbrs.getSQLJSON("default");
Map actualMap = jsonVal1.getMap();
if(actualMap != null){
jsonObjNew.putAll(actualMap);
}
assertEquals(obj, jsonObjNew);
}
}
finally{
stmt.executeUpdate("delete from default");
Thread.sleep(10000);
}
}
}
}
Example 18
| Source File: ProjectionJDBCDriverTests.java From jdbc-cb with Apache License 2.0 | 4 votes |
|
@SuppressWarnings("unchecked")
@Test
public void testNulls() throws Exception
{
JDBCTestUtils.setConnection(null);
JDBCTestUtils.deleteDataFromBucket("default");
JSONObject obj = new JSONObject();
JSONObject jsonObjNew = new JSONObject();
HashMap<String, Object> map = new HashMap<String, Object>();
map.put("name", null);
obj.putAll(map);
JSONArray expectedArray = new JSONArray();
HashMap<String,JSONObject> objMap = new HashMap<String,JSONObject>();
objMap.put("1_testNulls", obj);
expectedArray.add(obj);
JDBCTestUtils.insertData(objMap, "default");
Thread.sleep(5000);
String query = "select * from default";
JDBCTestUtils.resetConnection();
try ( Connection con = JDBCTestUtils.con)
{
try (Statement stmt = con.createStatement())
{
try (ResultSet rs = stmt.executeQuery(query))
{
while(rs.next()){
CBResultSet cbrs = (CBResultSet) rs;
java.sql.ResultSetMetaData meta = cbrs.getMetaData();
SQLJSON jsonVal1 = cbrs.getSQLJSON("default");
Map actualMap = jsonVal1.getMap();
if(actualMap != null){
jsonObjNew.putAll(actualMap);
}
assertEquals(obj, jsonObjNew);
}
}
finally{
stmt.executeUpdate("delete from default");
Thread.sleep(10000);
}
}
}
}
Example 19
| Source File: ProtobufParquetFileReaderWriterFactoryTest.java From secor with Apache License 2.0 | 4 votes |
|
@Test
public void testJsonParquetReadWriteRoundTrip() throws Exception {
Map<String, String> classPerTopic = new HashMap<String, String>();
classPerTopic.put("test-pb-topic", UnitTestMessage3.class.getName());
Map<String, String> formatForAll = new HashMap<String, String>();
formatForAll.put("*", "JSON");
Mockito.when(config.getProtobufMessageClassPerTopic()).thenReturn(classPerTopic);
Mockito.when(config.getMessageFormatPerTopic()).thenReturn(formatForAll);
Mockito.when(config.getFileReaderWriterFactory())
.thenReturn(ProtobufParquetFileReaderWriterFactory.class.getName());
Mockito.when(ParquetUtil.getParquetBlockSize(config))
.thenReturn(ParquetWriter.DEFAULT_BLOCK_SIZE);
Mockito.when(ParquetUtil.getParquetPageSize(config))
.thenReturn(ParquetWriter.DEFAULT_PAGE_SIZE);
Mockito.when(ParquetUtil.getParquetEnableDictionary(config))
.thenReturn(ParquetWriter.DEFAULT_IS_DICTIONARY_ENABLED);
Mockito.when(ParquetUtil.getParquetValidation(config))
.thenReturn(ParquetWriter.DEFAULT_IS_VALIDATING_ENABLED);
LogFilePath tempLogFilePath = new LogFilePath(Files.createTempDir().toString(), "test-pb-topic",
new String[] { "part-1" }, 0, 1, 23232, ".log");
FileWriter fileWriter = ReflectionUtil.createFileWriter(config.getFileReaderWriterFactory(), tempLogFilePath,
null, config);
UnitTestMessage3 protomsg1 = UnitTestMessage3.newBuilder().setData("abc").setTimestamp(1467176315L).build();
UnitTestMessage3 protomsg2 = UnitTestMessage3.newBuilder().setData("XYZ").setTimestamp(1467176344L).build();
Map jsonValues = new HashMap();
jsonValues.put("data", "abc");
jsonValues.put("timestamp", 1467176315L);
JSONObject json = new JSONObject();
json.putAll(jsonValues);
String msg1 = json.toJSONString();
jsonValues.put("data", "XYZ");
jsonValues.put("timestamp", 1467176344L);
json.putAll(jsonValues);
String msg2 = json.toJSONString();
KeyValue kv1 = (new KeyValue(23232, msg1.getBytes()));
KeyValue kv2 = (new KeyValue(23233, msg2.getBytes()));
fileWriter.write(kv1);
fileWriter.write(kv2);
fileWriter.close();
FileReader fileReader = ReflectionUtil.createFileReader(config.getFileReaderWriterFactory(), tempLogFilePath,
null, config);
KeyValue kvout = fileReader.next();
assertEquals(kv1.getOffset(), kvout.getOffset());
assertEquals(protomsg1.getData(), UnitTestMessage3.parseFrom(kvout.getValue()).getData());
kvout = fileReader.next();
assertEquals(kv2.getOffset(), kvout.getOffset());
assertEquals(protomsg2.getData(), UnitTestMessage3.parseFrom(kvout.getValue()).getData());
}
Example 20
| Source File: GrokParser.java From metron with Apache License 2.0 | 4 votes |
|
@SuppressWarnings("unchecked")
private Optional<MessageParserResult<JSONObject>> parseSingleLine(byte[] rawMessage) {
List<JSONObject> messages = new ArrayList<>();
Map<Object,Throwable> errors = new HashMap<>();
String originalMessage = null;
try {
originalMessage = new String(rawMessage, StandardCharsets.UTF_8);
LOG.debug("Grok parser parsing message: {}",originalMessage);
Match gm = grok.match(originalMessage);
gm.captures();
JSONObject message = new JSONObject();
message.putAll(gm.toMap());
if (message.size() == 0) {
Throwable rte = new RuntimeException("Grok statement produced a null message. Original message was: "
+ originalMessage + " and the parsed message was: " + message + " . Check the pattern at: "
+ grokPath);
errors.put(originalMessage, rte);
} else {
message.put("original_string", originalMessage);
for (String timeField : timeFields) {
String fieldValue = (String) message.get(timeField);
if (fieldValue != null) {
message.put(timeField, toEpoch(fieldValue));
}
}
if (timestampField != null) {
message.put(Constants.Fields.TIMESTAMP.getName(), formatTimestamp(message.get(timestampField)));
}
message.remove(patternLabel);
postParse(message);
messages.add(message);
LOG.debug("Grok parser parsed message: {}", message);
}
} catch (Exception e) {
LOG.error(e.getMessage(), e);
Exception innerException = new IllegalStateException("Grok parser Error: "
+ e.getMessage()
+ " on "
+ originalMessage, e);
return Optional.of(new DefaultMessageParserResult<>(innerException));
}
return Optional.of(new DefaultMessageParserResult<JSONObject>(messages, errors));
}
