Java Code Examples for com.esotericsoftware.minlog.Log#error()

@Override
public List<EntityProfile> getEntityProfiles() {
    if (!entityProfiles.isEmpty()) {
        return entityProfiles;
    }

    if (inputFilePath == null) {
        Log.error("Input file path has not been set!");
        return null;
    }

    final SAXBuilder saxBuilder = new SAXBuilder();
    try {
        final Document document = saxBuilder.build(inputFilePath);
        readXMLdoc(document);
    } catch (JDOMException | IOException e) {
        Log.error("Error in entities reading!", e);
    }

    return entityProfiles;
}
 

protected float getARCSSimilarity(CharacterNGramsWithGlobalWeights oModel) {
    final Set<String> commonKeys = new HashSet(itemsFrequency.keySet());
    commonKeys.retainAll(oModel.getItemsFrequency().keySet());

    float similarity = 0;
    if (datasetId == DATASET_1 && datasetId == oModel.getDatasetId()) { // Dirty ER
        similarity = commonKeys.stream().map((key) -> DOC_FREQ[DATASET_1].get(key)).map((frequency) -> 1.0f / ((float) Math.log1p(frequency * (frequency - 1.0f) / 2.0f) / (float) Math.log(2))).reduce(similarity, (accumulator, _item) -> accumulator + _item);
    } else if (datasetId != oModel.getDatasetId()) { // Clean-Clean ER
        similarity = commonKeys.stream().map((key) -> 1.0f / ((float) Math.log1p(((float) DOC_FREQ[DATASET_1].get(key)) * DOC_FREQ[DATASET_2].get(key)) / (float) Math.log(2))).reduce(similarity, (accumulator, _item) -> accumulator + _item);
    } else {
        Log.error("Both models come from dataset 1!");
        System.exit(-1);
    }

    return similarity;
}
 

private Connection getPostgreSQLconnection(String dbURL) throws IOException {
    try {
        final Properties props = new Properties();
        if (!(dbuser == null)) {
            props.setProperty("user", dbuser);
        }
        if (!(dbpassword == null)) {
            props.setProperty("password", dbpassword);
        }
        if (ssl) {
            props.setProperty("ssl", "true");
        }
        return DriverManager.getConnection("jdbc:" + dbURL, props);
    } catch (SQLException ex) {
        Log.error("Error with database connection!", ex);
        return null;
    }
}
 

@Override
public float getSimilarity(ITextModel oModel) {
    switch (simMetric) {
        case ARCS_SIMILARITY:
            return getARCSSimilarity((CharacterNGramsWithGlobalWeights) oModel);
        case COSINE_SIMILARITY:
            return getTfIdfCosineSimilarity((CharacterNGramsWithGlobalWeights) oModel);
        case GENERALIZED_JACCARD_SIMILARITY:
            return getTfIdfGeneralizedJaccardSimilarity((CharacterNGramsWithGlobalWeights) oModel);
        case SIGMA_SIMILARITY:
            return getSigmaSimilarity((CharacterNGramsWithGlobalWeights) oModel);
        default:
            Log.error("The given similarity metric is incompatible with the bag representation model!");
            System.exit(-1);
            return -1;
    }
}
 

@Override
public List<EntityProfile> getEntityProfiles() {
    if (!entityProfiles.isEmpty()) {
        return entityProfiles;
    }

    if (inputFilePath == null) {
        Log.error("Input file path has not been set!");
        return null;
    }

    //load the rdf model from the input file
    try {
        final Model model = RDFDataMgr.loadModel(inputFilePath);
        readModel(model);
    } catch (IOException ex) {
        Log.error("Error in entities reading!", ex);
        return null;
    }

    return entityProfiles;
}
 

@Override
public void developBlockBasedSchedule(List<AbstractBlock> blocks) {
    if (blocks == null || blocks.isEmpty()) {
        Log.error("No blocks were given as input!");
        System.exit(-1);
    }

    if (blocks.get(0) instanceof DecomposedBlock) {
        Log.warn("Decomposed blocks were given as input!");
        Log.warn("The pre-computed comparison weights will be used!");
        
        compIterator = processDecomposedBlocks(blocks);
    } else {
        final ProgressiveCEP pcep = new ProgressiveCEP(comparisonsBudget, wScheme);
        pcep.refineBlocks(blocks);
        compIterator = pcep.getTopComparisons().iterator();
    }
}
 

@SuppressWarnings("deprecation")
public String processTemplate(String templateUrl, Map<String, Object> model) {
	try {
		if(templateUrl != null) {
			String mailBody = mailContentBuilderService.getRemoteMailContent(templateUrl);
			Configuration cfg = new Configuration();
			cfg.setObjectWrapper(new DefaultObjectWrapper());
			Template t = new Template(UUID.randomUUID().toString(), new StringReader(mailBody), cfg);
		    Writer out = new StringWriter();
			t.process(model, out);
			return out.toString();
		}
	} catch (Exception exception) {
		Log.error(exception.getMessage());
	}
	return null;
}
 

public void printToCSV(String filename) throws FileNotFoundException {
    final PrintWriter pw = new PrintWriter(new File(filename));
    final StringBuilder sb = new StringBuilder();

    sb.append("cluster_id,dataset,entity_url\n");
    int counter = 0;
    for (EquivalenceCluster eqc : entityClusters) {
        if (eqc.getEntityIdsD1().isEmpty()) {
            continue;
        }
        counter++;
        for (TIntIterator iterator = eqc.getEntityIdsD1().iterator(); iterator.hasNext(); ) {
            sb.append(counter).append(",").append(1).append(",")
                    .append(profilesD1.get(iterator.next()).getEntityUrl()).append("\n");
        }
        if (eqc.getEntityIdsD2().isEmpty()) {
            continue;
        }
        if (profilesD2 == null) {
            Log.error("The entity profiles of Dataset 2 are missing!");
            continue;
        }
        for (TIntIterator iterator = eqc.getEntityIdsD2().iterator(); iterator.hasNext(); ) {
            sb.append(counter).append(",").append(2).append(",")
                    .append(profilesD2.get(iterator.next()).getEntityUrl()).append("\n");
        }

    }
    pw.write(sb.toString());
    pw.close();
}
 

@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
    String alert = (String) tuple.getValue(0);
    Log.error("ALERT RECEIVED [" + alert + "]");
    Log.error("Dispatch the national guard!");
    System.exit(0);
}
 

@Override
public Set<IdDuplicates> getDuplicatePairs(List<EntityProfile> profilesD1,
        List<EntityProfile> profilesD2) {
    if (!idDuplicates.isEmpty()) {
        return idDuplicates;
    }

    if (profilesD1 == null) {
        Log.error("First list of entity profiles is null! "
                + "The first argument should always contain entities.");
        return null;
    }

    initializeDataStructures(profilesD1, profilesD2);
    try {
        performReading();
    } catch (NoSuchElementException ex) {
        Log.error("Error in duplicates reading!", ex);
        return null;
    }
    Log.info("Total edges in duplicates graph\t:\t" + duplicatesGraph.edgeSet().size());

    // get connected components
    final ConnectivityInspector ci = new ConnectivityInspector(duplicatesGraph);
    final List<Set<Integer>> connectedComponents = ci.connectedSets();
    Log.info("Total connected components in duplicate graph\t:\t" + connectedComponents.size());

    // transform connected components into pairs of duplicates
    if (profilesD2 != null) { // Clean-Clean ER
        getBilateralConnectedComponents(connectedComponents);
    } else { // Dirty ER
        getUnilateralConnectedComponents(connectedComponents);
    }
    Log.info("Total pair of duplicats\t:\t" + idDuplicates.size());

    return idDuplicates;
}
 

public void convertToRDFfile(List<EntityProfile> profiles, String outputPath, String basicURI) {
    try {
        FileWriter fileWriter = new FileWriter(outputPath);
        PrintWriter printWriter = new PrintWriter(fileWriter);
        printWriter.println("<?xml version=\"1.0\"?>");
        printWriter.println();
        printWriter.println("<rdf:RDF");
        printWriter.println("xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\"");
        printWriter.println("xmlns:obj=\"https://www.w3schools.com/rdf/\">");
        profiles.stream().map((profile) -> {
            printWriter.println("<rdf:Description rdf:about=\"" + basicURI + profile.getEntityUrl().replace("&", "") + "\">");
            return profile;
        }).map((profile) -> {
            profile.getAttributes().stream().map((attribute) -> {
                printWriter.print("<obj:" + attribute.getName().replace("&", "") + ">");
                return attribute;
            }).map((attribute) -> {
                printWriter.print(attribute.getValue().replace("&", ""));
                return attribute;
            }).forEachOrdered((attribute) -> {
                printWriter.println("</obj:" + attribute.getName().replace("&", "") + ">");
            });
            return profile;
        }).forEachOrdered((_item) -> {
            printWriter.println("</rdf:Description>");
        });
        printWriter.println("</rdf:RDF>");
        printWriter.close();
    } catch (IOException ioex) {
        Log.error("Error in converting to RDF", ioex);
    }
}
 

public ExtendedCanopyClustering(int inThr, int outThr, WeightingScheme scheme) {
    super(scheme);
    nodeCentric = true;
    exclusiveThreshold = outThr;
    inclusiveThreshold = inThr;
    if (inclusiveThreshold < exclusiveThreshold) {
        Log.error(getMethodName(), "The Exclusive Threshold cannot be larger than the Inclusive one.");
        System.exit(-1);
    }
}
 

/**
 * Load pretrained embedding weights autoresolving the embedding dimension
 */
private void loadWeights() {
    if (weightsLoaded) return;
    ClassLoader classLoader = getClass().getClassLoader();
    //String fileName = classLoader.getResource("embeddings/weights-full.txt").getFile();
    String fileName = Objects.requireNonNull(classLoader.getResource("embeddings/weights.txt")).getFile();
    elementMap = new HashMap<>();

    try {
        BufferedReader br = new BufferedReader(new FileReader(fileName));

        CSVReader reader = new CSVReader(new FileReader(fileName), dataSeparator, CSVParser.NULL_CHARACTER, 0);

        String[] components;
        int counter = 0;
        while ((components = reader.readNext()) != null) {
            counter++;
            if (counter > 1) {
                if (components.length != dimension + 1)
                    throw new IOException(String.format("Mismatch in embedding vector #%d length : %d.",
                            counter, components.length));
            } else {
                dimension = components.length - 1;
            }
            float[] value = new float[dimension];
            for (int i = 1; i <= dimension; ++i) {
                value[i - 1] = Float.parseFloat(components[i]);
            }
            elementMap.put(components[0], value);
        }
    } catch (IOException e) {
        Log.error("Problem loading embedding weights", e);
        System.exit(-1);
    }
}
 

protected float getIdfWeight(String keyValue) {
    int frequency = DOC_FREQ[datasetId].get(keyValue);
    if (frequency == 0) {
        return 0;
    }

    if (NO_OF_DOCUMENTS[datasetId] < frequency) {
        Log.error("Error in the computation of IDF weights!!!");
        return 0;
    }
    
    return (float) Math.log10(NO_OF_DOCUMENTS[datasetId] / (1.0f + frequency));
}
 

public void printToRDFNT(String filename) throws FileNotFoundException {
    final PrintWriter printWriter = new PrintWriter(new File(filename));

    String xmlnsrdf = "http://www.w3.org/1999/02/22/";
    String xmlnsobj = "https://www.w3schools.com/rdf/";

    int counter = 0;
    for (EquivalenceCluster eqc : entityClusters) {
        if (eqc.getEntityIdsD1().isEmpty()) {
            continue;
        }
        counter++;
        for (TIntIterator iterator = eqc.getEntityIdsD1().iterator(); iterator.hasNext(); ) {

            printWriter.print("<" + xmlnsrdf + counter + "> ");
            printWriter.print("<" + xmlnsobj + "cluster_id" + "> \"");
            printWriter.print(counter + "");
            printWriter.println("\"^^<http://www.w3.org/2001/XMLSchema#integer>");

            printWriter.print("<" + xmlnsrdf + counter + "> ");
            printWriter.print("<" + xmlnsobj + "dataset" + "> \"");
            printWriter.print("1");
            printWriter.println("\"^^<http://www.w3.org/2001/XMLSchema#integer>");

            printWriter.print("<" + xmlnsrdf + counter + "> ");
            printWriter.print("<" + xmlnsobj + "entity_url" + "> \"");
            printWriter.print(profilesD1.get(iterator.next()).getEntityUrl().replace("&", "") + "");
            printWriter.println("\"^^<http://www.w3.org/2001/XMLSchema#string>");

        }
        if (eqc.getEntityIdsD2().isEmpty()) {
            continue;
        }
        if (profilesD2 == null) {
            Log.error("The entity profiles of Dataset 2 are missing!");
            continue;
        }
        for (TIntIterator iterator = eqc.getEntityIdsD2().iterator(); iterator.hasNext(); ) {

            printWriter.print("<" + xmlnsrdf + counter + "> ");
            printWriter.print("<" + xmlnsobj + "cluster_id" + "> \"");
            printWriter.print(counter + "");
            printWriter.println("\"^^<http://www.w3.org/2001/XMLSchema#integer>");

            printWriter.print("<" + xmlnsrdf + counter + "> ");
            printWriter.print("<" + xmlnsobj + "dataset" + "> \"");
            printWriter.print("2");
            printWriter.println("\"^^<http://www.w3.org/2001/XMLSchema#integer>");

            printWriter.print("<" + xmlnsrdf + counter + "> ");
            printWriter.print("<" + xmlnsobj + "entity_url" + "> \"");
            printWriter.print(profilesD2.get(iterator.next()).getEntityUrl().replace("&", "") + "");
            printWriter.println("\"^^<http://www.w3.org/2001/XMLSchema#string>");

        }

    }

}
 

@Override
public Comparison next() {
    if (totalComparisons <= executedComparisons) {
        Log.error("All comparisons were already executed!");
        return null;
    }

    executedComparisons++;
    if (block instanceof BilateralBlock) {
        BilateralBlock bilBlock = (BilateralBlock) block;
        innerLoop++;
        if (innerLimit < innerLoop) {
            innerLoop = 0;
            outerLoop++;
            if (outerLimit < outerLoop) {
                Log.error("All comparisons were already executed!");
                return null;
            }
        }

        return new Comparison(true, bilBlock.getIndex1Entities()[outerLoop], bilBlock.getIndex2Entities()[innerLoop]);
    } else if (block instanceof UnilateralBlock) {
        UnilateralBlock uniBlock = (UnilateralBlock) block;
        innerLoop++;
        if (innerLimit < innerLoop) {
            outerLoop++;
            if (outerLimit < outerLoop) {
                Log.error("All comparisons were already executed!");
                return null;
            }
            innerLoop = outerLoop + 1;
        }

        if (uniBlock.getEntities()[outerLoop] < uniBlock.getEntities()[innerLoop]) {
            return new Comparison(false, uniBlock.getEntities()[outerLoop], uniBlock.getEntities()[innerLoop]);
        } else if (uniBlock.getEntities()[innerLoop] < uniBlock.getEntities()[outerLoop]) {
            return new Comparison(false, uniBlock.getEntities()[innerLoop], uniBlock.getEntities()[outerLoop]);
        }
    } else if (block instanceof DecomposedBlock) {
        DecomposedBlock deBlock = (DecomposedBlock) block;
        outerLoop++;
        Comparison c = null;
        if (deBlock.isCleanCleanER()) {
            c = new Comparison(deBlock.isCleanCleanER(), deBlock.getEntities1()[outerLoop], deBlock.getEntities2()[outerLoop]);
        } else {
            if (deBlock.getEntities1()[outerLoop] < deBlock.getEntities2()[outerLoop]) {
                c = new Comparison(deBlock.isCleanCleanER(), deBlock.getEntities1()[outerLoop], deBlock.getEntities2()[outerLoop]);
            } else if (deBlock.getEntities2()[outerLoop] < deBlock.getEntities1()[outerLoop]) {
                c = new Comparison(deBlock.isCleanCleanER(), deBlock.getEntities2()[outerLoop], deBlock.getEntities1()[outerLoop]);
            }
        }
        c.setUtilityMeasure(deBlock.getWeights()[outerLoop] / DISCRETIZATION_FACTOR);
        return c;
    }

    return null;
}
 

public void printToXML(String filename) throws FileNotFoundException {
    final PrintWriter printWriter = new PrintWriter(new File(filename));

    printWriter.println("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
    printWriter.println();

    printWriter.println("<general>");

    int counter = 0;
    for (EquivalenceCluster eqc : entityClusters) {
        if (eqc.getEntityIdsD1().isEmpty()) {
            continue;
        }
        counter++;
        for (TIntIterator iterator = eqc.getEntityIdsD1().iterator(); iterator.hasNext(); ) {
            printWriter.println();
            printWriter.println("<entity" + counter + ">");

            printWriter.print("<cluster_id" + ">");
            printWriter.print(counter + "");
            printWriter.println("</cluster_id>");

            printWriter.print("<dataset" + ">");
            printWriter.print("1");
            printWriter.println("</dataset>");

            printWriter.print("<entity_url" + ">");
            printWriter.print(profilesD1.get(iterator.next()).getEntityUrl().replace("&", "") + "");
            printWriter.println("</entity_url" + ">");

            printWriter.println("</entity" + counter + ">");

        }
        if (eqc.getEntityIdsD2().isEmpty()) {
            continue;
        }
        if (profilesD2 == null) {
            Log.error("The entity profiles of Dataset 2 are missing!");
            continue;
        }
        for (TIntIterator iterator = eqc.getEntityIdsD2().iterator(); iterator.hasNext(); ) {

            printWriter.println();

            printWriter.println("<entity" + counter + ">");

            printWriter.print("<cluster_id" + ">");
            printWriter.print(counter + "");
            printWriter.println("</cluster_id>");

            printWriter.print("<dataset" + ">");
            printWriter.print("2");
            printWriter.println("</dataset>");

            printWriter.print("<entity_url" + ">");
            printWriter.print(profilesD2.get(iterator.next()).getEntityUrl().replace("&", "") + "");
            printWriter.println("</entity_url" + ">");

            printWriter.println("</entity" + counter + ">");

        }

    }
    printWriter.println("</general>");

    printWriter.close();
}
 

public void printToDB(String dbURL) throws FileNotFoundException {
    final StringBuilder sb = new StringBuilder();

    String dbquery1 = "INSERT INTO " + dbtable + " (cluster_id, dataset, entity_url) VALUES ";
    sb.append(dbquery1);

    int counter = 0;
    for (EquivalenceCluster eqc : entityClusters) {
        if (eqc.getEntityIdsD1().isEmpty()) {
            continue;
        }
        counter++;
        for (TIntIterator iterator = eqc.getEntityIdsD1().iterator(); iterator.hasNext(); ) {

            sb.append("('").append(counter).append("', ");
            sb.append("'" + "1" + "', ");
            sb.append("'").append(profilesD1.get(iterator.next()).getEntityUrl().replace("&", "")).append("'), ");

        }
        if (eqc.getEntityIdsD2().isEmpty()) {
            continue;
        }
        if (profilesD2 == null) {
            Log.error("The entity profiles of Dataset 2 are missing!");
            continue;
        }
        for (TIntIterator iterator = eqc.getEntityIdsD2().iterator(); iterator.hasNext(); ) {

            sb.append("('").append(counter).append("', ");
            sb.append("'" + "2" + "', ");
            sb.append("'").append(profilesD2.get(iterator.next()).getEntityUrl().replace("&", "")).append("'), ");
        }
    }

    sb.setLength(sb.length() - 2);//remove last ","
    sb.append(";");
    String dbquery = sb.toString();

    try {
        if (dbuser == null) {
            Log.error("Database user has not been set!");
        }
        if (dbpassword == null) {
            Log.error("Database password has not been set!");
        }
        if (dbtable == null) {
            Log.error("Database table has not been set!");
        }

        Connection conn = null;
        if (dbURL.startsWith("mysql")) {
            conn = getMySQLconnection(dbURL);
        } else if (dbURL.startsWith("postgresql")) {
            conn = getPostgreSQLconnection(dbURL);
        } else {
            Log.error("Only MySQL and PostgreSQL are supported for the time being!");
        }

        final Statement stmt = conn.createStatement();
        stmt.executeQuery(dbquery);//retrieve the appropriate table
    } catch (IOException | SQLException ex) {
        Log.error("Error in db writing!", ex);
    }
}
 

@SuppressWarnings("unchecked")
public JSONObject parse(byte[] msg) {

	String raw_message = "";

	try {

		raw_message = new String(msg, "UTF-8");
		_LOG.debug("Received message: " + raw_message);
		
		/*
		 * Reinitialize Parser. It has the effect of calling the constructor again. 
		 */
		_parser.ReInit(new StringReader("header=" + raw_message.trim()));

		JSONObject payload = _parser.parseObject();

		String ip_src_addr = (String) payload.get("Device IP Address");
		String ip_src_port = (String) payload.get("Device Port");
		String ip_dst_addr = (String) payload.get("DestinationIPAddress");
		String ip_dst_port = (String) payload.get("DestinationPort");
		
		/*
		 * Standard Fields for OpenSoc.
		 */

		if(ip_src_addr != null)
			payload.put("ip_src_addr", ip_src_addr);
		if(ip_src_port != null)
			payload.put("ip_src_port", ip_src_port);
		if(ip_dst_addr != null)
			payload.put("ip_dst_addr", ip_dst_addr);
		if(ip_dst_port != null)
			payload.put("ip_dst_port", ip_dst_port);

		JSONObject message = new JSONObject();
		//message.put("message", payload);

		return payload;

	} catch (Exception e) {
		Log.error(e.toString());
		e.printStackTrace();
	}
	return null;
}
 

public void debugToDB(List<EntityProfile> profilesD1, List<EntityProfile> profilesD2, String dbURL) throws FileNotFoundException {
    if (blocks.isEmpty()) {
        Log.warn("Empty set of blocks was given as input!");
        return;
    }

    setType(); // Clean-Clean or Dirty ER?
    StringBuilder sb = new StringBuilder();
    String dbquery1 = "INSERT INTO " + dbtable + " (url1, url2, pairtype, Profile1, Profile2) VALUES ";
    sb.append(dbquery1);

    List<AbstractBlock> blocksToUse = blocks;
    if (!(blocks.get(0) instanceof DecomposedBlock)) {
        final ComparisonPropagation cp = new ComparisonPropagation();
        blocksToUse = cp.refineBlocks(blocks);
    }

    abstractDP.resetDuplicates();
    for (AbstractBlock block : blocksToUse) {
        final ComparisonIterator iterator = block.getComparisonIterator();
        while (iterator.hasNext()) {
            final Comparison comp = iterator.next();
            abstractDP.isSuperfluous(comp.getEntityId1(), comp.getEntityId2());
        }
    }

    for (IdDuplicates duplicatesPair : abstractDP.getFalseNegatives()) {
        final EntityProfile profile1 = profilesD1.get(duplicatesPair.getEntityId1());
        final EntityProfile profile2 = isCleanCleanER ? profilesD2.get(duplicatesPair.getEntityId2()) : profilesD1.get(duplicatesPair.getEntityId2());

        sb.append("('").append(profile1.getEntityUrl()).append("', ");
        sb.append("'").append(profile2.getEntityUrl()).append("', ");
        sb.append("'" + "FN" + "', "); // false negative
        sb.append("'").append(profile1).append("', ");
        sb.append("'").append(profile2).append("'), ");
    }

    String dbquery = sb.toString();

    try {
        if (dbuser == null) {
            Log.error("Database user has not been set!");
        }
        if (dbpassword == null) {
            Log.error("Database password has not been set!");
        }
        if (dbtable == null) {
            Log.error("Database table has not been set!");
        }

        Connection conn = null;
        if (dbURL.startsWith("mysql")) {
            conn = getMySQLconnection(dbURL);
        } else if (dbURL.startsWith("postgresql")) {
            conn = getPostgreSQLconnection(dbURL);
        } else {
            Log.error("Only MySQL and PostgreSQL are supported for the time being!");
        }

        final Statement stmt = conn.createStatement();
        stmt.executeQuery(dbquery);//retrieve the appropriate table
    } catch (Exception ex) {
        Log.error("Error in db writing!", ex);
    }
}