Java Code Examples for scala.collection.JavaConversions#asJavaCollection()

private void updateCollectorItemMetricDetail(CollectorItemMetricDetail collectorItemMetricDetail, Row itemRow) {
    Date timeWindowDt = itemRow.getAs("timeWindow");
    List<String> scaMetricList = Arrays.asList("coverage");
    Collection<Object> javaCollection = JavaConversions.asJavaCollection(((WrappedArray) itemRow.getAs("metrics")).toList());

    Optional.ofNullable(javaCollection)
            .orElseGet(Collections::emptyList)
            .forEach(m -> {
                GenericRowWithSchema genericRowWithSchema = (GenericRowWithSchema) m;
                String existingLabelName = genericRowWithSchema.getAs("name");
                    if (scaMetricList.contains(existingLabelName)) {
                        String valueStr = genericRowWithSchema.getAs("value");
                        try{
                        double value = Double.parseDouble(valueStr);
                        MetricCount mc = getMetricCount("", value, "unit-test-coverage");
                        if (mc != null) {
                            collectorItemMetricDetail.setStrategy(getCollectionStrategy());
                            collectorItemMetricDetail.addCollectorItemMetricCount(timeWindowDt, mc);
                            collectorItemMetricDetail.setLastScanDate(timeWindowDt);
                        }
                        }catch (Exception e){
                            LOGGER.info("Exception: Not a number, 'value' = "+valueStr,e);
                        }
                    }
            });
}
 

private void updateCollectorItemMetricDetail(CollectorItemMetricDetail collectorItemMetricDetail, Row itemRow) {
    Date timeWindowDt = itemRow.getAs("timeWindow");
    List<String> scaMetricList = Arrays.asList("blocker_violations", "critical_violations", "major_violations", "sqale_index");
    Collection<Object> javaCollection = JavaConversions.asJavaCollection(((WrappedArray) itemRow.getAs("metrics")).toList());

    Optional.ofNullable(javaCollection)
            .orElseGet(Collections::emptyList)
            .forEach(m -> {
                GenericRowWithSchema genericRowWithSchema = (GenericRowWithSchema) m;
                String existingLabelName = genericRowWithSchema.getAs("name");
                if (scaMetricList.contains(existingLabelName)) {
                    String valueStr = genericRowWithSchema.getAs("value");
                    try {
                        double value = Double.parseDouble(valueStr);
                        MetricCount mc = getMetricCount("", value, existingLabelName);
                        if (mc != null) {
                            collectorItemMetricDetail.setStrategy(getCollectionStrategy());
                            collectorItemMetricDetail.addCollectorItemMetricCount(timeWindowDt, mc);
                            collectorItemMetricDetail.setLastScanDate(timeWindowDt);
                        }
                    } catch (NumberFormatException e) {
                        LOGGER.info("Exception: Not a number, 'value' = "+valueStr,e);
                    }
                }
            });
}
 

private void updateCollectorItemMetricDetail(CollectorItemMetricDetail collectorItemMetricDetail, Row itemRow) {
    Date timeWindowDt = itemRow.getAs("timeWindow");
    List<String> metricList = Arrays.asList("High", "Medium", "Low");
    Collection<Object> javaCollection = JavaConversions.asJavaCollection(((WrappedArray) itemRow.getAs("metrics")).toList());

    Optional.ofNullable(javaCollection)
            .orElseGet(Collections::emptyList)
            .forEach(m -> {
                GenericRowWithSchema genericRowWithSchema = (GenericRowWithSchema) m;
                String existingLabelName = genericRowWithSchema.getAs("name");
                if (metricList.contains(existingLabelName)) {
                    String valueStr = genericRowWithSchema.getAs("value");
                    try {
                        double value = Double.parseDouble(valueStr);
                        MetricCount mc = getMetricCount("", value, existingLabelName);
                        if (mc != null) {
                            collectorItemMetricDetail.setStrategy(getCollectionStrategy());
                            collectorItemMetricDetail.addCollectorItemMetricCount(timeWindowDt, mc);
                            collectorItemMetricDetail.setLastScanDate(timeWindowDt);
                        }
                    } catch (NumberFormatException e) {
                        LOGGER.info("Exception: Not a number, 'value' = "+valueStr,e);
                    }
                }
            });
}
 

private void updateCollectorItemMetricDetail(CollectorItemMetricDetail collectorItemMetricDetail, Row itemRow, String type) {
    Date timeWindowDt = itemRow.getAs("timeWindow");
    Collection<Object> javaCollection = JavaConversions.asJavaCollection(((WrappedArray) itemRow.getAs(type)).toList());

    Optional.ofNullable(javaCollection)
        .orElseGet(Collections::emptyList)
        .forEach(m -> {
            GenericRowWithSchema genericRowWithSchema = (GenericRowWithSchema) m;
            String level = genericRowWithSchema.getAs("level");
            int value = genericRowWithSchema.getAs("count");
            MetricCount mc = getMetricCount(level, value, type);
            if (mc != null) {
                collectorItemMetricDetail.setStrategy(getCollectionStrategy());
                collectorItemMetricDetail.addCollectorItemMetricCount(timeWindowDt, mc);
                collectorItemMetricDetail.setLastScanDate(timeWindowDt);
            }
        });
}
 

private void mergeSegments(String cubeId, String segmentId) throws IOException {
    CubeManager mgr = CubeManager.getInstance(config);
    CubeInstance cube = mgr.getCubeByUuid(cubeId);
    CubeSegment mergedSeg = cube.getSegmentById(segmentId);
    SegmentInfo mergedSegInfo = ManagerHub.getSegmentInfo(config, getParam(MetadataConstants.P_CUBE_ID), mergedSeg.getUuid());

    Map<Long, DFLayoutMergeAssist> mergeCuboidsAssist = generateMergeAssist(mergingSegInfos, ss);
    for (DFLayoutMergeAssist assist : mergeCuboidsAssist.values()) {
        SpanningTree spanningTree = new ForestSpanningTree(JavaConversions.asJavaCollection(mergedSegInfo.toBuildLayouts()));
        Dataset<Row> afterMerge = assist.merge(config, cube.getName());
        LayoutEntity layout = assist.getLayout();

        Dataset<Row> afterSort;
        if (layout.isTableIndex()) {
            afterSort = afterMerge.sortWithinPartitions(NSparkCubingUtil.getColumns(layout.getOrderedDimensions().keySet()));
        } else {
            Column[] dimsCols = NSparkCubingUtil.getColumns(layout.getOrderedDimensions().keySet());
            Dataset<Row> afterAgg = CuboidAggregator.agg(ss, afterMerge, layout.getOrderedDimensions().keySet(),
                    layout.getOrderedMeasures(), spanningTree, false);
            afterSort = afterAgg.sortWithinPartitions(dimsCols);
        }
        buildLayoutWithUpdate.submit(new BuildLayoutWithUpdate.JobEntity() {
            @Override
            public String getName() {
                return "merge-layout-" + layout.getId();
            }

            @Override
            public LayoutEntity build() throws IOException {
                return saveAndUpdateCuboid(afterSort, mergedSegInfo, layout, assist);
            }
        }, config);

        buildLayoutWithUpdate.updateLayout(mergedSegInfo, config);
    }
}
 

private void addProductToPortfolio(Portfolio portfolio, Row productRow) {
    String productName = productRow.getAs("productName");
    String productDept = productRow.getAs("ownerDept");
    String commonName = productRow.getAs("commonName");
    String productId = (String) ((GenericRowWithSchema) productRow.getAs("productId")).values()[0];

    LOGGER.debug("    Product Name = " + productName + " ; Owner Dept = " + productDept);

    // For a given portfolio, check if the current product already exists in the product list for the portfolio
    // If not, add it to the product list
    Product product =
            Optional.ofNullable(portfolio.getProducts())
                    .orElseGet(Collections::emptyList).stream()
                    .filter(p -> p.getName().equalsIgnoreCase(productName)
                            && p.getLob().equalsIgnoreCase(productDept))
                    .findFirst().orElse(null);
    if (product == null) {
        product = new Product();
        product.setId(new ObjectId(productId));
        product.setLob(productDept);
        product.setName(productName);
        product.setCommonName(commonName);
        product.setMetricLevel(MetricLevel.PRODUCT);
    }
    if (productRow.getAs("environments") != null) {
        Collection<Object> environmentNames = JavaConversions.asJavaCollection(((WrappedArray) productRow.getAs("environments")).toStream().toList());
        addEnvironmentsToProduct(product, environmentNames);
    }
    if (productRow.getAs("components") != null) {
        Collection<Object> componentNames = JavaConversions.asJavaCollection(((WrappedArray) productRow.getAs("components")).toStream().toList());
        addComponentsToProduct(product, componentNames);
    }
    product.addOwner(new PeopleRoleRelation(getPeople(productRow.getAs("appServiceOwner"), "appServiceOwner"), RoleRelationShipType.AppServiceOwner));
    product.addOwner(new PeopleRoleRelation(getPeople(productRow.getAs("supportOwner"), "supportOwner"), RoleRelationShipType.SupportOwner));
    product.addOwner(new PeopleRoleRelation(getPeople(productRow.getAs("developmentOwner"), "developmentOwner"), RoleRelationShipType.DevelopmentOwner));

    portfolio.addProduct(product);
}
 

private void addProductToLob(Lob lob, Row productRow) {
    String productName = productRow.getAs("productName");
    String productDept = productRow.getAs("ownerDept");
    String commonName = productRow.getAs("commonName");
    String productId = (String) ((GenericRowWithSchema) productRow.getAs("productId")).values()[0];

    LOGGER.debug("    Product Name = " + productName + " ; Owner Dept = " + productDept);

    // For a given portfolio, check if the current product already exists in the product list for the portfolio
    // If not, add it to the product list
    Product product =
            Optional.ofNullable(lob.getProducts())
                    .orElseGet(Collections::emptyList).stream()
                    .filter(p -> p.getName().equalsIgnoreCase(productName)
                            && p.getLob().equalsIgnoreCase(productDept))
                    .findFirst().orElse(null);
    if (product == null) {
        product = new Product();
        product.setId(new ObjectId(productId));
        product.setLob(productDept);
        product.setName(productName);
        product.setCommonName(commonName);
        product.setMetricLevel(MetricLevel.PRODUCT);
    }
    if (productRow.getAs("environments") != null) {
        Collection<Object> environmentNames = JavaConversions.asJavaCollection(((WrappedArray) productRow.getAs("environments")).toStream().toList());
        addEnvironmentsToProduct(product, environmentNames);
    }
    if (productRow.getAs("components") != null) {
        Collection<Object> componentNames = JavaConversions.asJavaCollection(((WrappedArray) productRow.getAs("components")).toStream().toList());
        addComponentsToProduct(product, componentNames);
    }
    product.addOwner(new PeopleRoleRelation(getPeople(productRow.getAs("appServiceOwner"), "appServiceOwner"), RoleRelationShipType.AppServiceOwner));
    product.addOwner(new PeopleRoleRelation(getPeople(productRow.getAs("supportOwner"), "supportOwner"), RoleRelationShipType.SupportOwner));
    product.addOwner(new PeopleRoleRelation(getPeople(productRow.getAs("developmentOwner"), "developmentOwner"), RoleRelationShipType.DevelopmentOwner));

    lob.addProduct(product);
}
 

private void updateCollectorItemMetricDetail(CollectorItemMetricDetail collectorItemMetricDetail, Row itemRow) {
    Date timeWindowDt = itemRow.getAs("timeWindow");
    LOGGER.info("TimeWindow:" +timeWindowDt );
    LOGGER.info("itemRow :" + itemRow);
    Collection<Object> javaCollection = JavaConversions.asJavaCollection(((WrappedArray) itemRow.getAs("prodStageList")).toList());

    Optional.ofNullable(javaCollection)
            .orElseGet(Collections::emptyList).stream().map(m -> (GenericRowWithSchema) m).forEach(genericRowWithSchema -> {
        Long pipelineTimeL = genericRowWithSchema.getAs("timestamp");
        Date dateObj = new Timestamp(new Date(pipelineTimeL).getTime());
        LOGGER.info("Date Object :" + dateObj);
        Long scmTimeL = genericRowWithSchema.getAs("scmCommitTimestamp");
        Long pipelineTimeAfterIgnore = pipelineTimeL/1000;
        Long scmTimeAfterIgnore = scmTimeL/1000;
        try {
            Long diffTimestamp = Math.abs(pipelineTimeAfterIgnore - scmTimeAfterIgnore);
            String strTimestampInsec = Long.toString(diffTimestamp);
            double value = Double.parseDouble(strTimestampInsec);
            MetricCount mc = getMetricCount("", value, "pipeline-lead-time");
            if (mc != null) {
                collectorItemMetricDetail.setStrategy(getCollectionStrategy());
                collectorItemMetricDetail.addCollectorItemMetricCount(dateObj, mc);
                collectorItemMetricDetail.setLastScanDate(dateObj);
            }
        } catch (NumberFormatException e) {
            LOGGER.info("Exception: Not a number, 'value' = " + scmTimeAfterIgnore, e);
        }
    });
}
 

private List<Object> getInputFields(IndexedRecord inputRecord, String columnName) {
    // Adapt non-avpath syntax to avpath.
    // TODO: This should probably not be automatic, use the actual syntax.
    if (!columnName.startsWith("."))
        columnName = "." + columnName;
    Try<scala.collection.immutable.List<Evaluator.Ctx>> result = wandou.avpath.package$.MODULE$.select(inputRecord,
            columnName);
    List<Object> values = new ArrayList<Object>();
    if (result.isSuccess()) {
        for (Evaluator.Ctx ctx : JavaConversions.asJavaCollection(result.get())) {
            values.add(ctx.value());
        }
    } else {
        // Evaluating the expression failed, and we can handle the exception.
        Throwable t = result.failed().get();
        throw ProcessingErrorCode.createAvpathSyntaxError(t, columnName, -1);
    }
    return values;
}
 

/**
 * Use an AVPath to extract data from an indexed record
 * 
 * @param record an indexed record
 * @param avPath the path to elements to extract (can be one or multiples elements)
 * @return the extracted data as a list.
 */
public static List<Evaluator.Ctx> getInputFields(IndexedRecord record, String avPath) {
    // Adapt non-avpath syntax to avpath.
    // TODO: This should probably not be automatic, use the actual syntax.
    if (!avPath.startsWith("."))
        avPath = "." + avPath;
    Try<scala.collection.immutable.List<Evaluator.Ctx>> result =
            wandou.avpath.package$.MODULE$.select(record, avPath);
    List<Evaluator.Ctx> results = new ArrayList<Evaluator.Ctx>();
    if (result.isSuccess()) {
        for (Evaluator.Ctx ctx : JavaConversions.asJavaCollection(result.get())) {
            results.add(ctx);
        }
    } else {
        // Evaluating the expression failed, and we can handle the exception.
        throw ProcessingErrorCode.createAvpathSyntaxError(result.failed().get(), avPath, -1);
    }
    return results;
}
 

@Override
protected void doExecute() throws Exception {
    logger.info("Start detect resource before cube.");
    Set<String> segmentIds = Sets.newHashSet(StringUtils.split(getParam(MetadataConstants.P_SEGMENT_IDS)));
    for (String segId : segmentIds) {
        SegmentInfo seg = ManagerHub.getSegmentInfo(config, getParam(MetadataConstants.P_CUBE_ID), segId);
        spanningTree = new ForestSpanningTree(JavaConversions.asJavaCollection(seg.toBuildLayouts()));
        ResourceDetectUtils.write(new Path(config.getJobTmpShareDir(project, jobId), ResourceDetectUtils.countDistinctSuffix()),
                ResourceDetectUtils.findCountDistinctMeasure(JavaConversions.asJavaCollection(seg.toBuildLayouts())));
        ParentSourceChooser datasetChooser = new ParentSourceChooser(spanningTree, seg, jobId, ss, config, false);
        datasetChooser.decideSources();
        NBuildSourceInfo buildFromFlatTable = datasetChooser.flatTableSource();
        if (buildFromFlatTable != null) {
            sources.add(buildFromFlatTable);
        }
        Map<Long, NBuildSourceInfo> buildFromLayouts = datasetChooser.reuseSources();
        sources.addAll(buildFromLayouts.values());

        Map<String, List<String>> resourcePaths = Maps.newHashMap();
        Map<String, Integer> layoutLeafTaskNums = Maps.newHashMap();
        infos.clearSparkPlans();
        for (NBuildSourceInfo source : sources) {
            Dataset<Row> dataset = source.getParentDS();
            RDD actionRdd = dataset.queryExecution().toRdd();
            logger.info("leaf nodes is: {} ", SparkUtils.leafNodes(actionRdd));
            infos.recordSparkPlan(dataset.queryExecution().sparkPlan());
            List<Path> paths = JavaConversions
                    .seqAsJavaList(ResourceDetectUtils.getPaths(dataset.queryExecution().sparkPlan()));
            List<String> pathList = paths.stream().map(Path::toString).collect(Collectors.toList());
            resourcePaths.put(String.valueOf(source.getLayoutId()), pathList);
            layoutLeafTaskNums.put(String.valueOf(source.getLayoutId()), SparkUtils.leafNodePartitionNums(actionRdd));
        }
        ResourceDetectUtils.write(
                new Path(config.getJobTmpShareDir(project, jobId), segId + "_" + ResourceDetectUtils.fileName()),
                resourcePaths);
        ResourceDetectUtils.write(
                new Path(config.getJobTmpShareDir(project, jobId), segId + "_" + ResourceDetectUtils.cubingDetectItemFileSuffix()),
                layoutLeafTaskNums);

    }
}
 

@Override
protected void doExecute() throws Exception {

    long start = System.currentTimeMillis();
    logger.info("Start building cube job...");
    buildLayoutWithUpdate = new BuildLayoutWithUpdate();
    Set<String> segmentIds = Sets.newHashSet(StringUtils.split(getParam(MetadataConstants.P_SEGMENT_IDS)));
    cubeManager = CubeManager.getInstance(config);
    List<String> persistedFlatTable = new ArrayList<>();
    List<String> persistedViewFactTable = new ArrayList<>();
    Path shareDir = config.getJobTmpShareDir(project, jobId);
    try {
        //TODO: what if a segment is deleted during building?
        for (String segId : segmentIds) {
            SegmentInfo seg = ManagerHub.getSegmentInfo(config, getParam(MetadataConstants.P_CUBE_ID), segId);
            SpanningTree spanningTree = new ForestSpanningTree(
                    JavaConversions.asJavaCollection(seg.toBuildLayouts()));
            // choose source
            ParentSourceChooser sourceChooser = new ParentSourceChooser(spanningTree, seg, jobId, ss, config, true);
            sourceChooser.decideSources();
            NBuildSourceInfo buildFromFlatTable = sourceChooser.flatTableSource();
            Map<Long, NBuildSourceInfo> buildFromLayouts = sourceChooser.reuseSources();

            infos.clearCuboidsNumPerLayer(segId);

            // build cuboids from flat table
            if (buildFromFlatTable != null) {
                collectPersistedTablePath(persistedFlatTable, sourceChooser);
                build(Collections.singletonList(buildFromFlatTable), seg, spanningTree);
            }

            // build cuboids from reused layouts
            if (!buildFromLayouts.isEmpty()) {
                build(buildFromLayouts.values(), seg, spanningTree);
            }
            infos.recordSpanningTree(segId, spanningTree);

            logger.info("Updating segment info");
            updateSegmentInfo(getParam(MetadataConstants.P_CUBE_ID), seg, buildFromFlatTable.getCount());
        }
        updateSegmentSourceBytesSize(getParam(MetadataConstants.P_CUBE_ID),
                ResourceDetectUtils.getSegmentSourceSize(shareDir));
    } finally {
        FileSystem fs = HadoopUtil.getWorkingFileSystem();
        for (String viewPath : persistedViewFactTable) {
            fs.delete(new Path(viewPath), true);
            logger.info("Delete persisted view fact table: {}.", viewPath);
        }
        for (String path : persistedFlatTable) {
            fs.delete(new Path(path), true);
            logger.info("Delete persisted flat table: {}.", path);
        }
        logger.info("Building job takes {} ms", (System.currentTimeMillis() - start));
    }
}