org.apache.tinkerpop.gremlin.process.traversal.step.util.ProfileStep Java Examples

/**
 * Determines the first (non-local) barrier step in the provided traversal. This method is used by {@link GroupStep}
 * and {@link GroupSideEffectStep} to ultimately determine the reducing bi-operator.
 *
 * @param traversal The traversal to inspect.
 * @return The first non-local barrier step or {@code null} if no such step was found.
 */
public default Barrier determineBarrierStep(final Traversal.Admin<S, V> traversal) {
    final List<Step> steps = traversal.getSteps();
    for (int ix = 0; ix < steps.size(); ix++) {
        final Step step = steps.get(ix);
        if (step instanceof Barrier && !(step instanceof LocalBarrier)) {
            final Barrier b = (Barrier) step;

            // when profile() is enabled the step needs to be wrapped up with the barrier so that the timer on
            // the ProfileStep is properly triggered
            if (ix < steps.size() - 1 && steps.get(ix + 1) instanceof ProfileStep)
                return new ProfilingAware.ProfiledBarrier(b, (ProfileStep) steps.get(ix + 1));
            else
                return b;
        }
    }
    return null;
}
 

private void addTopLevelMetrics(final Traversal.Admin traversal, final boolean onGraphComputer) {
    this.totalStepDuration = 0;

    final List<ProfileStep> profileSteps = TraversalHelper.getStepsOfClass(ProfileStep.class, traversal);
    final List<Pair<Integer, MutableMetrics>> tempMetrics = new ArrayList<>(profileSteps.size());

    for (int ii = 0; ii < profileSteps.size(); ii++) {
        // The index is necessary to ensure that step order is preserved after a merge.
        final ProfileStep step = profileSteps.get(ii);
        final MutableMetrics stepMetrics = onGraphComputer ? traversal.getSideEffects().get(step.getId()) : step.getMetrics();

        this.totalStepDuration += stepMetrics.getDuration(MutableMetrics.SOURCE_UNIT);
        tempMetrics.add(Pair.with(ii, stepMetrics.clone()));
    }

    tempMetrics.forEach(m -> {
        final double dur = m.getValue1().getDuration(TimeUnit.NANOSECONDS) * 100.d / this.totalStepDuration;
        m.getValue1().setAnnotation(PERCENT_DURATION_KEY, dur);
    });

    tempMetrics.forEach(p -> {
        this.stepIndexedMetrics.put(p.getValue1().getId(), p.getValue1().getImmutableClone());
        this.positionIndexedMetrics.put(p.getValue0(), p.getValue1().getImmutableClone());
    });
}
 

@Override
public void workerIterationEnd(final Memory memory) {
    // store profile metrics in proper ProfileStep metrics
    if (this.profile) {
        final List<ProfileStep> profileSteps = TraversalHelper.getStepsOfAssignableClassRecursively(ProfileStep.class, this.traversal.get());
        // guess the profile step to store data
        int profileStepIndex = memory.getIteration();
        // if we guess wrongly write timing into last step
        profileStepIndex = profileStepIndex >= profileSteps.size() ? profileSteps.size() - 1 : profileStepIndex;
        this.iterationMetrics.finish(0);
        // reset counts
        this.iterationMetrics.setCount(TraversalMetrics.TRAVERSER_COUNT_ID,0);
        if (null != MemoryTraversalSideEffects.getMemorySideEffectsPhase(this.traversal.get())) {
            this.traversal.get().getSideEffects().add(profileSteps.get(profileStepIndex).getId(), this.iterationMetrics);
        }
        this.iterationMetrics = null;
    }
}
 

private void validate_g_V_out_out_profile_modern(final Traversal traversal, final TraversalMetrics traversalMetrics) {
    traversalMetrics.toString(); // ensure no exceptions are thrown

    assumeThat("The following assertions apply to TinkerGraph only as provider strategies can alter the steps to not comply with expectations",
            graph.getClass().getSimpleName(), equalTo("TinkerGraph"));

    Metrics metrics = traversalMetrics.getMetrics(0);
    assertEquals(6, metrics.getCount(TraversalMetrics.TRAVERSER_COUNT_ID).longValue());
    assertEquals(6, metrics.getCount(TraversalMetrics.ELEMENT_COUNT_ID).longValue());

    metrics = traversalMetrics.getMetrics(1);
    assertEquals(6, metrics.getCount(TraversalMetrics.ELEMENT_COUNT_ID).longValue());
    assertNotEquals(0, metrics.getCount(TraversalMetrics.TRAVERSER_COUNT_ID).longValue());

    metrics = traversalMetrics.getMetrics(2);
    assertEquals(2, metrics.getCount(TraversalMetrics.ELEMENT_COUNT_ID).longValue());
    assertNotEquals(0, metrics.getCount(TraversalMetrics.TRAVERSER_COUNT_ID).longValue());

    if (!onGraphComputer(traversal.asAdmin())) {
        // Every other step should be a Profile step
        List<Step> steps = traversal.asAdmin().getSteps();
        for (int ii = 1; ii <= 6; ii += 2) {
            assertEquals("Every other Step should be a ProfileStep.", ProfileStep.class, steps.get(ii).getClass());
        }
    }
}
 

/**
 * Many parent traversals drip feed their start vertices in one at a time. To best exploit
 * the multiQuery we need to load all possible starts in one go so this method will attempt
 * to find a JanusGraphMultiQueryStep with the starts of the parent, and if found cache it.
 */
private void setParentMultiQueryStep() {
    Step firstStep = traversal.getStartStep();
    while (firstStep instanceof StartStep || firstStep instanceof SideEffectStep) {
        // Want the next step if this is a side effect
        firstStep = firstStep.getNextStep();
    }
    if (this.equals(firstStep)) {
        Step<?, ?> parentStep = traversal.getParent().asStep();
        if (JanusGraphTraversalUtil.isMultiQueryCompatibleStep(parentStep)) {
            Step<?, ?> parentPreviousStep = parentStep.getPreviousStep();
            if (parentStep instanceof RepeatStep) {
                RepeatStep repeatStep = (RepeatStep) parentStep;
                List<RepeatEndStep> repeatEndSteps = TraversalHelper.getStepsOfClass(RepeatEndStep.class, repeatStep.getRepeatTraversal());
                if (repeatEndSteps.size() == 1) {
                    parentPreviousStep = repeatEndSteps.get(0).getPreviousStep();
                }
            }
            if (parentPreviousStep instanceof ProfileStep) {
                parentPreviousStep = parentPreviousStep.getPreviousStep();
            }
            if (parentPreviousStep instanceof JanusGraphMultiQueryStep) {
                parentMultiQueryStep = (JanusGraphMultiQueryStep) parentPreviousStep;
            }
        }
    }
}
 

@Override
protected Object map(final Traverser.Admin<S> traverser) {
    if (this.getTraversal().getEndStep() instanceof WhereEndStep)
        ((WhereEndStep) this.getTraversal().getEndStep()).processStartTraverser(traverser);
    else if (this.getTraversal().getEndStep() instanceof ProfileStep && this.getTraversal().getEndStep().getPreviousStep() instanceof WhereEndStep)     // TOTAL SUCKY HACK!
        ((WhereEndStep) this.getTraversal().getEndStep().getPreviousStep()).processStartTraverser(traverser);
    return null == this.selectKey ? traverser.get() : this.getSafeScopeValue(Pop.last, this.selectKey, traverser);
}
 

@Override
public void apply(final Traversal.Admin<?, ?> traversal) {
    if (!traversal.getEndStep().getLabels().contains(MARKER) &&
            (traversal.isRoot() || traversal.getParent() instanceof VertexProgramStep) &&
            TraversalHelper.hasStepOfAssignableClassRecursively(ProfileSideEffectStep.class, traversal))
        TraversalHelper.applyTraversalRecursively(t -> t.getEndStep().addLabel(MARKER), traversal);

    if (traversal.getEndStep().getLabels().contains(MARKER)) {
        traversal.getEndStep().removeLabel(MARKER);
        // Add .profile() step after every pre-existing step.
        final List<Step> steps = traversal.getSteps();
        final int numSteps = steps.size();
        for (int i = 0; i < numSteps; i++) {
            // Do not inject profiling after ProfileSideEffectStep as this will be the last step on the root traversal.
            if (steps.get(i * 2) instanceof ProfileSideEffectStep)
                break;
            // Create and inject ProfileStep
            final ProfileStep profileStepToAdd = new ProfileStep(traversal);
            traversal.addStep((i * 2) + 1, profileStepToAdd);

            final Step stepToBeProfiled = traversal.getSteps().get(i * 2);
            if (stepToBeProfiled instanceof ProfilingAware) {
                ((ProfilingAware) stepToBeProfiled).prepareForProfiling();
            }
        }
    }
}
 

@Override
public void apply(final Traversal.Admin<?, ?> traversal) {
    if (!TraversalHelper.onGraphComputer(traversal))
        return;

    if (traversal.getParent() instanceof TraversalVertexProgramStep) {
        if (TraversalHelper.getStepsOfAssignableClassRecursively(GraphStep.class, traversal).size() > 1)
            throw new VerificationException("Mid-traversal V()/E() is currently not supported on GraphComputer", traversal);
        if (TraversalHelper.hasStepOfAssignableClassRecursively(ProfileStep.class, traversal) && TraversalHelper.getStepsOfAssignableClass(VertexProgramStep.class, TraversalHelper.getRootTraversal(traversal)).size() > 1)
            throw new VerificationException("Profiling a multi-VertexProgramStep traversal is currently not supported on GraphComputer", traversal);
    }

    // this is a problem because sideEffect.merge() is transient on the OLAP reduction
    if (TraversalHelper.getRootTraversal(traversal).getTraverserRequirements().contains(TraverserRequirement.ONE_BULK))
        throw new VerificationException("One bulk is currently not supported on GraphComputer: " + traversal, traversal);

    // you can not traverse past the local star graph with localChildren (e.g. by()-modulators).
    if (!TraversalHelper.isGlobalChild(traversal) && !TraversalHelper.isLocalStarGraph(traversal))
        throw new VerificationException("Local traversals may not traverse past the local star-graph on GraphComputer: " + traversal, traversal);

    for (final Step<?, ?> step : traversal.getSteps()) {
        if (step instanceof PathProcessor && ((PathProcessor) step).getMaxRequirement() != PathProcessor.ElementRequirement.ID)
            throw new VerificationException("It is not possible to access more than a path element's id on GraphComputer: " + step + " requires " + ((PathProcessor) step).getMaxRequirement(), traversal);

        if (UNSUPPORTED_STEPS.stream().filter(c -> c.isAssignableFrom(step.getClass())).findFirst().isPresent())
            throw new VerificationException("The following step is currently not supported on GraphComputer: " + step, traversal);
    }

    Step<?, ?> nextParentStep = traversal.getParent().asStep();
    while (!(nextParentStep instanceof EmptyStep)) {
        if (nextParentStep instanceof PathProcessor && ((PathProcessor) nextParentStep).getMaxRequirement() != PathProcessor.ElementRequirement.ID)
            throw new VerificationException("The following path processor step requires more than the element id on GraphComputer: " + nextParentStep + " requires " + ((PathProcessor) nextParentStep).getMaxRequirement(), traversal);
        nextParentStep = nextParentStep.getNextStep();
    }
}
 

@Override
protected Object map(final Traverser.Admin<S> traverser) {
    if (this.getTraversal().getEndStep() instanceof SqlgWhereTraversalStepBarrier.SqlgWhereEndStep)
        ((SqlgWhereEndStep) this.getTraversal().getEndStep()).processStartTraverser(traverser);
    else if (this.getTraversal().getEndStep() instanceof ProfileStep && this.getTraversal().getEndStep().getPreviousStep() instanceof SqlgWhereTraversalStepBarrier.SqlgWhereEndStep)     // TOTAL SUCKY HACK!
        ((SqlgWhereEndStep) this.getTraversal().getEndStep().getPreviousStep()).processStartTraverser(traverser);
    return null == this.selectKey ? traverser.get() : this.getScopeValue(Pop.last, this.selectKey, traverser);
}
 

public static Optional<String> getEndLabel(final Traversal.Admin<Object, Object> traversal) {
    final Step<?, ?> endStep = traversal.getEndStep();
    return endStep instanceof ProfileStep ?           // TOTAL HACK
            ((MatchEndStep) endStep.getPreviousStep()).getMatchKey() :
            ((MatchEndStep) endStep).getMatchKey();
}
 

public ProfiledBarrier(final Barrier barrier, final ProfileStep profileStep) {
    this.barrier = barrier;
    this.profileStep = profileStep;
}
 

protected boolean isEndStep() {
    return this.getNextStep() instanceof ComputerResultStep || (this.getNextStep() instanceof ProfileStep && this.getNextStep().getNextStep() instanceof ComputerResultStep);
}
 

@Override
public void loadState(final Graph graph, final Configuration configuration) {
    if (!configuration.containsKey(TRAVERSAL))
        throw new IllegalArgumentException("The configuration does not have a traversal: " + TRAVERSAL);
    this.traversal = PureTraversal.loadState(configuration, TRAVERSAL, graph);
    if (!this.traversal.get().isLocked())
        this.traversal.get().applyStrategies();
    /// traversal is compiled and ready to be introspected
    this.traversalMatrix = new TraversalMatrix<>(this.traversal.get());
    // get any master-traversal halted traversers
    this.haltedTraversers = TraversalVertexProgram.loadHaltedTraversers(configuration);
    // if results will be serialized out, don't save halted traversers across the cluster
    this.returnHaltedTraversers =
            (this.traversal.get().getParent().asStep().getNextStep() instanceof ComputerResultStep || // if its just going to stream it out, don't distribute
                    this.traversal.get().getParent().asStep().getNextStep() instanceof EmptyStep ||  // same as above, but if using TraversalVertexProgramStep directly
                    (this.traversal.get().getParent().asStep().getNextStep() instanceof ProfileStep && // same as above, but needed for profiling
                            this.traversal.get().getParent().asStep().getNextStep().getNextStep() instanceof ComputerResultStep));

    // determine how to store halted traversers
    final Iterator<?> itty = IteratorUtils.filter(this.traversal.get().getStrategies(), strategy -> strategy instanceof HaltedTraverserStrategy).iterator();
    this.haltedTraverserStrategy = itty.hasNext() ? (HaltedTraverserStrategy) itty.next() : HaltedTraverserStrategy.reference();

    // register traversal side-effects in memory
    this.memoryComputeKeys.addAll(MemoryTraversalSideEffects.getMemoryComputeKeys(this.traversal.get()));
    // register MapReducer memory compute keys
    for (final MapReducer<?, ?, ?, ?, ?> mapReducer : TraversalHelper.getStepsOfAssignableClassRecursively(MapReducer.class, this.traversal.get())) {
        this.mapReducers.add(mapReducer.getMapReduce());
        this.memoryComputeKeys.add(MemoryComputeKey.of(mapReducer.getMapReduce().getMemoryKey(), Operator.assign, false, false));
    }
    // register memory computing steps that use memory compute keys
    for (final MemoryComputing<?> memoryComputing : TraversalHelper.getStepsOfAssignableClassRecursively(MemoryComputing.class, this.traversal.get())) {
        this.memoryComputeKeys.add(memoryComputing.getMemoryComputeKey());
    }
    // register profile steps (TODO: try to hide this)
    for (final ProfileStep profileStep : TraversalHelper.getStepsOfAssignableClassRecursively(ProfileStep.class, this.traversal.get())) {
        this.traversal.get().getSideEffects().register(profileStep.getId(), new MutableMetricsSupplier(profileStep.getPreviousStep()), ProfileStep.ProfileBiOperator.instance());
    }
    // register TraversalVertexProgram specific memory compute keys
    this.memoryComputeKeys.add(MemoryComputeKey.of(VOTE_TO_HALT, Operator.and, false, true));
    this.memoryComputeKeys.add(MemoryComputeKey.of(HALTED_TRAVERSERS, Operator.addAll, false, false));
    this.memoryComputeKeys.add(MemoryComputeKey.of(ACTIVE_TRAVERSERS, Operator.addAll, true, true));
    this.memoryComputeKeys.add(MemoryComputeKey.of(MUTATED_MEMORY_KEYS, Operator.addAll, false, true));
    this.memoryComputeKeys.add(MemoryComputeKey.of(COMPLETED_BARRIERS, Operator.addAll, true, true));

    // does the traversal need profile information
    this.profile = !TraversalHelper.getStepsOfAssignableClassRecursively(ProfileStep.class, this.traversal.get()).isEmpty();
}