org.apache.tinkerpop.gremlin.process.computer.GraphComputer Java Examples

protected void validateStatePriorToExecution() {
    // a graph computer can only be executed one time
    if (this.executed)
        throw Exceptions.computerHasAlreadyBeenSubmittedAVertexProgram();
    else
        this.executed = true;
    // it is not possible execute a computer if it has no vertex program nor mapreducers
    if (null == this.vertexProgram && this.mapReducers.isEmpty())
        throw GraphComputer.Exceptions.computerHasNoVertexProgramNorMapReducers();
    // it is possible to run mapreducers without a vertex program
    if (null != this.vertexProgram) {
        GraphComputerHelper.validateProgramOnComputer(this, vertexProgram);
        this.mapReducers.addAll(this.vertexProgram.getMapReducers());
    }
    // if the user didn't set desired persistence/resultgraph, then get from vertex program or else, no persistence
    this.persist = GraphComputerHelper.getPersistState(Optional.ofNullable(this.vertexProgram), Optional.ofNullable(this.persist));
    this.resultGraph = GraphComputerHelper.getResultGraphState(Optional.ofNullable(this.vertexProgram), Optional.ofNullable(this.resultGraph));
    // determine persistence and result graph options
    if (!this.features().supportsResultGraphPersistCombination(this.resultGraph, this.persist))
        throw GraphComputer.Exceptions.resultGraphPersistCombinationNotSupported(this.resultGraph, this.persist);
    // if too many workers are requested, throw appropriate exception
    if (this.workers > this.features().getMaxWorkers())
        throw GraphComputer.Exceptions.computerRequiresMoreWorkersThanSupported(this.workers, this.features().getMaxWorkers());
}
 

public static void validateProgramOnComputer(final GraphComputer computer, final VertexProgram vertexProgram) {
    if (vertexProgram.getMemoryComputeKeys().contains(null))
        throw Memory.Exceptions.memoryKeyCanNotBeNull();
    if (vertexProgram.getMemoryComputeKeys().contains(""))
        throw Memory.Exceptions.memoryKeyCanNotBeEmpty();

    final GraphComputer.Features graphComputerFeatures = computer.features();
    final VertexProgram.Features vertexProgramFeatures = vertexProgram.getFeatures();

    for (final Method method : VertexProgram.Features.class.getMethods()) {
        if (method.getName().startsWith("requires")) {
            final boolean supports;
            final boolean requires;
            try {
                supports = (boolean) GraphComputer.Features.class.getMethod(method.getName().replace("requires", "supports")).invoke(graphComputerFeatures);
                requires = (boolean) method.invoke(vertexProgramFeatures);
            } catch (final Exception e) {
                throw new IllegalStateException("A reflection exception has occurred: " + e.getMessage(), e);
            }
            if (requires && !supports)
                throw new IllegalStateException("The vertex program can not be executed on the graph computer: " + method.getName());
        }
    }
}
 

@Test
public void shouldReadFromWriteToArbitraryRDD() throws Exception {
    final Configuration configuration = new BaseConfiguration();
    configuration.setProperty("spark.master", "local[4]");
    configuration.setProperty("spark.serializer", GryoSerializer.class.getCanonicalName());
    configuration.setProperty(Graph.GRAPH, HadoopGraph.class.getName());
    configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, ExampleInputRDD.class.getCanonicalName());
    configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, ExampleOutputRDD.class.getCanonicalName());
    configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, TestHelper.makeTestDataDirectory(this.getClass(), "shouldReadFromWriteToArbitraryRDD"));
    configuration.setProperty(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false);
    ////////
    Graph graph = GraphFactory.open(configuration);
    graph.compute(SparkGraphComputer.class)
            .result(GraphComputer.ResultGraph.NEW)
            .persist(GraphComputer.Persist.EDGES)
            .program(TraversalVertexProgram.build()
                    .traversal(graph.traversal().withComputer(SparkGraphComputer.class),
                            "gremlin-groovy",
                            "g.V()").create(graph)).submit().get();
}
 

@Override
public void apply(final Traversal.Admin<?, ?> traversal) {
    if (TraversalHelper.getStepsOfAssignableClass(VertexProgramStep.class, traversal).size() > 1)  // do not do if there is an OLAP chain
        return;
    final Graph graph = traversal.getGraph().orElse(EmptyGraph.instance()); // given that this strategy only works for single OLAP jobs, the graph is the traversal graph
    for (final TraversalVertexProgramStep step : TraversalHelper.getStepsOfClass(TraversalVertexProgramStep.class, traversal)) {   // will be zero or one step
        final Traversal.Admin<?, ?> computerTraversal = step.generateProgram(graph, EmptyMemory.instance()).getTraversal().get().clone();
        if (!computerTraversal.isLocked())
            computerTraversal.applyStrategies();
        final Computer computer = step.getComputer();
        if (null == computer.getEdges() && !GraphComputer.Persist.EDGES.equals(computer.getPersist())) {  // if edges() already set, use it
            final Traversal.Admin<Vertex, Edge> edgeFilter = getEdgeFilter(computerTraversal);
            if (null != edgeFilter)  // if no edges can be filtered, then don't set edges()
                step.setComputer(computer.edges(edgeFilter));
        }
    }
}
 

@Override
public <V> VertexProperty<V> property(final String key, final V value, final Object... keyValues) {
    if (state.equals(State.MAP_REDUCE))
        throw GraphComputer.Exceptions.vertexPropertiesCanNotBeUpdatedInMapReduce();
    if (!computeKeys.contains(key))
        throw GraphComputer.Exceptions.providedKeyIsNotAnElementComputeKey(key);
    return new ComputerVertexProperty<>(this.getBaseVertex().property(key, value, keyValues));
}
 

@Override
public GraphTraversalSource traversal(final Graph graph) {
    return graph.traversal().withStrategies(VertexProgramStrategy.create(new MapConfiguration(new HashMap<String, Object>() {{
        put(VertexProgramStrategy.WORKERS, RANDOM.nextInt(Runtime.getRuntime().availableProcessors()) + 1);
        put(VertexProgramStrategy.GRAPH_COMPUTER, RANDOM.nextBoolean() ?
                GraphComputer.class.getCanonicalName() :
                TinkerGraphComputer.class.getCanonicalName());
    }})));
}
 

@Override
public <C extends GraphComputer> C compute(final Class<C> graphComputerClass) {
    try {
        if (AbstractHadoopGraphComputer.class.isAssignableFrom(graphComputerClass))
            return graphComputerClass.getConstructor(HadoopGraph.class).newInstance(this);
        else
            throw Graph.Exceptions.graphDoesNotSupportProvidedGraphComputer(graphComputerClass);
    } catch (final Exception e) {
        throw new IllegalArgumentException(e.getMessage(), e);
    }
}
 

public <V> Property<V> addProperty(final TinkerVertex vertex, final String key, final V value) {
    ElementHelper.validateProperty(key, value);
    if (isComputeKey(key)) {
        final TinkerVertexProperty<V> property = new TinkerVertexProperty<V>((TinkerVertex) vertex, key, value) {
            @Override
            public void remove() {
                removeProperty(vertex, key, this);
            }
        };
        this.addValue(vertex, key, property);
        return property;
    } else {
        throw GraphComputer.Exceptions.providedKeyIsNotAnElementComputeKey(key);
    }
}
 

public static TraversalStrategies getStrategies(final Class graphOrGraphComputerClass) {
    try {
        // be sure to load the class so that its static{} traversal strategy registration component is loaded.
        // this is more important for GraphComputer classes as they are typically not instantiated prior to
        // strategy usage like Graph classes.
        if (!LOADED.contains(graphOrGraphComputerClass)) {
            final String graphComputerClassName = null != graphOrGraphComputerClass.getDeclaringClass() ?
                    graphOrGraphComputerClass.getCanonicalName().replace("." + graphOrGraphComputerClass.getSimpleName(), "$" + graphOrGraphComputerClass.getSimpleName()) :
                    graphOrGraphComputerClass.getCanonicalName();
            Class.forName(graphComputerClassName);

            // keep track of stuff we already loaded once - stuff in this if/statement isn't cheap and this
            // method gets called a lot, basically every time a new traversal gets spun up (that includes
            // child traversals. perhaps it is possible to just check the cache keys for this information, but
            // it's not clear if this method will be called with something not in the cache and if it is and
            // it results in error, then we'd probably not want to deal with this block again anyway
            LOADED.add(graphOrGraphComputerClass);
        }
    } catch (final ClassNotFoundException e) {
        throw new IllegalStateException(e.getMessage(), e);
    }

    if (GRAPH_CACHE.containsKey(graphOrGraphComputerClass)) {
        return GRAPH_CACHE.get(graphOrGraphComputerClass);
    } else if (Graph.class.isAssignableFrom(graphOrGraphComputerClass)) {
        return GRAPH_CACHE.get(Graph.class);
    } else if (GRAPH_COMPUTER_CACHE.containsKey(graphOrGraphComputerClass)) {
        return GRAPH_COMPUTER_CACHE.get(graphOrGraphComputerClass);
    } else if (GraphComputer.class.isAssignableFrom(graphOrGraphComputerClass)) {
        return GRAPH_COMPUTER_CACHE.get(GraphComputer.class);
    } else {
        throw new IllegalArgumentException("The TraversalStrategies.GlobalCache only supports Graph and GraphComputer strategy caching: " + graphOrGraphComputerClass.getCanonicalName());
    }
}
 

@Test
@LoadGraphWith(MODERN)
public void shouldExecutePageRankWithIterationsBreak() throws Exception {
    if (graphProvider.getGraphComputer(graph).features().supportsResultGraphPersistCombination(GraphComputer.ResultGraph.NEW, GraphComputer.Persist.VERTEX_PROPERTIES)) {
        final ComputerResult result = graph.compute(graphProvider.getGraphComputer(graph).getClass()).
                program(PageRankVertexProgram.build().epsilon(0.0d).iterations(30).create(graph)).submit().get(); // by using epsilon 0.0, we guarantee iterations 30
        result.graph().traversal().V().forEachRemaining(v -> {
            assertEquals(3, v.keys().size()); // name, age/lang, pageRank
            assertTrue(v.keys().contains("name"));
            assertTrue(v.keys().contains(PageRankVertexProgram.PAGE_RANK));
            assertEquals(1, IteratorUtils.count(v.values("name")));
            assertEquals(1, IteratorUtils.count(v.values(PageRankVertexProgram.PAGE_RANK)));
            final String name = v.value("name");
            final Double pageRank = v.value(PageRankVertexProgram.PAGE_RANK);
            //System.out.println(name + "-----" + pageRank);
            if (name.equals("marko"))
                assertTrue(pageRank > 0.10 && pageRank < 0.12);
            else if (name.equals("vadas"))
                assertTrue(pageRank > 0.13 && pageRank < 0.15);
            else if (name.equals("lop"))
                assertTrue(pageRank > 0.29 && pageRank < 0.31);
            else if (name.equals("josh"))
                assertTrue(pageRank > 0.13 && pageRank < 0.15);
            else if (name.equals("ripple"))
                assertTrue(pageRank > 0.16 && pageRank < 0.18);
            else if (name.equals("peter"))
                assertTrue(pageRank > 0.10 && pageRank < 0.12);
            else
                throw new IllegalStateException("The following vertex should not exist in the graph: " + name);
        });
        assertEquals(result.memory().getIteration(), 30);
        assertEquals(result.memory().asMap().size(), 0);
    }
}
 

public static HadoopGraph getOutputGraph(final Configuration configuration, final GraphComputer.ResultGraph resultGraph, final GraphComputer.Persist persist) {
    final HadoopConfiguration hadoopConfiguration = new HadoopConfiguration(configuration);
    final BaseConfiguration newConfiguration = new BaseConfiguration();
    newConfiguration.copy(org.apache.tinkerpop.gremlin.hadoop.structure.io.InputOutputHelper.getOutputGraph(configuration, resultGraph, persist).configuration());
    if (resultGraph.equals(GraphComputer.ResultGraph.NEW) && hadoopConfiguration.containsKey(Constants.GREMLIN_HADOOP_GRAPH_WRITER)) {
        if (null != InputOutputHelper.getInputFormat(hadoopConfiguration.getGraphWriter()))
            newConfiguration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, InputOutputHelper.getInputFormat(hadoopConfiguration.getGraphWriter()).getCanonicalName());
    }
    return HadoopGraph.open(newConfiguration);
}
 

public void checkGryoV3d0IoRegistryCompliance(final HadoopGraph graph, final Class<? extends GraphComputer> graphComputerClass) throws Exception {
    final File input = TestHelper.generateTempFile(this.getClass(), "gryo-io-registry", ".kryo");
    graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName());
    graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, GryoOutputFormat.class.getCanonicalName());
    graph.configuration().setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, Storage.toPath(input));
    graph.configuration().setProperty(GryoPool.CONFIG_IO_GRYO_VERSION, GryoVersion.V3_0.name());
    graph.configuration().setProperty(IoRegistry.IO_REGISTRY, ToyIoRegistry.class.getCanonicalName());
    final GryoRecordWriter writer = new GryoRecordWriter(new DataOutputStream(new FileOutputStream(input)), ConfUtil.makeHadoopConfiguration(graph.configuration()));
    validateIoRegistryGraph(graph, graphComputerClass, writer);
    assertTrue(input.delete());
}
 

@Override
public GraphComputer compute() {
    if (this.configuration.containsKey(Constants.GREMLIN_HADOOP_DEFAULT_GRAPH_COMPUTER)) {
        try {
            return this.compute((Class<? extends GraphComputer>) Class.forName(this.configuration.getString(Constants.GREMLIN_HADOOP_DEFAULT_GRAPH_COMPUTER)));
        } catch (final Exception e) {
            throw new IllegalStateException(e.getMessage(), e);
        }
    } else
        throw new IllegalArgumentException("There is no default GraphComputer for HadoopGraph. Use HadoopGraph.compute(class) or gremlin.hadoop.defaultGraphComputer to specify the GraphComputer to use.");
}
 

public static void registerStrategies(final Class graphOrGraphComputerClass, final TraversalStrategies traversalStrategies) {
    if (Graph.class.isAssignableFrom(graphOrGraphComputerClass))
        GRAPH_CACHE.put(graphOrGraphComputerClass, traversalStrategies);
    else if (GraphComputer.class.isAssignableFrom(graphOrGraphComputerClass))
        GRAPH_COMPUTER_CACHE.put(graphOrGraphComputerClass, traversalStrategies);
    else
        throw new IllegalArgumentException("The TraversalStrategies.GlobalCache only supports Graph and GraphComputer strategy caching: " + graphOrGraphComputerClass.getCanonicalName());
}
 

@Override
public GraphTraversalSource traversal(final Graph graph) {
    return graph.traversal().withStrategies(VertexProgramStrategy.create(new MapConfiguration(new HashMap<String, Object>() {{
        put(VertexProgramStrategy.WORKERS, RANDOM.nextInt(Runtime.getRuntime().availableProcessors()) + 1);
        put(VertexProgramStrategy.GRAPH_COMPUTER, RANDOM.nextBoolean() ?
                GraphComputer.class.getCanonicalName() :
                TinkerGraphComputer.class.getCanonicalName());
    }})));
}
 

@Override
public <C extends GraphComputer> C compute(Class<C> graphComputerClass) throws IllegalArgumentException {
    if (!graphComputerClass.equals(FulgoraGraphComputer.class)) {
        throw Graph.Exceptions.graphDoesNotSupportProvidedGraphComputer(graphComputerClass);
    } else {
        return (C)compute();
    }
}
 

@Override
public GraphComputer compute() throws IllegalArgumentException {
    throw new UnsupportedOperationException();
}
 

@Override
public <C extends GraphComputer> C compute(final Class<C> graphComputerClass) {
    throw Graph.Exceptions.graphComputerNotSupported();
}
 

@Override
public <C extends GraphComputer> C compute(final Class<C> graphComputerClass) throws IllegalArgumentException {
    return null;
}
 

@Override
public GraphComputer.Persist getPreferredPersist() {
    return GraphComputer.Persist.EDGES;
}
 

@Override
public GraphComputer result(final ResultGraph resultGraph) {
    this.resultGraph = resultGraph;
    return this;
}
 

@Override
public void accessSetProperty() {
    throw GraphComputer.Exceptions.adjacentVertexPropertiesCanNotBeReadOrUpdated();
}
 

@Override
public final void accessProperties() {
    throw GraphComputer.Exceptions.adjacentVertexPropertiesCanNotBeReadOrUpdated();
}
 

@Override
public final void accessEdges() {
    throw GraphComputer.Exceptions.adjacentVertexPropertiesCanNotBeReadOrUpdated();
}
 

@Test
@LoadGraphWith(MODERN)
public void shouldDumpWholeGraph() throws Exception {
    if (graphProvider.getGraphComputer(graph).features().supportsResultGraphPersistCombination(GraphComputer.ResultGraph.NEW, GraphComputer.Persist.EDGES)) {
        final ComputerResult result = graphProvider.getGraphComputer(graph).program(CloneVertexProgram.build().create(graph)).submit().get();
        result.graph().traversal().V().forEachRemaining(v -> {
            assertEquals(2, v.keys().size());
            assertTrue(v.keys().contains("name"));
            assertTrue(v.keys().contains("age") || v.keys().contains("lang"));
            assertEquals(1, IteratorUtils.count(v.values("name")));
            assertEquals(1, IteratorUtils.count(v.values("age", "lang")));
            final String name = v.value("name");
            if (name.equals("marko")) {
                assertEquals("person", v.label());
                assertEquals(Integer.valueOf(29), v.value("age"));
                assertEquals(3, IteratorUtils.count(v.edges(Direction.OUT)));
                assertEquals(2, IteratorUtils.count(v.edges(Direction.OUT, "knows")));
                assertEquals(1, IteratorUtils.count(v.edges(Direction.OUT, "created")));
                assertEquals(0, IteratorUtils.count(v.edges(Direction.IN)));
            } else if (name.equals("vadas")) {
                assertEquals("person", v.label());
                assertEquals(Integer.valueOf(27), v.value("age"));
                assertEquals(0, IteratorUtils.count(v.edges(Direction.OUT)));
                assertEquals(1, IteratorUtils.count(v.edges(Direction.IN)));
                assertEquals(1, IteratorUtils.count(v.edges(Direction.IN, "knows")));
            } else if (name.equals("lop")) {
                assertEquals("software", v.label());
                assertEquals("java", v.value("lang"));
                assertEquals(0, IteratorUtils.count(v.edges(Direction.OUT)));
                assertEquals(3, IteratorUtils.count(v.edges(Direction.IN)));
                assertEquals(3, IteratorUtils.count(v.edges(Direction.IN, "created")));
            } else if (name.equals("josh")) {
                assertEquals("person", v.label());
                assertEquals(Integer.valueOf(32), v.value("age"));
                assertEquals(2, IteratorUtils.count(v.edges(Direction.OUT)));
                assertEquals(2, IteratorUtils.count(v.edges(Direction.OUT, "created")));
                assertEquals(1, IteratorUtils.count(v.edges(Direction.IN)));
                assertEquals(1, IteratorUtils.count(v.edges(Direction.IN, "knows")));
            } else if (name.equals("ripple")) {
                assertEquals("software", v.label());
                assertEquals("java", v.value("lang"));
                assertEquals(0, IteratorUtils.count(v.edges(Direction.OUT)));
                assertEquals(1, IteratorUtils.count(v.edges(Direction.IN)));
                assertEquals(1, IteratorUtils.count(v.edges(Direction.IN, "created")));
            } else if (name.equals("peter")) {
                assertEquals("person", v.label());
                assertEquals(Integer.valueOf(35), v.value("age"));
                assertEquals(1, IteratorUtils.count(v.edges(Direction.OUT)));
                assertEquals(1, IteratorUtils.count(v.edges(Direction.OUT, "created")));
                assertEquals(0, IteratorUtils.count(v.edges(Direction.IN)));
            } else
                throw new IllegalStateException("The following vertex should not exist in the graph: " + name);
        });
        assertEquals(3.5, (Double) result.graph().traversal().E().values("weight").sum().next(), 0.01);
        assertEquals(1.5, (Double) result.graph().traversal().E().hasLabel("knows").values("weight").sum().next(), 0.01);
        assertEquals(2.0, (Double) result.graph().traversal().E().hasLabel("created").values("weight").sum().next(), 0.01);
        assertEquals(result.memory().getIteration(), 0);
        assertEquals(result.memory().asMap().size(), 0);
    }
}
 

@Override
public void removeRelation(InternalRelation e) {
    throw GraphComputer.Exceptions.adjacentVertexPropertiesCanNotBeReadOrUpdated();
}
 

@Override
public TitanEdge addEdge(String s, Vertex vertex, Object... keyValues) {
    throw GraphComputer.Exceptions.adjacentVertexEdgesAndVerticesCanNotBeReadOrUpdated();
}
 

@Override
public GraphComputer.Persist getPreferredPersist() {
    return GraphComputer.Persist.NOTHING;
}
 

private final void checkKeyValue(final String key, final Object value) {
    if (!this.memoryComputeKeys.containsKey(key))
        throw GraphComputer.Exceptions.providedKeyIsNotAMemoryComputeKey(key);
    MemoryHelper.validateValue(value);
}
 

@Override
public GraphComputer getGraphComputer(final Graph graph) {
    return RANDOM.nextBoolean() ?
            graph.compute().workers(RANDOM.nextInt(AVAILABLE_PROCESSORS) + 1) :
            graph.compute(SparkGraphComputer.class);
}