Java Code Examples for org.apache.pig.PigServer#getPigContext()

@Test
public void testIllustrate() throws Throwable {
    Assume.assumeTrue("Skip this test for TEZ. See PIG-3993", Util.isMapredExecType(cluster.getExecType()));
    PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
    PigContext context = server.getPigContext();

    String strCmd =
        "rmf bla;"
        +"a = load '"
        + Util.generateURI("file:test/org/apache/pig/test/data/passwd", context) + "';"
        +"e = group a by $0;"
        +"f = foreach e generate group, COUNT($1);"
        +"store f into 'bla';"
        +"f1 = load 'bla' as (f:chararray);"
        +"g = order f1 by $0;"
        +"illustrate g;";

    ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
    InputStreamReader reader = new InputStreamReader(cmd);

    Grunt grunt = new Grunt(new BufferedReader(reader), context);

    grunt.exec();
}
 

@Test
public void testExecStatmentNested() throws Throwable {
    PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
    PigContext context = server.getPigContext();
    boolean caught = false;

    String strCmd = "a = load 'foo' as (foo, fast, regenerate); exec "
            + basedir + "/testsubnested_exec.pig; explain bar";

    ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
    InputStreamReader reader = new InputStreamReader(cmd);

    Grunt grunt = new Grunt(new BufferedReader(reader), context);

    try {
        grunt.exec();
    } catch (Exception e) {
        caught = true;
        assertTrue(e.getMessage().contains("alias bar"));
    }
    assertTrue(caught);
}
 

@Test
public void testRegisterWithQuotes() throws Throwable {
    PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
    PigContext context = server.getPigContext();
    String jarName = Util.findPigJarName();

    String strCmd = "register '" + jarName + "'\n";

    ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
    InputStreamReader reader = new InputStreamReader(cmd);

    Grunt grunt = new Grunt(new BufferedReader(reader), context);

    grunt.exec();
    assertEquals(context.extraJars+ " of size 1", 1, context.extraJars.size());
    assertTrue(context.extraJars.get(0)+" ends with /" + jarName, context.extraJars.get(0).toString().endsWith("/" + jarName));
}
 

@SuppressWarnings("unchecked")
@Test
public void testCommaSeparatedString3() throws Exception {
    PigServer pig = servers[0];
    pc = pig.getPigContext();
    boolean noConversionExpected = true;
    checkLoadPath("hdfs:/tmp/test,hdfs:/tmp/test2,hdfs:/tmp/test3",
            "hdfs:/tmp/test,hdfs:/tmp/test2,hdfs:/tmp/test3", noConversionExpected );

    // check if a location 'hdfs:<abs path>,hdfs:<abs path>' can actually be
    // read using PigStorage
    String[] inputFileNames = new String[] {
            "/tmp/TestLoad-testCommaSeparatedString3-input1.txt",
            "/tmp/TestLoad-testCommaSeparatedString3-input2.txt"};
    String inputString = "hdfs:" + inputFileNames[0] + ",hdfs:" +
    inputFileNames[1];
    testLoadingMultipleFiles(inputFileNames, inputString);

}
 

@Test
public void testParsingAsGenerateInForeachBlock() throws Throwable {
    PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
    PigContext context = server.getPigContext();

    String strCmd = "a = load 'foo' as (foo, fast, regenerate); "
            + "b = group a by foo; c = foreach b {generate "
            + "{(1, '1', 0.4f),(2, '2', 0.45)} "
            + "as b: bag{t:(i: int, cease:chararray, degenerate: double)}, "
            + "SUM(a.fast) as fast, a.regenerate as degenerated;};\n";

    ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
    InputStreamReader reader = new InputStreamReader(cmd);

    Grunt grunt = new Grunt(new BufferedReader(reader), context);

    grunt.exec();
}
 

@Test
public void testInvalidParam() throws Throwable {
    PigServer server = new PigServer(ExecType.LOCAL, cluster.getProperties());
    PigContext context = server.getPigContext();

    String strCmd =
        "run -param -param;";

    ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
    InputStreamReader reader = new InputStreamReader(cmd);

    Grunt grunt = new Grunt(new BufferedReader(reader), context);

    boolean caught = false;
    try {
        grunt.exec();
    } catch (ParseException e) {
        caught = true;
        assertTrue(e.getMessage().contains("Encountered"));
    }
    assertTrue(caught);
}
 

@Test
public void testParsingGenerateInForeachWithOutBlock() throws Throwable {
    PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
    PigContext context = server.getPigContext();

    String strCmd = "a = load 'foo' as (foo, fast, regenerate); "
            + "b = group a by foo; c = foreach b generate a.regenerate;\n";

    ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
    InputStreamReader reader = new InputStreamReader(cmd);

    Grunt grunt = new Grunt(new BufferedReader(reader), context);

    grunt.exec();
}
 

@Test
public void testAddArchiveToDistributedCache() throws IOException {
    final File textFile = File.createTempFile("file", ".txt");
    textFile.deleteOnExit();

    final List<File> zipArchives = createFiles(".zip");
    zipArchives.add(textFile);
    final List<File> tarArchives = createFiles(".tgz", ".tar.gz", ".tar");

    final PigServer pigServer = new PigServer(ExecType.MAPREDUCE);
    final PigContext pigContext = pigServer.getPigContext();
    pigContext.connect();
    pigContext.getProperties().put("pig.streaming.ship.files",
            StringUtils.join(zipArchives, ","));
    pigContext.getProperties().put("pig.streaming.cache.files",
            StringUtils.join(tarArchives, ","));

    final JobConf jobConf = compileTestJob(pigContext, CONF);

    URI[] uris = DistributedCache.getCacheFiles(jobConf);
    int sizeTxt = 0;
    for (int i = 0; i < uris.length; i++) {
        if (uris[i].toString().endsWith(".txt")) {
            sizeTxt++;
        }
    }
    Assert.assertTrue(sizeTxt == 1);
    assertFilesInDistributedCache(
            DistributedCache.getCacheArchives(jobConf), 4, ".zip", ".tgz",
            ".tar.gz", ".tar");
}
 

@Test
public void testCommaSeparatedString2() throws Exception {
    for (PigServer pig : servers) {
        pc = pig.getPigContext();
        checkLoadPath("t?s*,test","/tmp/t?s*,/tmp/test");
    }
}
 

@Test
public void testCommaSeparatedString4() throws Exception {
    for (PigServer pig : servers) {
        pc = pig.getPigContext();
        checkLoadPath("usr/pig/{a,c},usr/pig/b","/tmp/usr/pig/{a,c},/tmp/usr/pig/b");
    }
}
 

@Test
public void testBagConstant() throws Throwable {
    PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
    PigContext context = server.getPigContext();

    String strCmd = "a = load 'input1'; b = foreach a generate {(1, '1', 0.4f),(2, '2', 0.45)};\n";

    ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
    InputStreamReader reader = new InputStreamReader(cmd);

    Grunt grunt = new Grunt(new BufferedReader(reader), context);

    grunt.exec();
}
 

@Test
public void testCommaSeparatedString6() throws Exception {
    for (PigServer pig : servers) {
        pc = pig.getPigContext();
        checkLoadPath("usr/pig/{a,c},/usr/pig/b","/tmp/usr/pig/{a,c},/usr/pig/b");
    }
}
 

@Test
public void testLoadRemoteAbs() throws Exception {
    for (PigServer pig : servers) {
        pc = pig.getPigContext();
        boolean noConversionExpected = true;
        checkLoadPath("/tmp/test","/tmp/test", noConversionExpected);
    }
}
 

@Before
public void setUp() throws Exception {
    pig = new PigServer(cluster.getExecType(), cluster.getProperties());
    pc = pig.getPigContext();
    inputFileName = TESTDIR + "/TestStore-" + new Random().nextLong() + ".txt";
    outputFileName = TESTDIR + "/TestStore-output-" + new Random().nextLong() + ".txt";

}
 

@Test
public void testParsingAsInForeachWithOutBlock() throws Throwable {
    PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
    PigContext context = server.getPigContext();

    String strCmd = "a = load 'foo' as (foo, fast); "
            + "b = group a by foo; c = foreach b generate SUM(a.fast) as fast;\n";

    ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
    InputStreamReader reader = new InputStreamReader(cmd);

    Grunt grunt = new Grunt(new BufferedReader(reader), context);

    grunt.exec();
}
 

@Test
public void testBagConstantInForeachBlock() throws Throwable {
    PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
    PigContext context = server.getPigContext();

    String strCmd = "a = load 'input1'; "
            + "b = foreach a {generate {(1, '1', 0.4f),(2, '2', 0.45)};};\n";

    ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
    InputStreamReader reader = new InputStreamReader(cmd);

    Grunt grunt = new Grunt(new BufferedReader(reader), context);

    grunt.exec();
}
 

@Test
public void testSuccessiveUserFuncs1() throws Exception {
    String query = "a = load 'students.txt' as (c1,c2,c3,c4); " +
            "c = group a by c2; " +
            "f = foreach c generate COUNT(org.apache.pig.builtin.Distinct($1.$2)); " +
            "store f into 'out';";
    PigServer pigServer = new PigServer(cluster.getExecType(), properties);
    PigContext pc = pigServer.getPigContext();
    assertTrue((Util.buildMRPlan(Util.buildPp(pigServer, query), pc).getRoots().get(0).combinePlan
            .isEmpty()));
    pigServer.shutdown();
}
 

@Test
public void testSuccessiveUserFuncs2() throws Exception {
    String dummyUDF = JiraPig1030.class.getName();
    String query = "a = load 'students.txt' as (c1,c2,c3,c4); " +
            "c = group a by c2; " +
            "f = foreach c generate COUNT(" + dummyUDF + "" +
            "(org.apache.pig.builtin.Distinct($1.$2)," + dummyUDF + "())); " +
            "store f into 'out';";
    PigServer pigServer = new PigServer(cluster.getExecType(), properties);
    PigContext pc = pigServer.getPigContext();
    assertTrue((Util.buildMRPlan(Util.buildPp(pigServer, query), pc).getRoots().get(0).combinePlan
            .isEmpty()));
    pigServer.shutdown();
}
 

@Test
public void testExplainOut() throws Throwable {
    PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
    PigContext context = server.getPigContext();

    String strCmd = "a = load 'foo' as (foo, fast, regenerate); explain -out /tmp -script "
            + basedir + "/testsubnested_run.pig;";

    ByteArrayInputStream cmd = new ByteArrayInputStream(strCmd.getBytes());
    InputStreamReader reader = new InputStreamReader(cmd);

    Grunt grunt = new Grunt(new BufferedReader(reader), context);

    grunt.exec();
}
 

public BlackAndWhitelistFilter(PigServer pigServer) {
    this(pigServer.getPigContext());
}