Java Code Examples for org.apache.pig.ExecType#LOCAL

@Test
public void fieldsFullBareExampleTest() throws Exception {
    PigServer pigServer = new PigServer(ExecType.LOCAL);
    Storage.Data data = resetData(pigServer);

    pigServer.registerQuery(
        "Example = " +
        "    LOAD '" + getClass().getResource("/access.log").toString() + "' " +
        "    USING nl.basjes.pig.input.apachehttpdlog.Loader(" +
        "          '" + LOGFORMAT + "', " +
        "          '-map:request.firstline.uri.query.g:HTTP.URI'," +
        "          '-map:request.firstline.uri.query.r:HTTP.URI'," +
        "          '-map:request.firstline.uri.query.s:SCREENRESOLUTION'," +
        "          '-load:nl.basjes.parse.httpdlog.dissectors.ScreenResolutionDissector:x'" +
        "           );"
    );

    pigServer.registerQuery("STORE Example INTO 'Example' USING mock.Storage();");

    validateExampleResult(data.get("Example"));
}
 

@Test
public void testMockSchema() throws Exception {
  PigServer pigServer = new PigServer(ExecType.LOCAL);
  Data data = resetData(pigServer);

  data.set("foo", "blah:chararray",
      tuple("a"),
      tuple("b"),
      tuple("c")
      );

  pigServer.registerQuery("A = LOAD 'foo' USING mock.Storage();");
  pigServer.registerQuery("B = FOREACH A GENERATE blah as a, blah as b;");
  pigServer.registerQuery("STORE B INTO 'bar' USING mock.Storage();");

  assertEquals(schema("a:chararray,b:chararray"), data.getSchema("bar"));

  List<Tuple> out = data.get("bar");
  assertEquals(tuple("a", "a"), out.get(0));
  assertEquals(tuple("b", "b"), out.get(1));
  assertEquals(tuple("c", "c"), out.get(2));
}
 

@Test(expected = RuntimeException.class)
public void testLocationStrictCheck() throws ExecException, IOException {
    Properties properties = PropertiesUtil.loadDefaultProperties();
    properties.setProperty("pig.location.check.strict", "true");
    PigServer pigServer = new PigServer(ExecType.LOCAL, properties);
    Data data = resetData(pigServer);

    data.set("foo",
            tuple("a", 1, "b"),
            tuple("b", 2, "c"),
            tuple("c", 3, "d"));

    pigServer.registerQuery("A = LOAD 'foo' USING mock.Storage() AS (f1:chararray,f2:int,f3:chararray);");
    pigServer.registerQuery("B = order A by f1,f2,f3 DESC;");
    pigServer.registerQuery("C = order A by f1,f2,f3;");
    // Storing to same location 'bar' should throw a RuntimeException
    pigServer.registerQuery("STORE B INTO 'bar' USING mock.Storage();");
    pigServer.registerQuery("STORE C INTO 'bar' USING mock.Storage();");

    List<Tuple> out = data.get("bar");
    assertEquals(tuple("a", 1, "b"), out.get(0));
    assertEquals(tuple("b", 2, "c"), out.get(1));
    assertEquals(tuple("c", 3, "d"), out.get(2));
}
 

/**
 * Verify that alias is not assignable to the ASSERT operator
 * @throws Exception
 */
@Test(expected=FrontendException.class)
public void testNegativeWithAlias() throws Exception {
    PigServer pigServer = new PigServer(ExecType.LOCAL);
    Data data = resetData(pigServer);

    data.set("foo",
            tuple(1),
            tuple(2),
            tuple(3)
            );
    try {
        pigServer.registerQuery("A = LOAD 'foo' USING mock.Storage() AS (i:int);");
        pigServer.registerQuery("B = ASSERT A BY i > 1 , 'i should be greater than 1';");
    }
    catch (FrontendException fe) {
        Util.checkMessageInException(fe, "Syntax error, unexpected symbol at or near 'B'");
        throw fe;
    }
}
 

@Test
// See PIG-3967
public void testGruntValidation() throws IOException {
    PigServer pigServer = new PigServer(ExecType.LOCAL);
    Data data = resetData(pigServer);

    data.set("foo",
            tuple("a", 1, "b"),
            tuple("b", 2, "c"),
            tuple("c", 3, "d"));

    pigServer.setValidateEachStatement(true);
    pigServer.registerQuery("A = LOAD 'foo' USING mock.Storage() AS (f1:chararray,f2:int,f3:chararray);");
    pigServer.registerQuery("store A into '" + Util.generateURI(tempDir.toString(), pigServer.getPigContext()) + "/testGruntValidation1';");
    pigServer.registerQuery("B = LOAD 'foo' USING mock.Storage() AS (f1:chararray,f2:int,f3:chararray);");
    pigServer.registerQuery("store B into '" + Util.generateURI(tempDir.toString(), pigServer.getPigContext()) + "/testGruntValidation2';"); // This should pass
    boolean validationExceptionCaptured = false;
    try {
        // This should fail due to output validation
        pigServer.registerQuery("store A into '" + Util.generateURI(tempDir.toString(),pigServer.getPigContext()) + "/testGruntValidation1';");
    } catch (FrontendException e) {
        validationExceptionCaptured = true;
    }

    assertTrue(validationExceptionCaptured);
}
 

@Test
public void testCastingAfterUnion() throws Exception {
    File f1 = Util.createInputFile("tmp", "i1.txt", new String[] {"aaa\t111"});
    File f2 = Util.createInputFile("tmp", "i2.txt", new String[] {"bbb\t222"});

    PigServer ps = new PigServer(ExecType.LOCAL, new Properties());
    ps.registerQuery("A = load '" + Util.encodeEscape(f1.getAbsolutePath()) + "' as (a,b);");
    ps.registerQuery("B = load '" + Util.encodeEscape(f2.getAbsolutePath()) + "' as (a,b);");
    ps.registerQuery("C = union A,B;");
    ps.registerQuery("D = foreach C generate (chararray)a as a,(int)b as b;");

    Schema dumpSchema = ps.dumpSchema("D");
    Schema expected = new Schema ();
    expected.add(new Schema.FieldSchema("a", DataType.CHARARRAY));
    expected.add(new Schema.FieldSchema("b", DataType.INTEGER));
    assertEquals(expected, dumpSchema);

    Iterator<Tuple> itr = ps.openIterator("D");
    int recordCount = 0;
    while(itr.next() != null)
        ++recordCount;
    assertEquals(2, recordCount);

}
 

@Test
public void testPigScriptrForBagToStringUDF() throws Exception {
	PigServer pigServer = new PigServer(ExecType.LOCAL);
	Data data = resetData(pigServer);

	data.set("foo", "myBag:bag{t:(l:chararray)}",
			tuple(bag(tuple("a"), tuple("b"), tuple("c"))));
	pigServer.registerQuery("A = LOAD 'foo' USING mock.Storage();");
	pigServer.registerQuery("B = FOREACH A GENERATE BagToString(myBag) as myBag;");
    pigServer.registerQuery("STORE B INTO 'bar' USING mock.Storage();");

    pigServer.registerQuery("C = FOREACH A GENERATE BagToString(myBag, '==') as myBag;");
    pigServer.registerQuery("STORE C INTO 'baz' USING mock.Storage();");

    List<Tuple> out = data.get("bar");
    assertEquals(schema("myBag:chararray"), data.getSchema("bar"));
    assertEquals(tuple("a_b_c"), out.get(0));

    out = data.get("baz");
    assertEquals(tuple("a==b==c"), out.get(0));
}
 

private void checkSchemaEx(String query, String expectedErr) throws IOException {
    PigServer pig = new PigServer(ExecType.LOCAL);

    boolean foundEx = false;
    try{
        Util.registerMultiLineQuery(pig, query);
        pig.dumpSchema("u");
    }catch(FrontendException e){
        PigException pigEx = LogUtils.getPigException(e);
        foundEx = true;
        if(!pigEx.getMessage().contains(expectedErr)){
            String msg = "Expected exception message matching '" 
                + expectedErr + "' but got '" + pigEx.getMessage() + "'" ;
            fail(msg);
        }
    }
    
    if(!foundEx)
        fail("No exception thrown. Exception is expected.");
    
   
}
 

@Test
public void testPigTempDir() throws Throwable {
    Properties properties = PropertiesUtil.loadDefaultProperties();
    File pigTempDir = new File(tempDir, FILE_SEPARATOR + "tmp" + FILE_SEPARATOR + "test");
    properties.put("pig.temp.dir", pigTempDir.getPath());
    PigContext pigContext=new PigContext(ExecType.LOCAL, properties);
    pigContext.connect();
    FileLocalizer.setInitialized(false);

    String tempPath= FileLocalizer.getTemporaryPath(pigContext).toString();
    Path path = new Path(tempPath);
    assertTrue(tempPath.startsWith(pigTempDir.toURI().toString()));

    FileSystem fs = FileSystem.get(path.toUri(),
            ConfigurationUtil.toConfiguration(pigContext.getProperties()));
    FileStatus status = fs.getFileStatus(path.getParent());
    // Temporary root dir should have 700 as permission
    assertEquals("rwx------", status.getPermission().toString());
    pigTempDir.delete();
    FileLocalizer.setInitialized(false);
}
 

@Test
public void testPigScriptMultipleElmementsPerTupleForBagToStringUDF() throws Exception {
	PigServer pigServer = new PigServer(ExecType.LOCAL);
	Data data = resetData(pigServer);

	data.set("foo", "myBag:bag{t:(l:chararray)}",
			tuple(bag(tuple("a", "b"), tuple("c", "d"), tuple("e", "f"))));
	pigServer.registerQuery("A = LOAD 'foo' USING mock.Storage();");
	pigServer.registerQuery("B = FOREACH A GENERATE BagToString(myBag) as myBag;");
	pigServer.registerQuery("STORE B INTO 'bar' USING mock.Storage();");

	pigServer.registerQuery("C = FOREACH A GENERATE BagToString(myBag, '^') as myBag;");
	pigServer.registerQuery("STORE C INTO 'baz' USING mock.Storage();");

    List<Tuple> out = data.get("bar");
    assertEquals(tuple("a_b_c_d_e_f"), out.get(0));

    out = data.get("baz");
    assertEquals(tuple("a^b^c^d^e^f"), out.get(0));
}
 

@Test
public void testPigStatsGetList() {
    File outputFile = null;
    try {
        String filename = this.getClass().getSimpleName() + "_" + "testPigStatsGetList";
        outputFile = File.createTempFile(filename, ".out");
        String filePath = outputFile.getAbsolutePath();
        outputFile.delete();
        PigServer pigServer = new PigServer(ExecType.LOCAL);
        pigServer.registerQuery("a = load 'test/org/apache/pig/test/data/passwd';");
        pigServer.registerQuery("b = group a by $0;");
        pigServer.registerQuery("c = foreach b generate group, COUNT(a) as cnt;");
        pigServer.registerQuery("d = group c by cnt;");
        pigServer.registerQuery("e = foreach d generate group;");
        ExecJob job = pigServer.store("e", filePath);
        JobGraph jobGraph = job.getStatistics().getJobGraph();
        assertEquals(2, jobGraph.getJobList().size());

    } catch (IOException e) {
        LOG.error("IOException while creating file ", e);
        fail("Encountered IOException");
    } finally {
        if (outputFile != null) {
            // delete the directory before returning
            deleteDirectory(outputFile);
        }
    }
}
 

@Before
public void setup() throws IOException {
    pig = new PigServer(ExecType.LOCAL);

    Util.deleteDirectory(new File(dataDir));
    try {
        pig.mkdirs(dataDir);
    } catch (IOException e) {};
}
 

/**
 * Test UNION ONSCHEMA where a common column has additional 'namespace' part
 *  in the column name in both the inputs
 * @throws IOException
 * @throws ParserException
 */
@Test
public void testUnionOnSchemaScopedColumnNameBothInp1() throws IOException, ParserException {
    PigServer pig = new PigServer(ExecType.LOCAL);
    String query = 
    "  l1 = load '" + INP_FILE_2NUMS + "' as (i : int, j : int); " 
    + "g1 = group l1 by i; "
    + "f1 = foreach g1 generate group as gkey, flatten(l1); "
    + "l2 = load '" + INP_FILE_2NUMS + "' as (i : int, x : chararray); " 
    + "g2 = group l2 by i; "
    + "f2 = foreach g2 generate group as gkey, flatten(l2); "
    + "u = union onschema f1, f2; " ; 
    Util.registerMultiLineQuery(pig, query);
    
    Schema sch = pig.dumpSchema("u");
    Schema expectedSch = 
        Utils.getSchemaFromString("gkey: int, l1::i: int, l1::j: int, l2::i: int, l2::x: chararray");
    assertEquals("Checking expected schema",sch, expectedSch);

    Iterator<Tuple> it = pig.openIterator("u");
    List<Tuple> expectedRes = 
        Util.getTuplesFromConstantTupleStrings(
                new String[] {
                        "(1,1,2,null,null)",
                        "(5,5,3,null,null)",
                        "(1,null,null,1,'2')",
                        "(5,null,null,5,'3')"
                });
    Util.checkQueryOutputsAfterSort(it, expectedRes);
}
 

@BeforeClass
public static void setUpBeforeClass() throws Exception {
    Util.copyFromLocalToLocal(
            "test/org/apache/pig/test/data/passwd", "passwd");
    Util.copyFromLocalToLocal(
            "test/org/apache/pig/test/data/passwd2", "passwd2");
    Properties props = new Properties();
    props.setProperty(PigConfiguration.PIG_OPT_MULTIQUERY, ""+true);
    myPig = new PigServer(ExecType.LOCAL, props);
}
 

private PigServer setup(String script, String endpoint) throws Exception {
    Configuration conf = new HdfsConfiguration();
    Map<String, String> parameters = new HashMap<>();
    parameters.put("ENDPOINT", endpoint);

    PigServer ps = new PigServer(ExecType.LOCAL, conf);
    ps.setBatchOn();
    ps.registerScript(script, parameters);

    return ps;
}
 

@Test
public void testUDFReturnMap_LocalMode() throws Exception {
    PigServer pig = new PigServer(ExecType.LOCAL);
    pig.registerScript(TempScriptFile.getAbsolutePath());

    Iterator<Tuple> iterator = pig.openIterator("B");
    while (iterator.hasNext()) {
        Tuple tuple = iterator.next();
        @SuppressWarnings("unchecked")
        Map<Object, Object> result = (Map<Object, Object>) tuple.get(0);
        assertEquals(result, MyUDFReturnMap.map);
    }
}
 

@Test
public void testSplitWithNotEvalCondition() throws Exception {
    String defineQ = "define minelogs org.apache.pig.test.RegexGroupCount('www\\\\.xyz\\\\.com/sports');";
    String defineL = "a = load 'nosuchfile' " +
            " using PigStorage() as (source : chararray);";
    String defineSplit = "SPLIT a INTO a1 IF (minelogs(source) > 0 ), a2 IF (NOT (minelogs(source)>0));";//    (NOT ( minelogs(source) ) > 0);";
    PigServer ps = new PigServer(ExecType.LOCAL);
    ps.registerQuery(defineQ);
    ps.registerQuery(defineL);
    ps.registerQuery(defineSplit);
}
 

@Before
public void setUp() throws Exception {
    pigServer = new PigServer(ExecType.LOCAL, new Properties());
    pc = pigServer.getPigContext();
    pc.connect();
    GenPhyOp.setPc(pc);
    POLoad ld1 = GenPhyOp.topLoadOp();
    String curDir = System.getProperty("user.dir");
    String inpDir = curDir + File.separatorChar + "test/org/apache/pig/test/data/InputFiles/";
    FileSpec fSpec = new FileSpec(Util.generateURI(inpDir + "passwd", pc), new FuncSpec(PigStorage.class.getName() , new String[]{":"}));
    ld1.setLFile(fSpec);

    POLoad ld2 = GenPhyOp.topLoadOp();
    ld2.setLFile(fSpec);

    POFilter fl1 = GenPhyOp.topFilterOpWithProj(1, 50, GenPhyOp.LTE);

    POFilter fl2 = GenPhyOp.topFilterOpWithProj(1, 50, GenPhyOp.GT);

    int[] flds = {0,2};
    Tuple sample = new DefaultTuple();
    sample.append(new String("S"));
    sample.append(new String("x"));
    sample.append(new Integer("10"));
    sample.append(new Integer("20"));
    sample.append(new String("S"));
    sample.append(new String("x"));
    sample.append(new String("S"));
    sample.append(new String("x"));

    POForEach fe1 = GenPhyOp.topForEachOPWithPlan(flds , sample);

    POForEach fe2 = GenPhyOp.topForEachOPWithPlan(flds , sample);

    sp = GenPhyOp.topUnionOp();

    PhysicalPlan plan = new PhysicalPlan();

    plan.add(ld1);
    plan.add(ld2);
    plan.add(fl1);
    plan.add(fl2);
    plan.add(fe1);
    plan.add(fe2);
    plan.add(sp);

    plan.connect(ld1, fe1);
    plan.connect(fe1, fl1);
    plan.connect(ld2, fe2);
    plan.connect(fe2, fl2);
    plan.connect(fl1, sp);
    plan.connect(fl2, sp);

    /*PlanPrinter ppp = new PlanPrinter(plan);
    ppp.visit();*/


    POLoad ld3 = GenPhyOp.topLoadOp();
    ld3.setLFile(fSpec);
    DataBag fullBag = DefaultBagFactory.getInstance().newDefaultBag();
    Tuple t=null;
    for(Result res=ld3.getNextTuple();res.returnStatus!=POStatus.STATUS_EOP;res=ld3.getNextTuple()){
        fullBag.add((Tuple)res.result);
    }

    int[] fields = {0,2};
    expBag = TestHelper.projectBag(fullBag, fields);
}
 

public TestLocal2() throws Throwable {
    pig = new PigServer(ExecType.LOCAL) ;
}
 

@Before
public void setup() throws IOException {
    pig = new PigServer(ExecType.LOCAL);
}