org.apache.pig.pigunit.PigTest Java Examples

@Test
public void setIntersectTestExample() throws Exception
{    
  PigTest test = createPigTestFromString(setIntersectTestExample);    
  
  // document with words to check for
  writeLinesToFile("docs", 
                   "1,word1 word4 word2 word1",
                   "2,word2 word6 word1 word5 word3 word7",
                   "3,word1 word3 word4 word5");
  
  // query to apply to document
  writeLinesToFile("query", 
                   "word2 word7 word5");
                
  test.runScript();
  
  assertOutput(test, "result",
               "(1,1)",
               "(2,3)",
               "(3,1)");
}
 

@Test
public void varTestLong() throws Exception
{
  PigTest test = createPigTestFromString(varTest,
      "VAL_TYPE=long");

  String[] input = {"1","2","3","4","10","5","6","7","8","9"};
  writeLinesToFile("input", input);
      
  test.runScript();
  
  List<Tuple> output = getLinesForAlias(test, "data_out", true);
  
  assertEquals(output.size(),1);
  assertEquals(output.get(0).toString(), "(8.25)");
}
 

@Test
public void quantileSchemaTest4() throws Exception
{
  PigTest test = createPigTestFromString(quantileSchemaTest,
                               "UDF=StreamingQuantile", 
                               "QUANTILES='3'",
                               "EXPECTED_OUTPUT=quantiles::quantile_0, "+
                                               "quantiles::quantile_1, "+
                                               "quantiles::quantile_2");

  String[] input = {"1","5","3","4","2"};
  writeLinesToFile("input", input);
      
  test.runScript();
  
  List<Tuple> output = getLinesForAlias(test, "data_out", true);
  
  assertEquals(output.size(),1);
  assertEquals(output.get(0).toString(), "(1.0,3.0,5.0)");
}
 

@Test
public void hasherGoodTest() throws Exception
{
  PigTest test = createPigTestFromString(hasherGoodTest);

  writeLinesToFile("input",
                   "Of all who give and receive gifts, ",
                   "such as they are wisest. ",
                   "Everywhere they are wisest. They are the magi.");

  test.runScript();

  assertOutput(test, "good_hashes",
    "(y,y)",
    "(y,y)",
    "(y,y)");
}
 

@Test
public void testNtoN() throws ParseException, IOException {
    String[] args = {
                    "n=3",
                    "reducers=1",
                    "input=top_queries_input_data.txt",
                    "output=top_3_queries",
    };
    test = new PigTest(PIG_SCRIPT, args);

    String[] output = {
                    "(yahoo,25)",
                    "(facebook,15)",
                    "(twitter,7)",
    };

    test.assertOutput("queries_limit", output);
}
 

@Test
public void quantileSchemaTest3() throws Exception
{
  PigTest test = createPigTestFromString(quantileSchemaTest,
                               "UDF=StreamingQuantile", 
                               "QUANTILES='0.0','0.5','1.0'",
                               "EXPECTED_OUTPUT=quantiles::quantile_0_0, "+
                                               "quantiles::quantile_0_5, "+
                                               "quantiles::quantile_1_0");

  String[] input = {"1","5","3","4","2"};
  writeLinesToFile("input", input);
      
  test.runScript();
  
  List<Tuple> output = getLinesForAlias(test, "data_out", true);
  
  assertEquals(output.size(),1);
  assertEquals(output.get(0).toString(), "(1.0,3.0,5.0)");
}
 

@Test
public void transposeTest() throws Exception
{ 
  PigTest test = createPigTestFromString(transposeTest);
  writeLinesToFile("input", "1,10,11,12",
                            "2,20,21,22");
  test.runScript();
  
  List<Tuple> output = getLinesForAlias(test, "data3");
  for (Tuple tuple : output) {
    int testCase = (Integer)tuple.get(0);
    DataBag bag = (DataBag)tuple.get(1);
    Assert.assertEquals(bag.size(), 3);
    int i=0;
    for (Tuple t : bag) {
      String expectedKey = String.format("val%d",i+1);
      Assert.assertEquals((String)t.get(0), expectedKey);
      int actualValue = (Integer)t.get(1); 
      Assert.assertEquals(actualValue, testCase*10+i);
      i++;
    }
  }
}
 

/**
 * Test the RandomUUID UDF.  The main purpose is to make sure it can be used in a Pig script.
 * Also the range of length of values is tested.
 *
 * @throws Exception
 */
@Test
public void randomUUIDTest() throws Exception
{
    PigTest test = createPigTestFromString(randomUUIDTest);

    writeLinesToFile("input",
            "input1",
            "input2",
            "input3");

    List<Tuple> tuples = getLinesForAlias(test, "data2", true);
    Set<UUID> set = new HashSet<UUID>();
    for (Tuple tuple : tuples)
    {
        set.add(UUID.fromString((String)tuple.get(1)));
    }
    Assert.assertEquals(set.size(), 3);
}
 

@Test
public void sampleByKeyTest() throws Exception
{
  PigTest test = createPigTestFromString(sampleByKeyTest);

  writeLinesToFile("input",
                   "A1\tB1\t1","A1\tB1\t4","A1\tB3\t4","A1\tB4\t4",
                   "A2\tB1\t4","A2\tB2\t4",
                   "A3\tB1\t3","A3\tB1\t1","A3\tB3\t77",
                   "A4\tB1\t3","A4\tB2\t3","A4\tB3\t59","A4\tB4\t29",
                   "A5\tB1\t4",
                   "A6\tB2\t3","A6\tB2\t55","A6\tB3\t1",
                   "A7\tB1\t39","A7\tB2\t27","A7\tB3\t85",
                   "A8\tB1\t4","A8\tB2\t45",
                   "A9\tB3\t92", "A9\tB1\t42","A9\tB2\t1","A9\tB3\t0",
                   "A10\tB1\t7","A10\tB2\t23","A10\tB3\t1","A10\tB4\t41","A10\tB5\t52");

  test.runScript();
  assertOutput(test, "sampled",
               "(A4,B1,3)","(A4,B2,3)","(A4,B3,59)","(A4,B4,29)",
               "(A5,B1,4)",
               "(A6,B2,3)","(A6,B2,55)","(A6,B3,1)",
               "(A7,B1,39)","(A7,B2,27)","(A7,B3,85)",
               "(A10,B1,7)","(A10,B2,23)","(A10,B3,1)","(A10,B4,41)","(A10,B5,52)");
}
 

@Test
public void noOrderEmpiricalEntropoyTest() throws Exception
{
  PigTest test = createPigTestFromString(noOrderEntropy);
  
  writeLinesToFile("input",
                   "98.94791",
                   "38.61010",
                   "38.61010",
                   "37.10575",
                   "62.28313",
                   "38.61010",
                   "32.05370",
                   "96.10962",
                   "38.61010",
                   "96.10962",
                   "20.41135");

  try {
       test.runScript();
       List<Tuple> output = this.getLinesForAlias(test, "data_out");
       fail( "Testcase should fail");
  } catch(Exception ex) {
  }
}
 

@Test
public void hasherTest() throws Exception
{
  PigTest test = createPigTestFromString(hasherTest);

  writeLinesToFile("input",
                   "Of all who give and receive gifts, ",
                   "such as they are wisest. ",
                   "Everywhere they are wisest. They are the magi.",
                   "has_lo_md5_1065433");

  test.runScript();

  assertOutput(test, "most_hashes",
      "(y,y,y,5dd3faff,858e0486,70874abeba0d07a0848bde717968362e,89002495272c9240f228863c9d229253,ad146f93da1f3cbe,f23760b1a2ac49dd,ce2df99fc66f35f8d00abbb3e58a73dcdc46857d,a6114a2f7bad5016cd11b0f74eab219bf629a41184041bb0e3dce16463a39674,73a7c303812965c115f9190ffdff843bf79289e643dc57d1d35865462fe6059daac2cd9209f50547ce63a902dc10f659aa0a4786e338a6e69062d3eeeccee252,b4b0d2cb8da053680556da959b611614,163c13a8,270c9ada)",
      "(y,y,y,f292b84a,07f6261e,cf6c44df722abd72ccc7c485bd259ea3,f9e77d8ad1061430160b0d3a0db7d290,23506548545203c0,2a9323fe502784b0,3db8d76faa5ae6b700db8f083e3162f1ff723edd,156acc8aa21b5110140bb5201245bdfaff99cab38f7a030dc6af8e198687e789,42ae14b43e2e4d75166dec84137c1c157c97a2fbf35e380f4f3b015a0af01a3f868b8ea1566a9b7564fbaef490b4e25614823e811ab43339c14a6d2c2fd0f5d0,8d0e66419d96d4a677f69758a7cf17cf,79710b14,e708e576)",
      "(y,y,y,820c3879,70b0a439,db96091e79c70a61dd328f92a17a657e,45b006d1363e9667081e7667330ff970,b345eac42e6551ad,d9ed1275d9d17639,c9f8b9ec0c9f92a898c81972304ea221ee3b87d3,6b6c1d0e17aa96a8d9dd616e62dc5c00147ba5167bdbaf204b3b3d2a424040fa,30d738452fbd9caaef06f6c7920a02a73797eb7644a361bdf53d154e4f9b2a8fc6a80dc8d3de09706191c76bd87666584fb0150b3c0e8e9a70bf318320771ae3,506a07d334a7fa034550d839671f17f2,348d70a4,a510ed94)",
      "(y,y,y,d3af58ed,134a4b46,301d20747aca51c7b73d4ceaf622b7fa,4d0702752ff1a376475aa12ef72ea832,9081ffb69dfeafc6,82106544171ed2a7,3d08148fe48f0e00486667833fd7b8bdc63412cf,60019a95b6e67b47d7527ff814eeba1001261f54ce14684a14c3ab6f716bc934,7d4f10c90ec6ca2112ae2f92c34e0d8ff557ed3fabdcef6b4b99af72194e6a60f9df311558f6556c04ba220b5c402c4dbb6268158762c6aa91e4e0a6ef13f8ec,000008e5487b3abae7be88a1d4bad573,5c7ff6ae,a1058d3d)");
}
 

@Test
public void varTestNullEntry() throws Exception
{
  PigTest test = createPigTestFromString(varTest,
      "VAL_TYPE=double");

  String[] input = {"1","2","3","4","10","5","6","7","8","9","null"};
  writeLinesToFile("input", input);
      
  test.runScript();
  
  List<Tuple> output = getLinesForAlias(test, "data_out", true);
  
  assertEquals(output.size(),1);
  assertEquals(output.get(0).toString(), "(8.25)");
}
 

@Test
public void setIntersectEmptyBagsTest() throws Exception
{ 
  PigTest test = createPigTestFromString(setIntersectTest);    
  
  writeLinesToFile("input", 
                   "{(1,10),(2,20),(3,30),(4,40),(5,50),(6,60)}\t{(0,0),(2,20),(4,40),(8,80)}",
                   "{(1,10),(1,10),(2,20),(3,30),(3,30),(4,40),(4,40)}\t{(100,10),(300,30)}",
                   "{(1,10),(2,20)}\t{(1,10),(2,20)}",
                   "{(1,10),(2,20)}\t{}",
                   "{}\t{(1,10),(2,20)}",
                   "{}\t{}");
                
  test.runScript();
          
  assertOutput(test, "data2",
               "({(2,20),(4,40)})",
               "({})",
               "({(1,10),(2,20)})",
               "({})",
               "({})",
               "({})");
}
 

@Test
public void countDistinctTest() throws Exception
{
  PigTest test = createPigTestFromString(countDistinctTest);

  writeLinesToFile("input",
                   "A1\t1","A1\t4","A1\t4","A1\t4",
                   "A2\t4","A2\t4",
                   "A3\t3","A3\t1","A3\t77",
                   "A4\t3","A4\t3","A4\t59","A4\t29",
                   "A5\t4",
                   "A6\t3","A6\t55","A6\t1",
                   "A7\t39","A7\t27","A7\t85",
                   "A8\t4","A8\t45",
                   "A9\t92", "A9\t42","A9\t1","A9\t0",
                   "A10\t7","A10\t23","A10\t1","A10\t41","A10\t52");

  test.runScript();

  assertOutput(test, "cnt", "(10)");
}
 

@Test
public void quantile3Test() throws Exception {
  PigTest test = createPigTestFromString(quantileTest,
                               "QUANTILES='0.0013','0.0228','0.1587','0.5','0.8413','0.9772','0.9987'");

  List<String> input = new ArrayList<String>();
  for (int i=100000; i>=0; i--)
  {
    input.add(Integer.toString(i));
  }
  
  writeLinesToFile("input", input.toArray(new String[0]));
      
  test.runScript();
  
  List<Tuple> output = getLinesForAlias(test, "data_out", true);
  
  assertEquals(output.size(),1);
  assertEquals(output.get(0).toString(), "(130.0,2280.0,15870.0,50000.0,84130.0,97720.0,99870.0)");
}
 

@Test
public void accumulatedEntropyTest() throws Exception
{
  PigTest test = createPigTestFromString(accumulatedEntropy); 
  
  writeLinesToFile("input",
                   "98.94791",
                   "38.61010",
                   "97.10575",
                   "62.28313",
                   "38.61010",
                   "32.05370",
                   "96.10962",
                   "38.61010",
                   "96.10962",
                   "20.41135");
      
  test.runScript();
  
  List<Double> expectedOutput = new ArrayList<Double>();
  //the same output as @test dupValEntropyTest
  expectedOutput.add(1.834372);
  
  List<Tuple> output = this.getLinesForAlias(test, "data_out");
  verifyEqualEntropyOutput(expectedOutput, output, 5); 
}
 

@Test
public void emptyBagToNullFieldsTest() throws Exception
{
  PigTest test = createPigTestFromString(emptyBagToNullFieldsTest);

  writeLinesToFile("input",
                   "({(1,1),(2,2),(3,3),(4,4),(5,5)})",
                   "({})",
                   "{(4,4),(5,5)})");

  test.runScript();

  assertOutput(test, "data2",
               "({(1,1),(2,2),(3,3),(4,4),(5,5)})",
               "({(,)})",
               "({(4,4),(5,5)})");
}
 

@Test
public void simpleTFIDFTest() throws Exception
{
  PigTest test = createPigTestFromString(tfidfTest);

  writeLinesToFile(	"input",
				"text\tthis is a a sample",
				"text2\tthis is another another example example example");

  test.runScript();

  String expected[] = {
  		"(text,{(a,0.6931471805599453),(sample,0.5198603854199589),(this,0.0),(is,0.0)})",
  		"(text2,{(example,0.6931471805599453),(another,0.5776226780098154),(this,0.0),(is,0.0)})"};
  
  assertOutput(test, "vectors", expected);
}
 

@Test
public void inTopLevelTest() throws Exception
{
  PigTest test = createPigTestFromString(inTopLevelTest);
  
  writeLinesToFile("input", 
                   "alice\tred",
                   "bob\tblue",
                   "charlie\tgreen",
                   "dave\tred");
  test.runScript();
  
  assertOutput(test, "data2", 
               "(alice,red)",
               "(bob,blue)",
               "(dave,red)");
}
 

@Test
public void quantile5aTest() throws Exception
{
  PigTest test = createPigTestFromString(quantileTest,
                               "QUANTILES='10'");

  String[] input = {"1","2","3","4","10","5","6","7","8","9"};
  writeLinesToFile("input", input);
      
  test.runScript();
  
  List<Tuple> output = getLinesForAlias(test, "data_out", true);
  
  assertEquals(output.size(),1);
  assertEquals(output.get(0).toString(), "(1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0,10.0)");
}
 

@Test
public void unorderedPairsTest() throws Exception
{
  PigTest test = createPigTestFromString(unorderedPairsTest);

  String[] input = {
    "{(1),(2),(3),(4),(5)}"
  };

  String[] output = {
      "(1,2)",
      "(1,3)",
      "(1,4)",
      "(1,5)",
      "(2,3)",
      "(2,4)",
      "(2,5)",
      "(3,4)",
      "(3,5)",
      "(4,5)"
    };

  test.assertOutput("data",input,"data4",output);
}
 

@Test
public void unorderedPairsTest2() throws Exception
{
  PigTest test = createPigTestFromString(unorderedPairsTest2);

  this.writeLinesToFile("input", "1\t{(1),(2),(3),(4),(5)}");

  test.runScript();
  this.getLinesForAlias(test, "data3");

  this.assertOutput(test, "data3",
                    "(1,(1),(2))",
                    "(1,(1),(3))",
                    "(1,(1),(4))",
                    "(1,(1),(5))",
                    "(1,(2),(3))",
                    "(1,(2),(4))",
                    "(1,(2),(5))",
                    "(1,(3),(4))",
                    "(1,(3),(5))",
                    "(1,(4),(5))");
}
 

@Test
public void singleElemInputBagEmpiricalEntropoyTest() throws Exception
{
  PigTest test = createPigTestFromString(entropy);
  
  writeLinesToFile("input",
                   "98.94791");

  test.runScript();
  
  /* Add expected values, computed using R:
   * 
   * e.g.
   * 
   * > count=c(1)
   * > library(entropy)
   * > entropy(count)
   * [1] 0
   * 
   */
  List<Double> expectedOutput = new ArrayList<Double>();
  expectedOutput.add(0.0);
  
  List<Tuple> output = this.getLinesForAlias(test, "data_out");
  verifyEqualEntropyOutput(expectedOutput, output, 5);
}
 

@Test
public void coalesceCastIntToDoubleTest() throws Exception
{
  PigTest test = createPigTestFromString(coalesceIntAndDoubleTest);
  
  this.writeLinesToFile("input", "1,5",
                                 "2,");
  
  test.runScript();
  
  List<Tuple> lines = this.getLinesForAlias(test, "data4");
  
  Assert.assertEquals(2, lines.size());
  for (Tuple t : lines)
  {
    switch((Integer)t.get(0))
    {
    case 1:
      Assert.assertEquals(500.0, t.get(1)); break;
    case 2:
      Assert.assertEquals(10000.0, t.get(1)); break;
    default:
      Assert.fail("Did not expect: " + t.get(1));                    
    }
  }
}
 

@Test
public void varTestDouble() throws Exception
{
  PigTest test = createPigTestFromString(varTest,
      "VAL_TYPE=double");

  String[] input = {"1.0","2.0","3.0","4.0","10.0","5.0","6.0","7.0","8.0","9.0"};
  writeLinesToFile("input", input);
      
  test.runScript();
  
  List<Tuple> output = getLinesForAlias(test, "data_out", true);
  
  assertEquals(output.size(),1);
  assertEquals(output.get(0).toString(), "(8.25)");
}
 

@Test
public void enumerateTest2() throws Exception
{
  PigTest test = createPigTestFromString(enumerateTest);

  writeLinesToFile("input",
                   "({(10,{(1),(2),(3)}),(20,{(4),(5),(6)}),(30,{(7),(8)}),(40,{(9),(10),(11)}),(50,{(12),(13),(14),(15)})})",
                   "({(11,{(11),(12),(13),(14)}),(21,{(15),(16),(17),(18)}),(31,{(19),(20)}),(41,{(21),(22),(23),(24)}),(51,{(25),(26),(27)})})");

  test.runScript();

  assertOutput(test, "data4",
               "(10,{(1),(2),(3)},0)",
               "(20,{(4),(5),(6)},1)",
               "(30,{(7),(8)},2)",
               "(40,{(9),(10),(11)},3)",
               "(50,{(12),(13),(14),(15)},4)",
               "(11,{(11),(12),(13),(14)},0)",
               "(21,{(15),(16),(17),(18)},1)",
               "(31,{(19),(20)},2)",
               "(41,{(21),(22),(23),(24)},3)",
               "(51,{(25),(26),(27)},4)");
}
 

@Test
public void comprehensiveBagSplitAndEnumerate() throws Exception
{
  PigTest test = createPigTestFromString(comprehensiveBagSplitAndEnumerate);

  writeLinesToFile("input",
                   "({(A,1.0),(B,2.0),(C,3.0),(D,4.0),(E,5.0)})");

  test.runScript();

  assertOutput(test, "data_out",
               // bag #1
               "(A,1.0,1)",
               "(B,2.0,1)",
               "(C,3.0,1)",
               // bag #2
               "(D,4.0,2)",
               "(E,5.0,2)");
}
 

@Test
public void singleElemInputBagEmpiricalCondEntropoyTest() throws Exception
{
  PigTest test = createPigTestFromString(condEntropy);
  
  writeLinesToFile("input",
                   "98.94791	view");

  test.runScript();
   /*
   * library(infotheo)
   * X = c("98.94791")
   * Y = c("view")
   * condentropy(Y,X)
   * [1] 0
   */      
  List<Double> expectedOutput = new ArrayList<Double>();
  expectedOutput.add(0.0);
  
  List<Tuple> output = this.getLinesForAlias(test, "data_out");
  verifyEqualEntropyOutput(expectedOutput, output, 5);
}
 

@Test
public void bagJoinInnerTest() throws Exception {
    PigTest test = createPigTestFromString(bagJoinInnerTest);

    writeLinesToFile("input",
            "1\t{(K1,A1),(K2,B1),(K3,C1)}\t{(K1,A2),(K2,B2),(K2,B22)}\t{(K1,A3),(K3,C3),(K4,D3)}");

    try {
        test.runScript();
    } catch (Exception e) {
        e.printStackTrace();
        throw e;
    }

    assertOutput(test, "data2",
            "(1,{(K1,A1,K1,A2,K1,A3)},{(K1,A1,K1,A3,K1,A2)})");
}
 

@Test
public void bagLeftOuterJoinTest() throws Exception
{
  PigTest test = createPigTestFromString(bagLeftOuterJoinTest);

  writeLinesToFile("input",
                   "1\t{(K1,A1),(K2,B1),(K3,C1)}\t{(K1,A2),(K2,B2),(K2,B22)}\t{(K1,A3),(K3,C3),(K4,D3)}");

  test.runScript();
  
  List<Tuple> tuples = getLinesForAlias(test, "data2");
  assertEquals(tuples.size(), 1);
  Tuple tuple = tuples.get(0);
  DataBag joined1 = (DataBag)tuple.get(1);
  DataBag joined2 = (DataBag)tuple.get(2);
  
  String joined1Schema = "{(bag1::k: chararray,bag1::v: chararray,bag2::k: chararray,bag2::v: chararray,bag3::k3: chararray,bag3::v3: chararray)}";
  String joined2Schema = "{(bag1::k: chararray,bag1::v: chararray,bag3::k3: chararray,bag3::v3: chararray,bag2::k: chararray,bag2::v: chararray)}";
  String expectedJoined1 = "{(K1,A1,K1,A2,K1,A3),(K2,B1,K2,B2,,),(K2,B1,K2,B22,,),(K3,C1,,,K3,C3)}";
  String expectedJoined2 = "{(K1,A1,K1,A3,K1,A2),(K2,B1,,,K2,B2),(K2,B1,,,K2,B22),(K3,C1,K3,C3,,)}";
  
  // compare sorted bags because there is no guarantee on the order
  assertEquals(getSortedBag(joined1).toString(),getSortedBag(expectedJoined1, joined1Schema).toString());
  assertEquals(getSortedBag(joined2).toString(),getSortedBag(expectedJoined2, joined2Schema).toString());
}