Java Code Examples for org.apache.pig.LoadFunc#getNext()

The following examples show how to use org.apache.pig.LoadFunc#getNext() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestBuiltin.java    From spork with Apache License 2.0 6 votes vote down vote up 
/**
 * test {@link TextLoader} - this also tests that {@link TextLoader} is capable
 * of reading data a couple of dirs deep when the input specified is the top
 * level directory
 */
@Test
public void testLFText() throws Exception {
    String input1 = "This is some text.\nWith a newline in it.\n";
    String expected1 = "This is some text.";
    String expected2 = "With a newline in it.";
    Util.createInputFile(cluster,
            "testLFTest-input1.txt",
            new String[] {input1});
    // check that loading the top level dir still reading the file a couple
    // of subdirs below
    LoadFunc text1 = new ReadToEndLoader(new TextLoader(), ConfigurationUtil.
        toConfiguration(cluster.getProperties()), "testLFTest-input1.txt", 0);
    Tuple f1 = text1.getNext();
    Tuple f2 = text1.getNext();
    Util.deleteFile(cluster, "testLFTest-input1.txt");
    assertTrue(expected1.equals(f1.get(0).toString()) &&
        expected2.equals(f2.get(0).toString()));
    Util.createInputFile(cluster, "testLFTest-input2.txt", new String[] {});
    LoadFunc text2 = new ReadToEndLoader(new TextLoader(), ConfigurationUtil.
        toConfiguration(cluster.getProperties()), "testLFTest-input2.txt", 0);
    Tuple f3 = text2.getNext();
    Util.deleteFile(cluster, "testLFTest-input2.txt");
    assertTrue(f3 == null);
}
Example 2
Source File: TestBuiltin.java    From spork with Apache License 2.0 5 votes vote down vote up 
@Test
public void testSFPig() throws Exception {
    Util.resetStateForExecModeSwitch();
    PigServer mrPigServer = new PigServer(cluster.getExecType(), properties);
    String inputStr = "amy\tbob\tcharlene\tdavid\terin\tfrank";
    Util.createInputFile(cluster, "testSFPig-input.txt", new String[]
                                                                {inputStr});
    DataByteArray[] input = { new DataByteArray("amy"),
        new DataByteArray("bob"), new DataByteArray("charlene"),
        new DataByteArray("david"), new DataByteArray("erin"),
        new DataByteArray("frank") };
    Tuple f1 = Util.loadTuple(TupleFactory.getInstance().
            newTuple(input.length), input);
    String outputLocation = "testSFPig-output.txt";
    String query = "a = load 'testSFPig-input.txt';" +
            "store a into '" + outputLocation + "';";
    mrPigServer.setBatchOn();
    Util.registerMultiLineQuery(mrPigServer, query);
    mrPigServer.executeBatch();
    LoadFunc lfunc = new ReadToEndLoader(new PigStorage(), ConfigurationUtil.
        toConfiguration(cluster.getProperties()), outputLocation, 0);
    Tuple f2 = lfunc.getNext();
    Util.deleteFile(cluster, "testSFPig-input.txt");

    Util.deleteFile(cluster, outputLocation);
    assertEquals(f1, f2);
}
Example 3
Source File: TestBuiltin.java    From spork with Apache License 2.0 4 votes vote down vote up 
@Test
public void testLFPig() throws Exception {
    Util.createInputFile(cluster, "input.txt", new String[]
        {"this:is:delimited:by:a:colon\n"});
    int arity1 = 6;
    LoadFunc lf = new PigStorage(":");
    LoadFunc p1 = new ReadToEndLoader(lf, ConfigurationUtil.
        toConfiguration(cluster.getProperties()), "input.txt", 0);
    Tuple f1 = p1.getNext();
    assertTrue(f1.size() == arity1);
    Util.deleteFile(cluster, "input.txt");

    int LOOP_COUNT = 100;
    String[] input = new String[LOOP_COUNT * LOOP_COUNT];
    int n = 0;
    for (int i = 0; i < LOOP_COUNT; i++) {
        for (int j = 0; j < LOOP_COUNT; j++) {
            input[n++] = (i + "\t" + i + "\t" + j % 2);
        }
    }
    Util.createInputFile(cluster, "input.txt", input);

    LoadFunc p15 = new ReadToEndLoader(new PigStorage(), ConfigurationUtil.
        toConfiguration(cluster.getProperties()), "input.txt", 0);

    int count = 0;
    while (true) {
        Tuple f15 = p15.getNext();
        if (f15 == null)
            break;
        count++;
        assertEquals(3, f15.size());
    }
    assertEquals(LOOP_COUNT * LOOP_COUNT, count);
    Util.deleteFile(cluster, "input.txt");

    String input2 = ":this:has:a:leading:colon\n";
    int arity2 = 6;
    Util.createInputFile(cluster, "input.txt", new String[] {input2});
    LoadFunc p2 = new ReadToEndLoader(new PigStorage(":"), ConfigurationUtil.
        toConfiguration(cluster.getProperties()), "input.txt", 0);
    Tuple f2 = p2.getNext();
    assertTrue(f2.size() == arity2);
    Util.deleteFile(cluster, "input.txt");

    String input3 = "this:has:a:trailing:colon:\n";
    int arity3 = 6;
    Util.createInputFile(cluster, "input.txt", new String[] {input3});
    LoadFunc p3 = new ReadToEndLoader(new PigStorage(":"), ConfigurationUtil.
        toConfiguration(cluster.getProperties()), "input.txt", 0);
    Tuple f3 = p3.getNext();
    assertTrue(f3.size() == arity3);
    Util.deleteFile(cluster, "input.txt");
}