Java Code Examples for org.apache.pig.PigServer#registerJar()

/**
 * The jar file to register is not present
 */
@Test
public void testRegisterJarFileNotPresent() throws Throwable {
    // resister a jar file that does not exist

    String jarName = "BadFileNameTestJarNotPresent.jar";

    // jar name is not present to start with
    PigServer pig = new PigServer(cluster.getExecType(), properties);
    verifyStringContained(pig.getPigContext().extraJars, jarName, false);
    boolean raisedException = false;
    try {
        pig.registerJar(jarName);
    } catch (IOException e) {
        raisedException = true;
    }
    assertTrue("registerJar on jarName ["+jarName+"] should have raised an exception", raisedException);
    verifyStringContained(pig.getPigContext().extraJars, jarName, false);
}
 

/**
 * Jar file to register is not present in the system resources
 * in this case name of jar file is relative to current working dir
 */
@Test
public void testRegisterJarLocalDir() throws Throwable {
    String dir1 = "test1_register_jar_local";
    String dir2 = "test2_register_jar_local";
    String jarLocation = dir1 + FILE_SEPARATOR +
                          dir2 + FILE_SEPARATOR;
    String jarName = "TestRegisterJarLocal.jar";


    createFakeJarFile(jarLocation, jarName);

    PigServer pig = new PigServer(cluster.getExecType(), properties);
    verifyStringContained(pig.getPigContext().extraJars, jarName, false);

    pig.registerJar(jarLocation + jarName);
    verifyStringContained(pig.getPigContext().extraJars, jarName, true);

    // clean-up
    assertTrue((new File(jarLocation + jarName)).delete());
    (new File(dir1 + FILE_SEPARATOR + dir2)).delete();
    (new File(dir1)).delete();
}
 

@Test
public void testRegisterJarGlobbingRelative() throws Throwable {
    String dir = "test1_register_jar_globbing_relative";
    String jarLocation = dir + FILE_SEPARATOR;
    String jar1Name = "TestRegisterJarGlobbing1.jar";
    String jar2Name = "TestRegisterJarGlobbing2.jar";

    createFakeJarFile(jarLocation, jar1Name);
    createFakeJarFile(jarLocation, jar2Name);

    PigServer pig = new PigServer(cluster.getExecType(), properties);
    pig.registerJar(jarLocation + "TestRegisterJarGlobbing*.jar");
    verifyStringContained(pig.getPigContext().extraJars, jar1Name, true);
    verifyStringContained(pig.getPigContext().extraJars, jar2Name, true);

    // clean-up
    assertTrue((new File(jarLocation + jar1Name)).delete());
    assertTrue((new File(jarLocation + jar2Name)).delete());
    (new File(dir)).delete();
}
 

@Test
public void testRegisterJarGlobbingAbsolute() throws Throwable {
    String dir = "test1_register_jar_globbing_absolute";
    String jarLocation = dir + FILE_SEPARATOR;
    String jar1Name = "TestRegisterJarGlobbing1.jar";
    String jar2Name = "TestRegisterJarGlobbing2.jar";

    createFakeJarFile(jarLocation, jar1Name);
    createFakeJarFile(jarLocation, jar2Name);

    String currentDir = System.getProperty("user.dir");
    PigServer pig = new PigServer(cluster.getExecType(), properties);
    pig.registerJar(new File(currentDir, dir) + FILE_SEPARATOR + "TestRegisterJarGlobbing*.jar");
    verifyStringContained(pig.getPigContext().extraJars, jar1Name, true);
    verifyStringContained(pig.getPigContext().extraJars, jar2Name, true);

    // clean-up
    assertTrue((new File(jarLocation + jar1Name)).delete());
    assertTrue((new File(jarLocation + jar2Name)).delete());
    (new File(dir)).delete();
}
 

@Test
public void testRegisterRemoteGlobbingJar() throws Throwable {
    String dir = "test1_register_remote_jar_globbing";
    String jarLocation = dir + FILE_SEPARATOR;
    String jar1Name = "TestRegisterRemoteJarGlobbing1.jar";
    String jar2Name = "TestRegisterRemoteJarGlobbing2.jar";

    FileSystem fs = cluster.getFileSystem();
    createFakeJarFile(jarLocation, jar1Name, fs);
    createFakeJarFile(jarLocation, jar2Name, fs);

    // find the absolute path for the directory so that it does not
    // depend on configuration
    String absPath = fs.getFileStatus(new Path(jarLocation)).getPath().toString();

    PigServer pig = new PigServer(cluster.getExecType(), properties);
    pig.registerJar(absPath + FILE_SEPARATOR + "TestRegister{Remote}Jar*.jar");

    verifyStringContained(pig.getPigContext().extraJars, jar1Name, true);
    verifyStringContained(pig.getPigContext().extraJars, jar2Name, true);

    // clean-up
    assertTrue(fs.delete(new Path(jarLocation), true));
}
 

@Override
protected Map<String, List<PigStats>> main(PigContext pigContext, String scriptFile)
        throws IOException {
    if (System.getProperty(PySystemState.PYTHON_CACHEDIR_SKIP)==null)
        System.setProperty(PySystemState.PYTHON_CACHEDIR_SKIP, "false");
    
    PigServer pigServer = new PigServer(pigContext, false);

    // register dependencies
    String jythonJar = getJarPath(PythonInterpreter.class);
    if (jythonJar != null) {
        pigServer.registerJar(jythonJar);
    }

    File f = new File(scriptFile);

    if (!f.canRead()) {
        throw new IOException("Can't read file: " + scriptFile);
    }

    FileInputStream fis1 = new FileInputStream(scriptFile);
    try {
        if (hasFunction(fis1)) {
            registerFunctions(scriptFile, null, pigContext);
        }
    } finally {
        fis1.close();
    }

    Interpreter.setMain(true);
    FileInputStream fis = new FileInputStream(scriptFile);
    try {
        load(fis, scriptFile, pigServer.getPigContext());
    } finally {
        fis.close();
    }
    return getPigStatsMap();
}
 

/**
 * Jar file is located via system resources
 * Test verifies that even with multiple resources matching,
 * only one of them is registered.
 */
@Test
public void testRegisterJarFromResources () throws Throwable {
    String dir = "test_register_jar_res_dir";
    String subDir1 = "test_register_jar_res_sub_dir1";
    String subDir2 = "test_register_jar_res_sub_dir2";
    String jarName = "TestRegisterJarFromRes.jar";
    String jarLocation1 = dir + FILE_SEPARATOR + subDir1 + FILE_SEPARATOR;
    String jarLocation2 = dir + FILE_SEPARATOR + subDir2 + FILE_SEPARATOR;


    createFakeJarFile(jarLocation1, jarName);
    createFakeJarFile(jarLocation2, jarName);

    PigServer pig = new PigServer(cluster.getExecType(), properties);
    verifyStringContained(pig.getPigContext().extraJars, jarName, false);

    registerNewResource(jarLocation1);
    registerNewResource(jarLocation2);

    pig.registerJar(jarName);
    verifyStringContained(pig.getPigContext().extraJars, jarName, true);

    // clean-up
    assertTrue((new File(jarLocation1 + jarName)).delete());
    assertTrue((new File(jarLocation2 + jarName)).delete());
    (new File(jarLocation1)).delete();
    (new File(jarLocation2)).delete();
    (new File(dir)).delete();
}
 

@Test
// See PIG-4109
public void testRegisterJarRemoteScript() throws Throwable {
    if (Util.WINDOWS) {
        properties.setProperty("pig.jars.relative.to.dfs", "true");
        String jarName = JarManager.findContainingJar(org.codehaus.jackson.JsonParser.class);
        PigServer pig = new PigServer(cluster.getExecType(), properties);
        pig.registerJar(jarName);
    }
}
 

@Test
public void testRegisterResourceMacro() throws Throwable {
    PigServer pig = new PigServer(ExecType.LOCAL);

    String macrosFile = "test/pig/macros.pig";
    File macrosJarFile = File.createTempFile("macros", ".jar");

    System.out.println("Creating macros jar " + macrosJarFile);

    Manifest manifest = new Manifest();
    manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0");

    JarOutputStream jarStream = new JarOutputStream(new FileOutputStream(macrosJarFile), manifest);

    JarEntry jarEntry = new JarEntry(macrosFile);
    jarEntry.setTime(System.currentTimeMillis());
    jarStream.putNextEntry(jarEntry);

    PrintWriter pw = new PrintWriter(jarStream);
    pw.println("DEFINE row_count_in_jar(X) RETURNS Z { Y = group $X all; $Z = foreach Y generate COUNT($X); };");
    pw.close();

    jarStream.close();

    Storage.Data data = resetData(pig);
    data.set("some_path", "(l:int)", tuple(tuple("1")), tuple(tuple("2")), tuple(tuple("3")), tuple(tuple("10")), tuple(tuple("11")));

    System.out.println("Registering macros jar " + macrosJarFile);
    pig.registerJar(macrosJarFile.toString());

    pig.registerQuery("import '" + macrosFile + "';");
    pig.registerQuery("a = load 'some_path' USING mock.Storage();");
    pig.registerQuery("b = row_count_in_jar(a);");
    Iterator<Tuple> iter = pig.openIterator("b");

    assertTrue(((Long)iter.next().get(0))==5);

    pig.shutdown();
}
 

@Test
public void testRegisterJarOverridePigJarPackages() throws IOException, ClassNotFoundException {
    // When jackson jar is not registered, jackson-core from the first jar in
    // classpath (pig.jar) should be picked up (version 1.8.8 in this case).
    String jacksonJar = JarManager.findContainingJar(org.codehaus.jackson.JsonParser.class);
    Assert.assertTrue(new File(jacksonJar).getName().contains("1.8.8"));

    PigServer pigServer = new PigServer(ExecType.LOCAL, new Properties());
    pigServer.registerJar("test/resources/jackson-core-asl-1.9.9.jar");
    pigServer.registerJar("test/resources/jackson-mapper-asl-1.9.9.jar");
    jacksonJar = JarManager.findContainingJar(org.codehaus.jackson.JsonParser.class);
    Assert.assertTrue(new File(jacksonJar).getName().contains("1.9.9"));

}
 

/**
 * Use a resource inside a jar file.
 * Verify that the containing jar file is registered correctly.
 * @throws Exception
 */
@Test
public void testRegisterJarResourceInJar() throws Throwable {
    String dir = "test_register_jar_res_in_jar";
    String subDir = "sub_dir";
    String jarName = "TestRegisterJarNonEmpty.jar";
    String className = "TestRegisterJar";
    String javaSrc = "package " + subDir + "; class " + className + " { }";


    // create dirs
    (new File(dir + FILE_SEPARATOR + subDir)).mkdirs();

    // generate java file
    FileOutputStream outStream =
        new FileOutputStream(new File(dir + FILE_SEPARATOR + subDir +
                                FILE_SEPARATOR + className + ".java"));

    OutputStreamWriter outWriter = new OutputStreamWriter(outStream);
    outWriter.write(javaSrc);
    outWriter.close();

    // compile
    int status;
    status = Util.executeJavaCommand("javac " + dir + FILE_SEPARATOR + subDir +
                           FILE_SEPARATOR + className + ".java");
    assertEquals(0, status);

    // remove src file
    (new File(dir + FILE_SEPARATOR + subDir +
              FILE_SEPARATOR + className + ".java")).delete();

    // generate jar file
    status = Util.executeJavaCommand("jar -cf " + dir + FILE_SEPARATOR + jarName + " " +
                          "-C " + dir + " " + subDir);
    assertEquals(0, status);

    // remove class file and sub_dir
    (new File(dir + FILE_SEPARATOR + subDir +
              FILE_SEPARATOR + className + ".class")).delete();
    (new File(dir + FILE_SEPARATOR + subDir)).delete();

    // register resource
    registerNewResource(dir + FILE_SEPARATOR + jarName);

    // load the specific resource
    boolean exceptionRaised = false;
    PigServer pig = new PigServer(cluster.getExecType(), properties);
    try {
        pig.registerJar("sub_dir/TestRegisterJar.class");
    }
    catch (IOException e) {
        exceptionRaised = true;
    }

    // verify proper jar file is located
    assertFalse(exceptionRaised);
    verifyStringContained(pig.getPigContext().extraJars, jarName, true);

    // clean up Jar file and test dir
    (new File(dir + FILE_SEPARATOR + jarName)).delete();
    (new File(dir)).delete();
}
 

/**
 * Register a jar for use in Pig.  Once this is done this jar will be
 * registered for <b>all subsequent</b> Pig pipelines in this script.
 * If you wish to register it for only a single Pig pipeline, use
 * register within that definition.
 * @param jarfile Path of jar to include.
 * @throws IOException if the indicated jarfile cannot be found.
 */
public static void registerJar(String jarfile) throws IOException {
    LOG.info("Register jar: "+ jarfile);
    ScriptPigContext ctx = getScriptContext();
    PigServer pigServer = new PigServer(ctx.getPigContext(), false);
    pigServer.registerJar(jarfile);
}