Java Code Examples for org.apache.pig.data.DataType#toInteger()

/**
 * java level API
 * @param input expects a numeric value to round, a number of digits to keep, and an optional rounding mode.
 * @return output returns a single numeric value, the number with only those digits retained
 */
@Override
public Double exec(Tuple input) throws IOException {
    if (input == null || input.size() < 2)
        return null;

    try {
        Double       num    = DataType.toDouble(input.get(0));
        Integer      digits = DataType.toInteger(input.get(1));
        RoundingMode mode   = (input.size() >= 3) ?
            RoundingMode.valueOf(DataType.toInteger(input.get(2))) : RoundingMode.HALF_EVEN;
        if (num == null) return null;

        BigDecimal bdnum  = BigDecimal.valueOf(num);
        bdnum = bdnum.setScale(digits, mode);
        return bdnum.doubleValue();
    } catch (NumberFormatException nfe){
        System.err.println("Failed to process input; error - " + nfe.getMessage());
        return null;
    } catch (Exception e){
        throw new IOException("Caught exception processing input row ", e);
    }
}
 

@Test
public void testLargeFile () throws Exception {
    System.out.println("Running testLargeFile...");
    pig.registerQuery("A = load " + fileName + ";");
    pig.registerQuery("A = group A by $0;");
    pig.store("A", tmpFile1, "BinStorage()");
    pig.registerQuery("B = foreach A generate group, COUNT($1);");

    Iterator<Tuple> B = pig.openIterator("B");

    while (B.hasNext()) {
        Tuple temp = B.next();
        int index = DataType.toInteger(temp.get(0));
        int value = DataType.toInteger(temp.get(1));
        System.out.println("COUNT [" + index + "] = " + COUNT[index] + " B[" + index + "] = " + value);

        assertEquals(COUNT[index].intValue(), value);
    }
}
 

@Test
public void testOrder() throws Exception {
    System.out.println("Running testOrder...");
    int N = 0, Nplus1 = 0;
    pig.registerQuery("A = load " + fileName + ";");
    pig.registerQuery("B = order A by $0;");

    Iterator<Tuple> B = pig.openIterator("B");

    if (B.hasNext()) {
        N = DataType.toInteger(B.next().get(0));
    }

    while (B.hasNext()) {
        Nplus1 = DataType.toInteger(B.next().get(0));

        assertTrue("Expecting Nplus ["+Nplus1+"] to be greater than or equal to N ["+N+"]", Nplus1 >= N);

        N = Nplus1;
    }
}
 

/**
 * java level API
 * @param input expects a tuple containing two numeric DataAtom value
 * @param output returns a single numeric DataAtom value, which is 
 * fistArgument pow(2,secondArgument) rounded as if performed by a 
 * single correctly rounded floating-point multiply to a member of 
 * the double value set.
 */
public Double exec(Tuple input) throws IOException {
       if (input == null || input.size() < 2)
           return null;
       if (input.get(0) == null || input.get(1) == null) {
           return null;
       }
	try{
		Double first = DataType.toDouble(input.get(0));
		Integer second = DataType.toInteger(input.get(1));

           return Math.scalb(first, second);
	} catch (NumberFormatException nfe){
           System.err.println("Failed to process input; error - " + nfe.getMessage());
           return null;
       } catch(Exception e){
           throw new IOException("Caught exception in MAX.Initial", e);
       }
}
 

private void verify(String query, boolean descending) throws Exception {
    pig.registerQuery(query);
    Iterator<Tuple> it = pig.openIterator("myid");
    int col = (descending ? 1 : 0);
    for(int i = 0; i < DATALEN; i++) {
        Tuple t = (Tuple)it.next();
        int value = DataType.toInteger(t.get(1));
        assertEquals(Integer.parseInt(DATA[col][i]), value);
    }
    assertFalse(it.hasNext());
}
 

@Test
public void testDistinct() throws Exception {
    System.out.println("Running testDistinct...");
    pig.registerQuery("A = load " + fileName + ";");
    pig.registerQuery("B = distinct A;");

    Iterator <Tuple> B = pig.openIterator("B");

    Integer[] COUNT_Test = new Integer[max_rand];
    Integer[] COUNT_Data = new Integer[max_rand];

    for(int i = 0; i < max_rand; i++) {
        COUNT_Test[i] = 0;
        if (COUNT[i] > 0) {
            COUNT_Data[i] = 1;
        } else {
            COUNT_Data[i] = 0;
        }
    }

    while(B.hasNext()) {
        int temp = DataType.toInteger(B.next().get(0));
        COUNT_Test[temp] ++;
    }

    for(int i = 0; i < max_rand; i++) {
        assertEquals(COUNT_Test[i].intValue(), COUNT_Data[i].intValue());
    }
}
 

@Test(expected = ExecException.class)
public void testIntegerConversionErr() throws Exception {
    List list = new ArrayList();
    try {
        DataType.toInteger(list);
    } catch (ExecException ee) {
        assertEquals(1071, ee.getErrorCode());
        throw ee;
    }
}
 

@Test(expected = ExecException.class)
public void testIntegerConversionErr1() throws Exception {
    DataByteArray ba = new DataByteArray("hello world");
    try {
        DataType.toInteger(ba);
    } catch (ExecException ee) {
        assertEquals(1074, ee.getErrorCode());
        throw ee;
    }
}
 

@Test
public void testPi() throws Exception {

    pig.registerQuery("A = load 'foo' using mock.Storage() as (x:double, y:double);");
    pig.registerQuery("B = foreach A generate $0 - 0.5 as d1, $1 - 0.5 as d2;");
    pig.registerQuery("C = foreach B generate $0 * $0 as m1, $1 * $1 as m2;");
    pig.registerQuery("D = foreach C generate $0 + $1 as s1;");
    pig.registerQuery("D = foreach D generate $0, ARITY($0);");

    pig.registerQuery("E = filter D by $0 <= 0.25;");

    pig.registerQuery("F = cogroup D by $1, E by $1;");
    pig.registerQuery("G = foreach F generate COUNT($1) as total, COUNT($2) as in_circle;");

    Iterator<Tuple> values = pig.openIterator("G");
    assertTrue(values.hasNext());
    Tuple finalValues = values.next();
    int totalPoints = DataType.toInteger(finalValues.get(0));
    int inCirclePoints = DataType.toInteger(finalValues.get(1));

    log.info("Value of PI = " + 4 * (double)inCircle / (double)total);
    log.info("Value of PI (From Test data) = " + 4 * (double)inCirclePoints / (double)totalPoints);


    Iterator<Tuple> lengthTest = pig.openIterator("D");

    while(lengthTest.hasNext()) {
        Tuple temp = lengthTest.next();
        totalLengthTest += temp.get(0).toString().length();
    }

    assertEquals(totalPoints, total);
    assertEquals(inCirclePoints, inCircle);
    assertEquals(totalLengthTest, totalLength);
}
 

/***
 * Check if the given dataset is properly sorted
 * @param dataIter the dataset to be checked
 * @param sortCols list of sorted columns
 * @param descFlags flags (true=desc, false=asc)
 */
private void checkOrder(Iterator<Tuple> dataIter,
                        int[] sortCols,
                        boolean[] descFlags)
                                    throws ExecException {

    assertEquals("checkOrder params have to be of the same size",
                                    sortCols.length, descFlags.length);

    List<String> error = new ArrayList<String>() ;

    Tuple lastTuple = null ;

    while (dataIter.hasNext()) {

        Tuple current = dataIter.next() ;
        System.out.println(current.toString()) ;

        if (lastTuple != null) {
            // do the actual check
            for(int i=0; i < sortCols.length ; i++) {

                int colIdx = sortCols[i] ;
                int lastInt = DataType.toInteger(lastTuple.get(colIdx)) ;
                int curInt = DataType.toInteger(current.get(colIdx)) ;

                // If it's ascending
                if (!descFlags[i]) {
                    if (curInt < lastInt) {
                        error.add("Not ASC") ;
                    }
                    // if this happens, no need to check further
                    if (curInt > lastInt) {
                        break ;
                    }
                }
                // If it's descending
                else {
                    if (curInt > lastInt) {
                        error.add("Not DESC") ;
                    }
                    // if this happens, no need to check further
                    if (curInt < lastInt) {
                        break ;
                    }
                }
            }
        }

        lastTuple = current ;
    }

    assertEquals(0, error.size());

}
 

@Override
public Object exec(Tuple input) throws IOException
{    
  if (input == null || input.size() == 0)
  {
    return null;
  }
  
  Byte type = (Byte)getInstanceProperties().get("type");
              
  for (Object o : input)
  {
    if (o != null)
    {
      if (strict)
      {
        return o;
      }
      else
      {
        try
        {
          switch (type)
          {
          case DataType.INTEGER:
            return DataType.toInteger(o);
          case DataType.LONG:
            return DataType.toLong(o);
          case DataType.DOUBLE:
            return DataType.toDouble(o); 
          case DataType.FLOAT:
            return DataType.toFloat(o);      
          default:
            return o;
          }
        }
        catch (Exception e)
        {
          DataFuException dfe = new DataFuException(e.getMessage(),e);
          dfe.setData(o);
          dfe.setFieldAliases(getFieldAliases());
          throw dfe;
        }
      }
    }
  }
  
  return null;
}
 

public void testIndexes(boolean ascOrdering) throws Exception {

        String order = (ascOrdering) ? "ASC" : "DESC";

        String query = "A = load 'test' USING mock.Storage();" +
        "B = order A by index " + order + ";" +
        "store B into 'result' using mock.Storage();";

        Util.registerMultiLineQuery(pigServer, query);

        Iterator<Tuple> it = data.get("result").iterator();

        int toCompare, value;

        for (int i = 0; i < MAX; i++) {

            Tuple t = (Tuple) it.next();
            value = DataType.toInteger(t.get(0));
            toCompare = (ascOrdering) ? i : MAX - i - 1;

            System.out.println("RESULT: " + toCompare + "," + value);

            assertEquals(toCompare, value);
        }

        assertFalse(it.hasNext());
    }