Java Code Examples for org.apache.pig.data.DataType#toLong()
The following examples show how to use
org.apache.pig.data.DataType#toLong() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: TestAlgebraicEval.java From spork with Apache License 2.0 | 6 votes |
|
@Test
public void testSimpleCount() throws Exception {
File tmpFile = File.createTempFile("test", "txt");
for (int i = 0; i < nullFlags.length; i++) {
System.err.println("Testing testSimpleCount with null flag:" + nullFlags[i]);
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
generateInput(ps, nullFlags[i]);
String query = "myid = foreach (group (load '"
+ Util.generateURI(tmpFile.toString(), pig.getPigContext())
+ "') all) generate COUNT($1);";
System.out.println(query);
pig.registerQuery(query);
Iterator<Tuple> it = pig.openIterator("myid");
tmpFile.delete();
Tuple t = it.next();
Long count = DataType.toLong(t.get(0));
assertEquals(this.getClass().getName() + "with nullFlags set to: "
+ nullFlags[i], count.longValue(), LOOP_COUNT);
}
}
Example 2
| Source File: TestAlgebraicEval.java From spork with Apache License 2.0 | 6 votes |
|
@Test
public void testGroupCount() throws Throwable {
File tmpFile = File.createTempFile("test", "txt");
for (int i = 0; i < nullFlags.length; i++) {
System.err.println("Testing testGroupCount with null flag:" + nullFlags[i]);
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
generateInput(ps, nullFlags[i]);
String query = "myid = foreach (group (load '"
+ Util.generateURI(tmpFile.toString(), pig.getPigContext())
+ "') all) generate group, COUNT($1) ;";
System.out.println(query);
pig.registerQuery(query);
Iterator<Tuple> it = pig.openIterator("myid");
tmpFile.delete();
Tuple t = it.next();
Long count = DataType.toLong(t.get(1));
assertEquals(this.getClass().getName() + "with nullFlags set to: "
+ nullFlags[i], count.longValue(), LOOP_COUNT);
}
}
Example 3
| Source File: TestAlgebraicEval.java From spork with Apache License 2.0 | 6 votes |
|
@Test
public void testGroupReorderCount() throws Throwable {
File tmpFile = File.createTempFile("test", "txt");
for (int i = 0; i < nullFlags.length; i++) {
System.err.println("Testing testGroupCount with null flag:" + nullFlags[i]);
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
generateInput(ps, nullFlags[i]);
String query = "myid = foreach (group (load '"
+ Util.generateURI(tmpFile.toString(), pig.getPigContext())
+ "') all) generate COUNT($1), group ;";
System.out.println(query);
pig.registerQuery(query);
Iterator<Tuple> it = pig.openIterator("myid");
tmpFile.delete();
Tuple t = it.next();
Long count = DataType.toLong(t.get(0));
assertEquals(this.getClass().getName() + "with nullFlags set to: "
+ nullFlags[i], count.longValue(), LOOP_COUNT);
}
}
Example 4
| Source File: TestBuiltin.java From spork with Apache License 2.0 | 6 votes |
|
@Test
public void testCOUNTIntermed() throws Exception {
Integer input[] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
DataBag intermediateInputBag = bagFactory.newDefaultBag();
// call initial and then Intermed
for (Integer i : input) {
Tuple t = tupleFactory.newTuple(i);
DataBag b = bagFactory.newDefaultBag();
b.add(t);
Tuple initialInput = tupleFactory.newTuple(b);
EvalFunc<?> initial = new COUNT.Initial();
intermediateInputBag.add((Tuple)initial.exec(initialInput));
}
EvalFunc<Tuple> countIntermed = new COUNT.Intermediate();
Tuple intermediateInput = tupleFactory.newTuple(intermediateInputBag);
Tuple output = countIntermed.exec(intermediateInput);
Long f1 = DataType.toLong(output.get(0));
assertEquals("Expected count to be 10", 10, f1.longValue());
}
Example 5
| Source File: TestBuiltin.java From spork with Apache License 2.0 | 6 votes |
|
@Test
public void testCOUNT_STARIntermed() throws Exception {
Integer input[] = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
DataBag intermediateInputBag = bagFactory.newDefaultBag();
// call initial and then Intermed
for (Integer i : input) {
Tuple t = tupleFactory.newTuple(i);
DataBag b = bagFactory.newDefaultBag();
b.add(t);
Tuple initialInput = tupleFactory.newTuple(b);
EvalFunc<?> initial = new COUNT_STAR.Initial();
intermediateInputBag.add((Tuple)initial.exec(initialInput));
}
EvalFunc<Tuple> countIntermed = new COUNT_STAR.Intermediate();
Tuple intermediateInput = tupleFactory.newTuple(intermediateInputBag);
Tuple output = countIntermed.exec(intermediateInput);
Long f1 = DataType.toLong(output.get(0));
assertEquals("Expected count to be 10", 10, f1.longValue());
}
Example 6
| Source File: TestAlgebraicEvalLocal.java From spork with Apache License 2.0 | 6 votes |
|
@Test
public void testSimpleCount() throws Exception {
File tmpFile = File.createTempFile("test", "txt");
for (int i = 0; i < nullFlags.length; i++) {
System.err.println("Testing testSimpleCount with null flag:" + nullFlags[i]);
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
generateInput(ps, nullFlags[i]);
String query = "myid = foreach (group (load '"
+ Util.generateURI(tmpFile.toString(), pig.getPigContext())
+ "') all) generate COUNT($1);";
System.out.println(query);
pig.registerQuery(query);
Iterator<Tuple> it = pig.openIterator("myid");
tmpFile.delete();
Tuple t = it.next();
Long count = DataType.toLong(t.get(0));
assertEquals(this.getClass().getName() + "with nullFlags set to: "
+ nullFlags[i], count.longValue(), LOOP_COUNT);
}
}
Example 7
| Source File: TestAlgebraicEvalLocal.java From spork with Apache License 2.0 | 6 votes |
|
@Test
public void testGroupCount() throws Throwable {
File tmpFile = File.createTempFile("test", "txt");
for (int i = 0; i < nullFlags.length; i++) {
System.err.println("Testing testGroupCount with null flag:" + nullFlags[i]);
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
generateInput(ps, nullFlags[i]);
String query = "myid = foreach (group (load '"
+ Util.generateURI(tmpFile.toString(), pig.getPigContext())
+ "') all) generate group, COUNT($1) ;";
System.out.println(query);
pig.registerQuery(query);
Iterator<Tuple> it = pig.openIterator("myid");
tmpFile.delete();
Tuple t = it.next();
Long count = DataType.toLong(t.get(1));
assertEquals(this.getClass().getName() + "with nullFlags set to: "
+ nullFlags[i], count.longValue(), LOOP_COUNT);
}
}
Example 8
| Source File: TestAlgebraicEvalLocal.java From spork with Apache License 2.0 | 6 votes |
|
@Test
public void testGroupReorderCount() throws Throwable {
File tmpFile = File.createTempFile("test", "txt");
for (int i = 0; i < nullFlags.length; i++) {
System.err.println("Testing testGroupCount with null flag:" + nullFlags[i]);
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
generateInput(ps, nullFlags[i]);
String query = "myid = foreach (group (load '"
+ Util.generateURI(tmpFile.toString(), pig.getPigContext())
+ "') all) generate COUNT($1), group ;";
System.out.println(query);
pig.registerQuery(query);
Iterator<Tuple> it = pig.openIterator("myid");
tmpFile.delete();
Tuple t = it.next();
Long count = DataType.toLong(t.get(0));
assertEquals(this.getClass().getName() + "with nullFlags set to: "
+ nullFlags[i], count.longValue(), LOOP_COUNT);
}
}
Example 9
| Source File: UnixToISO.java From spork with Apache License 2.0 | 5 votes |
|
@Override
public String exec(Tuple input) throws IOException
{
if (input == null || input.size() < 1 || input.get(0) == null) {
return null;
}
DateTime result = new DateTime(DataType.toLong(input.get(0)));
return result.toString();
}
Example 10
| Source File: ToDate.java From spork with Apache License 2.0 | 4 votes |
|
public DateTime exec(Tuple input) throws IOException {
if (input == null || input.size() < 1 || input.get(0) == null) {
return null;
}
return new DateTime(DataType.toLong(input.get(0)));
}
Example 11
| Source File: TestAlgebraicEval.java From spork with Apache License 2.0 | 4 votes |
|
@Test
public void testGroupUniqueColumnCount() throws Throwable {
File tmpFile = File.createTempFile("test", "txt");
for (int i = 0; i < nullFlags.length; i++) {
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
long groupsize = 0;
if(nullFlags[i] == false) {
// generate data without nulls
for(int j = 0; j < LOOP_COUNT; j++) {
if(j%10 == 0) groupsize++;
ps.println(j%10 + ":" + j);
}
} else {
// generate data with nulls
for(int j = 0; j < LOOP_COUNT; j++) {
if(j%10 == 0) groupsize++;
if(j % 20 == 0) {
// for half the groups
// emit nulls
ps.println(j%10 + ":");
groupsize--;
} else {
ps.println(j%10 + ":" + j);
}
}
}
ps.close();
String query = "myid = foreach (group (load '"
+ Util.generateURI(tmpFile.toString(), pig.getPigContext())
+ "' using " + PigStorage.class.getName()
+ "(':')) by $0) generate group, COUNT($1.$1) ;";
System.out.println(query);
pig.registerQuery(query);
Iterator<Tuple> it = pig.openIterator("myid");
tmpFile.delete();
System.err.println("Output from testGroupUniqueColumnCount");
while(it.hasNext()) {
Tuple t = it.next();
System.err.println(t);
String a = t.get(0).toString();
Double group = Double.valueOf(a.toString());
if(group == 0.0) {
Long count = DataType.toLong(t.get(1));
assertEquals(this.getClass().getName() + "with nullFlags set to: "
+ nullFlags[i], groupsize, count.longValue());
}
}
}
}
Example 12
| Source File: TestAlgebraicEval.java From spork with Apache License 2.0 | 4 votes |
|
@Test
public void testGroupDuplicateColumnCount() throws Throwable {
File tmpFile = File.createTempFile("test", "txt");
for (int i = 0; i < nullFlags.length; i++) {
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
long groupsize = 0;
long nonNullCnt = 0;
if(nullFlags[i] == false) {
// generate data without nulls
for(int j = 0; j < LOOP_COUNT; j++) {
if(j%10 == 0) {groupsize++; nonNullCnt++;}
ps.println(j%10 + ":" + j);
}
} else {
// generate data with nulls
for(int j = 0; j < LOOP_COUNT; j++) {
if(j%10 == 0) {groupsize++; nonNullCnt++;}
if(j % 20 == 0) {
// for half the groups
// emit nulls
nonNullCnt--;
ps.println(j%10 + ":");
} else {
ps.println(j%10 + ":" + j);
}
}
}
ps.close();
String query = "myid = foreach (group (load '"
+ Util.generateURI(tmpFile.toString(), pig.getPigContext())
+ "' using " + PigStorage.class.getName()
+ "(':')) by $0) generate group, COUNT($1.$1), COUNT($1.$0) ;";
System.out.println(query);
pig.registerQuery(query);
Iterator<Tuple> it = pig.openIterator("myid");
tmpFile.delete();
System.err.println("Output from testGroupDuplicateColumnCount");
while(it.hasNext()) {
Tuple t = it.next();
System.err.println(t);
String a = t.get(0).toString();
Double group = Double.valueOf(a.toString());
if(group == 0.0) {
Long count = DataType.toLong(t.get(2));
assertEquals(this.getClass().getName() + "with nullFlags set to: "
+ nullFlags[i],groupsize, count.longValue());
count = DataType.toLong(t.get(1));
assertEquals(this.getClass().getName() + "with nullFlags set to: "
+ nullFlags[i],nonNullCnt, count.longValue());
}
}
}
}
Example 13
| Source File: TestAlgebraicEvalLocal.java From spork with Apache License 2.0 | 4 votes |
|
@Test
public void testGroupUniqueColumnCount() throws Throwable {
File tmpFile = File.createTempFile("test", "txt");
for (int i = 0; i < nullFlags.length; i++) {
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
long groupsize = 0;
if(nullFlags[i] == false) {
// generate data without nulls
for(int j = 0; j < LOOP_COUNT; j++) {
if(j%10 == 0) groupsize++;
ps.println(j%10 + ":" + j);
}
} else {
// generate data with nulls
for(int j = 0; j < LOOP_COUNT; j++) {
if(j%10 == 0) groupsize++;
if(j % 20 == 0) {
// for half the groups
// emit nulls
ps.println(j%10 + ":");
groupsize --;
} else {
ps.println(j%10 + ":" + j);
}
}
}
ps.close();
String query = "myid = foreach (group (load '"
+ Util.generateURI(tmpFile.toString(), pig.getPigContext())
+ "' using " + PigStorage.class.getName()
+ "(':')) by $0) generate group, COUNT($1.$1) ;";
System.out.println(query);
pig.registerQuery(query);
Iterator<Tuple> it = pig.openIterator("myid");
tmpFile.delete();
System.err.println("Output from testGroupUniqueColumnCount");
while(it.hasNext()) {
Tuple t = it.next();
System.err.println(t);
String a = t.get(0).toString();
Double group = Double.valueOf(a.toString());
if(group == 0.0) {
Long count = DataType.toLong(t.get(1));
assertEquals(this.getClass().getName() + "with nullFlags set to: "
+ nullFlags[i], groupsize, count.longValue());
}
}
}
}
Example 14
| Source File: TestAlgebraicEvalLocal.java From spork with Apache License 2.0 | 4 votes |
|
@Test
public void testGroupDuplicateColumnCount() throws Throwable {
File tmpFile = File.createTempFile("test", "txt");
for (int i = 0; i < nullFlags.length; i++) {
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
long groupsize = 0;
long nonNullCnt = 0;
if(nullFlags[i] == false) {
// generate data without nulls
for(int j = 0; j < LOOP_COUNT; j++) {
if(j%10 == 0) {groupsize++; nonNullCnt++;}
ps.println(j%10 + ":" + j);
}
} else {
// generate data with nulls
for(int j = 0; j < LOOP_COUNT; j++) {
if(j%10 == 0) {groupsize++; nonNullCnt++;}
if(j % 20 == 0) {
// for half the groups
// emit nulls
ps.println(j%10 + ":");
nonNullCnt--;
} else {
ps.println(j%10 + ":" + j);
}
}
}
ps.close();
String query = "myid = foreach (group (load '"
+ Util.generateURI(tmpFile.toString(), pig.getPigContext())
+ "' using "
+ PigStorage.class.getName()
+ "(':')) by $0) generate group, COUNT($1.$1), COUNT($1.$0) ;";
System.out.println(query);
pig.registerQuery(query);
Iterator<Tuple> it = pig.openIterator("myid");
tmpFile.delete();
System.err.println("Output from testGroupDuplicateColumnCount");
while(it.hasNext()) {
Tuple t = it.next();
System.err.println(t);
String a = t.get(0).toString();
Double group = Double.valueOf(a.toString());
if(group == 0.0) {
Long count = DataType.toLong(t.get(2));
assertEquals(this.getClass().getName() + "with nullFlags set to: "
+ nullFlags[i], groupsize, count.longValue());
count = DataType.toLong(t.get(1));
assertEquals(this.getClass().getName() + "with nullFlags set to: "
+ nullFlags[i], nonNullCnt, count.longValue());
}
}
}
}
Example 15
| Source File: Coalesce.java From datafu with Apache License 2.0 | 4 votes |
|
@Override
public Object exec(Tuple input) throws IOException
{
if (input == null || input.size() == 0)
{
return null;
}
Byte type = (Byte)getInstanceProperties().get("type");
for (Object o : input)
{
if (o != null)
{
if (strict)
{
return o;
}
else
{
try
{
switch (type)
{
case DataType.INTEGER:
return DataType.toInteger(o);
case DataType.LONG:
return DataType.toLong(o);
case DataType.DOUBLE:
return DataType.toDouble(o);
case DataType.FLOAT:
return DataType.toFloat(o);
default:
return o;
}
}
catch (Exception e)
{
DataFuException dfe = new DataFuException(e.getMessage(),e);
dfe.setData(o);
dfe.setFieldAliases(getFieldAliases());
throw dfe;
}
}
}
}
return null;
}
