Java Code Examples for org.apache.pig.data.DataType#compare()
The following examples show how to use
org.apache.pig.data.DataType#compare() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: TOP.java From spork with Apache License 2.0 | 6 votes |
|
@Override
public int compare(Tuple o1, Tuple o2) {
int ret = 0;
if (o1 == null) {
ret = -1;
} else if (o2 == null) {
ret = 1;
} else {
try {
Object field1 = o1.get(fieldNum);
Object field2 = o2.get(fieldNum);
if (!typeFound) {
datatype = DataType.findType(field1);
if(datatype != DataType.NULL) {
typeFound = true;
}
}
ret = DataType.compare(field1, field2, datatype, datatype);
} catch (ExecException e) {
throw new RuntimeException("Error while comparing o1:" + o1
+ " and o2:" + o2, e);
}
}
return isDescOrder ? ret : ret * -1;
}
Example 2
| Source File: PartitionSkewedKeys.java From spork with Apache License 2.0 | 6 votes |
|
private boolean hasSameKey(Tuple t1, Tuple t2) {
// Have to break the tuple down and compare it field to field.
int sz1 = t1 == null ? 0 : t1.size();
int sz2 = t2 == null ? 0 : t2.size();
if (sz2 != sz1) {
return false;
}
for (int i = 0; i < sz1 - 2; i++) {
try {
int c = DataType.compare(t1.get(i), t2.get(i));
if (c != 0) {
return false;
}
} catch (ExecException e) {
throw new RuntimeException("Unable to compare tuples", e);
}
}
return true;
}
Example 3
| Source File: PigBytesRawComparator.java From spork with Apache License 2.0 | 6 votes |
|
public int compare(Object o1, Object o2) {
NullableBytesWritable nbw1 = (NullableBytesWritable)o1;
NullableBytesWritable nbw2 = (NullableBytesWritable)o2;
int rc = 0;
// If either are null, handle differently.
if (!nbw1.isNull() && !nbw2.isNull()) {
rc = DataType.compare(nbw1.getValueAsPigType(), nbw2.getValueAsPigType());
} else {
// For sorting purposes two nulls are equal.
if (nbw1.isNull() && nbw2.isNull()) rc = 0;
else if (nbw1.isNull()) rc = -1;
else rc = 1;
}
if (!mAsc[0]) rc *= -1;
return rc;
}
Example 4
| Source File: PigTupleSortComparator.java From spork with Apache License 2.0 | 6 votes |
|
private int compareTuple(Tuple t1, Tuple t2) {
int sz1 = t1.size();
int sz2 = t2.size();
if (sz2 < sz1) {
return 1;
} else if (sz2 > sz1) {
return -1;
} else {
for (int i = 0; i < sz1; i++) {
try {
int c = DataType.compare(t1.get(i), t2.get(i));
if (c != 0) {
if (!mWholeTuple && !mAsc[i])
c *= -1;
else if (mWholeTuple && !mAsc[0])
c *= -1;
return c;
}
} catch (ExecException e) {
throw new RuntimeException("Unable to compare tuples", e);
}
}
return 0;
}
}
Example 5
| Source File: Top.java From spork with Apache License 2.0 | 6 votes |
|
@Override
public int compare(Tuple o1, Tuple o2) {
if (o1 == null)
return -1;
if (o2 == null)
return 1;
try {
Object field1 = o1.get(fieldNum);
Object field2 = o2.get(fieldNum);
if (!typeFound) {
datatype = DataType.findType(field1);
typeFound = true;
}
return DataType.compare(field1, field2, datatype, datatype);
} catch (ExecException e) {
throw new RuntimeException("Error while comparing o1:" + o1
+ " and o2:" + o2, e);
}
}
Example 6
| Source File: ColumnarTupleStore.java From Cubert with Apache License 2.0 | 5 votes |
|
/**
* Implementation is copied from DefaultTuple implementation in pig.
*
* @param obj the other object to be compared
* @return -1 if the other is less, 0 if equal or 1 if the other is greater to this object
*/
@Override
public int compareTo(Object obj)
{
if (obj instanceof Tuple)
{
Tuple other = (Tuple) obj;
int otherSize = other.size();
if (otherSize < nColumns) return 1;
else if (otherSize > nColumns) return -1;
else
{
for (int i = 0; i < nColumns; i++)
{
try
{
int c = DataType.compare(get(i), other.get(i));
if (c != 0) return c;
}
catch (ExecException e)
{
throw new RuntimeException("Unable to compare tuples", e);
}
}
return 0;
}
}
else
{
return DataType.compare(this, obj);
}
}
Example 7
| Source File: FindQuantiles.java From spork with Apache License 2.0 | 5 votes |
|
@Override
@SuppressWarnings("unchecked")
public int compare(Tuple t1, Tuple t2) {
switch (mState) {
case ALL_ASC:
return t1.compareTo(t2);
case ALL_DESC:
return t2.compareTo(t1);
case MIXED:
// Have to break the tuple down and compare it field to field.
int sz1 = t1.size();
int sz2 = t2.size();
if (sz2 < sz1) {
return 1;
} else if (sz2 > sz1) {
return -1;
} else {
for (int i = 0; i < sz1; i++) {
try {
int c = DataType.compare(t1.get(i), t2.get(i));
if (c != 0) {
if (!mAsc[i]) c *= -1;
return c;
}
} catch (ExecException e) {
throw new RuntimeException("Unable to compare tuples", e);
}
}
return 0;
}
}
return -1; // keep the compiler happy
}
Example 8
| Source File: AvroBagWrapper.java From spork with Apache License 2.0 | 5 votes |
|
@Override
public int compareTo(final Object o) {
if (this == o) return 0;
if (o instanceof AvroBagWrapper) {
@SuppressWarnings("rawtypes")
AvroBagWrapper bOther = (AvroBagWrapper) o;
return GenericData.get().compare(theArray, bOther.theArray, theArray.getSchema());
} else {
return DataType.compare(this, o);
}
}
Example 9
| Source File: POSort.java From spork with Apache License 2.0 | 5 votes |
|
@Override
public int compare(Tuple o1, Tuple o2) {
int count = 0;
int ret = 0;
if(sortPlans == null || sortPlans.size() == 0) {
return 0;
}
for(PhysicalPlan plan : sortPlans) {
try {
plan.attachInput(o1);
Result res1 = getResult(plan, ExprOutputTypes.get(count));
plan.attachInput(o2);
Result res2 = getResult(plan, ExprOutputTypes.get(count));
if(res1.returnStatus != POStatus.STATUS_OK || res2.returnStatus != POStatus.STATUS_OK) {
log.error("Error processing the input in the expression plan : " + plan.toString());
} else {
if(mAscCols.get(count++)) {
ret = DataType.compare(res1.result, res2.result);
// If they are not equal, return
// Otherwise, keep comparing the next one
if (ret != 0) {
return ret ;
}
}
else {
ret = DataType.compare(res2.result, res1.result);
if (ret != 0) {
return ret ;
}
}
}
} catch (ExecException e) {
log.error("Invalid result while executing the expression plan : " + plan.toString() + "\n" + e.getMessage());
}
}
return ret;
}
Example 10
| Source File: PigTupleDefaultRawComparator.java From spork with Apache License 2.0 | 5 votes |
|
private int compareTuple(Tuple t1, Tuple t2) {
int sz1 = t1.size();
int sz2 = t2.size();
if (sz2 < sz1) {
return 1;
} else if (sz2 > sz1) {
return -1;
} else {
for (int i = 0; i < sz1; i++) {
try {
Object o1 = t1.get(i);
Object o2 = t2.get(i);
if (o1==null || o2==null)
mHasNullField = true;
int c = DataType.compare(o1, o2);
if (c != 0) {
if (!mWholeTuple && !mAsc[i])
c *= -1;
else if (mWholeTuple && !mAsc[0])
c *= -1;
return c;
}
} catch (ExecException e) {
throw new RuntimeException("Unable to compare tuples", e);
}
}
return 0;
}
}
Example 11
| Source File: TestPOSort.java From spork with Apache License 2.0 | 5 votes |
|
public void poSortAscString(DataBag input) throws ExecException {
List<PhysicalPlan> sortPlans = new LinkedList<PhysicalPlan>();
POProject pr1 = new POProject(new OperatorKey("", r.nextLong()), -1, 0);
pr1.setResultType(DataType.CHARARRAY);
PhysicalPlan expPlan = new PhysicalPlan();
expPlan.add(pr1);
sortPlans.add(expPlan);
List<Boolean> mAscCols = new LinkedList<Boolean>();
mAscCols.add(true);
PORead read = new PORead(new OperatorKey("", r.nextLong()), input);
List<PhysicalOperator> inputs = new LinkedList<PhysicalOperator>();
inputs.add(read);
POSort sort = new POSort(new OperatorKey("", r.nextLong()), -1, inputs,
sortPlans, mAscCols, null);
//verify
Tuple t = null;
Result res1 = sort.getNextTuple();
Result res2 = sort.getNextTuple();
while (res2.returnStatus != POStatus.STATUS_EOP) {
Object i1 = ((Tuple) res1.result).get(0);
Object i2 = ((Tuple) res2.result).get(0);
//System.out.println("i1: " + i1.toString() + " i2: " + i2.toString());
int i = DataType.compare(i1, i2);
System.out.println("RESULT2=i : " + res2.result + " i = " + i);
assertEquals(true, (i <= 0));
res1 = res2;
res2 = sort.getNextTuple();
}
}
Example 12
| Source File: TestPOSort.java From spork with Apache License 2.0 | 5 votes |
|
public void poSortDescString(DataBag input) throws ExecException {
List<PhysicalPlan> sortPlans = new LinkedList<PhysicalPlan>();
POProject pr1 = new POProject(new OperatorKey("", r.nextLong()), -1, 0);
pr1.setResultType(DataType.CHARARRAY);
PhysicalPlan expPlan = new PhysicalPlan();
expPlan.add(pr1);
sortPlans.add(expPlan);
List<Boolean> mAscCols = new LinkedList<Boolean>();
mAscCols.add(false);
PORead read = new PORead(new OperatorKey("", r.nextLong()), input);
List<PhysicalOperator> inputs = new LinkedList<PhysicalOperator>();
inputs.add(read);
POSort sort = new POSort(new OperatorKey("", r.nextLong()), -1, inputs,
sortPlans, mAscCols, null);
Tuple t = null;
Result res1 = sort.getNextTuple();
// System.out.println(res1.result);
Result res2 = sort.getNextTuple();
while (res2.returnStatus != POStatus.STATUS_EOP) {
Object i1 = ((Tuple) res1.result).get(0);
Object i2 = ((Tuple) res2.result).get(0);
int i = DataType.compare(i1, i2);
// System.out.println(res2.result + " i = " + i);
assertEquals(true, (i >= 0));
res1 = res2;
res2 = sort.getNextTuple();
}
}
Example 13
| Source File: TestPOSort.java From spork with Apache License 2.0 | 5 votes |
|
public void poSortAscInt( DataBag input) throws ExecException {
List<PhysicalPlan> sortPlans = new LinkedList<PhysicalPlan>();
POProject pr1 = new POProject(new OperatorKey("", r.nextLong()), -1, 1);
pr1.setResultType(DataType.INTEGER);
PhysicalPlan expPlan = new PhysicalPlan();
expPlan.add(pr1);
sortPlans.add(expPlan);
List<Boolean> mAscCols = new LinkedList<Boolean>();
mAscCols.add(true);
PORead read = new PORead(new OperatorKey("", r.nextLong()), input);
List<PhysicalOperator> inputs = new LinkedList<PhysicalOperator>();
inputs.add(read);
POSort sort = new POSort(new OperatorKey("", r.nextLong()), -1, inputs,
sortPlans, mAscCols, null);
Tuple t = null;
Result res1 = sort.getNextTuple();
// System.out.println(res1.result);
Result res2 = sort.getNextTuple();
while (res2.returnStatus != POStatus.STATUS_EOP) {
Object i1 = ((Tuple) res1.result).get(1);
Object i2 = ((Tuple) res2.result).get(1);
int i = DataType.compare(i1, i2);
assertEquals(true, (i <= 0));
// System.out.println(res2.result);
res1 = res2;
res2 = sort.getNextTuple();
}
}
Example 14
| Source File: TestPOSort.java From spork with Apache License 2.0 | 5 votes |
|
public void poSortDescInt(DataBag input) throws ExecException {
List<PhysicalPlan> sortPlans = new LinkedList<PhysicalPlan>();
POProject pr1 = new POProject(new OperatorKey("", r.nextLong()), -1, 1);
pr1.setResultType(DataType.INTEGER);
PhysicalPlan expPlan = new PhysicalPlan();
expPlan.add(pr1);
sortPlans.add(expPlan);
List<Boolean> mAscCols = new LinkedList<Boolean>();
mAscCols.add(false);
PORead read = new PORead(new OperatorKey("", r.nextLong()), input);
List<PhysicalOperator> inputs = new LinkedList<PhysicalOperator>();
inputs.add(read);
POSort sort = new POSort(new OperatorKey("", r.nextLong()), -1, inputs,
sortPlans, mAscCols, null);
Tuple t = null;
Result res1 = sort.getNextTuple();
// System.out.println(res1.result);
Result res2 = sort.getNextTuple();
while (res2.returnStatus != POStatus.STATUS_EOP) {
Object i1 = ((Tuple) res1.result).get(1);
Object i2 = ((Tuple) res2.result).get(1);
int i = DataType.compare(i1, i2);
assertEquals(true, (i >= 0));
// System.out.println(res2.result);
res1 = res2;
res2 = sort.getNextTuple();
}
}
Example 15
| Source File: CondEntropy.java From datafu with Apache License 2.0 | 5 votes |
|
@Override
public void accumulate(Tuple input) throws IOException
{
for (Tuple t : (DataBag) input.get(0)) {
if (this.xy != null)
{
int cmp = t.compareTo(this.xy);
//check if the comparison result is different from previous compare result
if ((cmp < 0 && this.lastCmp > 0)
|| (cmp > 0 && this.lastCmp < 0)) {
throw new ExecException("Out of order! previous tuple: " + this.xy + ", present tuple: " + t
+ ", comparsion: " + cmp + ", previous comparsion: " + this.lastCmp);
}
if (cmp != 0) {
//different <x,y>
this.combEstimator.accumulate(this.cxy);
this.cxy = 0;
this.lastCmp = cmp;
if(DataType.compare(this.xy.get(0), t.get(0)) != 0) {
//different x
this.condXEstimator.accumulate(this.cx);
this.cx = 0;
}
}
}
//set tuple t as the next tuple for comparison
this.xy = t;
//accumulate cx
this.cx++;
//accumulate cxy
this.cxy++;
}
}
