Java Code Examples for com.mongodb.BasicDBList#get()

public void setGrouyBy(DBObject gb) {
	this.groupby = gb;
	this.type = true;
	if (gb instanceof BasicDBList) {
		BasicDBList basicDBList = (BasicDBList)gb;
		if(!basicDBList.isEmpty()){
			Object gb2 = basicDBList.get(0);
			if (gb2 instanceof DBObject) {
				for (String field : ((DBObject) gb2).keySet()) {
					Object val = ((DBObject) gb2).get(field);
					setField(field, getObjectToType(val));
				}
			}
		}
	}
}
 

/**
 * Get the game data from database as a double value array.
 * If it does not exist in database, or is illegal, return an empty double array.
 * 
 * @param key
 * @param defaultValue
 * @return
 */
public double[] getGameDataAsDoubleArray(GameDataKey key) {
	Object obj = getValueFromDatabase(key);
	if ( obj == NULL ) {
		return new double[0];
	} else if ( obj instanceof double[] ) {
		return (double[])obj;
	} else {
		BasicDBList list = (BasicDBList)obj;
		double[] array = new double[list.size()];
		for ( int i=0; i<array.length; i++ ) {
			array[i] = (Double)list.get(i);
		}
		return array;
	}
}
 

/**
 * Get the game data from database as a two dimension double value array.
 * 
 * @param key
 * @return
 */
public double[][] getGameDataAsDoubleArrayArray(GameDataKey key) {
	Object obj = getValueFromDatabase(key);
	if ( obj == NULL ) {
		return new double[0][0];
	} else if ( obj instanceof double[] ) {
		return (double[][])obj;
	} else {
		BasicDBList list = (BasicDBList)obj;
		double[][] array = new double[list.size()][];
		for ( int i=0; i<array.length; i++ ) {
			BasicDBList dbList = (BasicDBList)list.get(i);
			array[i] = new double[dbList.size()];
			for ( int j=0; j<array[i].length; j++ ) {
				array[i][j] = (Double)dbList.get(j);
			}
		}
		return array;
	}
}
 

/**
 * Get the game data from database as a int value array.
 * If it does not exist in database, or is illegal, return an empty double array.
 * 
 * @param key
 * @param defaultValue
 * @return
 */
public int[] getGameDataAsIntArray(GameDataKey key) {
	Object obj = getValueFromDatabase(key);
	if ( obj == NULL ) {
		return new int[0];
	} else if ( obj instanceof double[] ) {
		return (int[])obj;
	} else {
		BasicDBList list = (BasicDBList)obj;
		int[] array = new int[list.size()];
		for ( int i=0; i<array.length; i++ ) {
			array[i] = (Integer)list.get(i);
		}
		return array;
	}
}
 

/**
 * Get the game data from database as a two dimension double value array.
 * 
 * @param key
 * @return
 */
public int[][] getGameDataAsIntArrayArray(GameDataKey key) {
	Object obj = getValueFromDatabase(key);
	if ( obj == NULL ) {
		return new int[0][0];
	} else if ( obj instanceof double[] ) {
		return (int[][])obj;
	} else {
		BasicDBList list = (BasicDBList)obj;
		int[][] array = new int[list.size()][];
		for ( int i=0; i<array.length; i++ ) {
			BasicDBList dbList = (BasicDBList)list.get(i);
			array[i] = new int[dbList.size()];
			for ( int j=0; j<array[i].length; j++ ) {
				array[i][j] = (Integer)dbList.get(j);
			}
		}
		return array;
	}
}
 

static DBObject[] getSecondaryObjectSets( DBObject exprObj )
{
    if( !(exprObj instanceof BasicDBList) )
        return null;    // no secondary element(s)

    BasicDBList objList = (BasicDBList)exprObj;
    if( objList.size() <= 1 )
        return null;
    
    // return the second and remaining DBObject(s) from the list
    List<DBObject> secondaryObjList = new ArrayList<DBObject>(objList.size()-1);
    for( int i=1; i < objList.size(); i++ )
    {
        Object value = objList.get(i);
        if( value instanceof DBObject )
            secondaryObjList.add( (DBObject)value );
        else // ignore elements that are not DBObject
            logInvalidTagValue( value );
    }

    if( secondaryObjList.isEmpty() )
        return null;
    return (DBObject[])secondaryObjList.toArray( new DBObject[secondaryObjList.size()] );
}
 

/**
 * Create a group based on a Mongo DB Object
 *
 * @param group The existing Mongo DB Object
 */
public Group(DBObject group) {
  this.id = ((ObjectId) group.get(DB_ID)).toString();
  this.name = (String) group.get(JSON_KEY_GROUP_NAME);

  BasicDBList dbMembers = ((BasicDBList) group.get(JSON_KEY_MEMBERS_LIST));
  this.members = new String[dbMembers.size()];
  for (int i = 0; i < dbMembers.size(); i++) {
    members[i] = (String) dbMembers.get(i);
  }
}
 

public static JSONArray conv(BasicDBList a) {
  JSONArray result = new JSONArray();
  for (int i = 0; i < a.size(); ++i) {
    Object o = a.get(i);
    if (o instanceof DBObject) {
      result.put(conv((DBObject)o));
    } else if (o instanceof BasicDBList) {
      result.put(conv((BasicDBList)o));
    } else {
      result.put(o);
    }
  }
  return result;
}
 

@Test
public void findDocumentFromANode() throws Exception {
    BasicDBList dbList = (BasicDBList) JSON.parse(IOUtils.toString(getClass().getResourceAsStream("arrayOfDocuments.json")));

    DBObject first = (DBObject) dbList.get(0);
    first.put("_id", new ObjectId(String.valueOf(first.get("_id"))));

    DBObject second = (DBObject) dbList.get(1);
    second.put("_id", new ObjectId(String.valueOf(second.get("_id"))));

    NoSqlTreeNode treeNode = (NoSqlTreeNode) JsonTreeModel.buildJsonTree(dbList);

    assertEquals(first, JsonTreeModel.findDocument((NoSqlTreeNode) treeNode.getChildAt(0)));
}
 

private static BasicDBList compare(BasicDBList l, BasicDBList refl, boolean listsAreSet) {
  boolean different = false;
  BasicDBList diff = new BasicDBList();

  if (!listsAreSet) {
    // lists are to be treated as ordered sets and so we can compare element by element
    for (int i = 0; i < l.size(); i++){
      Object val = l.get(i);
      Object refv = refl.get(i);
      Object d;
      if ((d = compare(val, refv, listsAreSet)) != null) {
        different = true;
        diff.add(d);
      } else {
        // elements at this index are identical, but we don't want to muck up the order
        // in case future elements are not identical... so add a null to the diff,
        // BUT IMP: do not set the flag that the list is different
        diff.add(null);
      }
    }
  } else {
    // lists are to be treated as unordered sets: we try to match each element best
    // effort to any one of the list elements, and if it does proceed greedily

    // we keep this as a list as opposed to a true set because the original (ref)
    // and the current (new) might have (identical) replicates, and so should not
    // be flagged different because of that.
    List<Object> refset = new ArrayList<Object>();
    refset.addAll(refl);

    for (Object e : l) {
      boolean matches_some = false;
      for (Object eref : refset) {
        if (compare(e, eref, listsAreSet) == null) {
          // this object matches something, great, lets move to the next object
          // also remove the matched object from the ref list, so that we have
          // a 1-1 mapping between this and the ref list object
          matches_some = true;
          refset.remove(eref);
          break;
        }
      }
      if (!matches_some) {
        // if this object in new list could not be matched against something,
        // the lists are different
        different = true;
        diff.add(e);
      }
    }

    if (refset.size() != 0) {
      // still some elements remain in the ref list, i.e., sets different
      different = true;
      diff.addAll(refset);
    }

  }

  return different ? diff : null;
}
 

private void updateBatch(BasicDBList sample) {
	
	// The batch may span collection splits, so maintain
	// a current collection and batch operation
	BulkWriteOperation currentOp = null;
	int currentOpOffset = 0;
	int sampleIdx = 0;
	DBCollection currentColl = null;	
	
	logger.debug("Received batch of size : {}", sample.size());
	
	try{
		for(; sampleIdx < sample.size(); ++sampleIdx){
			
			// prepare the sample to batch
			BasicDBObject doc = (BasicDBObject) (sample.get(sampleIdx));
			long timestamp = this.rollupPeriod * (doc.getLong(Sample.TS_KEY) / this.rollupPeriod);			
			DBCollection collection = collectionAllocator.getCollection(timestamp);
			
			// if the collection has changed, commit the current
			// batch to the collection and start new
			if(collection.equals(currentColl) == false){
				executeBatchUpdate(currentOp, sample);
				currentColl = collection;
				currentOp = collection.initializeUnorderedBulkOperation();
				currentOpOffset = sampleIdx;
			}
			
			// put the doc insert into the batch
			// Ask the id allocator for the query
			BasicDBObject query = this.idFactory.getQuery(doc.get(Sample.SOURCE_KEY), timestamp);
			
			// Build the update clause using the ops list
			BasicDBObject update = new BasicDBObject();
			for(RollupOperation rollupOp : this.rollupOps){
				
				DBObject updateClause = rollupOp.getUpdateClause(doc);
				
				// Check for top level operators that already exist so they dont overwrite
				for(String key : updateClause.keySet()){
					BasicDBObject existingClause = (BasicDBObject) update.get(key);
					if(existingClause != null){
						// Merge the arguments to the top level op
						existingClause.putAll((DBObject)updateClause.get(key));
					} else {
						update.put(key, updateClause.get(key));
					}
				}
			}
			
			currentOp.find(query).upsert().updateOne(update);
		}		
		
		// Finalize the last batch
		executeBatchUpdate(currentOp, sample);		
		
	} catch(Exception ex){
		
		// One of the bulk writes has failed
		BasicDBList failedDocs = new BasicDBList();
		if(ex instanceof BulkWriteException){
			
			// We need to figure out the failures and remove the writes
			// that worked from the batch
			int batchSize = sampleIdx - currentOpOffset;
			BulkWriteException bwex = (BulkWriteException)ex;
			int errorCount = bwex.getWriteErrors().size(); 
			if(errorCount < batchSize){
				
				for(BulkWriteError we : bwex.getWriteErrors()){
					failedDocs.add(sample.get(currentOpOffset + we.getIndex()));
				}
				
				// since we have accounted for the failures in the current
				// batch, move the offset forward to the last sample
				currentOpOffset = sampleIdx;					
			}
		}
		
		// If this happened part way through the batch, send remaining 
		// docs to failed list and update sample to contain only failed docs
		if(currentOpOffset > 0){
			for(; currentOpOffset < sample.size(); ++currentOpOffset)
				failedDocs.add(sample.get(currentOpOffset));
			sample.clear();
			sample.addAll(failedDocs);	
		}
		
		throw ex;
	}
}
 

private void storeBatch(BasicDBList sample, BasicDBList resultList) {
	
	// The batch may span collection splits, so maintain
	// a current collection and batch operation
	BulkWriteOperation currentOp = null;
	int currentOpOffset = 0;
	int sampleIdx = 0;
	DBCollection currentColl = null;	
	
	logger.debug("Received batch of size : {}", sample.size());
	
	try{
		for(; sampleIdx < sample.size(); ++sampleIdx){
			
			// prepare the sample to batch
			BasicDBObject doc = (BasicDBObject) (sample.get(sampleIdx));
			SampleId _id = this.idFactory.createId(doc);
			doc.put(Sample.ID_KEY, _id.toObject());
			resultList.add(_id.toObject());
			long timestamp = doc.getLong(Sample.TS_KEY);
			DBCollection collection = collectionAllocator.getCollection(timestamp);
			
			// if the collection has changed, commit the current
			// batch to the collection and start new
			if(collection.equals(currentColl) == false){
				executeBatchWrite(currentOp, sample);
				currentColl = collection;
				currentOp = collection.initializeUnorderedBulkOperation();
				currentOpOffset = sampleIdx;
			}
			
			// put the doc insert into the batch
			currentOp.insert(doc);
		}		
		
		// Finalize the last batch
		executeBatchWrite(currentOp, sample);		
		
	} catch(Exception ex){
		
		// One of the bulk writes has failed
		BasicDBList failedDocs = new BasicDBList();
		if(ex instanceof BulkWriteException){
			
			// We need to figure out the failures and remove the writes
			// that worked from the batch
			int batchSize = sampleIdx - currentOpOffset;
			BulkWriteException bwex = (BulkWriteException)ex;
			int errorCount = bwex.getWriteErrors().size(); 
			if(errorCount < batchSize){
				
				for(BulkWriteError we : bwex.getWriteErrors()){
					failedDocs.add(sample.get(currentOpOffset + we.getIndex()));
				}
				
				// since we have accounted for the failures in the current
				// batch, move the offset forward to the last sample
				currentOpOffset = sampleIdx;					
			}
		}
		
		// If this happened part way through the batch, send remaining 
		// docs to failed list and update sample to contain only failed docs
		if(currentOpOffset > 0){
			for(; currentOpOffset < sample.size(); ++currentOpOffset)
				failedDocs.add(sample.get(currentOpOffset));
			sample.clear();
			sample.addAll(failedDocs);	
		}
		
		// TODO : we also need to handle the result Ids here as well,
		// the failed doc Ids must be pulled from the resultList
		throw ex;
	}
}
 

private static void processList( BasicDBList list, String path, String name, Map<String, MongoField> lookup ) {

    if ( list.size() == 0 ) {
      return; // can't infer anything about an empty list
    }

    String nonPrimitivePath = path + "[-]"; //$NON-NLS-1$
    String primitivePath = path;

    for ( int i = 0; i < list.size(); i++ ) {
      Object element = list.get( i );

      if ( element instanceof BasicDBObject ) {
        processRecord( (BasicDBObject) element, nonPrimitivePath, name + "[" + i + //$NON-NLS-1$
            ":" + i + "]", lookup ); //$NON-NLS-1$ //$NON-NLS-2$
      } else if ( element instanceof BasicDBList ) {
        processList( (BasicDBList) element, nonPrimitivePath, name + "[" + i + //$NON-NLS-1$
            ":" + i + "]", lookup ); //$NON-NLS-1$ //$NON-NLS-2$
      } else {
        // some sort of primitive
        String finalPath = primitivePath + "[" + i + "]"; //$NON-NLS-1$ //$NON-NLS-2$
        String finalName = name + "[" + i + "]"; //$NON-NLS-1$ //$NON-NLS-2$
        if ( !lookup.containsKey( finalPath ) ) {
          MongoField newField = new MongoField();
          int kettleType = mongoToKettleType( element );
          // Following suit of mongoToKettleType by interpreting null as String type
          newField.m_mongoType = String.class;
          if ( element != null ) {
            newField.m_mongoType = element.getClass();
          }
          newField.m_fieldName = finalPath;
          newField.m_fieldPath = finalName;
          newField.m_kettleType = ValueMeta.getTypeDesc( kettleType );
          newField.m_percentageOfSample = 1;

          lookup.put( finalPath, newField );
        } else {
          // update max indexes in array parts of name
          MongoField m = lookup.get( finalPath );
          Class<?> elementClass = String.class;
          if ( element != null ) {
            elementClass = element.getClass();
          }
          if ( !m.m_mongoType.isAssignableFrom( elementClass ) ) {
            m.m_disparateTypes = true;
          }
          m.m_percentageOfSample++;
          updateMinMaxArrayIndexes( m, finalName );
        }
      }
    }
  }
 

/**
 * Convert a mongo array object to a Kettle field value (for the field defined in this path)
 * 
 * @param mongoList
 *          the array to convert
 * @return the kettle field value
 * @throws KettleException
 *           if a problem occurs
 */
public Object convertToKettleValue( BasicDBList mongoList ) throws KettleException {

  if ( mongoList == null ) {
    return null;
  }

  if ( m_tempParts.size() == 0 ) {
    throw new KettleException( BaseMessages.getString( PKG, "MongoDbInput.ErrorMessage.MalformedPathArray" ) ); //$NON-NLS-1$
  }

  String part = m_tempParts.remove( 0 );
  if ( !( part.charAt( 0 ) == '[' ) ) {
    // we're expecting an array at this point - this document does not
    // contain our field
    return null;
  }

  String index = part.substring( 1, part.indexOf( ']' ) );
  int arrayI = 0;
  try {
    arrayI = Integer.parseInt( index.trim() );
  } catch ( NumberFormatException e ) {
    throw new KettleException( BaseMessages.getString( PKG,
        "MongoDbInput.ErrorMessage.UnableToParseArrayIndex", index ) ); //$NON-NLS-1$
  }

  if ( part.indexOf( ']' ) < part.length() - 1 ) {
    // more dimensions to the array
    part = part.substring( part.indexOf( ']' ) + 1, part.length() );
    m_tempParts.add( 0, part );
  }

  if ( arrayI >= mongoList.size() || arrayI < 0 ) {
    return null;
  }

  Object element = mongoList.get( arrayI );

  if ( element == null ) {
    return null;
  }

  if ( m_tempParts.size() == 0 ) {
    // we're expecting a leaf primitive - let's see if that's what we have
    // here...
    return getKettleValue( element );
  }

  if ( element instanceof BasicDBObject ) {
    return convertToKettleValue( ( (BasicDBObject) element ) );
  }

  if ( element instanceof BasicDBList ) {
    return convertToKettleValue( ( (BasicDBList) element ) );
  }

  // must mean we have a primitive here, but we're expecting to process more
  // path so this doesn't match us - return null
  return null;
}