gnu.trove.map.hash.TIntDoubleHashMap Java Examples

The following examples show how to use gnu.trove.map.hash.TIntDoubleHashMap. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: Graph.java    From ambiverse-nlu with Apache License 2.0 6 votes vote down vote up
public Graph(String name, int nodesCount, double alpha) {
  this.name = name;
  this.nodesCount = nodesCount;
  this.alpha = alpha;

  nodes = new GraphNode[nodesCount];
  mentionNodesIds = new TObjectIntHashMap<Mention>();
  entityNodesIds = new TIntIntHashMap();
  localSimilarities = new HashMap<Mention, TIntDoubleHashMap>();
  mentionPriorSimL1 = new TIntDoubleHashMap(50, 0.5f, -1, -1.0);

  isRemoved = new boolean[nodesCount];
  Arrays.fill(isRemoved, false);
  nodesOutdegrees = new int[nodesCount];
  Arrays.fill(nodesOutdegrees, 0);
  nodesWeightedDegrees = new double[nodesCount];
  Arrays.fill(nodesWeightedDegrees, 0);

  removalSteps = new TIntArrayList();
}
 
Example #2
Source File: DataAccessForTesting.java    From ambiverse-nlu with Apache License 2.0 6 votes vote down vote up
@Override
public TIntDoubleHashMap getEntityPriors(String mention, boolean isNamedentity) throws EntityLinkingDataAccessException {
  if (mention.equals("PAGE")) {
    TIntDoubleHashMap pagePriors = new TIntDoubleHashMap();
    pagePriors.put(DataAccess.getInternalIdForKBEntity(getTestKBEntity("Jimmy_Page")), 0.3);
    pagePriors.put(DataAccess.getInternalIdForKBEntity(getTestKBEntity("Larry_Page")), 0.7);
    return pagePriors;
  } else if (mention.equals("KASHMIR")) {
    TIntDoubleHashMap kashmirPriors = new TIntDoubleHashMap();
    kashmirPriors.put(DataAccess.getInternalIdForKBEntity(getTestKBEntity("Kashmir")), 0.9);
    kashmirPriors.put(DataAccess.getInternalIdForKBEntity(getTestKBEntity("Kashmir_(song)")), 0.1);
    return kashmirPriors;
  } else if (mention.equals("KNEBWORTH")) {
    TIntDoubleHashMap knebworthPriors = new TIntDoubleHashMap();
    knebworthPriors.put(DataAccess.getInternalIdForKBEntity(getTestKBEntity("Knebworth_Festival")), 1.0);
    return knebworthPriors;
  } else if (mention.equals("LES PAUL")) {
    return new TIntDoubleHashMap();
  } else {
    return new TIntDoubleHashMap();
  }
}
 
Example #3
Source File: CocktailParty.java    From ambiverse-nlu with Apache License 2.0 6 votes vote down vote up
/**
 * Get the best candidate and (normalized) score from the given entity-score map.
 *
 */
private Pair<Integer, Double> getBestLocalCandidateAndScore(TIntDoubleHashMap entityCandidates) {
  if (entityCandidates.size() == 0) {
    return new Pair<Integer, Double>(-100, 0.0);
  }
  double bestScore = -1.0;
  int bestCandidate = -10;
  for (TIntDoubleIterator itr = entityCandidates.iterator(); itr.hasNext(); ) {
    itr.advance();
    int entityId = itr.key();
    double score = itr.value();
    if (score > bestScore) {
      bestScore = score;
      bestCandidate = entityId;
    }
  }

  if (computeConfidence) {
    TIntDoubleHashMap normalizedScores = CollectionUtils.normalizeValuesToSum(entityCandidates);
    bestScore = normalizedScores.get(bestCandidate);
  }

  return new Pair<>(new Integer(bestCandidate), new Double(bestScore));
}
 
Example #4
Source File: CocktailParty.java    From ambiverse-nlu with Apache License 2.0 6 votes vote down vote up
private TIntLinkedList getEntityMentionsNodesIds(Graph graph, int entityNodeId) {
  TIntLinkedList mentions = new TIntLinkedList();

  GraphNode entityNode = graph.getNode(entityNodeId);
  TIntDoubleHashMap successorsMap = entityNode.getSuccessors();
  TIntDoubleIterator successorsIterator = successorsMap.iterator();
  for (int i = successorsMap.size(); i-- > 0; ) {
    successorsIterator.advance();

    int successorId = successorsIterator.key();
    GraphNode successorNode = graph.getNode(successorId);
    if (successorNode.getType() == GraphNodeTypes.MENTION) {
      mentions.add(successorId);
    }
  }
  return mentions;
}
 
Example #5
Source File: CocktailParty.java    From ambiverse-nlu with Apache License 2.0 6 votes vote down vote up
private Map<Integer, Double> getConnectedEntities(Graph graph, int nodeId) {
  Map<Integer, Double> entities = new HashMap<Integer, Double>();

  GraphNode entityNode = graph.getNode(nodeId);

  TIntDoubleHashMap successorsMap = entityNode.getSuccessors();
  TIntDoubleIterator successorsIterator = successorsMap.iterator();
  for (int i = successorsMap.size(); i-- > 0; ) {
    successorsIterator.advance();

    int successorId = successorsIterator.key();
    GraphNode successorNode = graph.getNode(successorId);

    if (successorNode.getType() == GraphNodeTypes.ENTITY) {
      int entity = (int) successorNode.getNodeData();
      double weight = successorsIterator.value();

      entities.put(entity, weight);
    }
  }
  return entities;
}
 
Example #6
Source File: PriorProbability.java    From ambiverse-nlu with Apache License 2.0 6 votes vote down vote up
/**
 * Returns the prior probability for the given mention-entity pair.
 * If smoothing is true, it will return the lowest prior among all entities if
 * there is no real prior.
 *
 * @param mention
 * @param entity
 * @param smoothing
 * @return
 */
public double getPriorProbability(Mention mention, Entity entity, boolean smoothing) {
  Integer id = RunningTimer.recordStartTime("PriorProbability");
  TIntDoubleHashMap allMentionPriors = priors.get(mention);
  double entityPrior = allMentionPriors.get(entity.getId());

  if (smoothing && entityPrior == 0.0) {
    double smallestPrior = 1.0;

    for (TIntDoubleIterator it = allMentionPriors.iterator(); it.hasNext(); ) {
      it.advance();
      double currentPrior = it.value();
      if (currentPrior < smallestPrior) {
        smallestPrior = currentPrior;
      }
    }
    entityPrior = smallestPrior;
  }
  RunningTimer.recordEndTime("PriorProbability", id);
  return entityPrior;
}
 
Example #7
Source File: CocktailParty.java    From ambiverse-nlu with Apache License 2.0 6 votes vote down vote up
private boolean isNodeRemovable(Graph graph, int nodeId) {
  GraphNode node = graph.getNode(nodeId);
  if (node.getType() == GraphNodeTypes.MENTION) // this is a mention node
    return false;
  // Check if the entity is removable

  TIntDoubleHashMap successorsMap = node.getSuccessors();
  TIntDoubleIterator successorsIterator = successorsMap.iterator();
  for (int i = successorsMap.size(); i-- > 0; ) {
    successorsIterator.advance();

    int successorNodeId = successorsIterator.key();
    GraphNode successorNode = graph.getNode(successorNodeId);
    // if mention and mention connected to only one entity
    if (successorNode.getType() == GraphNodeTypes.MENTION && mentionDegrees.get(successorNodeId) == 1) {
      return false;
    }

  }
  return true;
}
 
Example #8
Source File: GraphConfidenceEstimator.java    From ambiverse-nlu with Apache License 2.0 6 votes vote down vote up
private Map<Integer, Double> getMentionEntityWeightedDegreeScores(Graph g, Configuration configuration, int mentionId) {
  Map<Integer, Double> scores = new HashMap<Integer, Double>();
  // When coherence robustness is enabled, all but one candidate entities
  // are dropped and the (normalized) score is always 1 (and thus useless).
  // Use the original scores if they are available.
  GraphNode mentionNode = g.getNode(mentionId);
  TIntDoubleHashMap successors = mentionNode.getSuccessors();
  if (successors.size() > 1) {
    for (TIntDoubleIterator itr = g.getNode(mentionId).getSuccessors().iterator(); itr.hasNext(); ) {
      itr.advance();
      int entity = itr.key();
      scores.put(entity, computeWeightedDegree(g, configuration, entity));
    }
  } else {
    // Use local scores.
    scores = getMentionEntityLocalScores(g, mentionId);
  }
  return scores;
}
 
Example #9
Source File: CollectionUtils.java    From ambiverse-nlu with Apache License 2.0 6 votes vote down vote up
/**
 * Normalizes values so that they sum up to 1.
 *
 * @param values
 * @return Normalized values.
 */
public static TIntDoubleHashMap normalizeValuesToSum(TIntDoubleHashMap values) {
  TIntDoubleHashMap normalizedScores = new TIntDoubleHashMap();
  double total = 0;
  for (TIntDoubleIterator itr = values.iterator(); itr.hasNext(); ) {
    itr.advance();
    total += itr.value();
  }
  if (total == 0) {
    return values;
  }
  for (TIntDoubleIterator itr = values.iterator(); itr.hasNext(); ) {
    itr.advance();
    Double normalizedScore = itr.value() / total;
    normalizedScores.put(itr.key(), normalizedScore);
  }
  return normalizedScores;
}
 
Example #10
Source File: CollectionUtilsTest.java    From ambiverse-nlu with Apache License 2.0 6 votes vote down vote up
@Test public void testConditionalElement() {
  TIntDoubleHashMap elements = new TIntDoubleHashMap();
  elements.put(0, 0.1);
  elements.put(1, 0.3);
  elements.put(2, 0.05);
  elements.put(3, 0.15);
  elements.put(4, 0.4);

  Random rand = new Random(1337);
  TIntDoubleHashMap counts = new TIntDoubleHashMap();
  for (int i = 0; i < 1000000; ++i) {
    int chosen = CollectionUtils.getConditionalElement(elements, rand);
    counts.adjustOrPutValue(chosen, 1.0, 1.0);
  }
  TIntDoubleHashMap actualProbs = CollectionUtils.normalizeValuesToSum(counts);
  assertEquals(0.1, actualProbs.get(0), 0.001);
  assertEquals(0.3, actualProbs.get(1), 0.001);
  assertEquals(0.05, actualProbs.get(2), 0.001);
  assertEquals(0.15, actualProbs.get(3), 0.001);
  assertEquals(0.4, actualProbs.get(4), 0.001);
}
 
Example #11
Source File: KeyphrasesMeasureTracer.java    From ambiverse-nlu with Apache License 2.0 5 votes vote down vote up
private String buildKeyhraseHTMLEntry(int[] keyphraseTokens, TIntDoubleHashMap matchedKeywords, TIntObjectHashMap<String> id2word) {
  StringBuilder sb = new StringBuilder();
  for (int token : keyphraseTokens) {
    if (matchedKeywords.containsKey(token)) {
      sb.append("<span class='matchedKeyword' style='BACKGROUND-COLOR: #FFAA70;'><strong>" + id2word.get(token) + "</strong> <small>(" + formatter
          .format(matchedKeywords.get(token)) + ")</small></span> ");
    } else {
      sb.append("<strong>" + id2word.get(token) + "</strong> ");
    }
  }

  return sb.toString();
}
 
Example #12
Source File: GraphGenerator.java    From ambiverse-nlu with Apache License 2.0 5 votes vote down vote up
private double calcL1(TIntDoubleHashMap priorDistribution, TIntDoubleHashMap simDistribution) {
  double l1 = 0.0;

  for (TIntDoubleIterator itr = priorDistribution.iterator(); itr.hasNext(); ) {
    itr.advance();
    double prior = itr.value();
    double sim = simDistribution.get(itr.key());
    double diff = Math.abs(prior - sim);
    l1 += diff;
  }

  return l1;
}
 
Example #13
Source File: GraphGenerator.java    From ambiverse-nlu with Apache License 2.0 5 votes vote down vote up
private TIntDoubleHashMap calcSimDistribution(Mention mention, EnsembleMentionEntitySimilarity combSimMeasure) throws Exception {
  TIntDoubleHashMap sims = new TIntDoubleHashMap();
  for (Entity e : mention.getCandidateEntities()) {
    sims.put(e.getId(), combSimMeasure.calcSimilarity(mention, context, e));
  }
  return CollectionUtils.normalizeValuesToSum(sims);
}
 
Example #14
Source File: DataAccess.java    From ambiverse-nlu with Apache License 2.0 5 votes vote down vote up
/**
 * Computes all entity occurrence probabilities based on their incoming links.
 *
 * @return Map of Entity->Probability.
 * @throws EntityLinkingDataAccessException
 */
public static TIntDoubleHashMap getAllEntityProbabilities() throws EntityLinkingDataAccessException {
  TIntObjectHashMap<int[]> entityInlinks = getAllInlinks();

  TIntDoubleHashMap entityProbabilities = new TIntDoubleHashMap(entityInlinks.size(), 0.5f);

  // Get the total number of links.
  long totalLinkCount = 0;

  TIntObjectIterator<int[]> itr = entityInlinks.iterator();

  while (itr.hasNext()) {
    itr.advance();
    totalLinkCount += itr.value().length;
  }

  // Derive probabilities from counts.
  itr = entityInlinks.iterator();

  while (itr.hasNext()) {
    itr.advance();
    double probability = (double) itr.value().length / (double) totalLinkCount;
    entityProbabilities.put(itr.key(), probability);
  }

  return entityProbabilities;
}
 
Example #15
Source File: GraphGenerator.java    From ambiverse-nlu with Apache License 2.0 5 votes vote down vote up
/**
 * Checks if the confidence of a mention disambiguation by local sim alone is 
 * high enough to fix it.
 *
 * @param mention
 * @return best candidate or null if check failed.
 */
private Entity doConfidenceThresholdCheck(Mention mention, TIntDoubleHashMap normalizedEntityLocalSims) {
  Entity bestEntity = null;
  if (settings.getGraphSettings().shouldUseConfidenceThresholdTest()) {
    double max = CollectionUtils.getMaxValue(normalizedEntityLocalSims);
    if (DoubleMath.fuzzyCompare(max, settings.getGraphSettings().getConfidenceTestThreshold(), EPSILON) > 0) {
      bestEntity = getBestCandidate(mention);
    }
  }
  return bestEntity;
}
 
Example #16
Source File: CocktailParty.java    From ambiverse-nlu with Apache License 2.0 5 votes vote down vote up
private void traceIntitialGraphStructure(Graph graph) {
  for (int menNodeId : mentionDegrees.keySet()) {
    GraphNode menNode = graph.getNode(menNodeId);
    Mention mention = (Mention) menNode.getNodeData();

    TIntDoubleHashMap successorsMap = menNode.getSuccessors();
    TIntDoubleIterator successorsIterator = successorsMap.iterator();
    for (int i = successorsMap.size(); i-- > 0; ) {
      successorsIterator.advance();

      int successorNodeId = successorsIterator.key();
      double sim = successorsIterator.value();

      double weight = 0.0;

      if (entityWeightedDegrees.containsKey(successorNodeId)) {
        weight = entityWeightedDegrees.get(successorNodeId);
      } else {
        weight = notRemovableEntityWeightedDegrees.get(successorNodeId);
      }

      GraphNode entityNode = graph.getNode(successorNodeId);
      int entity = (int) entityNode.getNodeData();

      GraphTracer.gTracer.addCandidateEntityToOriginalGraph(graph.getName(), mention.getIdentifiedRepresentation(), entity, weight, sim,
          getConnectedEntities(graph, successorNodeId));

    }
  }
}
 
Example #17
Source File: CocktailParty.java    From ambiverse-nlu with Apache License 2.0 5 votes vote down vote up
private void traceCleanedGraphStructure(Graph graph) {

    for (int menNodeId : mentionDegrees.keySet()) {

      GraphNode menNode = graph.getNode(menNodeId);
      Mention mention = (Mention) menNode.getNodeData();

      TIntDoubleHashMap successorsMap = menNode.getSuccessors();
      TIntDoubleIterator successorsIterator = successorsMap.iterator();
      for (int i = successorsMap.size(); i-- > 0; ) {
        successorsIterator.advance();

        int successorNodeId = successorsIterator.key();

        if (!graph.isRemoved(successorNodeId)) {
          double sim = 0;
          double weight = 0.0;
          if (entityWeightedDegrees.containsKey(successorNodeId)) {
            weight = entityWeightedDegrees.get(successorNodeId);
          } else {
            weight = notRemovableEntityWeightedDegrees.get(successorNodeId);
          }

          GraphNode entityNode = graph.getNode(successorNodeId);
          int entity = (int) entityNode.getNodeData();

          GraphTracer.gTracer.addCandidateEntityToCleanedGraph(graph.getName(), mention.getIdentifiedRepresentation(), entity, weight, sim);
        }
      }
    }

  }
 
Example #18
Source File: SqlToKeyValueConverterGeneric.java    From ambiverse-nlu with Apache License 2.0 5 votes vote down vote up
private Object getValue(Class clazz) {
  if (clazz.equals(TIntDoubleHashMap.class)) {
    return new TIntDoubleHashMap(20000);
  } else if (clazz.equals(int[].class)) {
    return new ArrayList<Integer>();
  } else if (clazz.equals(KeyValueStoreRow[].class)) {
    return new ArrayList<KeyValueStoreRow>();
  } else if (clazz.equals(double[].class)) {
    return new ArrayList<Double>();
  } else if (clazz.equals(String[].class)) {
    return new ArrayList<String>();
  } else {
    return null;
  }
}
 
Example #19
Source File: CocktailParty.java    From ambiverse-nlu with Apache License 2.0 5 votes vote down vote up
private void traceFinalGraphStructure(Graph graph) {
  for (int menNodeId : mentionDegrees.keySet()) {

    GraphNode menNode = graph.getNode(menNodeId);
    Mention mention = (Mention) menNode.getNodeData();

    TIntDoubleHashMap successorsMap = menNode.getSuccessors();
    TIntDoubleIterator successorsIterator = successorsMap.iterator();
    for (int i = successorsMap.size(); i-- > 0; ) {
      successorsIterator.advance();

      int successorNodeId = successorsIterator.key();
      if (!bestRemoved[successorNodeId]) {

        double sim = 0;
        double weight = 0.0;

        if (bestWeightedDegrees.containsKey(successorNodeId)) {
          weight = bestWeightedDegrees.get(successorNodeId);
        } else if (notRemovableEntityWeightedDegrees.containsKey(successorNodeId)) {
          weight = notRemovableEntityWeightedDegrees.get(successorNodeId);
        } else {
          weight = GraphTracer.gTracer.getRemovedEntityDegree(graph.getName(), (int) graph.getNode(successorNodeId).getNodeData());
        }

        GraphNode entityNode = graph.getNode(successorNodeId);
        int entity = (int) entityNode.getNodeData();

        GraphTracer.gTracer.addCandidateEntityToFinalGraph(graph.getName(), mention.getIdentifiedRepresentation(), entity, weight, sim);
      }

    }
  }


  GraphTracer.gTracer.cleanRemovalSteps(graph.getName());
}
 
Example #20
Source File: SparseArrayOfDoubles.java    From morpheus-core with Apache License 2.0 5 votes vote down vote up
/**
 * Constructor
 * @param length    the length for this array
 * @param defaultValue  the default value for array
 */
SparseArrayOfDoubles(int length, Double defaultValue) {
    super(Double.class, ArrayStyle.SPARSE, false);
    this.length = length;
    this.defaultValue = defaultValue != null ? defaultValue : Double.NaN;
    this.values = new TIntDoubleHashMap((int)Math.max(length * 0.5, 10d), 0.8f, -1, this.defaultValue);
}
 
Example #21
Source File: MetricTable.java    From monsoon with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
public MetricTable(int timestampsSize) {
    this.timestampsSize = timestampsSize;
    this.t_bool = new TIntByteHashMap(timestampsSize, 1, -1, (byte) -1);
    this.t_16bit = new TIntShortHashMap(timestampsSize, 1, -1, (short) -1);
    this.t_32bit = new TIntIntHashMap(timestampsSize, 1, -1, -1);
    this.t_64bit = new TIntLongHashMap(timestampsSize, 1, -1, -1);
    this.t_dbl = new TIntDoubleHashMap(timestampsSize, 1, -1, -1);
    this.t_str = new TIntIntHashMap(timestampsSize, 1, -1, -1);
    this.t_hist = new TIntObjectHashMap<>(timestampsSize, 1, -1);
    this.t_empty = new TIntHashSet(timestampsSize, 1, -1);
    this.t_other = new TIntObjectHashMap<>(timestampsSize, 1, -1);
}
 
Example #22
Source File: GraphGenerator.java    From ambiverse-nlu with Apache License 2.0 5 votes vote down vote up
private TIntDoubleHashMap calcPriorDistribution(Mention mention, MaterializedPriorProbability pp) {
  TIntDoubleHashMap priors = new TIntDoubleHashMap();

  for (Entity entity : mention.getCandidateEntities()) {
    priors.put(entity.getId(), pp.getPriorProbability(mention, entity));
  }

  return priors;
}
 
Example #23
Source File: GraphConfidenceEstimator.java    From ambiverse-nlu with Apache License 2.0 5 votes vote down vote up
/**
 * Returns all local scores computed during the graph creation. As some
 * entities are dropped (graph coherence), they will be missing from the
 * current graph. These entities will still be present in the returned
 * map with negative scores. This is necessary for proper normalization.
 */
private Map<Integer, Double> getMentionEntityLocalScores(Graph g, int mentionId) {
  Map<Integer, Double> scores = new HashMap<Integer, Double>();
  // Don't get the local scores from the graph edges, they are incomplete when
  // candidate entities are dropped due to the coherence robustness test.
  // The graph contains all scores as well in a different variable.
  Mention mention = (Mention) g.getNode(mentionId).getNodeData();
  TIntDoubleHashMap entitySims = g.getMentionEntitySims(mention);
  if (entitySims == null) {
    return new HashMap<Integer, Double>();
  }
  TIntIntHashMap entity2id = g.getEntityNodesIds();
  for (TIntDoubleIterator itr = entitySims.iterator(); itr.hasNext(); ) {
    itr.advance();
    // If the entity is not present in the graph anymore, assign a new, 
    // negative one. The negative ids will never be queried, they are
    // just there for the score normalization.
    Integer entityId = 0;
    if (!entity2id.contains(itr.key())) {
      entityId = outOfGraphEntityId;
      --outOfGraphEntityId;
    } else {
      entityId = entity2id.get(itr.key());
    }
    scores.put(entityId, itr.value());
  }
  return scores;
}
 
Example #24
Source File: CollectionUtils.java    From ambiverse-nlu with Apache License 2.0 5 votes vote down vote up
/**
 * Convenience method for the call above.
 *
 * @param elementProbabilities Map with elements as keys and their 
 *  probabilities as values. Values are expected to sum up to 1.
 * @param rand  Random generator to use.
 * @return Randomly selected element according to probabilities.
 */
public static Integer getConditionalElement(TIntDoubleHashMap elementProbabilities, Random rand) {
  Integer[] elements = new Integer[elementProbabilities.size()];
  double[] probs = new double[elementProbabilities.size()];
  double currentProb = 0.0;
  int i = 0;
  for (TIntDoubleIterator itr = elementProbabilities.iterator(); itr.hasNext(); ) {
    itr.advance();
    elements[i] = itr.key();
    currentProb += itr.value();
    probs[i] = currentProb;
    ++i;
  }
  return getConditionalElement(elements, probs, rand);
}
 
Example #25
Source File: CollectionUtils.java    From ambiverse-nlu with Apache License 2.0 5 votes vote down vote up
public static double getMaxValue(TIntDoubleHashMap map) {
  if (map.isEmpty()) {
    return 0.0;
  }

  double max = -Double.MAX_VALUE;
  for (TIntDoubleIterator itr = map.iterator(); itr.hasNext(); ) {
    itr.advance();
    max = Math.max(itr.value(), max);
  }
  return max;
}
 
Example #26
Source File: DataAccessKeyValueStore.java    From ambiverse-nlu with Apache License 2.0 5 votes vote down vote up
@Override public TIntDoubleHashMap getEntityPriors(String mention, boolean isNamedentity) throws EntityLinkingDataAccessException {
  TIntDoubleHashMap entityPriors = new TIntDoubleHashMap();
  try {
    DatabaseKeyValueStore db = DatabaseKeyValueStore.DICTIONARY_MENTION;
    if(db.getPartitions() != 1) {
      throw new IllegalArgumentException("Multiple partitions not supported for this key-value store");
    }
    Codec codec = DataAccessKeyValueStoreHandler.singleton().getCodec(db);
    mention = EntityLinkingManager.conflateToken(mention, isNamedentity);
    KeyValueStore<byte[],byte[]> keyValueStore = DataAccessKeyValueStoreHandler.singleton().getKeyValueStore(db);
    byte[] resultBytes = keyValueStore.get(codec.encodeKey(mention));
    if (resultBytes != null) {
      TIntDoubleHashMap tempResults = (TIntDoubleHashMap) codec.decodeValue(resultBytes);
      TIntObjectHashMap<EntityType> entityClasses = DataAccessCache.singleton().getEntityClasses(tempResults.keys());

      for(int entityId: entityClasses.keys()) {
        if (isNamedentity) {
          if (!(entityClasses.get(entityId) == EntityType.CONCEPT)) {
            entityPriors.put(entityId, tempResults.get(entityId));
          }
        }
        else {
          if (!(entityClasses.get(entityId) == EntityType.NAMED_ENTITY)) {
            entityPriors.put(entityId, tempResults.get(entityId));
          }
        }
      }
    }
  } catch (Exception e) {
    throw new EntityLinkingDataAccessException(e);
  }
  return entityPriors;
}
 
Example #27
Source File: DataAccessKeyValueStore.java    From ambiverse-nlu with Apache License 2.0 5 votes vote down vote up
public TIntDoubleHashMap getEntitiesImportances(int[] entitiesIds) throws EntityLinkingDataAccessException {
  if (entitiesIds == null || entitiesIds.length == 0) {
    return new TIntDoubleHashMap();
  }

  TIntDoubleHashMap entitiesImportances = new TIntDoubleHashMap(getCapacity(entitiesIds.length), troveLoadFactor);
  try {
    DatabaseKeyValueStore db = DatabaseKeyValueStore.ENTITY_RANK_ENTITY;
    if(db.getPartitions() != 1) {
      throw new IllegalArgumentException("Multiple partitions not supported for this key-value store");
    }
    Codec entityRankCodec = DataAccessKeyValueStoreHandler.singleton().getCodec(db);
    KeyValueStore<byte[], byte[]> keyValueStore = DataAccessKeyValueStoreHandler.singleton().getKeyValueStore(db);

    List<byte[]> encodedKeys = new ArrayList<>();
    for (int id : entitiesIds) {
      encodedKeys.add(entityRankCodec.encodeKey(id));
    }
    Map<byte[], byte[]> keyValueMap = keyValueStore.getAll(encodedKeys);

    for (Map.Entry<byte[], byte[]> entry : keyValueMap.entrySet()) {
      if (entry.getKey() == null || entry.getValue() == null) continue;
      //More than one value, not sure why?
      int entity = (int) entityRankCodec.decodeKey(entry.getKey());
      double[] rank = (double[]) entityRankCodec.decodeValue(entry.getValue());
      entitiesImportances.put(entity, rank[0]);
    }
  } catch (Exception e) {
    throw new EntityLinkingDataAccessException(e);
  }
  return entitiesImportances;
}
 
Example #28
Source File: InlinkCountImportance.java    From ambiverse-nlu with Apache License 2.0 5 votes vote down vote up
@Override protected void setupEntities(Entities e) throws IOException, EntityLinkingDataAccessException {
  inlinkImportance = new TIntDoubleHashMap();
  TIntObjectHashMap<int[]> neighbors = DataAccess.getInlinkNeighbors(e);
  double collectionSize = (double) DataAccess.getCollectionSize();
  for (int eId : e.getUniqueIds()) {
    double importance = (double) neighbors.get(eId).length / (double) collectionSize;
    inlinkImportance.put(eId, importance);
  }
}
 
Example #29
Source File: GraphConfidenceEstimator.java    From ambiverse-nlu with Apache License 2.0 5 votes vote down vote up
private Configuration getRandomConfiguration(Graph g, Map<Integer, Integer> solution, float mentionFlipPercentage) {
  Configuration flippedConfiguration = new Configuration();

  // Solution has at least 2 mentions, other case is handled in estimate().
  // Decide number of mentions to switch - at least 1, at most 20%.
  int mentionSize = Math.round(solution.size() * mentionFlipPercentage);
  mentionSize = Math.max(1, mentionSize);
  int numFlips = Math.max(1, random_.nextInt(mentionSize));
  TIntSet flipCandidates = getFlipCandidates(g, solution);
  TIntSet flippedMentions = getRandomElements(flipCandidates, numFlips);
  flippedConfiguration.flippedMentions_ = flippedMentions;
  Map<Integer, Integer> flippedSolution = new HashMap<Integer, Integer>(solution);
  for (TIntIterator itr = flippedMentions.iterator(); itr.hasNext(); ) {
    int mentionId = itr.next();
    TIntDoubleHashMap entityCandidates = new TIntDoubleHashMap(getConnectedEntitiesWithScores(g_, mentionId));
    // Remove correct solution from candidates - it should not be chosen
    // when flipping.
    entityCandidates.remove(solution.get(mentionId));
    // Put placeholder if resembling a missing entity (will not contribute
    // to coherence at all).
    Integer flippedEntity = -1;
    if (entityCandidates.size() > 0) {
      TIntDoubleHashMap entityCandidateProbabilities = CollectionUtils.normalizeValuesToSum(entityCandidates);
      flippedEntity = getRandomEntity(mentionId, entityCandidateProbabilities, random_);
    }
    flippedSolution.put(mentionId, flippedEntity);
  }
  flippedConfiguration.mapping_ = flippedSolution;
  // Store active nodes in graph for faster lookup.
  flippedConfiguration.presentGraphNodes_ = new TIntHashSet();
  for (Entry<Integer, Integer> entry : flippedSolution.entrySet()) {
    flippedConfiguration.presentGraphNodes_.add(entry.getKey());
    flippedConfiguration.presentGraphNodes_.add(entry.getValue());
  }
  //    logger_.debug("Flipped " + flippedMentions.size() + " mentions: " +
  //                  flippedMentions);
  return flippedConfiguration;
}
 
Example #30
Source File: GraphNode.java    From ambiverse-nlu with Apache License 2.0 4 votes vote down vote up
public void setSuccessors(TIntDoubleHashMap successors) {
  this.successors = successors;
}