com.microsoft.azure.storage.table.DynamicTableEntity Java Examples

@Override
public void deleteVreAuthorizations(String vreId) throws AuthorizationUnavailableException {
  String condition = TableQuery.generateFilterCondition(
    "PartitionKey",
    TableQuery.QueryComparisons.EQUAL,
    vreId
  );

  TableBatchOperation deletes = new TableBatchOperation();
  for (DynamicTableEntity entity : table.execute(TableQuery.from(DynamicTableEntity.class).where(condition))) {
    deletes.delete(entity);
  }

  try {
    table.execute(deletes);
  } catch (StorageException e) {
    LOG.error("deleteVreAuthorizations failed", e);
    throw new AuthorizationUnavailableException("Could not delete authorizations");
  }
}
 

/**
 * Instantiates a new AzureStorageAvroRegistry().
 */
protected AzureStorageAvroRegistry() {

    registerSchemaInferrer(DynamicTableEntity.class, new SerializableFunction<DynamicTableEntity, Schema>() {

        private static final long serialVersionUID = 6026115557764489022L;

        @Override
        public Schema apply(DynamicTableEntity t) {
            try {
                return inferSchemaDynamicTableEntity(t);
            } catch (Exception e) {
                throw new ComponentException(e);
            }
        }
    });
}
 

@Override
public void write(Object object) throws IOException {
    if (object == null) {
        return;
    }
    // initialize feedback collections for the write operation
    cleanWrites();

    result.totalCount++;
    IndexedRecord inputRecord = (IndexedRecord) object;
    // This for dynamic which would get schema from the first record
    if (writeSchema == null) {
        writeSchema = ((IndexedRecord) object).getSchema();
    }

    if (processOperationInBatch) {
        DynamicTableEntity entity = createDynamicEntityFromInputRecord(inputRecord, writeSchema);
        addOperationToBatch(entity, inputRecord);
    } else {
        recordToEnqueue.add(inputRecord);
        if (recordToEnqueue.size() >= MAX_RECORDS_TO_ENQUEUE) {
            processParallelRecords();
        }
    }
}
 

@Override
public Schema getEndpointSchema(RuntimeContainer container, String schemaName) throws IOException {

    try {
        AzureStorageTableService tableService = new AzureStorageTableService(getAzureConnection(container));
        TableQuery<DynamicTableEntity> partitionQuery;
        partitionQuery = TableQuery.from(DynamicTableEntity.class).take(1);
        Iterable<DynamicTableEntity> entities = tableService.executeQuery(schemaName, partitionQuery);
        if (entities.iterator().hasNext()) {
            DynamicTableEntity result = entities.iterator().next();
            return AzureStorageAvroRegistry.get().inferSchema(result);
        } else {
            return null;
        }

    } catch (InvalidKeyException | URISyntaxException | StorageException e) {
        LOGGER.error(e.getLocalizedMessage());
        throw new ComponentException(e);
    }
}
 

@Override
public Object convertToAvro(DynamicTableEntity value) {
    try {
        if (COL_TIMESTAMP.equals(f.name()) || COL_TIMESTAMP.equals(mappedName)) {
            return value.getTimestamp();
        }

        if (!value.getProperties().containsKey(mappedName) || value.getProperties().get(mappedName) == null) {
            return null;
        }

        return value.getProperties().get(mappedName).getValueAsLong();

    } catch (Exception e) {
        LOGGER.error(i18nMessages.getMessage("error.ConversionError", e));
        throw new ComponentException(e);
    }
}
 

@Override
public Object convertToAvro(DynamicTableEntity value) {
    try {
        if (COL_TIMESTAMP.equals(f.name()) || COL_TIMESTAMP.equals(mappedName)) {
            return value.getTimestamp();
        }

        if (!value.getProperties().containsKey(mappedName) || value.getProperties().get(mappedName) == null) {
            return null;
        }

        return value.getProperties().get(mappedName).getValueAsDate();
    } catch (Exception e) {
        LOGGER.error(i18nMessages.getMessage("error.ConversionError", e));
        throw new ComponentException(e);
    }
}
 

@Override
public Optional<User> getUserForTimLocalId(String userId) throws AuthenticationUnavailableException {
  String query = "(PartitionKey eq 'users') and (id eq '" + userId + "')";
  Iterator<DynamicTableEntity> users = table.execute(TableQuery.from(DynamicTableEntity.class)
    .where(query)
    .take(2)).iterator();
  if (users.hasNext()) {
    Optional<User> result = Optional.of(propsToObject(users.next()));
    if (users.hasNext()) {
      LOG.error("Multiple items found for query " + query);
    }
    return result;
  } else {
    return Optional.empty();
  }
}
 

@Override
public Set<Permission> getPermissionsForRoles(Collection<String> roles) {
  Set<Permission> permissions = Sets.newHashSet();

  try {
    for (String role : roles) {
      Optional<DynamicTableEntity> rolePermissions = retrieve(TABLE, role);
      if (rolePermissions.isPresent()) {
        String[] permissionStrings = getStringArrayOrEmpty(rolePermissions.get(), "permissions");
        for (String permissionString : permissionStrings) {
          permissions.add(Permission.valueOf(permissionString));
        }

      }
    }
  } catch (StorageException e) {
    LOG.error("Permission could not be retrieved", e);
  }

  return permissions;
}
 

public User propsToObject(DynamicTableEntity entity) {
  return User.create(
    getStringOrNull(entity, "displayName"),
    getStringOrNull(entity, "persistentId"),
    getStringOrNull(entity, "id")
  );
}
 

private Login makeLogin(DynamicTableEntity entity) {
  return Login.create(
    getStringOrNull(entity, "username"),
    getStringOrNull(entity, "userPid"),
    getByteArrayOrEmpty(entity, "password"),
    getByteArrayOrEmpty(entity, "salt"),
    getStringOrNull(entity, "givenName"),
    getStringOrNull(entity, "surName"),
    getStringOrNull(entity, "emailAddress"),
    getStringOrNull(entity, "organization")
  );
}
 

protected static String getStringOrNull(DynamicTableEntity entity, String key) {
  EntityProperty property = entity.getProperties().get(key);
  String result = null;
  if (property != null) {
    result = property.getValueAsString();
  }
  return result;
}
 

protected static byte[] getByteArrayOrEmpty(DynamicTableEntity entity, String key) {
  EntityProperty property = entity.getProperties().get(key);
  byte[] result = new byte[0];
  if (property != null) {
    result = property.getValueAsByteArray();
  }
  return result;
}
 

protected static String[] getStringArrayOrEmpty(DynamicTableEntity entity, String key) {
  String content = getStringOrNull(entity, key);
  if (content == null) {
    return new String[0];
  } else {
    return content.split(",");
  }
}
 

@Test
@Category({ DevFabricTests.class, DevStoreTests.class, SecondaryTests.class })
public void testTableMaximumExecutionTime() throws URISyntaxException, StorageException {
    OperationContext opContext = new OperationContext();
    setDelay(opContext, 2500);
    
    opContext.getResponseReceivedEventHandler().addListener(new StorageEvent<ResponseReceivedEvent>() {

        @Override
        public void eventOccurred(ResponseReceivedEvent eventArg) {
            // Set status code to 500 to force a retry
            eventArg.getRequestResult().setStatusCode(500);
        }
    });

    // set the maximum execution time
    TableRequestOptions options = new TableRequestOptions();
    options.setMaximumExecutionTimeInMs(2000);
    options.setTimeoutIntervalInMs(1000);

    CloudTableClient tableClient = TestHelper.createCloudTableClient();
    CloudTable table = tableClient.getTableReference(generateRandomName("share"));

    try {
        // 1. insert entity will fail as the table does not exist
        // 2. the executor will attempt to retry as we set the status code to 500
        // 3. maximum execution time should prevent the retry from being made
        DynamicTableEntity ent = new DynamicTableEntity("partition", "row");
        TableOperation insert = TableOperation.insert(ent);
        table.execute(insert, options, opContext);
        fail("Maximum execution time was reached but request did not fail.");
    }
    catch (StorageException e) {
        assertEquals(SR.MAXIMUM_EXECUTION_TIMEOUT_EXCEPTION, e.getMessage());
    }
}
 

@Test
public void testDynamicTableEntityConversion() {

    DynamicTableEntity entity = new DynamicTableEntity();
    entity.setPartitionKey(pk_test1);
    entity.setRowKey(rk_test1);
    entity.getProperties().put("a_bool", new EntityProperty(true));
    entity.getProperties().put("a_int", new EntityProperty(1000));
    entity.getProperties().put("a_string", new EntityProperty(RandomStringUtils.random(10)));

    Schema s = registry.inferSchemaDynamicTableEntity(entity);
    assertEquals(6, s.getFields().size());

    recordConv.setSchema(s);
    IndexedRecord record = recordConv.convertToAvro(entity);
    assertEquals(pk_test1, record.get(0));
    assertEquals(rk_test1, record.get(1));
    assertTrue(record.get(2) instanceof Date);
    //
    assertEquals(true, record.get(s.getField("a_bool").pos()));
    assertEquals(1000, record.get(s.getField("a_int").pos()));
    assertTrue(record.get(s.getField("a_string").pos()) instanceof String);

    Map<String, String> nameMappings = new HashMap<>();
    nameMappings.put("a_bool", "booly");
    AzureStorageTableAdaptorFactory adaptor = new AzureStorageTableAdaptorFactory(nameMappings);
    adaptor.setSchema(s);

    //
    registry.inferSchemaDynamicTableEntity(entity);
    assertEquals(DynamicTableEntity.class, recordConv.getDatumClass());
    assertNull(recordConv.convertToDatum(record));
}
 

@Override
public boolean start() throws IOException {

    String tableName = properties.tableName.getValue();
    String filter = "";
    if (properties.useFilterExpression.getValue()) {
        filter = properties.filterExpression.generateCombinedFilterConditions();
        LOGGER.debug(i18nMessages.getMessage("debug.FilterApplied", filter));
    }
    try {
        TableQuery<DynamicTableEntity> partitionQuery;
        if (filter.isEmpty()) {
            partitionQuery = TableQuery.from(DynamicTableEntity.class);
        } else {
            partitionQuery = TableQuery.from(DynamicTableEntity.class).where(filter);
        }
        // Using execute will automatically and lazily follow the continuation tokens from page to page of results.
        // So, we bypass the 1000 entities limit.
        Iterable<DynamicTableEntity> entities = tableService.executeQuery(tableName, partitionQuery);
        recordsIterator = entities.iterator();
        if (recordsIterator.hasNext()) {
            started = true;
            result.totalCount++;
            current = recordsIterator.next();
        }
    } catch (InvalidKeyException | URISyntaxException | StorageException e) {
        LOGGER.error(e.getLocalizedMessage());
        if (properties.dieOnError.getValue()) {
            throw new ComponentException(e);
        }
    }

    return started;
}
 

public VreAuthorization propsToObject(DynamicTableEntity entity) {
  return VreAuthorization.create(
    getStringOrNull(entity, "vreId"),
    getStringOrNull(entity, "userId"),
    getStringArrayOrEmpty(entity, "roles")
  );
}
 

private TableOperation getTableOperation(DynamicTableEntity entity) {
    TableOperation tableOpe = null;
    switch (actionData) {
    case Insert:
        tableOpe = TableOperation.insert(entity);
        break;
    case Insert_Or_Merge:
        tableOpe = TableOperation.insertOrMerge(entity);
        break;
    case Insert_Or_Replace:
        tableOpe = TableOperation.insertOrReplace(entity);
        break;
    case Merge:
        tableOpe = TableOperation.merge(entity);
        break;
    case Replace:
        tableOpe = TableOperation.replace(entity);
        break;
    case Delete:
        tableOpe = TableOperation.delete(entity);
        break;
    default:
        LOGGER.error("No specified operation for table");
    }

    return tableOpe;
}
 

protected Schema inferSchemaDynamicTableEntity(DynamicTableEntity entity) {
    List<Field> fields = new ArrayList<>();
    fields.add(new Field("PartitionKey", AvroUtils._string(), null, (Object) null));
    fields.add(new Field("RowKey", AvroUtils._string(), null, (Object) null));
    fields.add(new Field("Timestamp", AvroUtils._date(), null, (Object) null));

    // FIXME set tableName properly and manage nameMappings
    String tableName = "schemaInfered";
    for (Entry<String, EntityProperty> f : entity.getProperties().entrySet()) {
        String fieldName = f.getKey();
        Field field = getAvroMapping(fieldName, f.getValue());
        fields.add(field);
    }
    return Schema.createRecord(tableName, null, null, false, fields);
}
 

@SuppressWarnings("unchecked")
public AzureStorageTableIndexedRecord(DynamicTableEntity record, Schema schema) {
    try {
        values = new Object[schema.getFields().size()];
        for (int i = 0; i < values.length; i++) {
            values[i] = fieldConverter[i].convertToAvro(record);
        }
    } catch (Exception e) {
        LOGGER.error(e.getLocalizedMessage());
        throw new ComponentException(e);
    }
}
 

@Override
public Object convertToAvro(DynamicTableEntity value) {
    try {

        if (COL_PARITION_KEY.equals(f.name()) || COL_PARITION_KEY.equals(mappedName)) {
            return value.getPartitionKey();
        }

        if (COL_ROW_KEY.equals(f.name()) || COL_ROW_KEY.equals(mappedName)) {
            return value.getRowKey();
        }

        if (f.name().equals(COL_TIMESTAMP) || mappedName.equals(COL_TIMESTAMP)) { // should be set to DT
                                                                                  // but...
            String pattern = f.getProp(SchemaConstants.TALEND_COLUMN_PATTERN);
            if (pattern != null && !pattern.isEmpty())
                return new SimpleDateFormat(pattern).format(value.getTimestamp());
            else
                return value.getTimestamp();
        }

        if (!value.getProperties().containsKey(mappedName) || value.getProperties().get(mappedName) == null) {
            return null;
        }

        return value.getProperties().get(mappedName).getValueAsString();

    } catch (Exception e) {
        LOGGER.error(i18nMessages.getMessage("error.ConversionError", e));
        throw new ComponentException(e);
    }
}
 

@Override
public Object convertToAvro(DynamicTableEntity value) {
    try {

        if (!value.getProperties().containsKey(mappedName) || value.getProperties().get(mappedName) == null) {
            return null;
        }

        return value.getProperties().get(mappedName).getValueAsInteger();
    } catch (Exception e) {
        LOGGER.error(i18nMessages.getMessage("error.ConversionError", e));
        throw new ComponentException(e);
    }
}
 

@Override
public Object convertToAvro(DynamicTableEntity value) {
    try {

        if (!value.getProperties().containsKey(mappedName) || value.getProperties().get(mappedName) == null) {
            return null;
        }

        return value.getProperties().get(mappedName).getValueAsDouble();
    } catch (Exception e) {
        LOGGER.error(i18nMessages.getMessage("error.ConversionError", e));
        throw new ComponentException(e);
    }
}
 

private void addOperationToBatch(DynamicTableEntity entity, IndexedRecord record) throws IOException {
    if (latestPartitionKey == null || latestPartitionKey.isEmpty()) {
        latestPartitionKey = entity.getPartitionKey();
    }
    // we reached the threshold for batch OR changed PartitionKey
    if (batchOperationsCount == 100 || !entity.getPartitionKey().equals(latestPartitionKey)) {
        processBatch();
        latestPartitionKey = entity.getPartitionKey();
    }
    TableOperation to = getTableOperation(entity);
    batchOperations.add(to);
    batchRecords.add(record);
    batchOperationsCount++;
    latestPartitionKey = entity.getPartitionKey();
}
 

@Override
public Object convertToAvro(DynamicTableEntity value) {
    try {

        if (!value.getProperties().containsKey(mappedName) || value.getProperties().get(mappedName) == null) {
            return null;
        }

        return value.getProperties().get(mappedName).getValueAsByteArray();
    } catch (Exception e) {
        LOGGER.error(i18nMessages.getMessage("error.ConversionError", e));
        throw new ComponentException(e);
    }
}
 

@Override
public Object convertToAvro(DynamicTableEntity value) {
    try {
        if (!value.getProperties().containsKey(mappedName)) {
            return null;
        }

        return value.getProperties().get(mappedName).getValueAsBoolean();
    } catch (Exception e) {
        LOGGER.error(i18nMessages.getMessage("error.ConversionError", e));
        throw new ComponentException(e);
    }
}
 

@Override
public Class<DynamicTableEntity> getDatumClass() {
    return DynamicTableEntity.class;
}
 

@Override
public Class<DynamicTableEntity> getDatumClass() {
    return null;
}
 

@Override
public DynamicTableEntity convertToDatum(Object value) {
    return null;
}
 

protected void delete(String partitionKey, String rowKey) throws StorageException {
  table.execute(TableOperation.delete(new DynamicTableEntity(partitionKey, rowKey)));
}