com.google.privacy.dlp.v2.Table Java Examples
The following examples show how to use
com.google.privacy.dlp.v2.Table.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: DeIdentifyTableConditionMasking.java From java-docs-samples with Apache License 2.0 | 6 votes |
public static void deIdentifyTableConditionMasking() throws IOException { // TODO(developer): Replace these variables before running the sample. String projectId = "your-project-id"; Table tableToDeIdentify = Table.newBuilder() .addHeaders(FieldId.newBuilder().setName("AGE").build()) .addHeaders(FieldId.newBuilder().setName("PATIENT").build()) .addHeaders(FieldId.newBuilder().setName("HAPPINESS SCORE").build()) .addRows(Row.newBuilder() .addValues(Value.newBuilder().setStringValue("101").build()) .addValues(Value.newBuilder().setStringValue("Charles Dickens").build()) .addValues(Value.newBuilder().setStringValue("95").build()) .build()) .addRows(Row.newBuilder() .addValues(Value.newBuilder().setStringValue("22").build()) .addValues(Value.newBuilder().setStringValue("Jane Austen").build()) .addValues(Value.newBuilder().setStringValue("21").build()) .build()) .addRows(Row.newBuilder() .addValues(Value.newBuilder().setStringValue("55").build()) .addValues(Value.newBuilder().setStringValue("Mark Twain").build()) .addValues(Value.newBuilder().setStringValue("75").build()) .build()) .build(); deIdentifyTableConditionMasking(projectId, tableToDeIdentify); }
Example #2
Source File: DLPTextToBigQueryStreaming.java From DataflowTemplates with Apache License 2.0 | 6 votes |
@ProcessElement public void processElement(ProcessContext c) { Table tokenizedData = c.element().getValue(); List<String> headers = tokenizedData.getHeadersList().stream() .map(fid -> fid.getName()) .collect(Collectors.toList()); List<Table.Row> outputRows = tokenizedData.getRowsList(); if (outputRows.size() > 0) { for (Table.Row outputRow : outputRows) { if (outputRow.getValuesCount() != headers.size()) { throw new IllegalArgumentException( "CSV file's header count must exactly match with data element count"); } c.output( KV.of( c.element().getKey(), createBqRow(outputRow, headers.toArray(new String[headers.size()])))); } } }
Example #3
Source File: DLPTextToBigQueryStreaming.java From DataflowTemplates with Apache License 2.0 | 6 votes |
private static TableRow createBqRow(Table.Row tokenizedValue, String[] headers) { TableRow bqRow = new TableRow(); AtomicInteger headerIndex = new AtomicInteger(0); List<TableCell> cells = new ArrayList<>(); tokenizedValue .getValuesList() .forEach( value -> { String checkedHeaderName = checkHeaderName(headers[headerIndex.getAndIncrement()].toString()); bqRow.set(checkedHeaderName, value.getStringValue()); cells.add(new TableCell().set(checkedHeaderName, value.getStringValue())); }); bqRow.setF(cells); return bqRow; }
Example #4
Source File: ReIdentifyTableWithFpe.java From java-docs-samples with Apache License 2.0 | 6 votes |
public static void main(String[] args) throws Exception { // TODO(developer): Replace these variables before running the sample. String projectId = "your-project-id"; String kmsKeyName = "projects/YOUR_PROJECT/" + "locations/YOUR_KEYRING_REGION/" + "keyRings/YOUR_KEYRING_NAME/" + "cryptoKeys/YOUR_KEY_NAME"; String wrappedAesKey = "YOUR_ENCRYPTED_AES_256_KEY"; Table tableToReIdentify = Table.newBuilder() .addHeaders(FieldId.newBuilder().setName("Employee ID").build()) .addRows( Row.newBuilder().addValues( Value.newBuilder().setStringValue("28777").build()) .build()) .build(); reIdentifyTableWithFpe(projectId, tableToReIdentify, kmsKeyName, wrappedAesKey); }
Example #5
Source File: CSVStreamingPipelineTest.java From dlp-dataflow-deidentification with Apache License 2.0 | 6 votes |
@Test public void testCSVContentProcessorDoFn() { List<String> sampleStringList = new ArrayList<String>(); sampleStringList.add("A,a"); sampleStringList.add("B,b"); sampleStringList.add("C,c"); sampleStringList.add("D,c"); PCollection<KV<String, List<String>>> input = pipeline.apply(Create.of(KV.of("test", sampleStringList))); PCollection<KV<String, Table>> outputTables = input.apply( "ContentHandler", ParDo.of(new CSVContentProcessorDoFn(ValueProvider.StaticValueProvider.of(1)))); PCollection<String> outputKeys = outputTables.apply(Keys.create()); PAssert.that(outputKeys).containsInAnyOrder("test_1", "test_2", "test_3"); pipeline.run(); }
Example #6
Source File: DeIdentifyTableBucketing.java From java-docs-samples with Apache License 2.0 | 6 votes |
public static void deIdentifyTableBucketing() throws IOException { // TODO(developer): Replace these variables before running the sample. String projectId = "your-project-id"; Table tableToDeIdentify = Table.newBuilder() .addHeaders(FieldId.newBuilder().setName("AGE").build()) .addHeaders(FieldId.newBuilder().setName("PATIENT").build()) .addHeaders(FieldId.newBuilder().setName("HAPPINESS SCORE").build()) .addRows(Row.newBuilder() .addValues(Value.newBuilder().setStringValue("101").build()) .addValues(Value.newBuilder().setStringValue("Charles Dickens").build()) .addValues(Value.newBuilder().setStringValue("95").build()) .build()) .addRows(Row.newBuilder() .addValues(Value.newBuilder().setStringValue("22").build()) .addValues(Value.newBuilder().setStringValue("Jane Austen").build()) .addValues(Value.newBuilder().setStringValue("21").build()) .build()) .addRows(Row.newBuilder() .addValues(Value.newBuilder().setStringValue("55").build()) .addValues(Value.newBuilder().setStringValue("Mark Twain").build()) .addValues(Value.newBuilder().setStringValue("75").build()) .build()) .build(); deIdentifyTableBucketing(projectId, tableToDeIdentify); }
Example #7
Source File: DLPTokenizationDoFnTest.java From dlp-dataflow-deidentification with Apache License 2.0 | 6 votes |
@Test public void testConvertTableRowToRow() { DLPTokenizationDoFn dlp = new DLPTokenizationDoFn( "Project Name", ValueProvider.StaticValueProvider.of("DeidentifyTemplateName"), ValueProvider.StaticValueProvider.of("IdentifyTemplateName")); String[] header = {"header0", "header1"}; String key = "Key name"; Table.Row.Builder tableRowBuilder = Table.Row.newBuilder(); tableRowBuilder.addValues(0, Value.newBuilder().setStringValue("value0")); tableRowBuilder.addValues(1, Value.newBuilder().setStringValue("value1")); Table.Row row = tableRowBuilder.build(); Row result = dlp.convertTableRowToRow(header, key, row); assertEquals(result.getTableId(), key); assertEquals(result.getHeader()[0], "header0"); assertEquals(result.getValue()[1], "value1"); }
Example #8
Source File: DLPDeidentifyText.java From beam with Apache License 2.0 | 6 votes |
@ProcessElement public void processElement(ProcessContext c) throws IOException { String fileName = c.element().getKey(); List<FieldId> dlpTableHeaders; if (headerColumns != null) { dlpTableHeaders = c.sideInput(headerColumns).stream() .map(header -> FieldId.newBuilder().setName(header).build()) .collect(Collectors.toList()); } else { // handle unstructured input dlpTableHeaders = new ArrayList<>(); dlpTableHeaders.add(FieldId.newBuilder().setName("value").build()); } Table table = Table.newBuilder() .addAllHeaders(dlpTableHeaders) .addAllRows(c.element().getValue()) .build(); ContentItem contentItem = ContentItem.newBuilder().setTable(table).build(); this.requestBuilder.setItem(contentItem); DeidentifyContentResponse response = dlpServiceClient.deidentifyContent(this.requestBuilder.build()); c.output(KV.of(fileName, response)); }
Example #9
Source File: MapStringToDlpRowTest.java From beam with Apache License 2.0 | 6 votes |
@Test public void mapsDelimitedStringToRow() { PCollection<KV<String, Table.Row>> rowCollection = testPipeline .apply(Create.of(KV.of("key", "value,secondValue"))) .apply(ParDo.of(new MapStringToDlpRow(","))); PAssert.that(rowCollection) .containsInAnyOrder( KV.of( "key", Table.Row.newBuilder() .addValues(Value.newBuilder().setStringValue("value").build()) .addValues(Value.newBuilder().setStringValue("secondValue").build()) .build())); testPipeline.run().waitUntilFinish(); }
Example #10
Source File: DeIdentifyTableRowSuppress.java From java-docs-samples with Apache License 2.0 | 6 votes |
public static void deIdentifyTableRowSuppress() throws IOException { // TODO(developer): Replace these variables before running the sample. String projectId = "your-project-id"; Table tableToDeIdentify = Table.newBuilder() .addHeaders(FieldId.newBuilder().setName("AGE").build()) .addHeaders(FieldId.newBuilder().setName("PATIENT").build()) .addHeaders(FieldId.newBuilder().setName("HAPPINESS SCORE").build()) .addRows(Row.newBuilder() .addValues(Value.newBuilder().setStringValue("101").build()) .addValues(Value.newBuilder().setStringValue("Charles Dickens").build()) .addValues(Value.newBuilder().setStringValue("95").build()) .build()) .addRows(Row.newBuilder() .addValues(Value.newBuilder().setStringValue("22").build()) .addValues(Value.newBuilder().setStringValue("Jane Austen").build()) .addValues(Value.newBuilder().setStringValue("21").build()) .build()) .addRows(Row.newBuilder() .addValues(Value.newBuilder().setStringValue("55").build()) .addValues(Value.newBuilder().setStringValue("Mark Twain").build()) .addValues(Value.newBuilder().setStringValue("75").build()) .build()) .build(); deIdentifyTableRowSuppress(projectId, tableToDeIdentify); }
Example #11
Source File: DLPInspectText.java From beam with Apache License 2.0 | 6 votes |
@ProcessElement public void processElement(ProcessContext c) throws IOException { List<FieldId> tableHeaders; if (headerColumns != null) { tableHeaders = c.sideInput(headerColumns).stream() .map(header -> FieldId.newBuilder().setName(header).build()) .collect(Collectors.toList()); } else { tableHeaders = new ArrayList<>(); tableHeaders.add(FieldId.newBuilder().setName("value").build()); } Table table = Table.newBuilder().addAllHeaders(tableHeaders).addAllRows(c.element().getValue()).build(); ContentItem contentItem = ContentItem.newBuilder().setTable(table).build(); this.requestBuilder.setItem(contentItem); InspectContentResponse response = dlpServiceClient.inspectContent(this.requestBuilder.build()); c.output(KV.of(c.element().getKey(), response)); }
Example #12
Source File: DLPTextToBigQueryStreaming.java From dlp-dataflow-deidentification with Apache License 2.0 | 6 votes |
private static TableRow createBqRow(Table.Row tokenizedValue, String[] headers) { TableRow bqRow = new TableRow(); AtomicInteger headerIndex = new AtomicInteger(0); List<TableCell> cells = new ArrayList<>(); tokenizedValue .getValuesList() .forEach( value -> { String checkedHeaderName = checkHeaderName(headers[headerIndex.getAndIncrement()].toString()); bqRow.set(checkedHeaderName, value.getStringValue()); cells.add(new TableCell().set(checkedHeaderName, value.getStringValue())); }); bqRow.setF(cells); return bqRow; }
Example #13
Source File: DLPTextToBigQueryStreaming.java From dlp-dataflow-deidentification with Apache License 2.0 | 6 votes |
@ProcessElement public void processElement(ProcessContext c) { Table tokenizedData = c.element().getValue(); List<String> headers = tokenizedData.getHeadersList().stream() .map(fid -> fid.getName()) .collect(Collectors.toList()); List<Table.Row> outputRows = tokenizedData.getRowsList(); if (outputRows.size() > 0) { for (Table.Row outputRow : outputRows) { if (outputRow.getValuesCount() != headers.size()) { throw new IllegalArgumentException( "CSV file's header count must exactly match with data element count"); } c.output( KV.of( c.element().getKey(), createBqRow(outputRow, headers.toArray(new String[headers.size()])))); } } }
Example #14
Source File: DLPReidentifyText.java From beam with Apache License 2.0 | 6 votes |
@ProcessElement public void processElement(ProcessContext context) throws IOException { List<FieldId> tableHeaders; if (headerColumns != null) { tableHeaders = context.sideInput(headerColumns).stream() .map(header -> FieldId.newBuilder().setName(header).build()) .collect(Collectors.toList()); } else { // handle unstructured input. tableHeaders = new ArrayList<>(); tableHeaders.add(FieldId.newBuilder().setName("value").build()); } Table table = Table.newBuilder() .addAllHeaders(tableHeaders) .addAllRows(context.element().getValue()) .build(); ContentItem contentItem = ContentItem.newBuilder().setTable(table).build(); this.requestBuilder.setItem(contentItem); ReidentifyContentResponse response = dlpServiceClient.reidentifyContent(requestBuilder.build()); context.output(KV.of(context.element().getKey(), response)); }
Example #15
Source File: MapStringToDlpRowTest.java From beam with Apache License 2.0 | 5 votes |
@Test public void mapsStringToRow() { PCollection<KV<String, Table.Row>> rowCollection = testPipeline .apply(Create.of(KV.of("key", "value"))) .apply(ParDo.of(new MapStringToDlpRow(null))); PAssert.that(rowCollection) .containsInAnyOrder( KV.of( "key", Table.Row.newBuilder() .addValues(Value.newBuilder().setStringValue("value").build()) .build())); testPipeline.run().waitUntilFinish(); }
Example #16
Source File: BatchRequestForDlpTest.java From beam with Apache License 2.0 | 5 votes |
@Override public Void apply(Iterable<KV<String, Iterable<Table.Row>>> input) { List<KV<String, Iterable<Table.Row>>> itemList = new ArrayList<>(); input.forEach(itemList::add); assertEquals(1, itemList.size()); return null; }
Example #17
Source File: BatchRequestForDlpTest.java From beam with Apache License 2.0 | 5 votes |
@Test public void batchesRequests() { PCollection<KV<String, Iterable<Table.Row>>> batchedRows = testPipeline .apply(Create.of(KV.of("key", "value1"), KV.of("key", "value2"))) .apply(ParDo.of(new MapStringToDlpRow(null))) .apply(ParDo.of(new BatchRequestForDLP(524000))); PAssert.that(batchedRows).satisfies(new VerifyPCollectionSize()); testPipeline.run().waitUntilFinish(); }
Example #18
Source File: DeIdentifyTableWithFpe.java From java-docs-samples with Apache License 2.0 | 5 votes |
public static void main(String[] args) throws Exception { // TODO(developer): Replace these variables before running the sample. String projectId = "your-project-id"; String kmsKeyName = "projects/YOUR_PROJECT/" + "locations/YOUR_KEYRING_REGION/" + "keyRings/YOUR_KEYRING_NAME/" + "cryptoKeys/YOUR_KEY_NAME"; String wrappedAesKey = "YOUR_ENCRYPTED_AES_256_KEY"; Table tableToDeIdentify = Table.newBuilder() .addHeaders(FieldId.newBuilder().setName("Employee ID").build()) .addHeaders(FieldId.newBuilder().setName("Date").build()) .addHeaders(FieldId.newBuilder().setName("Compensation").build()) .addRows(Row.newBuilder() .addValues(Value.newBuilder().setStringValue("11111").build()) .addValues(Value.newBuilder().setStringValue("2015").build()) .addValues(Value.newBuilder().setStringValue("$10").build()) .build()) .addRows(Row.newBuilder() .addValues(Value.newBuilder().setStringValue("11111").build()) .addValues(Value.newBuilder().setStringValue("2016").build()) .addValues(Value.newBuilder().setStringValue("$20").build()) .build()) .addRows(Row.newBuilder() .addValues(Value.newBuilder().setStringValue("22222").build()) .addValues(Value.newBuilder().setStringValue("2016").build()) .addValues(Value.newBuilder().setStringValue("$15").build()) .build()) .build(); deIdentifyTableWithFpe(projectId, tableToDeIdentify, kmsKeyName, wrappedAesKey); }
Example #19
Source File: DeIdentifyWithDateShift.java From java-docs-samples with Apache License 2.0 | 5 votes |
public static Table.Row parseLineAsRow(String line) { List<String> values = Splitter.on(",").splitToList(line); Value name = Value.newBuilder().setStringValue(values.get(0)).build(); Value birthDate = Value.newBuilder().setDateValue(parseAsDate(values.get(1))).build(); Value creditCardNumber = Value.newBuilder().setStringValue(values.get(2)).build(); Value registerDate = Value.newBuilder().setDateValue(parseAsDate(values.get(3))).build(); return Table.Row.newBuilder() .addValues(name) .addValues(birthDate) .addValues(creditCardNumber) .addValues(registerDate) .build(); }
Example #20
Source File: DeIdentifyTableInfoTypes.java From java-docs-samples with Apache License 2.0 | 5 votes |
public static void deIdentifyTableInfoTypes() throws IOException { // TODO(developer): Replace these variables before running the sample. String projectId = "your-project-id"; Table tableToDeIdentify = Table.newBuilder() .addHeaders(FieldId.newBuilder().setName("AGE").build()) .addHeaders(FieldId.newBuilder().setName("PATIENT").build()) .addHeaders(FieldId.newBuilder().setName("HAPPINESS SCORE").build()) .addHeaders(FieldId.newBuilder().setName("FACTOID").build()) .addRows(Row.newBuilder() .addValues(Value.newBuilder().setStringValue("101").build()) .addValues(Value.newBuilder().setStringValue("Charles Dickens").build()) .addValues(Value.newBuilder().setStringValue("95").build()) .addValues(Value.newBuilder().setStringValue( "Charles Dickens name was a curse, possibly invented by Shakespeare.").build()) .build()) .addRows(Row.newBuilder() .addValues(Value.newBuilder().setStringValue("22").build()) .addValues(Value.newBuilder().setStringValue("Jane Austen").build()) .addValues(Value.newBuilder().setStringValue("21").build()) .addValues(Value.newBuilder().setStringValue( "There are 14 kisses in Jane Austen's novels.").build()) .build()) .addRows(Row.newBuilder() .addValues(Value.newBuilder().setStringValue("55").build()) .addValues(Value.newBuilder().setStringValue("Mark Twain").build()) .addValues(Value.newBuilder().setStringValue("75").build()) .addValues(Value.newBuilder().setStringValue("Mark Twain loved cats.").build()) .build()) .build(); deIdentifyTableInfoTypes(projectId, tableToDeIdentify); }
Example #21
Source File: DeIdentifyTableConditionInfoTypes.java From java-docs-samples with Apache License 2.0 | 5 votes |
public static void deIdentifyTableConditionInfoTypes() throws IOException { // TODO(developer): Replace these variables before running the sample. String projectId = "your-project-id"; Table tableToDeIdentify = Table.newBuilder() .addHeaders(FieldId.newBuilder().setName("AGE").build()) .addHeaders(FieldId.newBuilder().setName("PATIENT").build()) .addHeaders(FieldId.newBuilder().setName("HAPPINESS SCORE").build()) .addHeaders(FieldId.newBuilder().setName("FACTOID").build()) .addRows(Row.newBuilder() .addValues(Value.newBuilder().setStringValue("101").build()) .addValues(Value.newBuilder().setStringValue("Charles Dickens").build()) .addValues(Value.newBuilder().setStringValue("95").build()) .addValues(Value.newBuilder().setStringValue( "Charles Dickens name was a curse, possibly invented by Shakespeare.").build()) .build()) .addRows(Row.newBuilder() .addValues(Value.newBuilder().setStringValue("22").build()) .addValues(Value.newBuilder().setStringValue("Jane Austen").build()) .addValues(Value.newBuilder().setStringValue("21").build()) .addValues(Value.newBuilder().setStringValue( "There are 14 kisses in Jane Austen's novels.").build()) .build()) .addRows(Row.newBuilder() .addValues(Value.newBuilder().setStringValue("55").build()) .addValues(Value.newBuilder().setStringValue("Mark Twain").build()) .addValues(Value.newBuilder().setStringValue("75").build()) .addValues(Value.newBuilder().setStringValue("Mark Twain loved cats.").build()) .build()) .build(); deIdentifyTableConditionInfoTypes(projectId, tableToDeIdentify); }
Example #22
Source File: InspectTable.java From java-docs-samples with Apache License 2.0 | 5 votes |
public static void main(String[] args) throws Exception { // TODO(developer): Replace these variables before running the sample. String projectId = "your-project-id"; Table tableToInspect = Table.newBuilder() .addHeaders(FieldId.newBuilder().setName("name").build()) .addHeaders(FieldId.newBuilder().setName("phone").build()) .addRows(Row.newBuilder() .addValues(Value.newBuilder().setStringValue("John Doe").build()) .addValues(Value.newBuilder().setStringValue("(206) 555-0123").build())) .build(); inspectTable(projectId, tableToInspect); }
Example #23
Source File: DeIdentificationTests.java From java-docs-samples with Apache License 2.0 | 5 votes |
@Test public void testDeIdentifyTableWithFpe() throws IOException { Table tableToDeIdentify = Table.newBuilder() .addHeaders(FieldId.newBuilder().setName("Employee ID").build()) .addHeaders(FieldId.newBuilder().setName("Date").build()) .addHeaders(FieldId.newBuilder().setName("Compensation").build()) .addRows(Row.newBuilder() .addValues(Value.newBuilder().setStringValue("11111").build()) .addValues(Value.newBuilder().setStringValue("2015").build()) .addValues(Value.newBuilder().setStringValue("$10").build()) .build()) .addRows(Row.newBuilder() .addValues(Value.newBuilder().setStringValue("11111").build()) .addValues(Value.newBuilder().setStringValue("2016").build()) .addValues(Value.newBuilder().setStringValue("$20").build()) .build()) .addRows(Row.newBuilder() .addValues(Value.newBuilder().setStringValue("22222").build()) .addValues(Value.newBuilder().setStringValue("2016").build()) .addValues(Value.newBuilder().setStringValue("$15").build()) .build()) .build(); DeIdentifyTableWithFpe.deIdentifyTableWithFpe( PROJECT_ID, tableToDeIdentify, kmsKeyName, wrappedKey); String output = bout.toString(); assertThat(output).contains("Table after format-preserving encryption:"); }
Example #24
Source File: DeIdentificationTests.java From java-docs-samples with Apache License 2.0 | 5 votes |
@Test public void testReIdentifyTableWithFpe() throws IOException { Table tableToReIdentify = Table.newBuilder() .addHeaders(FieldId.newBuilder().setName("Employee ID").build()) .addRows(Row.newBuilder() .addValues(Value.newBuilder().setStringValue("28777").build()).build()) .build(); ReIdentifyTableWithFpe.reIdentifyTableWithFpe( PROJECT_ID, tableToReIdentify, kmsKeyName, wrappedKey); String output = bout.toString(); assertThat(output).contains("Table after re-identification:"); }
Example #25
Source File: InspectTests.java From java-docs-samples with Apache License 2.0 | 5 votes |
@Test public void testInspectTable() { Table tableToInspect = Table.newBuilder() .addHeaders(FieldId.newBuilder().setName("name").build()) .addHeaders(FieldId.newBuilder().setName("phone").build()) .addRows(Row.newBuilder() .addValues(Value.newBuilder().setStringValue("John Doe").build()) .addValues(Value.newBuilder().setStringValue("(206) 555-0123").build())) .build(); InspectTable.inspectTable(PROJECT_ID, tableToInspect); String output = bout.toString(); assertThat(output).contains("Info type: PHONE_NUMBER"); }
Example #26
Source File: Util.java From dlp-dataflow-deidentification with Apache License 2.0 | 5 votes |
public static Table createDLPTable(List<FieldId> headers, List<String> lines) { List<Table.Row> rows = new ArrayList<>(); lines.forEach( line -> { rows.add(convertCsvRowToTableRow(line)); }); Table table = Table.newBuilder().addAllHeaders(headers).addAllRows(rows).build(); return table; }
Example #27
Source File: DLPTextToBigQueryStreaming.java From dlp-dataflow-deidentification with Apache License 2.0 | 5 votes |
private Table.Row convertCsvRowToTableRow(CSVRecord csvRow) { /** convert from CSV row to DLP Table Row */ Iterator<String> valueIterator = csvRow.iterator(); Table.Row.Builder tableRowBuilder = Table.Row.newBuilder(); while (valueIterator.hasNext()) { String value = valueIterator.next(); if (value != null) { tableRowBuilder.addValues(Value.newBuilder().setStringValue(value.toString()).build()); } else { tableRowBuilder.addValues(Value.newBuilder().setStringValue("").build()); } } return tableRowBuilder.build(); }
Example #28
Source File: DLPTextToBigQueryStreaming.java From dlp-dataflow-deidentification with Apache License 2.0 | 5 votes |
@ProcessElement public void processElement(ProcessContext c) { String key = c.element().getKey(); Table nonEncryptedData = c.element().getValue(); ContentItem tableItem = ContentItem.newBuilder().setTable(nonEncryptedData).build(); this.requestBuilder.setItem(tableItem); DeidentifyContentResponse response = dlpServiceClient.deidentifyContent(this.requestBuilder.build()); Table tokenizedData = response.getItem().getTable(); numberOfRowsTokenized.update(tokenizedData.getRowsList().size()); numberOfBytesTokenized.update(tokenizedData.toByteArray().length); c.output(KV.of(key, tokenizedData)); }
Example #29
Source File: DLPTokenizationDoFn.java From dlp-dataflow-deidentification with Apache License 2.0 | 5 votes |
public Row convertTableRowToRow(String[] header, String key, Table.Row outputRow) { String dlpRow = outputRow.getValuesList().stream() .map(value -> value.getStringValue()) .collect(Collectors.joining(",")); String[] values = dlpRow.split(","); Row row = new Row(key, header, values); return row; }
Example #30
Source File: DLPTokenizationDoFn.java From dlp-dataflow-deidentification with Apache License 2.0 | 5 votes |
@ProcessElement public void processElement(ProcessContext c) { String key = c.element().getKey(); Table nonEncryptedData = c.element().getValue(); setInspectTemplateExist(); ContentItem tableItem = ContentItem.newBuilder().setTable(nonEncryptedData).build(); DeidentifyContentResponse response; DeidentifyContentRequest request = buildDeidentifyContentRequest(tableItem); response = dlpServiceClient.deidentifyContent(request); Table encryptedData = response.getItem().getTable(); LOG.info( "Request Size Successfully Tokenized:{} rows {} bytes ", encryptedData.getRowsList().size(), request.toByteString().size()); List<String> outputHeaders = encryptedData.getHeadersList().stream().map(FieldId::getName).collect(Collectors.toList()); String[] header = new String[outputHeaders.size()]; for (int i = 0; i < header.length; i++) { header[i] = Util.checkHeaderName(outputHeaders.get(i)); } List<Table.Row> outputRows = encryptedData.getRowsList(); for (Table.Row outputRow : outputRows) { Row row = convertTableRowToRow(header, key, outputRow); c.output(row); } }