org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions Java Examples
The following examples show how to use
org.apache.beam.sdk.io.fs.ResolveOptions.StandardResolveOptions.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: WriteFilesTest.java From beam with Apache License 2.0 | 6 votes |
@Test public void testShardedDisplayData() { DynamicDestinations<String, Void, String> dynamicDestinations = DynamicFileDestinations.constant( DefaultFilenamePolicy.fromParams( new Params() .withBaseFilename( getBaseOutputDirectory() .resolve("file", StandardResolveOptions.RESOLVE_FILE)) .withShardTemplate("-SS-of-NN"))); SimpleSink<Void> sink = new SimpleSink<Void>( getBaseOutputDirectory(), dynamicDestinations, Compression.UNCOMPRESSED) { @Override public void populateDisplayData(DisplayData.Builder builder) { builder.add(DisplayData.item("foo", "bar")); } }; WriteFiles<String, ?, String> write = WriteFiles.to(sink).withNumShards(1); DisplayData displayData = DisplayData.from(write); assertThat(displayData, hasDisplayItem("sink", sink.getClass())); assertThat(displayData, includesDisplayDataFor("sink", sink)); assertThat(displayData, hasDisplayItem("numShards", 1)); }
Example #2
Source File: HadoopResourceId.java From beam with Apache License 2.0 | 6 votes |
@Override public ResourceId resolve(String other, ResolveOptions resolveOptions) { checkState( isDirectory(), String.format("Expected this resource is a directory, but had [%s].", uri)); if (resolveOptions == StandardResolveOptions.RESOLVE_DIRECTORY) { if (!other.endsWith("/")) { other += "/"; } return new HadoopResourceId(uri.resolve(other)); } else if (resolveOptions == StandardResolveOptions.RESOLVE_FILE) { checkArgument(!other.endsWith("/"), "Resolving a file with a directory path: %s", other); return new HadoopResourceId(uri.resolve(other)); } else { throw new UnsupportedOperationException( String.format("Unexpected StandardResolveOptions %s", resolveOptions)); } }
Example #3
Source File: WriteToText.java From beam with Apache License 2.0 | 6 votes |
@Override public ResourceId windowedFilename( int shardNumber, int numShards, BoundedWindow window, PaneInfo paneInfo, OutputFileHints outputFileHints) { IntervalWindow intervalWindow = (IntervalWindow) window; String filename = String.format( "%s-%s-of-%s%s", filenamePrefixForWindow(intervalWindow), shardNumber, numShards, outputFileHints.getSuggestedFilenameSuffix()); return prefix.getCurrentDirectory().resolve(filename, StandardResolveOptions.RESOLVE_FILE); }
Example #4
Source File: WordCountIT.java From beam with Apache License 2.0 | 6 votes |
@Test public void testE2EWordCount() throws Exception { WordCountITOptions options = TestPipeline.testingPipelineOptions().as(WordCountITOptions.class); options.setInputFile(DEFAULT_INPUT); options.setOutput( FileSystems.matchNewResource(options.getTempRoot(), true) .resolve( String.format("WordCountIT-%tF-%<tH-%<tM-%<tS-%<tL", new Date()), StandardResolveOptions.RESOLVE_DIRECTORY) .resolve("output", StandardResolveOptions.RESOLVE_DIRECTORY) .resolve("results", StandardResolveOptions.RESOLVE_FILE) .toString()); WordCount.runWordCount(options); assertThat( new NumberedShardedFile(options.getOutput() + "*-of-*"), fileContentsHaveChecksum(DEFAULT_OUTPUT_CHECKSUM)); }
Example #5
Source File: WindowedFilenamePolicy.java From DataflowTemplates with Apache License 2.0 | 6 votes |
/** * The windowed filename method will construct filenames per window according to the baseFile, * suffix, and shardTemplate supplied. Directories with date templates in them will automatically * have their values resolved. For example the outputDirectory of /YYYY/MM/DD would resolve to * /2017/01/08 on January 8th, 2017. */ @Override public ResourceId windowedFilename( int shardNumber, int numShards, BoundedWindow window, PaneInfo paneInfo, OutputFileHints outputFileHints) { ResourceId outputFile = resolveWithDateTemplates(outputDirectory, window) .resolve(outputFilenamePrefix.get(), StandardResolveOptions.RESOLVE_FILE); DefaultFilenamePolicy policy = DefaultFilenamePolicy.fromStandardParameters( StaticValueProvider.of(outputFile), shardTemplate.get(), suffix.get(), true); ResourceId result = policy.windowedFilename(shardNumber, numShards, window, paneInfo, outputFileHints); LOG.debug("Windowed file name policy created: {}", result.toString()); return result; }
Example #6
Source File: WindowedFilenamePolicy.java From DataflowTemplates with Apache License 2.0 | 6 votes |
/** * The windowed filename method will construct filenames per window according to the baseFile, * suffix, and shardTemplate supplied. Directories with date templates in them will automatically * have their values resolved. For example the outputDirectory of /YYYY/MM/DD would resolve to * /2017/01/08 on January 8th, 2017. */ @Override public ResourceId windowedFilename( int shardNumber, int numShards, BoundedWindow window, PaneInfo paneInfo, OutputFileHints outputFileHints) { ResourceId outputFile = resolveWithDateTemplates(outputDirectory, window) .resolve(outputFilenamePrefix.get(), StandardResolveOptions.RESOLVE_FILE); DefaultFilenamePolicy policy = DefaultFilenamePolicy.fromStandardParameters( StaticValueProvider.of(outputFile), shardTemplate.get(), suffix.get(), true); ResourceId result = policy.windowedFilename(shardNumber, numShards, window, paneInfo, outputFileHints); LOG.debug("Windowed file name policy created: {}", result.toString()); return result; }
Example #7
Source File: PackageUtil.java From beam with Apache License 2.0 | 6 votes |
public static PackageAttributes forBytesToStage( byte[] bytes, String targetName, String stagingPath) { HashCode hashCode = Hashing.sha256().newHasher().putBytes(bytes).hash(); long size = bytes.length; String target = Environments.createStagingFileName(new File(targetName), hashCode); String resourcePath = FileSystems.matchNewResource(stagingPath, true) .resolve(target, StandardResolveOptions.RESOLVE_FILE) .toString(); DataflowPackage targetPackage = new DataflowPackage(); targetPackage.setName(target); targetPackage.setLocation(resourcePath); return new AutoValue_PackageUtil_PackageAttributes( null, bytes, targetPackage, size, hashCode.toString()); }
Example #8
Source File: WriteOneFilePerWindow.java From deployment-examples with MIT License | 6 votes |
@Override public ResourceId windowedFilename( int shardNumber, int numShards, BoundedWindow window, PaneInfo paneInfo, OutputFileHints outputFileHints) { IntervalWindow intervalWindow = (IntervalWindow) window; String filename = String.format( "%s-%s-of-%s%s", filenamePrefixForWindow(intervalWindow), shardNumber, numShards, outputFileHints.getSuggestedFilenameSuffix()); return baseFilename .getCurrentDirectory() .resolve(filename, StandardResolveOptions.RESOLVE_FILE); }
Example #9
Source File: DynamicOneFilePerWindow.java From dlp-dataflow-deidentification with Apache License 2.0 | 6 votes |
@Override public ResourceId windowedFilename( int shardNumber, int numShards, BoundedWindow window, PaneInfo paneInfo, OutputFileHints outputFileHints) { IntervalWindow intervalWindow = (IntervalWindow) window; String filename = String.format( "%s-%s-of-%s%s", filenamePrefixForWindow(intervalWindow), shardNumber, numShards, outputFileHints.getSuggestedFilenameSuffix()); return baseFilename .getCurrentDirectory() .resolve(filename, StandardResolveOptions.RESOLVE_FILE); }
Example #10
Source File: WriteToText.java From deployment-examples with MIT License | 6 votes |
@Override public ResourceId windowedFilename( int shardNumber, int numShards, BoundedWindow window, PaneInfo paneInfo, OutputFileHints outputFileHints) { IntervalWindow intervalWindow = (IntervalWindow) window; String filename = String.format( "%s-%s-of-%s%s", filenamePrefixForWindow(intervalWindow), shardNumber, numShards, outputFileHints.getSuggestedFilenameSuffix()); return prefix.getCurrentDirectory().resolve(filename, StandardResolveOptions.RESOLVE_FILE); }
Example #11
Source File: WriteOneFilePerWindow.java From dlp-dataflow-deidentification with Apache License 2.0 | 6 votes |
@Override public ResourceId windowedFilename( int shardNumber, int numShards, BoundedWindow window, PaneInfo paneInfo, OutputFileHints outputFileHints) { IntervalWindow intervalWindow = (IntervalWindow) window; String filename = String.format( "%s-%s-of-%s%s", filenamePrefixForWindow(intervalWindow), shardNumber, numShards, outputFileHints.getSuggestedFilenameSuffix()); return baseFilename .getCurrentDirectory() .resolve(filename, StandardResolveOptions.RESOLVE_FILE); }
Example #12
Source File: WriteOneFilePerWindow.java From beam with Apache License 2.0 | 6 votes |
@Override public ResourceId windowedFilename( int shardNumber, int numShards, BoundedWindow window, PaneInfo paneInfo, OutputFileHints outputFileHints) { IntervalWindow intervalWindow = (IntervalWindow) window; String filename = String.format( "%s-%s-of-%s%s", filenamePrefixForWindow(intervalWindow), shardNumber, numShards, outputFileHints.getSuggestedFilenameSuffix()); return baseFilename .getCurrentDirectory() .resolve(filename, StandardResolveOptions.RESOLVE_FILE); }
Example #13
Source File: WindowedWordCountIT.java From beam with Apache License 2.0 | 6 votes |
private WindowedWordCountITOptions defaultOptions() throws Exception { WindowedWordCountITOptions options = TestPipeline.testingPipelineOptions().as(WindowedWordCountITOptions.class); options.setInputFile(DEFAULT_INPUT); options.setTestTimeoutSeconds(1200L); options.setMinTimestampMillis(0L); options.setMinTimestampMillis(Duration.standardHours(1).getMillis()); options.setWindowSize(10); options.setOutput( FileSystems.matchNewResource(options.getTempRoot(), true) .resolve( String.format( "WindowedWordCountIT.%s-%tFT%<tH:%<tM:%<tS.%<tL+%s", testName.getMethodName(), new Date(), ThreadLocalRandom.current().nextInt()), StandardResolveOptions.RESOLVE_DIRECTORY) .resolve("output", StandardResolveOptions.RESOLVE_DIRECTORY) .resolve("results", StandardResolveOptions.RESOLVE_FILE) .toString()); return options; }
Example #14
Source File: TopWikipediaSessionsIT.java From beam with Apache License 2.0 | 6 votes |
@Test public void testE2ETopWikiPages() throws Exception { TopWikipediaSessionsITOptions options = TestPipeline.testingPipelineOptions().as(TopWikipediaSessionsITOptions.class); options.setWikiInput(DEFAULT_INPUT_10_FILES); options.setOutput( FileSystems.matchNewResource(options.getTempRoot(), true) .resolve( String.format("topwikisessions-it-%tF-%<tH-%<tM-%<tS-%<tL", new Date()), StandardResolveOptions.RESOLVE_DIRECTORY) .resolve("output", StandardResolveOptions.RESOLVE_DIRECTORY) .resolve("results", StandardResolveOptions.RESOLVE_FILE) .toString()); TopWikipediaSessions.run(options); assertThat( new NumberedShardedFile(options.getOutput() + "*-of-*"), fileContentsHaveChecksum(DEFAULT_OUTPUT_CHECKSUM)); }
Example #15
Source File: GcsResourceId.java From beam with Apache License 2.0 | 6 votes |
@Override public GcsResourceId resolve(String other, ResolveOptions resolveOptions) { checkState( isDirectory(), String.format("Expected the gcsPath is a directory, but had [%s].", gcsPath)); checkArgument( resolveOptions.equals(StandardResolveOptions.RESOLVE_FILE) || resolveOptions.equals(StandardResolveOptions.RESOLVE_DIRECTORY), String.format("ResolveOptions: [%s] is not supported.", resolveOptions)); if (resolveOptions.equals(StandardResolveOptions.RESOLVE_FILE)) { checkArgument( !other.endsWith("/"), "The resolved file: [%s] should not end with '/'.", other); return fromGcsPath(gcsPath.resolve(other)); } else { // StandardResolveOptions.RESOLVE_DIRECTORY if (other.endsWith("/")) { // other already contains the delimiter for gcs. // It is not recommended for callers to set the delimiter. // However, we consider it as a valid input. return fromGcsPath(gcsPath.resolve(other)); } else { return fromGcsPath(gcsPath.resolve(other + "/")); } } }
Example #16
Source File: WriteFilesTest.java From beam with Apache License 2.0 | 6 votes |
@Override public ResourceId windowedFilename( int shardNumber, int numShards, BoundedWindow window, PaneInfo paneInfo, OutputFileHints outputFileHints) { DecimalFormat df = new DecimalFormat("0000"); IntervalWindow intervalWindow = (IntervalWindow) window; String filename = String.format( "%s-%s-of-%s%s%s", filenamePrefixForWindow(intervalWindow), df.format(shardNumber), df.format(numShards), outputFileHints.getSuggestedFilenameSuffix(), suffix); return baseFilename .getCurrentDirectory() .resolve(filename, StandardResolveOptions.RESOLVE_FILE); }
Example #17
Source File: WriteFilesTest.java From beam with Apache License 2.0 | 6 votes |
@Test public void testDisplayData() { DynamicDestinations<String, Void, String> dynamicDestinations = DynamicFileDestinations.constant( DefaultFilenamePolicy.fromParams( new Params() .withBaseFilename( getBaseOutputDirectory() .resolve("file", StandardResolveOptions.RESOLVE_FILE)) .withShardTemplate("-SS-of-NN"))); SimpleSink<Void> sink = new SimpleSink<Void>( getBaseOutputDirectory(), dynamicDestinations, Compression.UNCOMPRESSED) { @Override public void populateDisplayData(DisplayData.Builder builder) { builder.add(DisplayData.item("foo", "bar")); } }; WriteFiles<String, ?, String> write = WriteFiles.to(sink); DisplayData displayData = DisplayData.from(write); assertThat(displayData, hasDisplayItem("sink", sink.getClass())); assertThat(displayData, includesDisplayDataFor("sink", sink)); }
Example #18
Source File: WriteFilesTest.java From beam with Apache License 2.0 | 6 votes |
@Override public ResourceId unwindowedFilename( int shardNumber, int numShards, OutputFileHints outputFileHints) { DecimalFormat df = new DecimalFormat("0000"); String prefix = baseFilename.isDirectory() ? "" : firstNonNull(baseFilename.getFilename(), ""); String filename = String.format( "%s-%s-of-%s%s%s", prefix, df.format(shardNumber), df.format(numShards), outputFileHints.getSuggestedFilenameSuffix(), suffix); return baseFilename .getCurrentDirectory() .resolve(filename, StandardResolveOptions.RESOLVE_FILE); }
Example #19
Source File: FileBasedSinkTest.java From beam with Apache License 2.0 | 6 votes |
/** * Writer opens the correct file, writes the header, footer, and elements in the correct order, * and returns the correct filename. */ @Test public void testWriter() throws Exception { String testUid = "testId"; ResourceId expectedTempFile = getBaseTempDirectory().resolve(testUid, StandardResolveOptions.RESOLVE_FILE); List<String> values = Arrays.asList("sympathetic vulture", "boresome hummingbird"); List<String> expected = new ArrayList<>(); expected.add(SimpleSink.SimpleWriter.HEADER); expected.addAll(values); expected.add(SimpleSink.SimpleWriter.FOOTER); SimpleSink.SimpleWriter<Void> writer = buildWriteOperationWithTempDir(getBaseTempDirectory()).createWriter(); writer.open(testUid); for (String value : values) { writer.write(value); } writer.close(); assertEquals(expectedTempFile, writer.getOutputFile()); assertFileContains(expected, expectedTempFile); }
Example #20
Source File: NumberedShardedFileTest.java From beam with Apache License 2.0 | 6 votes |
@Test public void testReadMultipleShards() throws Exception { String contents1 = "To be or not to be, ", contents2 = "it is not a question.", contents3 = "should not be included"; File tmpFile1 = tmpFolder.newFile("result-000-of-002"); File tmpFile2 = tmpFolder.newFile("result-001-of-002"); File tmpFile3 = tmpFolder.newFile("tmp"); Files.write(contents1, tmpFile1, StandardCharsets.UTF_8); Files.write(contents2, tmpFile2, StandardCharsets.UTF_8); Files.write(contents3, tmpFile3, StandardCharsets.UTF_8); filePattern = LocalResources.fromFile(tmpFolder.getRoot(), true) .resolve("result-*", StandardResolveOptions.RESOLVE_FILE) .toString(); NumberedShardedFile shardedFile = new NumberedShardedFile(filePattern); assertThat(shardedFile.readFilesWithRetries(), containsInAnyOrder(contents1, contents2)); }
Example #21
Source File: FilePatternMatchingShardedFileTest.java From beam with Apache License 2.0 | 6 votes |
@Test public void testReadMultipleShards() throws Exception { String contents1 = "To be or not to be, ", contents2 = "it is not a question.", contents3 = "should not be included"; File tmpFile1 = tmpFolder.newFile("result-000-of-002"); File tmpFile2 = tmpFolder.newFile("result-001-of-002"); File tmpFile3 = tmpFolder.newFile("tmp"); Files.write(contents1, tmpFile1, StandardCharsets.UTF_8); Files.write(contents2, tmpFile2, StandardCharsets.UTF_8); Files.write(contents3, tmpFile3, StandardCharsets.UTF_8); filePattern = LocalResources.fromFile(tmpFolder.getRoot(), true) .resolve("result-*", StandardResolveOptions.RESOLVE_FILE) .toString(); FilePatternMatchingShardedFile shardedFile = new FilePatternMatchingShardedFile(filePattern); assertThat(shardedFile.readFilesWithRetries(), containsInAnyOrder(contents1, contents2)); }
Example #22
Source File: LocalFileSystemTest.java From beam with Apache License 2.0 | 6 votes |
@Test public void testMatchNewResource() throws Exception { LocalResourceId fileResource = localFileSystem.matchNewResource("/some/test/resource/path", false /* isDirectory */); LocalResourceId dirResource = localFileSystem.matchNewResource("/some/test/resource/path", true /* isDirectory */); assertNotEquals(fileResource, dirResource); assertThat( fileResource .getCurrentDirectory() .resolve("path", StandardResolveOptions.RESOLVE_DIRECTORY), equalTo(dirResource.getCurrentDirectory())); assertThat( fileResource .getCurrentDirectory() .resolve("path", StandardResolveOptions.RESOLVE_DIRECTORY), equalTo(dirResource.getCurrentDirectory())); assertThat(dirResource.toString(), equalTo("/some/test/resource/path/")); }
Example #23
Source File: LocalResourceId.java From beam with Apache License 2.0 | 6 votes |
@Override public LocalResourceId resolve(String other, ResolveOptions resolveOptions) { checkState(isDirectory, "Expected the path is a directory, but had [%s].", pathString); checkArgument( resolveOptions.equals(StandardResolveOptions.RESOLVE_FILE) || resolveOptions.equals(StandardResolveOptions.RESOLVE_DIRECTORY), "ResolveOptions: [%s] is not supported.", resolveOptions); checkArgument( !(resolveOptions.equals(StandardResolveOptions.RESOLVE_FILE) && other.endsWith("/")), "The resolved file: [%s] should not end with '/'.", other); if (SystemUtils.IS_OS_WINDOWS) { return resolveLocalPathWindowsOS(other, resolveOptions); } else { return resolveLocalPath(other, resolveOptions); } }
Example #24
Source File: LocalResourceIdTest.java From beam with Apache License 2.0 | 6 votes |
@Test public void testResolveInWindowsOS() { if (!SystemUtils.IS_OS_WINDOWS) { // Skip tests return; } assertEquals( toResourceIdentifier("C:\\my home\\out put"), toResourceIdentifier("C:\\my home\\") .resolve("out put", StandardResolveOptions.RESOLVE_FILE)); assertEquals( toResourceIdentifier("C:\\out put"), toResourceIdentifier("C:\\my home\\") .resolve("..", StandardResolveOptions.RESOLVE_DIRECTORY) .resolve(".", StandardResolveOptions.RESOLVE_DIRECTORY) .resolve("out put", StandardResolveOptions.RESOLVE_FILE)); assertEquals( toResourceIdentifier("C:\\my home\\**\\*"), toResourceIdentifier("C:\\my home\\") .resolve("**", StandardResolveOptions.RESOLVE_DIRECTORY) .resolve("*", StandardResolveOptions.RESOLVE_FILE)); }
Example #25
Source File: SimpleSink.java From beam with Apache License 2.0 | 6 votes |
public static SimpleSink<Void> makeSimpleSink( ResourceId baseDirectory, String prefix, String shardTemplate, String suffix, WritableByteChannelFactory writableByteChannelFactory) { DynamicDestinations<String, Void, String> dynamicDestinations = DynamicFileDestinations.constant( DefaultFilenamePolicy.fromParams( new Params() .withBaseFilename( baseDirectory.resolve(prefix, StandardResolveOptions.RESOLVE_FILE)) .withShardTemplate(shardTemplate) .withSuffix(suffix))); return new SimpleSink<>(baseDirectory, dynamicDestinations, writableByteChannelFactory); }
Example #26
Source File: TfIdfIT.java From beam with Apache License 2.0 | 6 votes |
@Test public void testE2ETfIdf() throws Exception { TfIdfITOptions options = TestPipeline.testingPipelineOptions().as(TfIdfITOptions.class); options.setInput(DEFAULT_INPUT); options.setOutput( FileSystems.matchNewResource(options.getTempRoot(), true) .resolve( String.format("TfIdfIT-%tF-%<tH-%<tM-%<tS-%<tL", new Date()), StandardResolveOptions.RESOLVE_DIRECTORY) .resolve("output", StandardResolveOptions.RESOLVE_DIRECTORY) .resolve("results", StandardResolveOptions.RESOLVE_FILE) .toString()); TfIdf.runTfIdf(options); assertThat( new NumberedShardedFile(options.getOutput() + "*-of-*.csv", DEFAULT_SHARD_TEMPLATE), fileContentsHaveChecksum(EXPECTED_OUTPUT_CHECKSUM)); }
Example #27
Source File: FileBasedSinkTest.java From beam with Apache License 2.0 | 5 votes |
/** Reject non-distinct output filenames. */ @Test public void testCollidingOutputFilenames() throws Exception { ResourceId root = getBaseOutputDirectory(); SimpleSink<Void> sink = SimpleSink.makeSimpleSink(root, "file", "-NN", "test", Compression.UNCOMPRESSED); SimpleSink.SimpleWriteOperation<Void> writeOp = new SimpleSink.SimpleWriteOperation<>(sink); try { List<FileResult<Void>> results = Lists.newArrayList(); for (int i = 0; i < 3; ++i) { results.add( new FileResult<>( root.resolve("temp" + i, StandardResolveOptions.RESOLVE_FILE), 1 /* shard - should be different, but is the same */, GlobalWindow.INSTANCE, PaneInfo.ON_TIME_AND_ONLY_FIRING, null)); } writeOp.finalizeDestination(null, GlobalWindow.INSTANCE, 5 /* numShards */, results); fail("Should have failed."); } catch (IllegalArgumentException exn) { assertThat(exn.getMessage(), containsString("generated the same name")); assertThat(exn.getMessage(), containsString("temp0")); assertThat(exn.getMessage(), containsString("temp1")); } }
Example #28
Source File: FileBasedSinkTest.java From beam with Apache License 2.0 | 5 votes |
/** Output filenames are generated correctly when an extension is supplied. */ @Test public void testGenerateOutputFilenames() { List<ResourceId> expected; List<ResourceId> actual; ResourceId root = getBaseOutputDirectory(); SimpleSink<Void> sink = SimpleSink.makeSimpleSink( root, "file", ".SSSSS.of.NNNNN", ".test", Compression.UNCOMPRESSED); FilenamePolicy policy = sink.getDynamicDestinations().getFilenamePolicy(null); expected = Arrays.asList( root.resolve("file.00000.of.00003.test", StandardResolveOptions.RESOLVE_FILE), root.resolve("file.00001.of.00003.test", StandardResolveOptions.RESOLVE_FILE), root.resolve("file.00002.of.00003.test", StandardResolveOptions.RESOLVE_FILE)); actual = generateDestinationFilenames(policy, 3); assertEquals(expected, actual); expected = Collections.singletonList( root.resolve("file.00000.of.00001.test", StandardResolveOptions.RESOLVE_FILE)); actual = generateDestinationFilenames(policy, 1); assertEquals(expected, actual); expected = new ArrayList<>(); actual = generateDestinationFilenames(policy, 0); assertEquals(expected, actual); }
Example #29
Source File: PackageUtilTest.java From beam with Apache License 2.0 | 5 votes |
@Test public void testPackageUploadIsSkippedWithNonExistentResource() throws Exception { String nonExistentFile = FileSystems.matchNewResource(tmpFolder.getRoot().getPath(), true) .resolve("non-existent-file", StandardResolveOptions.RESOLVE_FILE) .toString(); assertEquals( Collections.EMPTY_LIST, defaultPackageUtil.stageClasspathElements( ImmutableList.of(makeStagedFile(nonExistentFile)), STAGING_PATH, createOptions)); }
Example #30
Source File: FhirIO.java From beam with Apache License 2.0 | 5 votes |
/** * Init batch. * * @throws IOException the io exception */ @StartBundle public void initFile() throws IOException { // Write each bundle to newline delimited JSON file. String filename = String.format("fhirImportBatch-%s.ndjson", UUID.randomUUID().toString()); ResourceId tempDir = FileSystems.matchNewResource(this.tempGcsPath.get(), true); this.resourceId = tempDir.resolve(filename, StandardResolveOptions.RESOLVE_FILE); this.ndJsonChannel = FileSystems.create(resourceId, "application/ld+json"); if (mapper == null) { this.mapper = new ObjectMapper(); } }