Java Code Examples for org.apache.commons.io.IOUtils#writeLines()

/**
 * Generate an EC keypair on the prime256v1 curve and save them to a file for later usage.
 *
 * Some code borrowed from
 * <a href=
 * "https://github.com/web-push-libs/webpush-java/blob/master/src/main/java/nl/martijndwars/webpush/cli/handlers/GenerateKeyHandler.java">webpush-java</a>.
 *
 * @author Martijn Dwars
 *
 * @throws InvalidAlgorithmParameterException
 * @throws NoSuchProviderException
 * @throws NoSuchAlgorithmException
 * @throws IOException
 * @throws FileNotFoundException
 */
private void generateVAPIDKeyPair() throws InvalidAlgorithmParameterException, NoSuchProviderException,
        NoSuchAlgorithmException, FileNotFoundException, IOException {
    ECNamedCurveParameterSpec parameterSpec = ECNamedCurveTable.getParameterSpec(Utils.CURVE);

    KeyPairGenerator keyPairGenerator = KeyPairGenerator.getInstance(Utils.ALGORITHM, PROVIDER_NAME);
    keyPairGenerator.initialize(parameterSpec);

    KeyPair keyPair = keyPairGenerator.generateKeyPair();
    byte[] publicKey = Utils.savePublicKey((ECPublicKey) keyPair.getPublic());
    byte[] privateKey = Utils.savePrivateKey((ECPrivateKey) keyPair.getPrivate());

    List<String> encodedKeys = new ArrayList<String>();
    encodedKeys.add(BaseEncoding.base64Url().encode(publicKey));
    encodedKeys.add(BaseEncoding.base64Url().encode(privateKey));

    // write the public key, then the private key in encoded form on separate lines in the file
    File file = new File(ConfigConstants.getUserDataFolder() + File.separator + VAPID_KEYS_FILE_NAME);
    file.getParentFile().mkdirs();
    IOUtils.writeLines(encodedKeys, System.lineSeparator(), new FileOutputStream(file));

    this.publicVAPIDKey = encodedKeys.get(0);
    this.privateVAPIDKey = encodedKeys.get(1);
}
 

private static File createKrb5Configuration(String domain) throws IOException {
        File tempFile = File.createTempFile("krb", "kdc");
        tempFile.deleteOnExit();
        ArrayList<String> lines = new ArrayList<>();
        lines.add("[libdefaults]");
        lines.add("\tdefault_realm = " + domain.toUpperCase());
        lines.add("[realms]");
        lines.add("\t" + domain.toUpperCase() + " = {");
        lines.add("\t\tkdc = " + domain);
        lines.add("\t\tadmin_server = " + domain);
        lines.add("\t}");
        FileWriter writer = null;
        try {
            writer = new FileWriter(tempFile);
            IOUtils.writeLines(lines, System.lineSeparator(), writer);
        } finally {
            if (writer != null) {
//                IOUtils.closeQuietly(writer);
                safeClose(writer);
            }
        }

        return tempFile;
    }
 

public TextIOJobBuilder writeInputs()
    throws IOException {

  if (fs.exists(outputPath)) {
    fs.delete(outputPath, true);
  }
  if (fs.exists(inputPath)) {
    fs.delete(inputPath, true);
  }
  fs.mkdirs(inputPath);

  DataOutputStream stream = fs.create(new Path(inputPath, "part-0"));

  IOUtils.writeLines(inputs, String.format("%n"), stream);

  stream.close();

  return this;
}
 

private File createTempConfig(Map<String, String> values) throws IOException {
    File configTestFile = new File(UUID.randomUUID().toString());
    String path = configTestFile.getAbsolutePath();
    
    List<String> lines = new ArrayList<>();
    for (Entry<String, String> entry : values.entrySet()) {
        lines.add(String.valueOf(entry.getKey()) + " = " + String.valueOf(entry.getValue()));
    }
    
    OutputStream outputStream = new FileOutputStream(configTestFile);
    IOUtils.writeLines(lines, null, outputStream, Charsets.UTF_8);
    outputStream.close();
    
    System.setProperty(Key.APPLICATION_CONFIG.toString(), path);
    
    return configTestFile;
}
 

private static File createLoginConfig() throws IOException {
        File tempFile = File.createTempFile("krb", "loginConf");
        tempFile.deleteOnExit();
        ArrayList<String> lines = new ArrayList<>();
        lines.add("com.sun.security.jgss.initiate {\n" +
                "  " + KrbHttpLoginModule.class.getCanonicalName() + " required\n" +
                "  doNotPrompt=true\n" +
                "  useFirstPass=true\n" +
                "  debug=true ;\n" +
                "};");
        FileWriter writer = null;
        try {
            writer = new FileWriter(tempFile);
            IOUtils.writeLines(lines, System.lineSeparator(), writer);
        } finally {
            if (writer != null) {
//                IOUtils.closeQuietly(writer);
                safeClose(writer);
            }
        }
        return tempFile;
    }
 

@Test
public void testWithMultipleThreads() throws Exception {
  // set up multiple test files
  File f = new File("target", UUID.randomUUID().toString());
  Assert.assertTrue(f.mkdirs());

  final int numFiles = 10;
  for (int i = 0; i < numFiles; i++) {
    FileOutputStream outputStream = new FileOutputStream(new File(f.getAbsolutePath(), "-file-" + i + ".log"));
    // each file has 5 lines
    IOUtils.writeLines(ImmutableList.of("1", "2", "3", "4", "5"), "\n", outputStream);
    outputStream.close();
  }

  readFilesMultipleThreads(f.getAbsolutePath(), 1);
  readFilesMultipleThreads(f.getAbsolutePath(), 2);
  readFilesMultipleThreads(f.getAbsolutePath(), 5);
  readFilesMultipleThreads(f.getAbsolutePath(), 10);

}
 

private void writeServices(
    final Filer filer, final String resourceFile, final Collection<String> services
) throws Exception {
    final FileObject file = createResource(filer, resourceFile);
    final OutputStream outputStream = file.openOutputStream();
    try {
        IOUtils.writeLines(services, "\n", outputStream, StandardCharsets.UTF_8);
    } finally {
        IOUtils.closeQuietly(outputStream);
    }
}
 

private Map<String, String> store(Map<String, String> config, File file) throws IOException {
    FileOutputStream os = new FileOutputStream(file);

    List<String> lines = new ArrayList<String>();
    for (Entry<String, String> line : config.entrySet()) {
        String value = isBlank(line.getValue()) ? "" : "=" + line.getValue();
        lines.add(line.getKey() + value);
    }

    IOUtils.writeLines(lines, System.getProperty("line.separator"), os);
    IOUtils.closeQuietly(os);
    return config;
}
 

@Override
public void tailContent(Path folder, String filename, OutputStream stream, int lines) throws IOException {
    try (ReversedLinesFileReader reader = new ReversedLinesFileReader(getFile(folder, filename))) {
        int i = 0;
        String line;
        List<String> content = new ArrayList<>();
        while ((line = reader.readLine()) != null && i++ < lines) {
            content.add(line);
        }
        Collections.reverse(content);
        IOUtils.writeLines(content, System.lineSeparator(), stream);
    }
}
 

private Path createFile(List<String> lines) throws IOException {
  Path file = new File(testDir, UUID.randomUUID().toString()).toPath();
  try (Writer writer = new FileWriter(file.toFile())) {
    IOUtils.writeLines(lines, "", writer);
  }
  return file;
}
 

public static void main(String[] args) throws IOException, Exception {
	List<List<Integer>> statsHistory = new ArrayList<>();
	List<Integer> lastResults;
	
	System.out.println("[AVG, P90%, P99%]");
	while (true) {
		lastResults = executeParallelLoadTest();
		List<Integer> stats = getStats(lastResults);
		System.out.println(stats + " ("+lastResults.size()+" requests)");
		
		if (statsHistory.size() >= 2) {
			List<Integer> lastStats1 = statsHistory.get(statsHistory.size()-1);
			List<Integer> lastStats2 = statsHistory.get(statsHistory.size()-2);
			if (similar(stats, lastStats1) && 
				similar(lastStats2, lastStats1) && 
				similar(stats, lastStats2)) {
				System.out.println("Found 3 similar consecutive stats. STOP");
				break;
			}
		}
		statsHistory.add(stats);
	}
	
	try (BufferedWriter writer = new BufferedWriter(new FileWriter("data.csv"))) {
		IOUtils.writeLines(lastResults, "\n", writer);
		System.out.println("File written");
	}
	System.out.println("Shuting down executor");
	executor.shutdown();
	executor.awaitTermination(1, TimeUnit.SECONDS);
	System.out.println("Program terminated");		
}
 

@SuppressWarnings("unchecked")
static boolean undoFile(File sourceFile, File destination, boolean dryRun, boolean curatEntityAnnot) {
	try {
		List<String> inLines;
		try (FileReader reader = new FileReader(sourceFile)) {
			inLines = IOUtils.readLines(reader);
		}
		if (inLines.size() >= 1 && inLines.get(0).contains("SOLUTION")) {
			if (!dryRun) {
				sourceFile.delete();
			} 
			return true;
		}
		List<String> outLines = stripSolutionLines(inLines);
		
		if (curatEntityAnnot) {
			outLines = eliminaAdnotariDinEntitati(sourceFile, outLines);
		}
		
		if (!dryRun) {
			if (destination.isFile()) {
				destination.delete();
			}
			destination.getParentFile().mkdirs();		
			destination.createNewFile();
			FileWriter writer = new FileWriter(destination);
			IOUtils.writeLines(outLines, "\n", writer);
			writer.close();
		}
		return !outLines.equals(inLines);
	} catch (Exception e) {
		e.printStackTrace();
		return false;
	}
}
 

@Test
public void testLineReaderWithInputStreamInputSplit() throws Exception {
    String tempDir = System.getProperty("java.io.tmpdir");
    File tmpdir = new File(tempDir, "tmpdir");
    tmpdir.mkdir();

    File tmp1 = new File(tmpdir, "tmp1.txt.gz");

    OutputStream os = new GZIPOutputStream(new FileOutputStream(tmp1, false));
    IOUtils.writeLines(Arrays.asList("1", "2", "3", "4", "5", "6", "7", "8", "9"), null, os);
    os.flush();
    os.close();

    InputSplit split = new InputStreamInputSplit(new GZIPInputStream(new FileInputStream(tmp1)));

    RecordReader reader = new LineRecordReader();
    reader.initialize(split);

    int count = 0;
    while (reader.hasNext()) {
        assertEquals(1, reader.next().size());
        count++;
    }

    assertEquals(9, count);

    try {
        FileUtils.deleteDirectory(tmpdir);
    } catch (Exception e) {
        e.printStackTrace();
    }
}
 

private static void chopOff(File f, String annoPtn) throws IOException {
    
    System.out.println("Processing " + f);
    
    FileInputStream is = new FileInputStream(f);
    List<String> lines = IOUtils.readLines(is, "UTF-8");
    is.close();
    List<String> outLines = new ArrayList<>(lines.size());
    
    boolean del = false;
    for (String l : lines) {
        if (l.startsWith("    @") && l.contains(annoPtn))
            del = true;
        
        if (del)
            System.out.println("x " + l);
        else
            outLines.add(l);
        
        if (del && l.startsWith("    }"))
            del = false;
    }
    
    if (!dryRun && outLines.size() < lines.size()) {
        FileOutputStream os = new FileOutputStream(f);
        IOUtils.writeLines(outLines, "\n", os, "UTF-8");
        os.close();
        System.out.println("UPDATED " + f);
    } else {
        System.out.println("skipped");
    }
    
    System.out.println("============================================================================");
}
 

private void processSynapseConfig(MxisdConfig cfg) {
    String synapseRegFile = cfg.getAppsvc().getRegistration().getSynapse().getFile();

    if (StringUtils.isBlank(synapseRegFile)) {
        log.info("No synapse registration file path given - skipping generation...");
        return;
    }

    SynapseRegistrationYaml syncCfg = SynapseRegistrationYaml.parse(cfg.getAppsvc(), cfg.getMatrix().getDomain());

    Representer rep = new Representer();
    rep.getPropertyUtils().setBeanAccess(BeanAccess.FIELD);
    Yaml yaml = new Yaml(rep);

    // SnakeYAML set the type of object on the first line, which can fail to be parsed on synapse
    // We therefore need to split the resulting string, remove the first line, and then write it
    List<String> lines = new ArrayList<>(Arrays.asList(yaml.dump(syncCfg).split("\\R+")));
    if (StringUtils.equals(lines.get(0), "!!" + SynapseRegistrationYaml.class.getCanonicalName())) {
        lines.remove(0);
    }

    try (FileOutputStream os = new FileOutputStream(synapseRegFile)) {
        IOUtils.writeLines(lines, System.lineSeparator(), os, StandardCharsets.UTF_8);
    } catch (IOException e) {
        throw new RuntimeException("Unable to write synapse appservice registration file", e);
    }
}
 

private static void chopOff(File f, String annoPtn) throws IOException {
    
    System.out.println("Processing " + f);
    
    FileInputStream is = new FileInputStream(f);
    List<String> lines = IOUtils.readLines(is, "UTF-8");
    is.close();
    List<String> outLines = new ArrayList<>(lines.size());
    
    boolean del = false;
    for (String l : lines) {
        if (l.startsWith("    @") && l.contains(annoPtn))
            del = true;
        
        if (del)
            System.out.println("x " + l);
        else
            outLines.add(l);
        
        if (del && l.startsWith("    }"))
            del = false;
    }
    
    if (!dryRun && outLines.size() < lines.size()) {
        FileOutputStream os = new FileOutputStream(f);
        IOUtils.writeLines(outLines, "\n", os, "UTF-8");
        os.close();
        System.out.println("UPDATED " + f);
    } else {
        System.out.println("skipped");
    }
    
    System.out.println("============================================================================");
}
 

@Test
public void testRecoverableDataParserError() throws Exception {
  File f = new File("target", UUID.randomUUID().toString());
  f.mkdir();

  SpoolDirConfigBean conf = new SpoolDirConfigBean();
  conf.dataFormat = DataFormat.DELIMITED;
  conf.spoolDir = f.getAbsolutePath();
  conf.batchSize = 10;
  conf.overrunLimit = 100;
  conf.poolingTimeoutSecs = 10;
  conf.filePattern = "file-[0-9].log";
  conf.pathMatcherMode = PathMatcherMode.GLOB;
  conf.maxSpoolFiles = 10;
  conf.initialFileToProcess = "file-0.log";
  conf.dataFormatConfig.compression = Compression.NONE;
  conf.dataFormatConfig.filePatternInArchive = "*";
  conf.dataFormatConfig.csvHeader = CsvHeader.WITH_HEADER;
  conf.errorArchiveDir = null;
  conf.postProcessing = PostProcessingOptions.NONE;
  conf.retentionTimeMins = 10;
  conf.allowLateDirectory = false;
  conf.dataFormatConfig.textMaxLineLen = 10;
  conf.dataFormatConfig.onParseError = OnParseError.ERROR;
  conf.dataFormatConfig.maxStackTraceLines = 0;

  FileOutputStream outputStream = new FileOutputStream(new File(conf.spoolDir, "file-0.log"));
  IOUtils.writeLines(ImmutableList.of("A,B", "a,b,c", "a,b"), "\n", outputStream);
  outputStream.close();

  SpoolDirSource source = new SpoolDirSource(conf);
  PushSourceRunner runner = new PushSourceRunner.Builder(SpoolDirDSource.class, source)
    .setOnRecordError(OnRecordError.TO_ERROR)
    .addOutputLane("lane")
    .build();
  final List<Record> records = Collections.synchronizedList(new ArrayList<>(10));

  runner.runInit();
  try {
    runner.runProduce(new HashMap<>(), 10, output -> {
      synchronized (records) {
        records.addAll(output.getRecords().get("lane"));
      }
      runner.setStop();
    });

    runner.waitOnProduce();

    // Verify proper record
    Assert.assertNotNull(records);
    Assert.assertEquals(1, records.size());

    // And error record
    List<Record> errorRecords = runner.getErrorRecords();
    Assert.assertEquals(1, errorRecords.size());

  } finally {
    source.destroy();
    runner.runDestroy();
  }
}
 

@Test
public void testWithMultipleThreadsInitialOffsets() throws Exception {
  // set up multiple test files
  File f = new File("target", UUID.randomUUID().toString());
  Assert.assertTrue(f.mkdirs());

  final int numFiles = 10;
  for (int i = 0; i < numFiles; i++) {
    FileOutputStream outputStream = new FileOutputStream(new File(f.getAbsolutePath(), "file-" + i + ".log"));
    // each file has 5 lines
    IOUtils.writeLines(ImmutableList.of("1", "2", "3", "4", "5"), "\n", outputStream);
    outputStream.close();
  }

  // let the first 2 files, file-0.log and file-3.log, were processed and
  // file-2.log was processed 1 line/record
  // file-1.log will be skipped since is less then file-3.log
  Map<String, String> lastSourceOffsetMap = ImmutableMap.of(
      SpoolDirSource.OFFSET_VERSION, OFFSET_VERSION_ONE,
      "file-0.log", "{\"POS\":\"-1\"}",
      "file-2.log", "{\"POS\":\"2\"}",
      "file-3.log", "{\"POS\":\"-1\"}"
  );

  SpoolDirConfigBean conf = new SpoolDirConfigBean();
  conf.dataFormat = DataFormat.TEXT;
  conf.spoolDir = f.getAbsolutePath();
  conf.batchSize = 10;
  conf.overrunLimit = 100;
  conf.poolingTimeoutSecs = 10;
  conf.filePattern = "file-[0-9].log";
  conf.pathMatcherMode = PathMatcherMode.GLOB;
  conf.maxSpoolFiles = 10;
  conf.initialFileToProcess = null;
  conf.dataFormatConfig.compression = Compression.NONE;
  conf.dataFormatConfig.filePatternInArchive = "*";
  conf.errorArchiveDir = null;
  conf.postProcessing = PostProcessingOptions.NONE;
  conf.retentionTimeMins = 10;
  conf.dataFormatConfig.textMaxLineLen = 10;
  conf.dataFormatConfig.onParseError = OnParseError.ERROR;
  conf.dataFormatConfig.maxStackTraceLines = 0;
  conf.allowLateDirectory = false;
  conf.numberOfThreads = 10;

  SpoolDirSource source = new SpoolDirSource(conf);
  PushSourceRunner runner = new PushSourceRunner.Builder(SpoolDirDSource.class, source).addOutputLane("lane").build();

  AtomicInteger batchCount = new AtomicInteger(0);
  final List<Record> records = Collections.synchronizedList(new ArrayList<>(10));
  runner.runInit();

  final int maxBatchSize = 10;

  try {
    runner.runProduce(lastSourceOffsetMap, maxBatchSize, output -> {
      batchCount.incrementAndGet();

      synchronized (records) {
        records.addAll(output.getRecords().get("lane"));
      }

       if (records.size() == 34 || batchCount.get() > 10) {
        runner.setStop();
      }
    });

    runner.waitOnProduce();
    Assert.assertTrue(batchCount.get() > 1);
    TestOffsetUtil.compare("file-9.log::-1", runner.getOffsets());
    Assert.assertTrue(records.size() == 34 || batchCount.get() > 10);
  } finally {
    source.destroy();
    runner.runDestroy();
  }

}
 

@Test
public void testBatchSizeLargerThanMax() throws Exception {
  final int maxBatchSize = 5;
  final String MINUS_ONE_JSON = "-1\"}";
  final String EXPECTED_ERROR = "SPOOLDIR_37 - " +
      "Batch size greater than maximal batch size allowed in sdc.properties, maxBatchSize: 5";

  File spoolDir = new File("target", UUID.randomUUID().toString());
  spoolDir.mkdir();

  SpoolDirConfigBean conf = new SpoolDirConfigBean();
  conf.dataFormat = DataFormat.TEXT;
  conf.spoolDir = spoolDir.getAbsolutePath();
  conf.batchSize = maxBatchSize + 1; // use a batch size greater than maxBatchSize
  conf.overrunLimit = 100;
  conf.poolingTimeoutSecs = 10;
  conf.filePattern = "file-[0-9].log";
  conf.pathMatcherMode = PathMatcherMode.GLOB;
  conf.maxSpoolFiles = 10;
  conf.initialFileToProcess = "file-0.log";
  conf.dataFormatConfig.compression = Compression.NONE;
  conf.dataFormatConfig.filePatternInArchive = "*";
  conf.dataFormatConfig.csvHeader = CsvHeader.WITH_HEADER;
  conf.postProcessing = PostProcessingOptions.NONE;
  conf.retentionTimeMins = 10;
  conf.allowLateDirectory = false;
  conf.dataFormatConfig.jsonContent = JsonMode.MULTIPLE_OBJECTS;
  conf.dataFormatConfig.onParseError = OnParseError.ERROR;

  List<String> messages = new ArrayList<>();
  for (int i = 0; i < 10; i++) messages. add("MESSAGE " + i);
  FileOutputStream outputStream = new FileOutputStream(new File(conf.spoolDir, "file-0.log"));
  IOUtils.writeLines(messages, "\n", outputStream);
  outputStream.close();

  SpoolDirSource source = new SpoolDirSource(conf);
  PushSourceRunner runner = new PushSourceRunner.Builder(SpoolDirDSource.class, source)
      .setOnRecordError(OnRecordError.TO_ERROR)
      .addOutputLane("lane")
      .build();
  AtomicInteger batchCount = new AtomicInteger();

  runner.runInit();
  try {
    runner.runProduce(new HashMap<>(), maxBatchSize, output -> {
      List<Record> records = output.getRecords().get("lane");
      int produceNum = batchCount.getAndIncrement();

      // expect 3rd batch to be offset -1, otherwise check max 5 batches to make sure we stop the runner
      if (!output.getNewOffset().endsWith(MINUS_ONE_JSON) && produceNum < 5) {
        Assert.assertNotNull(records);
        Assert.assertEquals(maxBatchSize, records.size());
        Assert.assertEquals(1, runner.getErrors().size());
        Assert.assertEquals(EXPECTED_ERROR, runner.getErrors().get(0));
      } else {
        runner.setStop();
      }
    });

    runner.waitOnProduce();
    Assert.assertEquals(3, batchCount.get());

  } finally {
    source.destroy();
    runner.runDestroy();
  }
}
 

private void truncateLastLine(File file) throws IOException {
    List<String> lines = IOUtils.readLines(new FileInputStream(file));
    IOUtils.writeLines(lines.subList(0, lines.size() - 1), "\n", new FileOutputStream(file));
}