Java Code Examples for com.google.common.io.Files#toByteArray()

@Test(expected=ProcessingException.class)
public void testCreateJobHistoryFileParserNullConf() throws IOException {

  final String JOB_HISTORY_FILE_NAME =
      "src/test/resources/job_1329348432655_0001-1329348443227-user-Sleep+job-1329348468601-10-1-SUCCEEDED-default.jhist";

  File jobHistoryfile = new File(JOB_HISTORY_FILE_NAME);
  byte[] contents = Files.toByteArray(jobHistoryfile);
  JobHistoryFileParser historyFileParser =
      JobHistoryFileParserFactory.createJobHistoryFileParser(contents, null);
  assertNotNull(historyFileParser);

  // confirm that we get back an object that can parse hadoop 2.0 files
  assertTrue(historyFileParser instanceof JobHistoryFileParserHadoop2);

  JobKey jobKey = new JobKey("cluster1", "user", "Sleep", 1, "job_1329348432655_0001");
  // pass in null as jobConf and confirm the exception thrown
  historyFileParser.parse(contents, jobKey);
}
 

@Test
public void testGrokEmail() throws Exception {
  morphline = createMorphline("test-morphlines/grokEmail");
  Record record = new Record();
  byte[] bytes = Files.toByteArray(new File(RESOURCES_DIR + "/test-documents/email.txt"));
  record.put(Fields.ATTACHMENT_BODY, bytes);
  assertTrue(morphline.process(record));
  Record expected = new Record();
  String msg = new String(bytes, "UTF-8"); //.replaceAll("(\r)?\n", "\n");
  expected.put(Fields.MESSAGE, msg);
  expected.put("message_id", "12345.6789.JavaMail.foo@bar");
  expected.put("date", "Wed, 6 Feb 2012 06:06:05 -0800");
  expected.put("from", "[email protected]");
  expected.put("to", "[email protected]");
  expected.put("subject", "WEDNESDAY WEATHER HEADLINES");
  expected.put("from_names", "Foo Bar <[email protected]>@xxx");
  expected.put("to_names", "'Weather News Distribution' <[email protected]>");    
  expected.put("text", 
      "Average 1 to 3- degrees above normal: Mid-Atlantic, Southern Plains.." +
  		"\nAverage 4 to 6-degrees above normal: Ohio Valley, Rockies, Central Plains");
  assertEquals(expected, collector.getFirstRecord());
  assertNotSame(record, collector.getFirstRecord());      
}
 

@Test
public void testGrokEmail() throws Exception {
  morphline = createMorphline("test-morphlines/grokEmail");
  Record record = new Record();
  byte[] bytes = Files.toByteArray(new File(RESOURCES_DIR + "/test-documents/email.txt"));
  record.put(Fields.ATTACHMENT_BODY, bytes);
  assertTrue(morphline.process(record));
  Record expected = new Record();
  String msg = new String(bytes, "UTF-8"); //.replaceAll("(\r)?\n", "\n");
  expected.put(Fields.MESSAGE, msg);
  expected.put("message_id", "12345.6789.JavaMail.foo@bar");
  expected.put("date", "Wed, 6 Feb 2012 06:06:05 -0800");
  expected.put("from", "[email protected]");
  expected.put("to", "[email protected]");
  expected.put("subject", "WEDNESDAY WEATHER HEADLINES");
  expected.put("from_names", "Foo Bar <[email protected]>@xxx");
  expected.put("to_names", "'Weather News Distribution' <[email protected]>");    
  expected.put("text", 
      "Average 1 to 3- degrees above normal: Mid-Atlantic, Southern Plains.." +
      "\nAverage 4 to 6-degrees above normal: Ohio Valley, Rockies, Central Plains");
  assertEquals(expected, collector.getFirstRecord());
  assertNotSame(record, collector.getFirstRecord());      
}
 

@Override
public byte[] retrieve(String identifier, int uniqueIdentifier)
{
  String normalizedFilename = normalizeFileName(identifier);
  File directory = new File(basePath, normalizedFilename);
  if (directory.exists()) {
    File identityFile = new File(directory, "identity");
    if (identityFile.isFile()) {
      try {
        byte[] stored = Files.toByteArray(identityFile);
        if (Arrays.equals(stored, identifier.getBytes())) {
          File filename = new File(directory, String.valueOf(uniqueIdentifier));
          if (filename.exists() && filename.isFile()) {
            return Files.toByteArray(filename);
          } else {
            throw new RuntimeException("File " + filename.getPath() + " either is non existent or not a file!");
          }
        } else {
          throw new RuntimeException("Collision in the identifier name," +
              " please ensure that the slugs for the identifiers [" + identifier + "], and [" +  new String(stored) +
              "] are different.");
        }
      } catch (IOException ex) {
        throw new RuntimeException(ex);
      }
    } else {
      throw new RuntimeException(identityFile + " is not a file!");
    }
  } else {
    throw new RuntimeException("directory " + directory.getPath() + " does not exist!");
  }
}
 

@Override
public byte[] load(String siddhiAppName, String revision) {
    File file = new File(folder + File.separator + siddhiAppName + File.separator + revision);
    try {
        byte[] bytes = Files.toByteArray(file);
        log.info("State loaded for " + siddhiAppName + " revision " + revision + " from the file system.");
        return bytes;
    } catch (IOException e) {
        log.error("Cannot load the revision " + revision + " of SiddhiApp: " + siddhiAppName +
                " from file system.", e);
    }
    return null;
}
 

@Override
public byte[] getBinary(){
	try {
		return Files.toByteArray(file);
	} catch (IOException e) {
		LoggerFactory.getLogger(getClass()).error("Could not access file", e);
	}
	return new byte[0];
}
 

@Test
  @Ignore
  // Before running this disable debug logging 
  // via log4j.logger.org.kitesdk.morphline=INFO in log4j.properties
  public void benchmark() throws Exception {
    String morphlineConfigFile = "test-morphlines/readCSVWithoutQuoting";
    //String morphlineConfigFile = "test-morphlines/readCSVWithoutQuotingUsingSplit";
    //String morphlineConfigFile = "test-morphlines/grokEmail";
    //String morphlineConfigFile = "test-morphlines/grokSyslogNgCisco";
    long durationSecs = 20;
    //File file = new File(RESOURCES_DIR + "/test-documents/email.txt");
    //File file = new File(RESOURCES_DIR + "/test-documents/emails.txt");
    File file = new File(RESOURCES_DIR + "/test-documents/cars3.csv");
    String msg = "<179>Jun 10 04:42:51 www.foo.com Jun 10 2013 04:42:51 : %myproduct-3-mysubfacility-251010: " +
        "Health probe failed for server 1.2.3.4 on port 8083, connection refused by server";
    System.out.println("Now benchmarking " + morphlineConfigFile + " ...");
    morphline = createMorphline(morphlineConfigFile);    
    byte[] bytes = Files.toByteArray(file);
    long start = System.currentTimeMillis();
    long duration = durationSecs * 1000;
    int iters = 0; 
    while (System.currentTimeMillis() < start + duration) {
      Record record = new Record();
      record.put(Fields.ATTACHMENT_BODY, bytes);      
//    record.put(Fields.MESSAGE, msg);      
      collector.reset();
      startSession();
      assertEquals(1, collector.getNumStartEvents());
      assertTrue(morphline.process(record));    
      iters++;
    }
    float secs = (System.currentTimeMillis() - start) / 1000.0f;
    System.out.println("Results: iters=" + iters + ", took[secs]=" + secs + ", iters/secs=" + (iters/secs));
  }
 

private ZeppelinhubRestApiHandler getMockedZeppelinHandler() throws HttpException, IOException {
  ZeppelinhubRestApiHandler mockedZeppelinhubHandler = mock(ZeppelinhubRestApiHandler.class);

  byte[] listOfNotesResponse = Files.toByteArray(pathOfNotebooks);
  when(mockedZeppelinhubHandler.get("AAA-BBB-CCC-00", ""))
    .thenReturn(new String(listOfNotesResponse));

  byte[] noteResponse =  Files.toByteArray(pathOfNotebook);
  when(mockedZeppelinhubHandler.get("AAA-BBB-CCC-00", "AAAAA"))
    .thenReturn(new String(noteResponse));

  return mockedZeppelinhubHandler;
}
 

private synchronized void refresh() {
  try {
    byte[] newBytes = Files.toByteArray(file);
    String newETag = Hashing.murmur3_128().hashBytes(newBytes).toString();

    bytes = newBytes;
    etag = '"' + newETag + '"';
    lastModifiedTime = file.lastModified();
  } catch (IOException e) {
    // Ignored, don't update anything
  }
}
 

private MakeAttachmentEvent makeAttachmentEvent() {
    if (assertions != null && assertions.size() > 0) {
        return makeAttachmentEventWithAssertions();
    }
    try {
        File file = new File(fileName);
        MakeAttachmentEvent event = new MakeAttachmentEvent(Files.toByteArray(file), message, "image/png");
        file.delete();
        return event;
    } catch (IOException e) {
        e.printStackTrace();
    }
    return null;
}
 

@Test
public void testSealedBoxFile() throws IOException, URISyntaxException {
    Sodium sodium= NaCl.sodium();
    int ret=0;

    long publickeylen=Sodium.crypto_box_publickeybytes();
    long privatekeylen=Sodium.crypto_box_secretkeybytes();
    final byte[] public_key=new byte[(int)publickeylen];
    final byte[] private_key=new byte[(int)privatekeylen];

    Sodium.crypto_box_keypair(public_key,private_key);

    File public_key_file=File.createTempFile("SealedBoxTest","box_public.key",TemporaryFile.temporaryFileDirectory());
    public_key_file.deleteOnExit();
    Files.write(public_key,public_key_file);

    File private_key_file=File.createTempFile("SealedBoxTest","box_private.key",TemporaryFile.temporaryFileDirectory());
    private_key_file.deleteOnExit();
    Files.write(private_key,private_key_file);

    byte[] public_key_fromfile = Files.toByteArray(public_key_file);
    byte[] private_key_fromfile = Files.toByteArray(private_key_file);

    String message="testmessage";

    byte[] ciphertext=new byte[Sodium.crypto_box_sealbytes()+message.length()];
    Sodium.crypto_box_seal(ciphertext,message.getBytes(),message.length(),public_key_fromfile);

    byte[] plaintext=new byte[message.length()];
    ret=Sodium.crypto_box_seal_open(plaintext,ciphertext,ciphertext.length,public_key_fromfile,private_key_fromfile);
    Assert.assertEquals(0,ret);
}
 

@Test
/**
 * Test that schema caching in readAvroContainer works even if the Avro writer schema of each input
 * file is different (yet compatible). Test writer schema A before B and B before A.
 */
public void testReadAvroContainerWithMultipleSchemas() throws IOException {
  for (int reverse = 0; reverse < 2; reverse++) {
    morphline = createMorphline("test-morphlines/readAvroContainer");
    for (int run = 0; run < 10; run++) {
      collector.reset();
      int version = run % 2;
      version = (version + reverse) % 2; // reverse direction with reverse == 1: 0 -> 1  as well as 1 -> 0
      byte[] fileContents = Files.toByteArray(
          new File(RESOURCES_DIR + "/test-documents/avroContainerWithWriterschema" + version + ".avro"));
      Record inputRecord = new Record();
      inputRecord.put(Fields.ATTACHMENT_BODY, fileContents);
      assertTrue(morphline.process(inputRecord));

      int numRecords = 5;
      assertEquals(numRecords, collector.getRecords().size());
      
      String[] expectedUids = new String[] {"sdfsdf", "fhgfgh", "werwer", "345trgt", "dfgdg"};
      for (int i = 0; i < numRecords; i++) {
        Record record = collector.getRecords().get(i);
        GenericData.Record avroRecord = (GenericData.Record)record.getFirstValue(Fields.ATTACHMENT_BODY);
        assertEquals(expectedUids[i], avroRecord.get("sc_uid").toString());
      }
    }
  }
}
 

private static byte[] read(final File file) {
    try {
        return Files.toByteArray(file);
    } catch (IOException e) {
        throw new Error(e);
    }
}
 

/** command-line main for testing */
public static void main(String[] argv) {

  if (argv.length < 3) {
    System.err.println("Usage: RobotRulesParser <robots-file> <url-file> <agent-names>\n");
    System.err.println("\tThe <robots-file> will be parsed as a robots.txt file,");
    System.err.println("\tusing the given <agent-name> to select rules.  URLs ");
    System.err.println("\twill be read (one per line) from <url-file>, and tested");
    System.err.println("\tagainst the rules. Multiple agent names can be specified using spaces.");
    System.exit(-1);
  }

  try {
    StringBuilder agentNames = new StringBuilder();
    for(int counter = 2; counter < argv.length; counter++) 
      agentNames.append(argv[counter]).append(",");

    agentNames.deleteCharAt(agentNames.length()-1);

    byte[] robotsBytes = Files.toByteArray(new File(argv[0]));
    BaseRobotRules rules = robotParser.parseContent(argv[0], robotsBytes, "text/plain", agentNames.toString());

    LineNumberReader testsIn = new LineNumberReader(new FileReader(argv[1]));
    String testPath = testsIn.readLine().trim();
    while (testPath != null) {
      System.out.println( (rules.isAllowed(testPath) ? "allowed" : "not allowed") +
          ":\t" + testPath);
      testPath = testsIn.readLine();
    }
    testsIn.close();
  } catch (Exception e) {
    e.printStackTrace();
  }
}
 

public static void main(String[] args) throws Exception {

        if(args.length == 0){
            System.out.println("args[0] must be data file path");
            return;
        }
        LZ4Factory factory = LZ4Factory.fastestInstance();

        byte[] data = Files.toByteArray(new File(args[0]));
        final int decompressedLength = data.length;

        // compress data
        LZ4Compressor compressor = factory.fastCompressor();
        long start = System.currentTimeMillis();
        int maxCompressedLength = compressor.maxCompressedLength(decompressedLength);
        byte[] compressed = new byte[maxCompressedLength];
        int compressedLength = compressor.compress(data, 0, decompressedLength, compressed, 0, maxCompressedLength);
        System.out.println("compress take:" + (System.currentTimeMillis() - start));
        System.out.println(compressedLength);

        // decompress data
        // - method 1: when the decompressed length is known
        LZ4FastDecompressor decompressor = factory.fastDecompressor();
        start = System.currentTimeMillis();
        byte[] restored = new byte[decompressedLength];
        int compressedLength2 = decompressor.decompress(compressed, 0, restored, 0, decompressedLength);
        System.out.println("decompress take:" + (System.currentTimeMillis() - start));
        System.out.println(decompressedLength);
        // compressedLength == compressedLength2

        // - method 2: when the compressed length is known (a little slower)
        // the destination buffer needs to be over-sized
        LZ4SafeDecompressor decompressor2 = factory.safeDecompressor();
        int decompressedLength2 = decompressor2.decompress(compressed, 0, compressedLength, restored, 0);
    }
 

@Test
public void testSealedBoxFileXChaCha20() throws IOException, URISyntaxException {
    Sodium sodium= NaCl.sodium();
    int ret=0;

    long publickeylen=Sodium.crypto_box_curve25519xchacha20poly1305_publickeybytes();
    long privatekeylen=Sodium.crypto_box_curve25519xchacha20poly1305_secretkeybytes();
    final byte[] public_key=new byte[(int)publickeylen];
    final byte[] private_key=new byte[(int)privatekeylen];

    Sodium.crypto_box_curve25519xchacha20poly1305_keypair(public_key,private_key);

    File public_key_file=File.createTempFile("SealedBoxTest","box_public_xchacha.key",TemporaryFile.temporaryFileDirectory());
    public_key_file.deleteOnExit();
    Files.write(public_key,public_key_file);

    File private_key_file=File.createTempFile("SealedBoxTest","box_private_xchacha.key",TemporaryFile.temporaryFileDirectory());
    private_key_file.deleteOnExit();
    Files.write(private_key,private_key_file);

    byte[] public_key_fromfile = Files.toByteArray(public_key_file);
    byte[] private_key_fromfile = Files.toByteArray(private_key_file);

    String message="testmessage";

    byte[] ciphertext=new byte[Sodium.crypto_box_curve25519xchacha20poly1305_sealbytes()+message.length()];
    Sodium.crypto_box_curve25519xchacha20poly1305_seal(ciphertext,message.getBytes(),message.length(),public_key_fromfile);

    byte[] plaintext=new byte[message.length()];
    ret=Sodium.crypto_box_curve25519xchacha20poly1305_seal_open(plaintext,ciphertext,ciphertext.length,public_key_fromfile,private_key_fromfile);
    Assert.assertEquals(0,ret);
}
 

private static SecretKey loadKey(File secretFile) throws IOException {
    byte[] bytes = Files.toByteArray(secretFile);
    return new SecretKeySpec(bytes, "AES");
}
 

@Test
public void testSignatureFromFile() throws IOException, URISyntaxException {
    Sodium sodium= NaCl.sodium();
    int ret=0;

    long publickeylen=Sodium.crypto_sign_publickeybytes();
    long privatekeylen=Sodium.crypto_sign_secretkeybytes();
    System.out.println("PublicKeyLen="+publickeylen);
    System.out.println("PrivateKeyLen="+privatekeylen);
    final byte[] public_key=new byte[(int)publickeylen];
    final byte[] private_key=new byte[(int)privatekeylen];
    System.out.println("Generating keypair");
    Sodium.randombytes(public_key,(int)publickeylen);
    Sodium.randombytes(private_key,(int)privatekeylen);
    ret=Sodium.crypto_sign_keypair(public_key,private_key);
    System.out.println(ret);
    System.out.println("Generated keypair");

    File public_key_file=File.createTempFile("SignatureTest","public.key",TemporaryFile.temporaryFileDirectory());
    public_key_file.deleteOnExit();
    Files.write(public_key,public_key_file);

    File private_key_file=File.createTempFile("SignatureTest","private.key",TemporaryFile.temporaryFileDirectory());
    private_key_file.deleteOnExit();
    Files.write(private_key,private_key_file);

    byte[] public_key_fromfile = Files.toByteArray(public_key_file);

    byte[] originalmessage="test".getBytes();
    System.out.println("Original message="+new String(originalmessage));
    long signaturelen=Sodium.crypto_sign_bytes();
    byte[] signed_message=new byte[(int)signaturelen+originalmessage.length];
    final int[] signed_message_len = new int[1];
    System.out.println("Signing message");
    ret=Sodium.crypto_sign(signed_message,signed_message_len,originalmessage,originalmessage.length,private_key);
    System.out.println(ret);
    System.out.println("byte length="+signed_message.length);
    System.out.println("signed message size="+signed_message_len[0]);
    Assert.assertEquals(signed_message.length,signed_message_len[0]);
    System.out.println("Signed message=");

    byte[] message=new byte[originalmessage.length];
    final int[] messageSize=new int[1];
    System.out.println("Verifying message");
    ret=Sodium.crypto_sign_open(message, messageSize, signed_message, signed_message_len[0], public_key_fromfile);
    System.out.println(ret);
    System.out.println("Recovered message="+new String(message));
    Assert.assertEquals(0,ret);
}
 

@Test
public void testVersion2_4() throws IOException {
  final String JOB_HISTORY_FILE_NAME = "src/test/resources/" +
      "job_1410289045532_259974-1411647985641-user35-SomeJobName-1411647999554-1-0-SUCCEEDED-root.someQueueName-1411647995323.jhist";
  File jobHistoryfile = new File(JOB_HISTORY_FILE_NAME);
  byte[] contents = Files.toByteArray(jobHistoryfile);
  final String JOB_CONF_FILE_NAME =
      "src/test/resources/job_1329348432655_0001_conf.xml";
  Configuration jobConf = new Configuration();
  jobConf.addResource(new Path(JOB_CONF_FILE_NAME));

  JobHistoryFileParser historyFileParser =
      JobHistoryFileParserFactory.createJobHistoryFileParser(contents, jobConf);
  assertNotNull(historyFileParser);

  // confirm that we get back an object that can parse hadoop 2.x files
  assertTrue(historyFileParser instanceof JobHistoryFileParserHadoop2);
  JobKey jobKey = new JobKey("cluster1", "user", "Sleep", 1, "job_1329348432655_0001");
  historyFileParser.parse(contents, jobKey);

  List<Put> jobPuts = historyFileParser.getJobPuts();

  // now confirm that 2.4 constructs have been parsed
  // check that queue name is found, since
  // it's part of the JobQueueChange object in the history file
  boolean foundQueue = false;
  for (Put p : jobPuts) {
    List<Cell> kv2 =
        p.get(Constants.INFO_FAM_BYTES,
          Bytes.toBytes(JobHistoryKeys.JOB_QUEUE.toString().toLowerCase()));
    if (kv2.size() == 0) {
      // we are interested in queue put only
      // hence continue
      continue;
    } else {
      for (Cell kv : kv2) {
        // ensure we have a hadoop2 version as the value
        assertEquals(Bytes.toString(kv.getValue()), "root.someQueueName");
        // ensure we don't see the same put twice
        assertFalse(foundQueue);
        // now set this to true
        foundQueue = true;
      }
      break;
    }

  }
  // ensure that we got the hadoop2 version put
  assertTrue(foundQueue);
}
 

@Test
public void file_to_byte_array_guava () throws IOException, URISyntaxException {
	
	File file = new File(fileLocation);
	
	byte[] fileInBytes = Files.toByteArray(file);
	
	assertEquals(18, fileInBytes.length);
}