jetbrains.buildServer.util.FileUtil Java Examples

private void maybeCreate(@NotNull final DiskShare diskShare, @NotNull final String pathInShare) {
  String existingPrefix = FileUtil.normalizeRelativePath(pathInShare).replace('/', '\\');
  final Stack<String> toCreate = new Stack<>();

  while (existingPrefix.length() > 0 && !diskShare.folderExists(existingPrefix)) {
    final int endIndex = existingPrefix.lastIndexOf('\\');
    if (endIndex > -1) {
      toCreate.push(existingPrefix.substring(endIndex + 1));
      existingPrefix = existingPrefix.substring(0, endIndex);
    } else {
      toCreate.push(existingPrefix);
      existingPrefix = "";
    }
  }

  while (!toCreate.empty()) {
    existingPrefix = (existingPrefix.length() > 0 ? existingPrefix + "\\" : "") + toCreate.pop();
    diskShare.mkdir(existingPrefix);
  }
}
 

private synchronized void loadIdxes(@NotNull final File idxStorage) {
  final File[] idxes = idxStorage.listFiles();
  if (idxes == null){

  }
  for (File idxFile : idxes) {
    if (!idxFile.getName().endsWith(".idx"))
      continue;
    String idxName = idxFile.getName().substring(0, idxFile.getName().length()-4);
    try {
      int val = StringUtil.parseInt(FileUtil.readText(idxFile), -1);
      if (val > 0){
        myCounters.putIfAbsent(idxName, new AtomicInteger(0));
        myCounters.get(idxName).set(val);
      }
    } catch (IOException e) {}
  }
}
 

private void storeIdxes() {
  // wait for generation operations to finish:
  try {
    if (!myLock.writeLock().tryLock(100, TimeUnit.MILLISECONDS)) {
      Loggers.AGENT.warn("Waited more than 100ms to store Kube indexes");
    }
    for (Map.Entry<String, AtomicBoolean> entry : myIdxTouchedMaps.entrySet()) {
      if (entry.getValue().compareAndSet(true, false)) {
        final AtomicInteger counter = myCounters.get(entry.getKey());
        try {
          final File idxFile = new File(myIdxStorage, entry.getKey() + ".idx");
          FileUtil.writeViaTmpFile(idxFile, new ByteArrayInputStream(String.valueOf(counter.get()).getBytes()), FileUtil.IOAction.DO_NOTHING);
        } catch (IOException ignored) {
        }
      }
    }
  } catch (InterruptedException e) {
    e.printStackTrace();
  } finally {
    myLock.writeLock().unlock();
  }
}
 

private <T> T withKubernetesClient(boolean retrying, Function<KubernetesClient, T> function){
    try {
        return function.apply(myKubernetesClient);
    } catch (KubernetesClientException kce){
        final String operation = getOperation(kce);
        LOG.warnAndDebugDetails(String.format("An error occurred at %s, ProfileId: %s, Code: %d, Status: %s", operation, myProfileId, kce.getCode(), kce.getStatus()),
                                KubernetesClientException.launderThrowable(kce));
        if (!retrying && kce.getCode()==401){
            final KubeApiConnectionCheckResult result = testConnection();
            LOG.info(String.format("Test connection for %s: %s", myProfileId, result));
            if (result.isNeedRefresh()){
                LOG.info("Will now invalidate and recreate client for " + myProfileId);
                invalidate();
                KubernetesClient oldClient = myKubernetesClient;
                myKubernetesClient = createClient(createConfig(myConnectionSettings, myAuthStrategy));
                FileUtil.close(oldClient);
            }
            return withKubernetesClient(true, function);
        }
        throw kce;
    }
}
 

@NotNull
@Override
public BuildProcess executeCommandLine(@NotNull BuildRunnerContext hostContext,
                                       @NotNull Collection<String> argz,
                                       @NotNull File workingDir,
                                       @NotNull Map<String, String> additionalEnvironment) throws RunBuildException {
  final List<String> processed = new ArrayList<String>();
  for (String arg : argz) {
    processed.add(
            arg
                    .replace(home.getPath(), "!HOME!")
                    .replace(work.getPath(), "!WORK!")
                    .replace(FileUtil.getRelativePath(home, work), "!REL_WORK!")
    );
  }

  System.out.println("cmd>> " + processed);
  return cmd.executeCommandLine(hostContext, processed, workingDir, additionalEnvironment);
}
 

private String getCustomScriptExecutable(AnsibleRunConfig config) throws RunBuildException {
    String content = null;
    File scriptFile = null;
    if (config.getSourceCode() != null) {
        content = config.getSourceCode().replace("\r\n", "\n").replace("\r", "\n");
    }
    if (StringUtil.isEmptyOrSpaces(content)) {
        throw new RunBuildException("Custom script source code cannot be empty");
    }
    try {
        scriptFile = File.createTempFile("ansible_custom_exe", null, getBuildTempDirectory());
        FileUtil.writeFileAndReportErrors(scriptFile, content);
    } catch (IOException e) {
        throw new RunBuildException("Failed to create a tmp file for custom ansible execution script");
    }
    boolean executable = scriptFile.setExecutable(true, true);
    if (!executable) {
        throw new RunBuildException("Failed to set executable permissions to " + scriptFile.getAbsolutePath());
    }
    return scriptFile.getAbsolutePath();
}
 

@TestFor(issues = "TW-54729")
public void check_name_generator_doesnt_use_disk() throws IOException, InterruptedException {
  final Set<String> generatedNames = new HashSet<>();
  Thread nameGenerator = new Thread(()->{
    for (int i=0; i<10000; i++){
      if (Thread.currentThread().isInterrupted())
        break;
      generatedNames.add(myImage.generateNewVmName());
    }
  });
  nameGenerator.start();
  new WaitFor(500){

    @Override
    protected boolean condition() {
      return generatedNames.size() == 10000;
    }
  };
  assertEquals(10000, generatedNames.size());
  nameGenerator.join();
  assertEquals("1", FileUtil.readText(new File(myIdxStorage, myImage.getImageDetails().getSourceId() + ".idx")));
  myImage.storeIdx();
  assertEquals("10001", FileUtil.readText(new File(myIdxStorage, myImage.getImageDetails().getSourceId() + ".idx")));
}
 

@Override
public void dispose() {
    LOG.debug("Disposing " + myCloudProfileId);
    myUpdater.unregisterClient(this);
    List<Runnable> runnables = myExecutorService.shutdownNow();
    if (runnables.size() > 0) {
        LOG.info(String.format("Forced shutdown of executor for '%s'. %d tasks might have been cancelled", myCloudProfileId, runnables.size()));
    }
    FileUtil.close(myApiConnector);
}
 

@Test
public void request_file_with_plus_sign() throws Exception{
  myFixture.getServerSettings().setPerProjectPermissionsEnabled(true);

  SUser user = myFixture.getUserModel().getGuestUser();
  user.addRole(RoleScope.projectScope(myProject.getProjectId()), getProjectDevRole());
  assertTrue(user.isPermissionGrantedForProject(myProject.getProjectId(), Permission.VIEW_BUILD_RUNTIME_DATA));

  final File artDirectory = createTempDir();
  final String fileName = "file++.pdb";
  File file = new File(artDirectory, fileName);
  assertTrue(file.createNewFile());
  FileUtil.writeFile(file, "text", "UTF-8");

  myBuildType.setArtifactPaths(artDirectory.getAbsolutePath());
  RunningBuildEx build = startBuild();
  build.publishArtifact(fileName, file);
  finishBuild(build, false);

  final String fileSignature = "8EF4E863187C45E78F4632152CC82FEB";
  final String guid = "8EF4E863187C45E78F4632152CC82FE";

  myBuildMetadataStorage.addEntry(build.getBuildId(), guid.toLowerCase(), fileName, fileName);
  myRequest.setRequestURI("mock", String.format("/app/symbols/%s/%s/%s", fileName, fileSignature, fileName));

  doGet();

  assertEquals(-1, myResponse.getStatus());
  assertEquals("text", myResponse.getReturnedContent());

  myRequest.setRequestURI("mock", String.format("/app/symbols/%s/%s/%s", "file%2b%2b.pdb", fileSignature, fileName));

  doGet();

  assertEquals(-1, myResponse.getStatus());
  assertEquals("text", myResponse.getReturnedContent());
}
 

@Override
public void execute(@NotNull final OutputStream out,
                    @NotNull final InputStream in) throws IOException {
  final String command = "C0755 " + myFile.length() + " " + myFile.getName() + "\n";
  out.write(command.getBytes());
  out.flush();
  ScpExecUtil.checkScpAck(in);

  // send the content
  FileInputStream fis = null;
  byte[] buf = new byte[1024];
  try {
    fis = new FileInputStream(myFile);
    while (true) {
      int len = fis.read(buf, 0, buf.length);
      if (len <= 0) break;
      out.write(buf, 0, len); //out.flush();
    }
  } finally {
    FileUtil.close(fis);
  }

  // send '\0'
  buf[0] = 0;
  out.write(buf, 0, 1);
  out.flush();
  ScpExecUtil.checkScpAck(in);
}
 

private void deployTestResourceAsArtifact(String testResourceName, String artifactName) throws IOException, RunBuildException {
  FileUtil.copy(getTestResource(testResourceName), new File(workingDir, artifactName));
  InetAddress addr = NetworkUtil.getSelfAddresses(null)[0];
  String hostname = addr.getHostName();

  final BuildProcess process = getProcess(hostname + ":" + testPort, artifactName);
  DeployTestUtils.runProcess(process, 5000);
}
 

public VmwareCloudImage(@NotNull final VMWareApiConnector apiConnector,
                        @NotNull final VmwareCloudImageDetails imageDetails,
                        @NotNull final CloudAsyncTaskExecutor asyncTaskExecutor,
                        @NotNull final File idxStorage,
                        @NotNull final CloudProfile profile) {
  super(imageDetails.getSourceId(), imageDetails.getSourceId());
  myImageDetails = imageDetails;
  myApiConnector = apiConnector;
  myAsyncTaskExecutor = asyncTaskExecutor;
  myProfile = profile;
  myActualSourceState = new AtomicReference<>();
  myIdxFile = new File(idxStorage, imageDetails.getSourceId() + ".idx");
  try {
    if (!myIdxFile.exists()) {
      myIdxCounter.set(1);
      myIdxTouched.set(true);
      storeIdx();
    } else {
      myIdxCounter.set(Integer.parseInt(FileUtil.readText(myIdxFile)));
    }
  } catch (Exception e) {
    LOG.warnAndDebugDetails(String.format("Unable to process idx file '%s'. Will reset the index for " + imageDetails.getSourceId(), myIdxFile.getAbsolutePath()), e);
    myIdxCounter.set(1);
  }

  asyncTaskExecutor.scheduleWithFixedDelay("Store idx", ()->{
    storeIdx();
  }, 5000, 5000, TimeUnit.MILLISECONDS);
}
 

void storeIdx() {
  if (myIdxTouched.compareAndSet(true, false)){
    synchronized (myIdxFile) {
      try {
        FileUtil.writeViaTmpFile(myIdxFile, new ByteArrayInputStream(String.valueOf(myIdxCounter.get()).getBytes()), FileUtil.IOAction.DO_NOTHING);
      } catch (IOException ignored) {}
    }
  }
}
 

public void should_generate_unique_name() throws IOException {
  VmwareCloudImage sameImage = new VmwareCloudImage(myApiConnector, myImageDetails, myTaskExecutor, myIdxStorage, myProfile){
    @Override
    protected Set<String> getInstanceIds() {
      return createSet("imageNickname-1", "imageNickname-3");
    }
  };
  assertEquals("1", FileUtil.readText(new File(myIdxStorage, myImage.getImageDetails().getSourceId() + ".idx")));
  assertEquals("imageNickname-2", sameImage.generateNewVmName());
  assertEquals("imageNickname-4", sameImage.generateNewVmName());

}
 

public void should_store_idx_on_dispose() throws IOException {
  new WaitFor(500){

    @Override
    protected boolean condition() {
      return myCloudClient.isInitialized();
    }
  };
  VmwareCloudImage img = myCloudClient.getImages().iterator().next();
  assertEquals("imageNickname-1", img.generateNewVmName());
  File file = new File(myIdxStorage, img.getImageDetails().getSourceId() + ".idx");
  assertEquals("1", FileUtil.readText(file));
  myCloudClient.dispose();
  assertEquals( "2", FileUtil.readText(file));
}
 

public void reset_idx_when_file_unreadable() throws IOException {
  final File idxFile = new File(myIdxStorage, myImage.getImageDetails().getSourceId() + ".idx");
  FileUtil.writeFileAndReportErrors(idxFile, "aaaaaaaaaasdasfdfsgfsgeaweewq");
  VmwareCloudImage sameImage = new VmwareCloudImage(myApiConnector, myImageDetails, myTaskExecutor, myIdxStorage, myProfile){
    @Override
    protected Set<String> getInstanceIds() {
      return createSet("imageNickname-1", "imageNickname-3");
    }
  };
  assertEquals("imageNickname-2", sameImage.generateNewVmName());
  assertEquals("imageNickname-4", sameImage.generateNewVmName());

}
 

@NotNull
public static String resolveRelativePath(@NotNull final File baseDir, @NotNull final File path) {
  String result = FileUtil.getRelativePath(baseDir, path);
  if (result == null) return "";

  result = result.replace('\\', '/');

  while (result.endsWith(".") || result.endsWith("/")) result = result.substring(0, result.length() - 1);
  return result;
}
 

public void test() throws IOException, RunBuildException {
  FileUtil.createDir(home);
  FileUtil.createDir(work);

  setupRunnerParameters();

  final BuildProcess process = run.createBuildProcess(build, context);
  process.start();
  process.waitFor();

  m.assertIsSatisfied();
}
 

private void reloadConfiguration() {
	Loggers.ACTIVITIES.info("Loading configuration file: " + this.myConfigFile.getAbsolutePath());

	myConfigDir.mkdirs();
	FileUtil.copyResourceIfNotExists(getClass(), "/config_templates/slack-config.xml", new File(this.myConfigDir, "slack-config.xml"));

	Document document = parseFile(this.myConfigFile);
	if (document != null)
	{
		Element rootElement = document.getRootElement();
		readConfigurationFromXmlElement(rootElement);
	}
}
 

@Test
public void testSpacesInPaths() throws Exception {
  final File output = FileUtil.createTempFile("test spaces in paths", ".out");
  final File input = FileUtil.createTempFile("test spaces in paths", ".in");
  final StringBuilder warnings = new StringBuilder();
  final int exitCode = myExe.dumpPdbGuidsToFile(Collections.singleton(input), output, new NullBuildProgressLogger() {
    @Override
    public void warning(String message) {
      warnings.append(message).append("\n");
    }
  });
  assertEquals(1, exitCode);
  assertTrue(warnings.toString(),warnings.indexOf("Nothing to dump.") > 0);
}
 

@Nullable
private String getDefaultServiceAccountAuthToken() {
    try {
        return FileUtil.readText(new File(DEFAULT_SERVICE_ACCOUNT_TOKEN_FILE));
    } catch (IOException e) {
        return null;
    }
}
 

/**
 * Save configuration on disk
 * @param newSettings {@link TelegramSettings}
 */
public synchronized void saveConfiguration(@NotNull TelegramSettings newSettings) {
  changeObserver.runActionWithDisabledObserver(() ->
      FileUtil.processXmlFile(configFile.toFile(), (root) -> {
        root.setAttribute(BOT_TOKEN_ATTR, scramble(newSettings.getBotToken()));
        root.setAttribute(PAUSE_ATTR, Boolean.toString(newSettings.isPaused()));
        root.setAttribute(USE_PROXY_ATTR, Boolean.toString(newSettings.isUseProxy()));
        root.setAttribute(PROXY_SERVER_ATTR, newSettings.getProxyServer());
        root.setAttribute(PROXY_PORT_ATTR, storeInteger(newSettings.getProxyPort()));
        root.setAttribute(PROXY_USERNAME_ATTR, newSettings.getProxyUsername());
        root.setAttribute(PROXY_PASSWORD_ATTR, scramble(newSettings.getProxyPassword()));
      }));
  settings = newSettings;
  botManager.reloadIfNeeded(settings);
}
 

public static String normalizeArtifactPath(final String path, final File file) {
  if (StringUtil.isEmpty(path)) {
    return file.getName();
  } else {
    return FileUtil.normalizeRelativePath(String.format("%s/%s", path, file.getName()));
  }
}
 

@Nullable
public String getFileUrl(File file) {
  final String canonicalFilePath = FileUtil.getCanonicalFile(file).getPath();
  if (!StringUtil.startsWithIgnoreCase(canonicalFilePath, mySourcesRootDirectoryCanonicalPath)) {
    LOG.debug(String.format("Failed to construct URL for file %s. It locates outside of source root directory %s.",
      canonicalFilePath, mySourcesRootDirectoryCanonicalPath));
    return null;
  }
  return canonicalFilePath.substring(mySourcesRootDirectoryCanonicalPath.length() + 1)
    .replace(File.separator, "/");
}
 

@NotNull
private PdbSignatureIndexEntry getPdbSignature(File pdbFile) throws Exception {
  final File guidDumpFile = FileUtil.createTempFile(myBuildTempDirectory, "symbol-signature-local-", ".xml", false);
  myJetSymbolsExe.dumpPdbGuidsToFile(Collections.singleton(pdbFile), guidDumpFile, myProgressLogger);
  if(guidDumpFile.isFile())
    return PdbSignatureIndexUtil.read(new FileInputStream(guidDumpFile), true).iterator().next();
  else
    throw new Exception("Failed to get signature of " + pdbFile.getPath());
}
 

@NotNull
private PdbSignatureIndexEntry getBinarySignature(File binaryFile) throws Exception {
  final File guidDumpFile = FileUtil.createTempFile(myBuildTempDirectory, "binary-signature-local-", ".xml", false);
  BinaryGuidDumper.dumpBinaryGuidsToFile(Collections.singleton(binaryFile), guidDumpFile, myProgressLogger);
  if(guidDumpFile.isFile())
    return PdbSignatureIndexUtil.read(new FileInputStream(guidDumpFile), true).iterator().next();
  else
    throw new Exception("Failed to get signature of " + binaryFile.getPath());
}
 

private Map<File, String> getArtifactPathsByFileExtension(List<ArtifactsCollection> artifactsCollections, String fileExtension){
  final Map<File, String> result = new HashMap<>();
  for(ArtifactsCollection artifactsCollection : artifactsCollections){
    if(artifactsCollection.isEmpty()) continue;
    final Map<File, String> filePathMap = artifactsCollection.getFilePathMap();
    for (final File artifact : filePathMap.keySet()){
      if(FileUtil.getExtension(artifact.getPath()).equalsIgnoreCase(fileExtension))
        result.put(artifact, filePathMap.get(artifact));
    }
  }
  return result;
}
 

private File dumpPathsToFile(Collection<File> files) throws IOException {
  final File result = FileUtil.createTempFile(DUMP_SYMBOL_SIGN_CMD, ".input");
  StringBuilder contentBuilder = new StringBuilder();
  for(File file : files){
    contentBuilder.append(file.getPath()).append("\n");
  }
  FileUtil.writeToFile(result, contentBuilder.toString().getBytes());
  return result;
}
 

/**
 * Executes patching process.
 *
 * @param symbolsFile is a source PDB file.
 * @param buildLogger is a build logger.
 * @return true if file was patched, otherwise false.
 * @throws Exception is error has happen during patching process.
 */
public boolean patch(File symbolsFile) throws Exception {
  final PdbType pdbType = myJetSymbolsExe.getPdbType(symbolsFile, myProgressLogger);
  final PdbFilePatcherAdapter patherAdapter = myPatcheAdapterFactory.create(pdbType);

  final Collection<File> sourceFiles = patherAdapter.getReferencedSourceFiles(symbolsFile);
  final String symbolsFileCanonicalPath = symbolsFile.getCanonicalPath();
  if (sourceFiles.isEmpty()) {
    final String message = "No source information found in pdb file " + symbolsFileCanonicalPath;
    myProgressLogger.warning(message);
    LOG.warn(message);
    return false;
  }

  final File tmpFile = FileUtil.createTempFile(myWorkingDir, "pdb-", ".patch", false);
  try {
    int processedFilesCount = patherAdapter.serializeSourceLinks(tmpFile, sourceFiles);
    if (processedFilesCount == 0) {
      myProgressLogger.message(String.format("No local source files were found for pdb file %s. Looks like it was not produced during the current build.", symbolsFileCanonicalPath));
      return false;
    } else {
      myProgressLogger.message(String.format("Information about %d source files will be updated.", processedFilesCount));
    }

    final ExecResult result = patherAdapter.updatePdbSourceLinks(symbolsFile, tmpFile);
    if (result.getExitCode() != 0) {
      throw new IOException(String.format("Failed to update symbols file %s: %s", symbolsFile, result.getStderr()));
    }
  } finally {
    FileUtil.delete(tmpFile);
  }
  return true;
}
 

@Test
public void testCmdParametersLengthLimit() throws Exception {
  final File output = FileUtil.createTempFile("testCmdParametersLengthLimit", ".out");
  final StringBuilder warnings = new StringBuilder();
  final int dumpExitCode = myExe.dumpPdbGuidsToFile(getFilesCollection(500), output, new NullBuildProgressLogger() {
    @Override
    public void warning(String message) {
      warnings.append(message).append("\n");
    }
  });
  assertEquals(1, dumpExitCode);
  assertTrue(warnings.toString(),warnings.indexOf("Nothing to dump.") > 0);
}