org.apache.commons.exec.Executor Java Examples

@Test
void willScheduleOnUnixWithoutSudo() throws IOException {
    Assumptions.assumeTrue(SystemUtils.IS_OS_UNIX);
    final JobsProperties properties = JobsProperties.getJobsPropertiesDefaults();
    properties.getUsers().setRunAsUserEnabled(false);
    final TaskScheduler scheduler = Mockito.mock(TaskScheduler.class);
    final Resource jobsDir = Mockito.mock(Resource.class);
    Mockito.when(jobsDir.exists()).thenReturn(true);
    final DataServices dataServices = Mockito.mock(DataServices.class);
    Mockito.when(dataServices.getPersistenceService()).thenReturn(Mockito.mock(PersistenceService.class));
    Assertions.assertThat(
        new DiskCleanupTask(
            new DiskCleanupProperties(),
            scheduler,
            jobsDir,
            dataServices,
            properties,
            Mockito.mock(Executor.class),
            new SimpleMeterRegistry()
        )
    ).isNotNull();
    Mockito.verify(scheduler, Mockito.times(1)).schedule(Mockito.any(Runnable.class), Mockito.any(Trigger.class));
}
 

@Test
void willScheduleOnUnixWithSudo() throws IOException {
    Assumptions.assumeTrue(SystemUtils.IS_OS_UNIX);
    final TaskScheduler scheduler = Mockito.mock(TaskScheduler.class);
    final Resource jobsDir = Mockito.mock(Resource.class);
    Mockito.when(jobsDir.exists()).thenReturn(true);
    final DataServices dataServices = Mockito.mock(DataServices.class);
    Mockito.when(dataServices.getPersistenceService()).thenReturn(Mockito.mock(PersistenceService.class));
    Assertions.assertThat(
        new DiskCleanupTask(
            new DiskCleanupProperties(),
            scheduler,
            jobsDir,
            dataServices,
            JobsProperties.getJobsPropertiesDefaults(),
            Mockito.mock(Executor.class),
            new SimpleMeterRegistry()
        )
    ).isNotNull();
    Mockito.verify(scheduler, Mockito.times(1)).schedule(Mockito.any(Runnable.class), Mockito.any(Trigger.class));
}
 

@Test
void wontScheduleOnNonUnixWithSudo() throws IOException {
    Assumptions.assumeTrue(!SystemUtils.IS_OS_UNIX);
    final TaskScheduler scheduler = Mockito.mock(TaskScheduler.class);
    final Resource jobsDir = Mockito.mock(Resource.class);
    Mockito.when(jobsDir.exists()).thenReturn(true);
    final DataServices dataServices = Mockito.mock(DataServices.class);
    Mockito.when(dataServices.getPersistenceService()).thenReturn(Mockito.mock(PersistenceService.class));
    Assertions.assertThat(
        new DiskCleanupTask(
            new DiskCleanupProperties(),
            scheduler,
            jobsDir,
            dataServices,
            JobsProperties.getJobsPropertiesDefaults(),
            Mockito.mock(Executor.class),
            new SimpleMeterRegistry()
        )
    ).isNotNull();
    Mockito.verify(scheduler, Mockito.never()).schedule(Mockito.any(Runnable.class), Mockito.any(Trigger.class));
}
 

@Test
public void cantConstruct() {
    final JobsProperties properties = JobsProperties.getJobsPropertiesDefaults();
    properties.getUsers().setRunAsUserEnabled(false);
    final Resource jobsDir = Mockito.mock(Resource.class);
    Mockito.when(jobsDir.exists()).thenReturn(false);
    final DataServices dataServices = Mockito.mock(DataServices.class);
    Mockito.when(dataServices.getPersistenceService()).thenReturn(Mockito.mock(PersistenceService.class));
    Assertions
        .assertThatIOException()
        .isThrownBy(
            () -> new DiskCleanupTask(
                new DiskCleanupProperties(),
                Mockito.mock(TaskScheduler.class),
                jobsDir,
                dataServices,
                properties,
                Mockito.mock(Executor.class),
                new SimpleMeterRegistry()
            )
        );
}
 

/**
 * Can get a bean for killing V3 jobs.
 */
@Test
public void canGetJobKillServiceV3Bean() {
    final DataServices dataServices = Mockito.mock(DataServices.class);
    Mockito.when(dataServices.getPersistenceService()).thenReturn(Mockito.mock(PersistenceService.class));
    Assert.assertNotNull(
        this.servicesAutoConfiguration.jobKillServiceV3(
            new GenieHostInfo("localhost"),
            dataServices,
            Mockito.mock(Executor.class),
            JobsProperties.getJobsPropertiesDefaults(),
            Mockito.mock(GenieEventBus.class),
            Mockito.mock(FileSystemResource.class),
            GenieObjectMapper.getMapper(),
            Mockito.mock(ProcessChecker.Factory.class)
        )
    );
}
 

/**
 * All the expected beans exist.
 */
@Test
void expectedBeansExist() {
    this.contextRunner.run(
        context -> {
            Assertions.assertThat(context).hasSingleBean(TasksExecutorPoolProperties.class);
            Assertions.assertThat(context).hasSingleBean(TasksSchedulerPoolProperties.class);

            Assertions.assertThat(context).hasSingleBean(Executor.class);

            Assertions.assertThat(context).hasBean("genieTaskScheduler");
            Assertions.assertThat(context).hasBean("genieAsyncTaskExecutor");
            Assertions.assertThat(context).hasBean("genieSyncTaskExecutor");
        }
    );
}
 

/**
 * Constructor.
 *
 * @param runAsUserEnabled                   Flag that tells if job should be run as user specified in the request
 * @param userCreationEnabled                Flag that tells if the user specified should be created
 * @param executor                           An executor object used to run jobs
 * @param hostname                           Hostname for the node the job is running on
 * @param registry                           The metrics registry to use
 * @param jobDirectoryManifestCreatorService The manifest creator service
 */
public JobKickoffTask(
    final boolean runAsUserEnabled,
    final boolean userCreationEnabled,
    @NotNull final Executor executor,
    @NotNull final String hostname,
    @NotNull final MeterRegistry registry,
    final JobDirectoryManifestCreatorService jobDirectoryManifestCreatorService
) {
    super(registry);
    this.isRunAsUserEnabled = runAsUserEnabled;
    this.isUserCreationEnabled = userCreationEnabled;
    this.executor = executor;
    this.hostname = hostname;
    this.jobDirectoryManifestCreatorService = jobDirectoryManifestCreatorService;
    this.retryTemplate = new RetryTemplate();
    this.retryTemplate.setBackOffPolicy(new ExponentialBackOffPolicy());
}
 

/**
 * If required get a {@link DiskCleanupTask} instance for use.
 *
 * @param properties      The disk cleanup properties to use.
 * @param scheduler       The scheduler to use to schedule the cron trigger.
 * @param jobsDir         The resource representing the location of the job directory
 * @param dataServices    The {@link DataServices} instance to use
 * @param jobsProperties  The jobs properties to use
 * @param processExecutor The process executor to use to delete directories
 * @param registry        The metrics registry
 * @return The {@link DiskCleanupTask} instance
 * @throws IOException When it is unable to open a file reference to the job directory
 */
@Bean
@ConditionalOnProperty(value = DiskCleanupProperties.ENABLED_PROPERTY, havingValue = "true")
@ConditionalOnMissingBean(DiskCleanupTask.class)
public DiskCleanupTask diskCleanupTask(
    final DiskCleanupProperties properties,
    @Qualifier("genieTaskScheduler") final TaskScheduler scheduler,
    @Qualifier("jobsDir") final Resource jobsDir,
    final DataServices dataServices,
    final JobsProperties jobsProperties,
    final Executor processExecutor,
    final MeterRegistry registry
) throws IOException {
    return new DiskCleanupTask(
        properties,
        scheduler,
        jobsDir,
        dataServices,
        jobsProperties,
        processExecutor,
        registry
    );
}
 

public int unload(String loadedInputPath) throws Exception {
	String unloaderDir = platformConfig.getUnloaderPath();
	commandLine = new CommandLine(Paths.get(unloaderDir).toFile());

	commandLine.addArgument("--graph-name");
	commandLine.addArgument(formattedGraph.getName());

	commandLine.addArgument("--output-path");
	commandLine.addArgument(loadedInputPath);

	String commandString = StringUtils.toString(commandLine.toStrings(), " ");
	LOG.info(String.format("Execute graph unloader with command-line: [%s]", commandString));

	Executor executor = new DefaultExecutor();
	executor.setStreamHandler(new PumpStreamHandler(System.out, System.err));
	executor.setExitValue(0);

	return executor.execute(commandLine);
}
 

/**
 * Get an local implementation of the JobKillService.
 *
 * @param genieHostInfo         Information about the host the Genie process is running on
 * @param dataServices          The {@link DataServices} instance to use
 * @param executor              The executor to use to run system processes.
 * @param jobsProperties        The jobs properties to use
 * @param genieEventBus         The application event bus to use to publish system wide events
 * @param genieWorkingDir       Working directory for genie where it creates jobs directories.
 * @param objectMapper          The Jackson ObjectMapper used to serialize from/to JSON
 * @param processCheckerFactory The process checker factory
 * @return A job kill service instance.
 */
@Bean
@ConditionalOnMissingBean(JobKillServiceV3.class)
public JobKillServiceV3 jobKillServiceV3(
    final GenieHostInfo genieHostInfo,
    final DataServices dataServices,
    final Executor executor,
    final JobsProperties jobsProperties,
    final GenieEventBus genieEventBus,
    @Qualifier("jobsDir") final Resource genieWorkingDir,
    final ObjectMapper objectMapper,
    final ProcessChecker.Factory processCheckerFactory
) {
    return new JobKillServiceV3(
        genieHostInfo.getHostname(),
        dataServices,
        executor,
        jobsProperties.getUsers().isRunAsUserEnabled(),
        genieEventBus,
        genieWorkingDir,
        objectMapper,
        processCheckerFactory
    );
}
 

/**
 * Constructor.
 *
 * @param hostname              The name of the host this Genie node is running on
 * @param dataServices          The {@link DataServices} instance to use
 * @param executor              The executor to use to run system processes
 * @param runAsUser             True if jobs are run as the user who submitted the job
 * @param genieEventBus         The system event bus to use
 * @param genieWorkingDir       The working directory where all job directories are created.
 * @param objectMapper          The Jackson ObjectMapper used to serialize from/to JSON
 * @param processCheckerFactory The process checker factory
 */
public JobKillServiceV3(
    @NotBlank final String hostname,
    @NotNull final DataServices dataServices,
    @NotNull final Executor executor,
    final boolean runAsUser,
    @NotNull final GenieEventBus genieEventBus,
    @NotNull final Resource genieWorkingDir,
    @NotNull final ObjectMapper objectMapper,
    @NotNull final ProcessChecker.Factory processCheckerFactory
) {
    this.hostname = hostname;
    this.persistenceService = dataServices.getPersistenceService();
    this.executor = executor;
    this.runAsUser = runAsUser;
    this.genieEventBus = genieEventBus;
    this.objectMapper = objectMapper;
    this.processCheckerFactory = processCheckerFactory;

    try {
        this.baseWorkingDir = genieWorkingDir.getFile();
    } catch (IOException gse) {
        throw new RuntimeException("Could not load the base path from resource", gse);
    }
}
 

/**
 * Create an Job Kickoff Task bean that runs the job.
 *
 * @param jobsProperties                     The various jobs properties
 * @param executor                           An instance of an executor
 * @param genieHostInfo                      Info about the host Genie is running on
 * @param registry                           The metrics registry to use
 * @param jobDirectoryManifestCreatorService The manifest creation service
 * @return An application task object
 */
@Bean
@Order(value = 6)
@ConditionalOnMissingBean(JobKickoffTask.class)
public JobKickoffTask jobKickoffTask(
    final JobsProperties jobsProperties,
    final Executor executor,
    final GenieHostInfo genieHostInfo,
    final MeterRegistry registry,
    final JobDirectoryManifestCreatorService jobDirectoryManifestCreatorService
) {
    return new JobKickoffTask(
        jobsProperties.getUsers().isRunAsUserEnabled(),
        jobsProperties.getUsers().isCreationEnabled(),
        executor,
        genieHostInfo.getHostname(),
        registry,
        jobDirectoryManifestCreatorService
    );
}
 

/**
 * Create a {@link ProcessChecker.Factory} suitable for UNIX systems.
 *
 * @param executor       The executor where checks are executed
 * @param jobsProperties The jobs properties
 * @return a {@link ProcessChecker.Factory}
 */
@Bean
@ConditionalOnMissingBean(ProcessChecker.Factory.class)
public ProcessChecker.Factory processCheckerFactory(
    final Executor executor,
    final JobsProperties jobsProperties
) {
    if (SystemUtils.IS_OS_UNIX) {
        return new UnixProcessChecker.Factory(
            executor,
            jobsProperties.getUsers().isRunAsUserEnabled()
        );
    } else {
        throw new BeanCreationException("No implementation available for non-UNIX systems");
    }
}
 

/**
 * Constructor.
 *
 * @param pid           The process id to check.
 * @param executor      The executor to use for generating system commands.
 * @param timeout       The time which after this job should be killed due to timeout
 * @param checkWithSudo Whether the checker requires sudo
 */
UnixProcessChecker(
    @Min(1) final int pid,
    @NotNull final Executor executor,
    @NotNull final Instant timeout,
    final boolean checkWithSudo
) {
    if (!SystemUtils.IS_OS_UNIX) {
        throw new IllegalArgumentException("Not running on a Unix system.");
    }

    this.executor = executor;

    // Use POSIX compliant 'kill -0 <PID>' to check if process is still running.
    if (checkWithSudo) {
        this.commandLine = new CommandLine("sudo");
        this.commandLine.addArgument("kill");
        this.commandLine.addArgument("-0");
        this.commandLine.addArgument(Integer.toString(pid));
    } else {
        this.commandLine = new CommandLine("kill");
        this.commandLine.addArgument("-0");
        this.commandLine.addArgument(Integer.toString(pid));
    }

    this.timeout = timeout;
}
 

private Executor createExecutor(File workingDirectory, long timeoutInSeconds) {
    DefaultExecutor executor = new DefaultExecutor();
    executor.setWorkingDirectory(workingDirectory);
    executor.setProcessDestroyer(new ShutdownHookProcessDestroyer());   // Fixes #41

    if (timeoutInSeconds > 0) {
        executor.setWatchdog(new ExecuteWatchdog(timeoutInSeconds * 1000));
    }

    return executor;
}
 

/**
 * Get an {@link Executor} to use for executing processes from tasks.
 *
 * @return The executor to use
 */
// TODO: Remove once agent job runs complete
@Bean
@ConditionalOnMissingBean(Executor.class)
public Executor processExecutor() {
    final Executor executor = new DefaultExecutor();
    executor.setStreamHandler(new PumpStreamHandler(null, null));
    return executor;
}
 

/**
 * Constructor. Schedules this task to be run by the task scheduler.
 *
 * @param properties      The disk cleanup properties to use.
 * @param scheduler       The scheduler to use to schedule the cron trigger.
 * @param jobsDir         The resource representing the location of the job directory
 * @param dataServices    The {@link DataServices} instance to use
 * @param jobsProperties  The jobs properties to use
 * @param processExecutor The process executor to use to delete directories
 * @param registry        The metrics registry
 * @throws IOException When it is unable to open a file reference to the job directory
 */
public DiskCleanupTask(
    @NotNull final DiskCleanupProperties properties,
    @NotNull final TaskScheduler scheduler,
    @NotNull final Resource jobsDir,
    @NotNull final DataServices dataServices,
    @NotNull final JobsProperties jobsProperties,
    @NotNull final Executor processExecutor,
    @NotNull final MeterRegistry registry
) throws IOException {
    // Job Directory is guaranteed to exist by the MvcConfig bean creation but just in case someone overrides
    if (!jobsDir.exists()) {
        throw new IOException("Jobs dir " + jobsDir + " doesn't exist. Unable to create task to cleanup.");
    }

    this.properties = properties;
    this.jobsDir = jobsDir.getFile();
    this.persistenceService = dataServices.getPersistenceService();
    this.runAsUser = jobsProperties.getUsers().isRunAsUserEnabled();
    this.processExecutor = processExecutor;

    this.numberOfDeletedJobDirs = registry.gauge(
        "genie.tasks.diskCleanup.numberDeletedJobDirs.gauge",
        new AtomicLong()
    );
    this.numberOfDirsUnableToDelete = registry.gauge(
        "genie.tasks.diskCleanup.numberDirsUnableToDelete.gauge",
        new AtomicLong()
    );
    this.unableToGetJobCounter = registry.counter("genie.tasks.diskCleanup.unableToGetJobs.rate");
    this.unableToDeleteJobDirCounter = registry.counter("genie.tasks.diskCleanup.unableToDeleteJobsDir.rate");

    // Only schedule the task if we don't need sudo while on a non-unix system
    if (this.runAsUser && !SystemUtils.IS_OS_UNIX) {
        log.error("System is not UNIX like. Unable to schedule disk cleanup due to needing Unix commands");
    } else {
        final CronTrigger trigger = new CronTrigger(properties.getExpression(), JobConstants.UTC);
        scheduler.schedule(this, trigger);
    }
}
 

/**
 * Create a new {@link Executor} implementation instance.
 *
 * @param detached Whether the streams for processes run on this executor should be detached (ignored) or not
 * @return A {@link Executor} instance
 */
public Executor newInstance(final boolean detached) {
    final Executor executor = new DefaultExecutor();
    if (detached) {
        executor.setStreamHandler(new PumpStreamHandler(null, null));
    }
    return executor;
}
 

/**
 * Set up the tests.
 */
@Before
public void setUp() {
    new JobKickoffTask(
        false,
        false,
        Mockito.mock(Executor.class),
        "localhost",
        new SimpleMeterRegistry(),
        Mockito.mock(JobDirectoryManifestCreatorService.class)
    );
}
 

/**
 * Change the ownership of a directory (recursively).
 *
 * @param dir      The directory to change the ownership of.
 * @param user     Userid of the user.
 * @param executor the command executor
 * @throws IOException if the operation fails
 */
public static void changeOwnershipOfDirectory(
    final String dir,
    final String user,
    final Executor executor
) throws IOException {

    final CommandLine commandLine = new CommandLine(SUDO)
        .addArgument("chown")
        .addArgument("-R")
        .addArgument(user)
        .addArgument(dir);
    executor.execute(commandLine);
}
 

/**
 * Setup for the tests.
 *
 * @throws IOException if the job directory cannot be created
 */
@Before
public void setup() throws IOException {
    Assume.assumeTrue(SystemUtils.IS_OS_UNIX);
    final File tempDirectory = Files.createTempDir();
    this.genieWorkingDir = new FileSystemResource(tempDirectory);
    Files.createParentDirs(new File(tempDirectory.getPath() + "/" + ID + "/genie/x"));
    this.persistenceService = Mockito.mock(PersistenceService.class);
    this.executor = Mockito.mock(Executor.class);
    this.genieEventBus = Mockito.mock(GenieEventBus.class);
    this.processCheckerFactory = Mockito.mock(ProcessChecker.Factory.class);
    this.processChecker = Mockito.mock(ProcessChecker.class);
    this.dataServices = Mockito.mock(DataServices.class);
    Mockito.when(this.dataServices.getPersistenceService()).thenReturn(this.persistenceService);
    this.service = new JobKillServiceV3(
        HOSTNAME,
        this.dataServices,
        this.executor,
        false,
        this.genieEventBus,
        this.genieWorkingDir,
        GenieObjectMapper.getMapper(),
        this.processCheckerFactory
    );

    this.killCommand = new CommandLine("kill");
    this.killCommand.addArguments(Integer.toString(PID));
}
 

/**
 * Setup for the tests.
 */
@BeforeEach
void setup() {
    Assumptions.assumeTrue(SystemUtils.IS_OS_UNIX);
    this.executor = Mockito.mock(Executor.class);
    this.tomorrow = Instant.now().plus(1, ChronoUnit.DAYS);
    // For standard tests this will keep it from dying
}
 

/**
 * Using apache commons exec for cmd line execution
 *
 * @param command
 * @return exitCode
 * @throws ExecuteException
 * @throws IOException
 * @throws InterruptedException
 */
private void startDynamoLocal() throws ExecuteException, IOException, InterruptedException {
  // java -Djava.library.path=./DynamoDBLocal_lib -jar DynamoDBLocal.jar
  // -sharedDb
  final CommandLine cmdLine = new CommandLine("java");

  cmdLine.addArgument("-Djava.library.path=" + dynLocalDir + "/DynamoDBLocal_lib");
  cmdLine.addArgument("-jar");
  cmdLine.addArgument(dynLocalDir + "/DynamoDBLocal.jar");
  cmdLine.addArgument("-sharedDb");
  cmdLine.addArgument("-inMemory");
  cmdLine.addArgument("-port");
  cmdLine.addArgument(Integer.toString(port));
  System.setProperty("aws.accessKeyId", "dummy");
  System.setProperty("aws.secretKey", "dummy");

  // Using a result handler makes the emulator run async
  final DefaultExecuteResultHandler resultHandler = new DefaultExecuteResultHandler();

  // watchdog shuts down the emulator, later
  watchdog = new ExecuteWatchdog(ExecuteWatchdog.INFINITE_TIMEOUT);
  final Executor executor = new DefaultExecutor();
  executor.setWatchdog(watchdog);
  executor.execute(cmdLine, resultHandler);

  // we need to wait here for a bit, in case the emulator needs to update
  // itself
  Thread.sleep(EMULATOR_SPINUP_DELAY_MS);
}
 

private Executor build(Optional<String> workingDirectory, long timeoutInMilliSec) {
    ExecuteWatchdog ew = new ExecuteWatchdog(timeoutInMilliSec);
    DefaultExecutor executor = new DefaultExecutor();
    //executor.setWorkingDirectory(new File(workingDirectory.or(".")));
    executor.setWatchdog(ew);
    return executor;
}
 

private void fillSuccessCodes( Executor exec )
{
    if ( successCodes != null && successCodes.length > 0 )
    {
        exec.setExitValues( successCodes );
    }
}
 

private void stopProcess(String paragraphId) {
  if (runningProcesses.containsKey(paragraphId)) {
    final Executor executor = runningProcesses.get(paragraphId);
    final ExecuteWatchdog watchdog = executor.getWatchdog();
    watchdog.destroyProcess();
  }
}
 

protected int executeCommandLine( Executor exec, CommandLine commandLine, Map<String, String> enviro,
                                  OutputStream out, OutputStream err )
                                      throws ExecuteException, IOException
{
    // note: don't use BufferedOutputStream here since it delays the outputs MEXEC-138
    PumpStreamHandler psh = new PumpStreamHandler( out, err, System.in );
    return executeCommandLine( exec, commandLine, enviro, psh );
}
 

@Override
protected int executeCommandLine(Executor exec, CommandLine commandLine, Map enviro, OutputStream out,
                                 OutputStream err) throws IOException, ExecuteException {
    commandLines.add(commandLine);
    if (failureMsg != null) {
        throw new ExecuteException(failureMsg, executeResult);
    }
    return executeResult;
}
 

protected int executeCommandLine( Executor exec, CommandLine commandLine, Map<String, String> enviro,
                                  FileOutputStream outputFile )
                                      throws ExecuteException, IOException
{
    BufferedOutputStream bos = new BufferedOutputStream( outputFile );
    PumpStreamHandler psh = new PumpStreamHandler( bos );
    return executeCommandLine( exec, commandLine, enviro, psh );
}
 

protected int executeCommandLine( Executor exec, CommandLine commandLine, Map enviro, OutputStream out,
                                  OutputStream err )
                                      throws IOException, ExecuteException
{
    commandLines.add( commandLine );
    if ( failureMsg != null )
    {
        throw new ExecuteException( failureMsg, executeResult );
    }
    return executeResult;
}