Python luigi.ExternalTask() Examples

The following are 4 code examples of luigi.ExternalTask(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module luigi , or try the search function .
Example #1
Source File: amass.py    From recon-pipeline with MIT License 6 votes vote down vote up 
def requires(self):
        """ ParseAmassOutput depends on AmassScan to run.

        TargetList expects target_file as a parameter.
        AmassScan accepts exempt_list as an optional parameter.

        Returns:
            luigi.ExternalTask - TargetList
        """

        args = {
            "target_file": self.target_file,
            "exempt_list": self.exempt_list,
            "results_dir": self.results_dir,
            "db_location": self.db_location,
        }
        return AmassScan(**args)
Example #2
Source File: test_bulk_import.py    From luigi-td with Apache License 2.0 5 votes vote down vote up 
def test_local_path(self):
        class TestInput(luigi.ExternalTask):
            def output(self):
                return luigi.LocalTarget('test_input.tsv')
        class TestBulkImportFromLocalTarget(TestBulkImport):
            def requires(self):
                return TestInput()
        task = TestBulkImportFromLocalTarget('session-1')
        eq_(task.get_path(), os.path.realpath('test_input.tsv'))
Example #3
Source File: test_bulk_import.py    From luigi-td with Apache License 2.0 5 votes vote down vote up 
def test_s3_path(self):
        class TestInput(luigi.ExternalTask):
            def output(self):
                s3client = luigi.s3.S3Client(aws_access_key_id='test-key', aws_secret_access_key='test-secret')
                return luigi.s3.S3Target('s3://test-bucket/test_input.tsv', client=s3client)
        class TestBulkImportFromS3Target(TestBulkImport):
            def requires(self):
                return TestInput()
        task = TestBulkImportFromS3Target('session-1')
        eq_(task.get_path(), 's3://test-key:test-secret@/test-bucket/test_input.tsv')
Example #4
Source File: amass.py    From recon-pipeline with MIT License 5 votes vote down vote up 
def requires(self):
        """ AmassScan depends on TargetList to run.

        TargetList expects target_file as a parameter.

        Returns:
            luigi.ExternalTask - TargetList
        """
        args = {"target_file": self.target_file, "results_dir": self.results_dir, "db_location": self.db_location}
        return TargetList(**args)