Python boto.mturk.connection.MTurkConnection() Examples

def getHITType():
    # Changing this will add another hit type and might mess up later fetches...
    # Only change if you know what you are doing...

    _mtc = MTurkConnection( host = _host )

    _title = "Guess the Closest Part of a Person!"
    _description="Help us find out which body part of a person is closest to the camera that took the picture."
    _keywords = "person, people, image, images, object, objects, depth, comparisons, LSP dataset"
    
    _reward = _mtc.get_price_as_price(0.15)
    _duration = 60 * 15
    _approval_delay = 60 * 60 * 24 * 10
        
    _qualifications = Qualifications()
    _qualifications.add(PercentAssignmentsApprovedRequirement('GreaterThanOrEqualTo', 98, required_to_preview=True))
    _qualifications.add(NumberHitsApprovedRequirement('GreaterThanOrEqualTo', 100, required_to_preview=True))
    _qualifications.add(LocaleRequirement('EqualTo', 'US', required_to_preview=True))

    return _mtc.register_hit_type(title=_title, description=_description, reward=_reward, duration=_duration, keywords=_keywords, approval_delay=_approval_delay, qual_req=_qualifications) 

def getHITType():
    # Changing this will add another hit type and might mess up later fetches...
    # Only change if you know what you are doing...

    _mtc = MTurkConnection( host = _host )

    _title = "Guess the Closest Part of a Person!"
    _description="Help us find out which body part of a person is closest to the camera that took the picture."
    _keywords = "person, people, image, images, object, objects, depth, comparisons, human3.6m"
    
    _reward = _mtc.get_price_as_price(0.1)
    _duration = 60 * 15
    _approval_delay = 60 * 60 * 24 * 10
        
    _qualifications = Qualifications()
    _qualifications.add(PercentAssignmentsApprovedRequirement('GreaterThanOrEqualTo', 98, required_to_preview=True))
    _qualifications.add(NumberHitsApprovedRequirement('GreaterThanOrEqualTo', 100, required_to_preview=True))
    _qualifications.add(LocaleRequirement('EqualTo', 'US', required_to_preview=True))

    return _mtc.register_hit_type(title=_title, description=_description, reward=_reward, duration=_duration, keywords=_keywords, approval_delay=_approval_delay, qual_req=_qualifications) 

def connect_mturk(aws_access_key_id=None, aws_secret_access_key=None,
                  **kwargs):
    """
    :type aws_access_key_id: string
    :param aws_access_key_id: Your AWS Access Key ID

    :type aws_secret_access_key: string
    :param aws_secret_access_key: Your AWS Secret Access Key

    :rtype: :class:`boto.mturk.connection.MTurkConnection`
    :return: A connection to MTurk
    """
    from boto.mturk.connection import MTurkConnection
    return MTurkConnection(aws_access_key_id, aws_secret_access_key, **kwargs) 

def test():
    q = ExternalQuestion(external_url="http://websort.net/s/F3481C", frame_height=800)
    conn = MTurkConnection(host='mechanicalturk.sandbox.amazonaws.com')
    keywords=['boto', 'test', 'doctest']
    qualifications = Qualifications()
    qualifications.add(PercentAssignmentsApprovedRequirement(comparator="GreaterThan", integer_value="95"))
    create_hit_rs = conn.create_hit(question=q, lifetime=60*65,max_assignments=2,title="Boto External Question Test", keywords=keywords,reward = 0.05, duration=60*6,approval_delay=60*60, annotation='An annotation from boto external question test', qualifications=qualifications)
    assert(create_hit_rs.status == True)
    print create_hit_rs.HITTypeId 

def connect_mturk(aws_access_key_id=None, aws_secret_access_key=None, **kwargs):
    """
    :type aws_access_key_id: string
    :param aws_access_key_id: Your AWS Access Key ID

    :type aws_secret_access_key: string
    :param aws_secret_access_key: Your AWS Secret Access Key

    :rtype: :class:`boto.mturk.connection.MTurkConnection`
    :return: A connection to MTurk
    """
    from boto.mturk.connection import MTurkConnection
    return MTurkConnection(aws_access_key_id, aws_secret_access_key, **kwargs) 

def get_amt_connection(sandbox):
    """ Get a connection object to communicate with the AMT API. """
    host = (settings.AMT_SANDBOX_HOST
            if sandbox else settings.AMT_HOST)
    return MTurkConnection(aws_access_key_id=settings.AMT_ACCESS_KEY,
                           aws_secret_access_key=settings.AMT_SECRET_KEY,
                           host=host) 

def get_amt_connection(sandbox):
    """ Get a connection object to communicate with the AMT API. """
    host = (settings.AMT_SANDBOX_HOST
            if sandbox else settings.AMT_HOST)
    return MTurkConnection(aws_access_key_id=settings.AMT_ACCESS_KEY,
                           aws_secret_access_key=settings.AMT_SECRET_KEY,
                           host=host) 

def getReviewableAssignments():
    mtc = MTurkConnection( host = _host )
    # note: if there are more than 100 assignments per hit the function
    # must be modified to retrieve all pages of the assignments
    page_size = 100
    
    _assignments = []

    #_num_hits = sum(1 for _ in mtc.get_all_hits())
    #print "Total Number of HITs: [%d]" %(_num_hits)
    
    _num_reviewable = 0
    _num_hits = 0
    print "Analyzed [%d] HITs" %(_num_hits+1)
    for hit in mtc.get_all_hits():
        _num_hits += 1
	if _num_hits % 500 == 0:
            print "Analyzed [%d] HITs" %_num_hits
        
        tmp_assign = [_assignment for _assignment in mtc.get_assignments( hit.HITId, page_size = page_size )]
        if len( tmp_assign ) == NUMBER_HIT_ASSIGNMENTS:
            _num_reviewable += 1 
        
        _assignments.extend( tmp_assign )
    
    print "Total Number of HITs: [%d]" %( _num_hits )
    print "Total Number of Assignments: [%d]" %( len(_assignments) )
    print "Total Number of Reviewavle HITs: [%d]" %( _num_reviewable )
    
    return _assignments 

def getReviewableHITs( verbose = True ):
    mtc = MTurkConnection( host = _host )
    hitType = getHITType()[0]
    
    page_size = 100
    # this gets the first page and allows to check how many other pages
    hits = mtc.get_reviewable_hits( page_size = page_size )

    total_pages = float( hits.TotalNumResults ) / page_size
    int_total = int( total_pages )
    if( total_pages - int_total > 0 ):
        total_pages = int_total + 1
    else:
        total_pages = int_total
            
    if verbose:
        print "Total Reviewable HITs: [%s]" % hits.TotalNumResults
        print "Total Number of Pages: [%i]" % total_pages
   
    # first page was already retrieved
    pn = 1
    if verbose:
        print " -> request page [%i]" % pn    

    while pn < total_pages:
        pn = pn + 1
        if verbose:
            print " -> request page [%i]" % pn        
        temp_hits = mtc.get_reviewable_hits( hit_type=hitType.HITTypeId, page_size=page_size, page_number=pn )
        # extend the hit list
        hits.extend(temp_hits)
        
    return hits 

def deleteAllHits():
    # this function should probably take an input parameter 
    # of a pickle file with the hits to be disposed...
    mtc = MTurkConnection(host=_host)
    for hit in mtc.get_all_hits():
        mtc.disable_hit( hit.HITId ) 

def getReviewableAssignments():
    mtc = MTurkConnection( host = _host )
    # note: if there are more than 100 assignments per hit the function
    # must be modified to retrieve all pages of the assignments
    page_size = 100
    
    _assignments = []

    _num_reviewable = 0
    _num_hits = 0
    print "Analyzed [%d] HITs" %(_num_hits+1)
    for hit in mtc.get_all_hits():
        _num_hits += 1
	if _num_hits % 500 == 0:
            print "Analyzed [%d] HITs" %_num_hits
        
        tmp_assign = [_assignment for _assignment in mtc.get_assignments( hit.HITId, page_size = page_size )]
        if len( tmp_assign ) == NUMBER_HIT_ASSIGNMENTS:
            _num_reviewable += 1 
        
        _assignments.extend( tmp_assign )
    
    print "Total Number of HITs: [%d]" %( _num_hits )
    print "Total Number of Assignments: [%d]" %( len(_assignments) )
    print "Total Number of Reviewavle HITs: [%d]" %( _num_reviewable )
    
    return _assignments 

def getReviewableHITs( verbose = True ):
    mtc = MTurkConnection( host = _host )
    hitType = getHITType()[0]
    
    page_size = 100
    # this gets the first page and allows to check how many other pages
    hits = mtc.get_reviewable_hits( page_size = page_size )

    total_pages = float( hits.TotalNumResults ) / page_size
    int_total = int( total_pages )
    if( total_pages - int_total > 0 ):
        total_pages = int_total + 1
    else:
        total_pages = int_total
            
    if verbose:
        print "Total Reviewable HITs: [%s]" % hits.TotalNumResults
        print "Total Number of Pages: [%i]" % total_pages
   
    # first page was already retrieved
    pn = 1
    if verbose:
        print " -> request page [%i]" % pn    

    while pn < total_pages:
        pn = pn + 1
        if verbose:
            print " -> request page [%i]" % pn        
        temp_hits = mtc.get_reviewable_hits( hit_type=hitType.HITTypeId, page_size=page_size, page_number=pn )
        # extend the hit list
        hits.extend(temp_hits)
        
    return hits 

def deleteAllHits():
    # this function should probably take an input parameter 
    # of a pickle file with the hits to be disposed...
    mtc = MTurkConnection(host=_host)
    for hit in mtc.get_all_hits():
        mtc.disable_hit( hit.HITId ) 

def turk_connect():
    """
    Connect to Mechanical Turk and return a connection. This uses `AWS_ACCESS_KEY_ID` and `AWS_SECRET_KEY` from
    `secret_keys.py` (you must put these in yourself).

    Returns
    -------
    boto.MTurkConnection
    """
    return MTurkConnection(aws_access_key_id=AWS_ACCESS_KEY_ID,
                           aws_secret_access_key=AWS_SECRET_KEY,
                           host=app.config['MTURK_HOST']) 

def get_mturk_connection():
    return MTurkConnection(
        aws_access_key_id=settings.MTURK_AWS_ACCESS_KEY_ID,
        aws_secret_access_key=settings.MTURK_AWS_SECRET_ACCESS_KEY,
        host=settings.MTURK_HOST,
        debug=settings.MTURK_SANDBOX) 

def createHITs( savePath = '/home/ubuntu/amt_guis/cocoa_depth/hits/human/', hit_name = '' ):
    setIds = range( STARTING_HIT, STARTING_HIT + NUMBER_HITS )

    mtc = MTurkConnection( host = _host )
    hits = []

    hitType = getHITType()[0]
    hitLifeTime = 60 * 60 * 24 * 7

    count = 0
    for setId in setIds:
        
        external_url = HOST_DOMAIN + '/cocoa_depth/human/' + str( setId )
        print external_url
        
        q = ExternalQuestion( external_url=external_url, frame_height=1000 )
        hit = mtc.create_hit(hit_type=hitType.HITTypeId,
                             question=q,
                             max_assignments = NUMBER_HIT_ASSIGNMENTS,
                             lifetime=hitLifeTime)

        hits.append( hit[0] )
        
        count += 1
        if count >= MAX_HITS:
            # pass is just a place holder
            pass
            
    if savePath == '':
        if 'MTURK_STORAGE_PATH' in os.environ:
            savePath = os.environ['MTURK_STORAGE_PATH']
        else:
            savePath == './'

    if hit_name == '':
        hit_name = 'cocoa_test_' + str(NUMBER_HITS) + '_DepthHITS'
   
    time_stamp = time.strftime( "%Y-%m-%d_%H-%M-%S" )
        
    filename = os.path.join( savePath, hit_name + '_' + time_stamp + ".pkl")
        
    print "Storing created hit data at %s" % (filename)
    with open(filename, 'wb') as f:
        pickle.dump( hits, f ) 

def payTurkersAssignments():
    _mtc = MTurkConnection( host = _host )

    rejected = 0
    approved = 0
    failed_rejected = 0
    failed_approved = 0
    
    failed_approved_list = []
    failed_rejected_list = []

    return_dict = processAssignments( save=False )

    # list with the assignments that are not rejected nor flagged
    _good_assignments = return_dict['_good_assignments']
    for ass in _good_assignments:
        try:
            _mtc.approve_assignment( ass['assignment_id'] )
            approved += 1
        except MTurkRequestError:
            failed_approved += 1
            failed_approved_list.append( ass )

    # list containing the assignments that were flagged by the turkers
    _flagged_assignments = return_dict['_flagged_assignments']            
    for ass in _flagged_assignments:
        try:
            _mtc.approve_assignment( ass['assignment_id'] )
            approved += 1
        except MTurkRequestError:
            failed_approved += 1
            failed_approved_list.append( ass )

    # list with the assignments were something inexpected on my side happened
    _error_assignments = return_dict['_error_assignments']
    for ass in _error_assignments:
        try:
            _mtc.approve_assignment( ass['assignment_id'] )
            approved += 1
        except MTurkRequestError:
            failed_approved += 1
            failed_approved_list.append( ass )
                
    # list with the assignments that were rejected
    _rejected_assignments = return_dict['_rejected_assignments']
    for ass in _rejected_assignments:
        try:
            _mtc.reject_assignment( ass['assignment_id'] )
            rejected += 1
        except MTurkRequestError:
            failed_rejected += 1
            failed_rejected_list.append( ass )
            
    print "Approved:        [%d]"%approved
    print "Rejected:        [%d]"%rejected
    print "Not Approved:    [%d]"%failed_approved
    print "Not Rejected:    [%d]"%failed_rejected 
    
    return (failed_approved_list, failed_rejected_list) 

def createHITs( savePath = '/home/ubuntu/amt_guis/cocoa_depth/hits/lsp/', hit_name = '' ):
    setIds = range( STARTING_HIT, STARTING_HIT + NUMBER_HITS )

    mtc = MTurkConnection( host = _host )
    hits = []

    hitType = getHITType()[0]
    hitLifeTime = 60 * 60 * 24 * 7

    count = 0
    for setId in setIds:
        external_url = HOST_DOMAIN + '/cocoa_depth/lsp/' + str( setId )
        print external_url
        
        q = ExternalQuestion( external_url=external_url, frame_height=1000 )
        hit = mtc.create_hit(hit_type=hitType.HITTypeId,
                             question=q,
                             max_assignments = NUMBER_HIT_ASSIGNMENTS,
                             lifetime=hitLifeTime)

        hits.append( hit[0] )
        
        count += 1
        if count >= MAX_HITS:
            # pass is just a place holder
            pass
            
    if savePath == '':
        if 'MTURK_STORAGE_PATH' in os.environ:
            savePath = os.environ['MTURK_STORAGE_PATH']
        else:
            savePath == './'

    if hit_name == '':
        hit_name = 'cocoa_test_' + str(NUMBER_HITS) + '_DepthHITS'
   
    time_stamp = time.strftime( "%Y-%m-%d_%H-%M-%S" )
        
    filename = os.path.join( savePath, hit_name + '_' + time_stamp + ".pkl")
        
    print "Storing created hit data at %s" % (filename)
    with open(filename, 'wb') as f:
        pickle.dump( hits, f ) 

def qualify(qualification, value, worker):
    """Assign a qualification to a worker."""
    # create connection to AWS
    from boto.mturk.connection import MTurkConnection
    config = PsiturkConfig()
    config.load_config()
    aws_access_key_id = config.get('AWS Access', 'aws_access_key_id')
    aws_secret_access_key = config.get('AWS Access', 'aws_secret_access_key')
    conn = MTurkConnection(aws_access_key_id, aws_secret_access_key)

    def get_workers_with_qualification(qualification):
        """Get workers with the given qualification."""
        results = []
        continue_flag = True
        page = 1
        while(continue_flag):
            new_results = conn.get_qualifications_for_qualification_type(
                qualification,
                page_size=100,
                page_number=page)

            if(len(new_results) == 0):
                continue_flag = False
            else:
                results.extend(new_results)
                page = page + 1

        return results

    results = get_workers_with_qualification(qualification)
    workers = [x.SubjectId for x in results]

    # assign the qualification
    click.echo(
        "Assigning qualification {} with value {} to worker {}".format(
            qualification,
            value,
            worker))

    if worker in workers:
        result = conn.update_qualification_score(qualification, worker, value)
    else:
        result = conn.assign_qualification(qualification, worker, value)

    if result:
        click.echo(result)

    # print out the current set of workers with the qualification
    results = get_workers_with_qualification(qualification)

    click.echo("{} workers with qualification {}:".format(
        len(results),
        qualification))

    values = [r.IntegerValue for r in results]
    unique_values = list(set([r.IntegerValue for r in results]))
    for v in unique_values:
        click.echo("{} with value {}".format(
            len([val for val in values if val == v]),
            v)) 

def recruit_participants(self, n=1):
        """Recruit n participants."""
        auto_recruit = os.environ['auto_recruit'] == 'true'

        if auto_recruit:

            print "Starting Wallace's recruit_participants."

            hit_id = str(
                Participant.query.
                with_entities(Participant.hitid).first().hitid)

            print "hit_id is {}.".format(hit_id)

            is_sandbox = self.config.getboolean(
                'Shell Parameters', 'launch_in_sandbox_mode')

            if is_sandbox:
                host = 'mechanicalturk.sandbox.amazonaws.com'
            else:
                host = 'mechanicalturk.amazonaws.com'

            mturkparams = dict(
                aws_access_key_id=self.aws_access_key_id,
                aws_secret_access_key=self.aws_secret_access_key,
                host=host)

            self.mtc = MTurkConnection(**mturkparams)

            self.mtc.extend_hit(
                hit_id,
                assignments_increment=int(n or 0))

            expiration_increment = self.config.get('HIT Configuration',
                                                   'duration')

            self.mtc.extend_hit(
                hit_id,
                expiration_increment=int(
                    float(expiration_increment or 0) * 3600))
        else:
            print(">>>> auto_recruit set to {}: recruitment suppressed"
                  .format(auto_recruit))