Python pyrouge.Rouge155() Examples

def evaluate_summ(article):
    ref=''
    for k in wt(data['summaries'][article])[:20]:
        ref=ref+' '+k
    gen_sum = generateText(summarize(train_data["article"][article]))
    print("-----------------------------------------------------")
    print("Original summary")
    print(ref)
    print("-----------------------------------------------------")
    print("Generated summary")
    print(gen_sum)
    print("-----------------------------------------------------")
    rouge = Rouge155()
    score = rouge.score_summary(ref, gen_sum)
    print("Rouge1 Score: ",score)
        
#######################################################################################
################################ Train model and test##################################
####################################################################################### 

def _rouge(system_dir, gold_dir):
    # Run rouge
    r = Rouge155()
    r.system_dir = system_dir
    r.model_dir = gold_dir
    r.system_filename_pattern = '([a-zA-Z0-9]*).model'
    r.model_filename_pattern = '#ID#.gold'
    output = r.convert_and_evaluate(rouge_args="-e /address/to/rouge/data/directory/rouge/data -a -c 95 -m -n 4 -w 1.2")
    # print output
    output_dict = r.output_to_dict(output)
    # print output_dict
    
    # avg_rscore = 0
    # if FLAGS.rouge_reward_fscore:
    #     avg_rscore = (output_dict["rouge_1_f_score"]+output_dict["rouge_2_f_score"]+
    #                   output_dict["rouge_3_f_score"]+output_dict["rouge_4_f_score"]+
    #                   output_dict["rouge_l_f_score"])/5.0
    # else:
    #     avg_rscore = (output_dict["rouge_1_recall"]+output_dict["rouge_2_recall"]+
    #                   output_dict["rouge_3_recall"]+output_dict["rouge_4_recall"]+
    #                   output_dict["rouge_l_recall"])/5.0

    avg_rscore = (output_dict["rouge_1_f_score"]+output_dict["rouge_2_f_score"]+output_dict["rouge_l_f_score"])/3.0

    return avg_rscore 

def test_options(self):
        rouge = Rouge155()
        model_dir = add_data_path("models_plain")
        system_dir = add_data_path("systems_plain")
        config_file = add_data_path("config_test2.xml")
        command_part1 = (
            "pyrouge_evaluate_plain_text_files -m {} -s {} -sfp "
            "D(\d+).M.100.T.A -mfp D#ID#.M.100.T.[A-Z] -id 1 -rargs".format(
                model_dir, system_dir))

        command_part2 = [
            "\"-e {data} -c 90 -2 -1 -U -r 1000 -n 2 -w 1.2 "
            "-a -m {xml}\"".format(
                data=rouge.data_dir, xml=config_file)]

        pyrouge_command = command_part1.split() + command_part2
        pyrouge_output = check_output_clean(pyrouge_command)
        rouge_command = (
            "{bin} -e {data} -c 90 -2 -1 -U -r 1000 -n 2 -w 1.2 "
            "-a -m {xml}".format(
                bin=rouge.bin_path, data=rouge.data_dir, xml=config_file))
        orig_rouge_output = check_output_clean(rouge_command.split())
        self.assertEqual(pyrouge_output, orig_rouge_output) 

def test_rouge_for_plain_text(self):
        model_dir = add_data_path("models_plain")
        system_dir = add_data_path("systems_plain")
        pyrouge_command = (
            "pyrouge_evaluate_plain_text_files -m {} -s {} -sfp "
            "D(\d+).M.100.T.A -mfp D#ID#.M.100.T.[A-Z] -id 1".format(
                model_dir, system_dir))
        pyrouge_output = check_output_clean(pyrouge_command.split())
        rouge = Rouge155()
        config_file = add_data_path("config_test2.xml")
        rouge_command = (
            "{bin} -e {data} -c 95 -2 -1 -U -r 1000 -n 4 -w 1.2 "
            "-a -m {xml}".format(
                bin=rouge.bin_path,
                data=rouge.data_dir,
                xml=config_file))
        orig_rouge_output = check_output_clean(rouge_command.split())
        self.assertEqual(pyrouge_output, orig_rouge_output) 

def test_wrong_model_pattern(self):
        rouge = Rouge155()
        rouge.system_dir = add_data_path("systems")
        rouge.model_dir = add_data_path("models_plain")
        rouge.system_filename_pattern = "SL.P.10.R.11.SL062003-(\d+).html"
        rouge.model_filename_pattern = "SL.P.10.R.[A-D].SL062003-#ID#.html"
        with self.assertRaises(Exception) as context:
            rouge.evaluate()
        match_string = (
            r"Could not find any model summaries for the system "
            r"summary with ID " + "(\d+)" + r". Specified model filename "
            r"pattern was: " + re.escape(rouge.model_filename_pattern))
        try:
            assert_regex = self.assertRegex
        except AttributeError:
            assert_regex = self.assertRegexpMatches
        assert_regex(str(context.exception), re.compile(match_string)) 

def _rouge(system_dir, gold_dir):
    # Run rouge
    r = Rouge155()
    r.system_dir = system_dir
    r.model_dir = gold_dir
    r.system_filename_pattern = '([a-zA-Z0-9]*).model'
    r.model_filename_pattern = '#ID#.gold'
    output = r.convert_and_evaluate(rouge_args="-e Code/neuralsum/ROUGE_evaluation/rouge/data -a -c 95 -m -n 4 -w 1.2")
    # print output
    output_dict = r.output_to_dict(output)
    # print output_dict
    
    avg_rscore = 0
    
    avg_rscore = (output_dict["rouge_1_recall"]+output_dict["rouge_2_recall"]+
                  output_dict["rouge_3_recall"]+output_dict["rouge_4_recall"]+
                  output_dict["rouge_l_recall"])/5.0
    return avg_rscore 

def test_rouge(cand_file, ref_file):
    f_cand = open(cand_file, encoding="utf-8")
    f_ref = open(ref_file, encoding="utf-8")
    current_time = time.strftime('%Y-%m-%d-%H-%M-%S', time.localtime())
    tmp_dir = ".rouge-tmp-{}".format(current_time)
    try:
        if not os.path.isdir(tmp_dir):
            os.mkdir(tmp_dir)
            os.mkdir(tmp_dir + "/candidate")
            os.mkdir(tmp_dir + "/reference")
        candidates = [line.strip() for line in f_cand]
        references = [line.strip() for line in f_ref]
        assert len(candidates) == len(references)
        cnt = len(candidates)
        for i in range(cnt):
            if len(references[i]) < 1:
                continue
            with open(tmp_dir + "/candidate/cand.{}.txt".format(i), "w",
                      encoding="utf-8") as f:
                f.write(candidates[i])
            with open(tmp_dir + "/reference/ref.{}.txt".format(i), "w",
                      encoding="utf-8") as f:
                f.write(references[i])
        f_cand.close()
        f_ref.close()
        r = pyrouge.Rouge155()
        r.model_dir = tmp_dir + "/reference/"
        r.system_dir = tmp_dir + "/candidate/"
        r.model_filename_pattern = 'ref.#ID#.txt'
        r.system_filename_pattern = 'cand.(\d+).txt'
        rouge_results = r.convert_and_evaluate()
        results_dict = r.output_to_dict(rouge_results)
        return results_dict
    finally:
        pass
        if os.path.isdir(tmp_dir):
            shutil.rmtree(tmp_dir) 

def test_rouge(cand, ref):
    """Calculate ROUGE scores of sequences passed as an iterator
       e.g. a list of str, an open file, StringIO or even sys.stdin
    """
    current_time = time.strftime('%Y-%m-%d-%H-%M-%S', time.localtime())
    tmp_dir = ".rouge-tmp-{}".format(current_time)
    try:
        if not os.path.isdir(tmp_dir):
            os.mkdir(tmp_dir)
            os.mkdir(tmp_dir + "/candidate")
            os.mkdir(tmp_dir + "/reference")
        candidates = [line.strip() for line in cand]
        references = [line.strip() for line in ref]
        assert len(candidates) == len(references)
        cnt = len(candidates)
        for i in range(cnt):
            if len(references[i]) < 1:
                continue
            with open(tmp_dir + "/candidate/cand.{}.txt".format(i), "w",
                      encoding="utf-8") as f:
                f.write(candidates[i])
            with open(tmp_dir + "/reference/ref.{}.txt".format(i), "w",
                      encoding="utf-8") as f:
                f.write(references[i])
        r = pyrouge.Rouge155()
        r.model_dir = tmp_dir + "/reference/"
        r.system_dir = tmp_dir + "/candidate/"
        r.model_filename_pattern = 'ref.#ID#.txt'
        r.system_filename_pattern = 'cand.(\d+).txt'
        rouge_results = r.convert_and_evaluate()
        results_dict = r.output_to_dict(rouge_results)
        return results_dict
    finally:
        pass
        if os.path.isdir(tmp_dir):
            shutil.rmtree(tmp_dir) 

def test_rouge(cand, ref):
    """Calculate ROUGE scores of sequences passed as an iterator
       e.g. a list of str, an open file, StringIO or even sys.stdin
    """
    current_time = time.strftime('%Y-%m-%d-%H-%M-%S', time.localtime())
    tmp_dir = ".rouge-tmp-{}".format(current_time)
    try:
        if not os.path.isdir(tmp_dir):
            os.mkdir(tmp_dir)
            os.mkdir(tmp_dir + "/candidate")
            os.mkdir(tmp_dir + "/reference")
        candidates = [line.strip() for line in cand]
        references = [line.strip() for line in ref]
        assert len(candidates) == len(references)
        cnt = len(candidates)
        for i in range(cnt):
            if len(references[i]) < 1:
                continue
            with open(tmp_dir + "/candidate/cand.{}.txt".format(i), "w",
                      encoding="utf-8") as f:
                f.write(candidates[i])
            with open(tmp_dir + "/reference/ref.{}.txt".format(i), "w",
                      encoding="utf-8") as f:
                f.write(references[i])
        r = pyrouge.Rouge155()
        r.model_dir = tmp_dir + "/reference/"
        r.system_dir = tmp_dir + "/candidate/"
        r.model_filename_pattern = 'ref.#ID#.txt'
        r.system_filename_pattern = 'cand.(\d+).txt'
        rouge_results = r.convert_and_evaluate()
        results_dict = r.output_to_dict(rouge_results)
        return results_dict
    finally:
        pass
        if os.path.isdir(tmp_dir):
            shutil.rmtree(tmp_dir) 

def rouge():
    r = Rouge155()
    r.home_dir = '.'
    r.system_dir = 'hyp'
    r.model_dir =  'ref'

    r.system_filename_pattern = '(\d+).txt'
    r.model_filename_pattern = '#ID#.txt'

    command = '-e /YOUR/PATH/TO/ROUGE-1.5.5/data -a -c 95 -m -n 2 -b 75'
    output = r.convert_and_evaluate(rouge_args=command)
    print(output) 

def test_rouge(cand, ref):
    """Calculate ROUGE scores of sequences passed as an iterator
       e.g. a list of str, an open file, StringIO or even sys.stdin
    """
    current_time = time.strftime('%Y-%m-%d-%H-%M-%S', time.localtime())
    tmp_dir = ".rouge-tmp-{}".format(current_time)
    try:
        if not os.path.isdir(tmp_dir):
            os.mkdir(tmp_dir)
            os.mkdir(tmp_dir + "/candidate")
            os.mkdir(tmp_dir + "/reference")
        candidates = [line.strip() for line in cand]
        references = [line.strip() for line in ref]
        assert len(candidates) == len(references)
        cnt = len(candidates)
        for i in range(cnt):
            if len(references[i]) < 1:
                continue
            with open(tmp_dir + "/candidate/cand.{}.txt".format(i), "w",
                      encoding="utf-8") as f:
                f.write(candidates[i])
            with open(tmp_dir + "/reference/ref.{}.txt".format(i), "w",
                      encoding="utf-8") as f:
                f.write(references[i])
        r = pyrouge.Rouge155()
        r.model_dir = tmp_dir + "/reference/"
        r.system_dir = tmp_dir + "/candidate/"
        r.model_filename_pattern = 'ref.#ID#.txt'
        r.system_filename_pattern = r'cand.(\d+).txt'
        rouge_results = r.convert_and_evaluate()
        results_dict = r.output_to_dict(rouge_results)
        return results_dict
    finally:
        pass
        if os.path.isdir(tmp_dir):
            shutil.rmtree(tmp_dir) 

def rouge_eval(ref_dir, dec_dir):
  """Evaluate the files in ref_dir and dec_dir with pyrouge, returning results_dict"""
  r = pyrouge.Rouge155()
  r.model_filename_pattern = '#ID#_reference.txt'
  r.system_filename_pattern = '(\d+)_decoded.txt'
  r.model_dir = ref_dir
  r.system_dir = dec_dir
  logging.getLogger('global').setLevel(logging.WARNING) # silence pyrouge logging
  rouge_results = r.convert_and_evaluate()
  return r.output_to_dict(rouge_results) 

def rouge_eval(ref_dir, dec_dir):
  """Evaluate the files in ref_dir and dec_dir with pyrouge, returning results_dict"""
  r = pyrouge.Rouge155()
  r.model_filename_pattern = '#ID#_reference.txt'
  r.system_filename_pattern = '(\d+)_decoded.txt'
  r.model_dir = ref_dir
  r.system_dir = dec_dir
  logging.getLogger('global').setLevel(logging.WARNING) # silence pyrouge logging
  rouge_results = r.convert_and_evaluate()
  return r.output_to_dict(rouge_results) 

def rouge_eval(ref_dir, dec_dir):
  """Evaluate the files in ref_dir and dec_dir with pyrouge, returning results_dict"""
  r = pyrouge.Rouge155()
  r.model_filename_pattern = '#ID#_reference.txt'
  r.system_filename_pattern = '(\d+)_decoded.txt'
  r.model_dir = ref_dir
  r.system_dir = dec_dir
  logging.getLogger('global').setLevel(logging.WARNING) # silence pyrouge logging
  rouge_results = r.convert_and_evaluate()
  return r.output_to_dict(rouge_results) 

def rouge_eval(ref_dir, dec_dir):
  """Evaluate the files in ref_dir and dec_dir with pyrouge, returning results_dict"""
  r = pyrouge.Rouge155()
  r.model_filename_pattern = '#ID#_reference.txt'
  r.system_filename_pattern = '(\d+)_decoded.txt'
  r.model_dir = ref_dir
  r.system_dir = dec_dir
  logging.getLogger('global').setLevel(logging.WARNING) # silence pyrouge logging
  rouge_results = r.convert_and_evaluate()
  return r.output_to_dict(rouge_results) 

def test_paths(self):
        rouge = Rouge155()

        def get_home_from_settings():
            with open(rouge.settings_file) as f:
                for line in f.readlines():
                    if line.startswith("home_dir"):
                        rouge_home = line.split("=")[1].strip()
            return rouge_home

        self.assertEqual(rouge.home_dir, get_home_from_settings())
        self.assertTrue(os.path.exists(rouge.bin_path))
        self.assertTrue(os.path.exists(rouge.data_dir))

        wrong_path = "/nonexisting/path/rewafafkljaerearjafankwe3"
        with self.assertRaises(Exception) as context:
            rouge.system_dir = wrong_path
        self.assertEqual(
            str(context.exception),
            "Cannot set {} directory because the path {} does not "
            "exist.".format("system", wrong_path))
        right_path = add_data_path("systems")
        rouge.system_dir = right_path
        self.assertEqual(rouge.system_dir, right_path)

        with self.assertRaises(Exception) as context:
            rouge.model_dir = wrong_path
        self.assertEqual(
            str(context.exception),
            "Cannot set {} directory because the path {} does not "
            "exist.".format("model", wrong_path))
        right_path = add_data_path("models")
        rouge.model_dir = right_path
        self.assertEqual(rouge.model_dir, right_path) 

def test_wrong_system_pattern(self):
        wrong_regexp = "adfdas454fd"
        rouge = Rouge155()
        rouge.system_dir = add_data_path("systems")
        rouge.model_dir = add_data_path("models")
        #rouge.system_filename_pattern = "SL.P.10.R.11.SL062003-(\d+).html"
        rouge.system_filename_pattern = wrong_regexp
        rouge.model_filename_pattern = "SL.P.10.R.[A-D].SL062003-#ID#.html"
        with self.assertRaises(Exception) as context:
            rouge.evaluate()
        self.assertEqual(
            str(context.exception),
            "Did not find any files matching the pattern {} in the system "
            "summaries directory {}.".format(wrong_regexp, rouge.system_dir)) 

def test_text_conversion(self):
        rouge = Rouge155()
        text = str_from_file(add_data_path("spl_test_doc"))
        html = rouge.convert_text_to_rouge_format(text, "D00000.M.100.A.C")
        target = str_from_file(add_data_path("spl_test_doc.html"))
        self.assertEqual(html, target)

    # only run this test if BeautifulSoup is installed 

def test_config_file(self):
        rouge = Rouge155()
        rouge.system_dir = add_data_path("systems")
        rouge.model_dir = add_data_path("models")
        rouge.system_filename_pattern = "SL.P.10.R.11.SL062003-(\d+).html"
        rouge.model_filename_pattern = "SL.P.10.R.[A-D].SL062003-#ID#.html"
        rouge.config_file = add_data_path("config_test.xml")
        rouge.write_config(system_id=11)
        self.assertTrue(xml_equal(
            rouge.config_file,
            add_data_path("ROUGE-test_11.xml")))
        os.remove(rouge.config_file) 

def test_rouge(cand, ref):
    """Calculate ROUGE scores of sequences passed as an iterator
       e.g. a list of str, an open file, StringIO or even sys.stdin
    """
    current_time = time.strftime('%Y-%m-%d-%H-%M-%S', time.localtime())
    tmp_dir = ".rouge-tmp-{}".format(current_time)
    try:
        if not os.path.isdir(tmp_dir):
            os.mkdir(tmp_dir)
            os.mkdir(tmp_dir + "/candidate")
            os.mkdir(tmp_dir + "/reference")
        candidates = [line.strip() for line in cand]
        references = [line.strip() for line in ref]
        assert len(candidates) == len(references)
        cnt = len(candidates)
        for i in range(cnt):
            if len(references[i]) < 1:
                continue
            with open(tmp_dir + "/candidate/cand.{}.txt".format(i), "w",
                      encoding="utf-8") as f:
                f.write(candidates[i])
            with open(tmp_dir + "/reference/ref.{}.txt".format(i), "w",
                      encoding="utf-8") as f:
                f.write(references[i])
        r = pyrouge.Rouge155()
        r.model_dir = tmp_dir + "/reference/"
        r.system_dir = tmp_dir + "/candidate/"
        r.model_filename_pattern = 'ref.#ID#.txt'
        r.system_filename_pattern = r'cand.(\d+).txt'
        rouge_results = r.convert_and_evaluate()
        results_dict = r.output_to_dict(rouge_results)
        return results_dict
    finally:
        pass
        if os.path.isdir(tmp_dir):
            shutil.rmtree(tmp_dir) 

def main(_):
  rouge = pyrouge.Rouge155()
  rouge.log.setLevel(logging.ERROR)
  rouge.system_filename_pattern = "rouge.(\\d+).txt"
  rouge.model_filename_pattern = "rouge.[A-Z].#ID#.txt"

  tf.logging.set_verbosity(tf.logging.INFO)

  tmpdir = mkdtemp()
  tf.logging.info("tmpdir: %s" % tmpdir)
  # system = decodes/predictions
  system_dir = os.path.join(tmpdir, "system")
  # model = targets/gold
  model_dir = os.path.join(tmpdir, "model")
  os.mkdir(system_dir)
  os.mkdir(model_dir)

  rouge.system_dir = system_dir
  rouge.model_dir = model_dir

  prep_data(rouge.system_dir, rouge.model_dir)

  rouge_scores = rouge.convert_and_evaluate()
  rouge_scores = rouge.output_to_dict(rouge_scores)
  for prefix in ["rouge_1", "rouge_2", "rouge_l"]:
    for suffix in ["f_score", "precision", "recall"]:
      key = "_".join([prefix, suffix])
      tf.logging.info("%s: %.4f" % (key, rouge_scores[key]))

  # clean up after pyrouge
  shutil.rmtree(tmpdir)
  shutil.rmtree(rouge._config_dir)  # pylint: disable=protected-access
  shutil.rmtree(os.path.split(rouge._system_dir)[0])  # pylint: disable=protected-access 

def test_rouge(cand, ref):
    """Calculate ROUGE scores of sequences passed as an iterator
       e.g. a list of str, an open file, StringIO or even sys.stdin
    """
    current_time = time.strftime('%Y-%m-%d-%H-%M-%S', time.localtime())
    tmp_dir = ".rouge-tmp-{}".format(current_time)
    try:
        if not os.path.isdir(tmp_dir):
            os.mkdir(tmp_dir)
            os.mkdir(tmp_dir + "/candidate")
            os.mkdir(tmp_dir + "/reference")
        candidates = [line.strip() for line in cand]
        references = [line.strip() for line in ref]
        assert len(candidates) == len(references)
        cnt = len(candidates)
        for i in range(cnt):
            if len(references[i]) < 1:
                continue
            with open(tmp_dir + "/candidate/cand.{}.txt".format(i), "w",
                      encoding="utf-8") as f:
                f.write(candidates[i])
            with open(tmp_dir + "/reference/ref.{}.txt".format(i), "w",
                      encoding="utf-8") as f:
                f.write(references[i])
        r = pyrouge.Rouge155()
        r.model_dir = tmp_dir + "/reference/"
        r.system_dir = tmp_dir + "/candidate/"
        r.model_filename_pattern = 'ref.#ID#.txt'
        r.system_filename_pattern = 'cand.(\d+).txt'
        rouge_results = r.convert_and_evaluate()
        results_dict = r.output_to_dict(rouge_results)
        return results_dict
    finally:
        pass
        if os.path.isdir(tmp_dir):
            shutil.rmtree(tmp_dir) 

def rouge_eval(ref_dir, dec_dir):
  """Evaluate the files in ref_dir and dec_dir with pyrouge, returning results_dict"""
  r = pyrouge.Rouge155()
  r.model_filename_pattern = '#ID#_reference.txt'
  r.system_filename_pattern = '(\d+)_decoded.txt'
  r.model_dir = ref_dir
  r.system_dir = dec_dir
  logging.getLogger('global').setLevel(logging.WARNING) # silence pyrouge logging
  rouge_results = r.convert_and_evaluate()
  return r.output_to_dict(rouge_results) 

def test_rouge(cand, ref):
    """Calculate ROUGE scores of sequences passed as an iterator
       e.g. a list of str, an open file, StringIO or even sys.stdin
    """
    current_time = time.strftime('%Y-%m-%d-%H-%M-%S', time.localtime())
    tmp_dir = ".rouge-tmp-{}".format(current_time)
    try:
        if not os.path.isdir(tmp_dir):
            os.mkdir(tmp_dir)
            os.mkdir(tmp_dir + "/candidate")
            os.mkdir(tmp_dir + "/reference")
        candidates = [line.strip() for line in cand]
        references = [line.strip() for line in ref]
        assert len(candidates) == len(references)
        cnt = len(candidates)
        for i in range(cnt):
            if len(references[i]) < 1:
                continue
            with open(tmp_dir + "/candidate/cand.{}.txt".format(i), "w",
                      encoding="utf-8") as f:
                f.write(candidates[i])
            with open(tmp_dir + "/reference/ref.{}.txt".format(i), "w",
                      encoding="utf-8") as f:
                f.write(references[i])
        r = pyrouge.Rouge155()
        r.model_dir = tmp_dir + "/reference/"
        r.system_dir = tmp_dir + "/candidate/"
        r.model_filename_pattern = 'ref.#ID#.txt'
        r.system_filename_pattern = 'cand.(\d+).txt'
        rouge_results = r.convert_and_evaluate()
        results_dict = r.output_to_dict(rouge_results)
        return results_dict
    finally:
        pass
        if os.path.isdir(tmp_dir):
            shutil.rmtree(tmp_dir) 

def main(_):
  rouge = pyrouge.Rouge155()
  rouge.log.setLevel(logging.ERROR)
  rouge.system_filename_pattern = "rouge.(\\d+).txt"
  rouge.model_filename_pattern = "rouge.[A-Z].#ID#.txt"

  tf.logging.set_verbosity(tf.logging.INFO)

  tmpdir = mkdtemp()
  tf.logging.info("tmpdir: %s" % tmpdir)
  # system = decodes/predictions
  system_dir = os.path.join(tmpdir, "system")
  # model = targets/gold
  model_dir = os.path.join(tmpdir, "model")
  os.mkdir(system_dir)
  os.mkdir(model_dir)

  rouge.system_dir = system_dir
  rouge.model_dir = model_dir

  prep_data(rouge.system_dir, rouge.model_dir)

  rouge_scores = rouge.convert_and_evaluate()
  rouge_scores = rouge.output_to_dict(rouge_scores)
  for prefix in ["rouge_1", "rouge_2", "rouge_l"]:
    for suffix in ["f_score", "precision", "recall"]:
      key = "_".join([prefix, suffix])
      tf.logging.info("%s: %.4f" % (key, rouge_scores[key]))

  # clean up after pyrouge
  shutil.rmtree(tmpdir)
  shutil.rmtree(rouge._config_dir)  # pylint: disable=protected-access
  shutil.rmtree(os.path.split(rouge._system_dir)[0])  # pylint: disable=protected-access 

def rouge_eval(ref_dir, dec_dir):
  """Evaluate the files in ref_dir and dec_dir with pyrouge, returning results_dict"""
  r = pyrouge.Rouge155()
  r.model_filename_pattern = '#ID#_reference.txt'
  r.system_filename_pattern = '(\d+)_decoded.txt'
  r.model_dir = ref_dir
  r.system_dir = dec_dir
  logging.getLogger('global').setLevel(logging.WARNING) # silence pyrouge logging
  rouge_results = r.convert_and_evaluate()
  return r.output_to_dict(rouge_results) 

def test_rouge(cand, ref):
    """Calculate ROUGE scores of sequences passed as an iterator
       e.g. a list of str, an open file, StringIO or even sys.stdin
    """
    current_time = time.strftime('%Y-%m-%d-%H-%M-%S', time.localtime())
    tmp_dir = ".rouge-tmp-{}".format(current_time)
    try:
        if not os.path.isdir(tmp_dir):
            os.mkdir(tmp_dir)
            os.mkdir(tmp_dir + "/candidate")
            os.mkdir(tmp_dir + "/reference")
        candidates = [line.strip() for line in cand]
        references = [line.strip() for line in ref]
        assert len(candidates) == len(references)
        cnt = len(candidates)
        for i in range(cnt):
            if len(references[i]) < 1:
                continue
            with open(tmp_dir + "/candidate/cand.{}.txt".format(i), "w",
                      encoding="utf-8") as f:
                f.write(candidates[i])
            with open(tmp_dir + "/reference/ref.{}.txt".format(i), "w",
                      encoding="utf-8") as f:
                f.write(references[i])
        r = pyrouge.Rouge155()
        r.model_dir = tmp_dir + "/reference/"
        r.system_dir = tmp_dir + "/candidate/"
        r.model_filename_pattern = 'ref.#ID#.txt'
        r.system_filename_pattern = r'cand.(\d+).txt'
        rouge_results = r.convert_and_evaluate()
        results_dict = r.output_to_dict(rouge_results)
        return results_dict
    finally:
        pass
        if os.path.isdir(tmp_dir):
            shutil.rmtree(tmp_dir) 

def main(_):
  rouge = pyrouge.Rouge155()
  rouge.log.setLevel(logging.ERROR)
  rouge.system_filename_pattern = "rouge.(\\d+).txt"
  rouge.model_filename_pattern = "rouge.[A-Z].#ID#.txt"

  tf.logging.set_verbosity(tf.logging.INFO)

  tmpdir = mkdtemp()
  tf.logging.info("tmpdir: %s" % tmpdir)
  # system = decodes/predictions
  system_dir = os.path.join(tmpdir, "system")
  # model = targets/gold
  model_dir = os.path.join(tmpdir, "model")
  os.mkdir(system_dir)
  os.mkdir(model_dir)

  rouge.system_dir = system_dir
  rouge.model_dir = model_dir

  prep_data(rouge.system_dir, rouge.model_dir)

  rouge_scores = rouge.convert_and_evaluate()
  rouge_scores = rouge.output_to_dict(rouge_scores)
  for prefix in ["rouge_1", "rouge_2", "rouge_l"]:
    for suffix in ["f_score", "precision", "recall"]:
      key = "_".join([prefix, suffix])
      tf.logging.info("%s: %.4f" % (key, rouge_scores[key]))

  # clean up after pyrouge
  shutil.rmtree(tmpdir)
  shutil.rmtree(rouge._config_dir)  # pylint: disable=protected-access
  shutil.rmtree(os.path.split(rouge._system_dir)[0])  # pylint: disable=protected-access 

def print_pyrouge(config):
    logging.getLogger('global').setLevel(logging.WARNING) # silence pyrouge logging

    r = pyrouge.Rouge155()
    r.system_filename_pattern = '(\d+).txt'
    r.model_filename_pattern = '#ID#.txt'
    
    r.system_dir = config.outdir
    r.model_dir = config.refdir

    rouge_results = r.convert_and_evaluate()
    rouge_dict = r.output_to_dict(rouge_results)
    
    print(rouge_results) 

def main(_):
  rouge = pyrouge.Rouge155()
  rouge.log.setLevel(logging.ERROR)
  rouge.system_filename_pattern = "rouge.(\\d+).txt"
  rouge.model_filename_pattern = "rouge.[A-Z].#ID#.txt"

  tf.logging.set_verbosity(tf.logging.INFO)

  tmpdir = mkdtemp()
  tf.logging.info("tmpdir: %s" % tmpdir)
  # system = decodes/predictions
  system_dir = os.path.join(tmpdir, "system")
  # model = targets/gold
  model_dir = os.path.join(tmpdir, "model")
  os.mkdir(system_dir)
  os.mkdir(model_dir)

  rouge.system_dir = system_dir
  rouge.model_dir = model_dir

  prep_data(rouge.system_dir, rouge.model_dir)

  rouge_scores = rouge.convert_and_evaluate()
  rouge_scores = rouge.output_to_dict(rouge_scores)
  for prefix in ["rouge_1", "rouge_2", "rouge_l"]:
    for suffix in ["f_score", "precision", "recall"]:
      key = "_".join([prefix, suffix])
      tf.logging.info("%s: %.4f" % (key, rouge_scores[key]))

  # clean up after pyrouge
  shutil.rmtree(tmpdir)
  shutil.rmtree(rouge._config_dir)  # pylint: disable=protected-access
  shutil.rmtree(os.path.split(rouge._system_dir)[0])  # pylint: disable=protected-access