com.ibm.icu.text.RuleBasedBreakIterator Java Examples

@Override
public String getType(int script, int ruleStatus) {
  switch (ruleStatus) {
    case RuleBasedBreakIterator.WORD_IDEO:
      return WORD_IDEO;
    case RuleBasedBreakIterator.WORD_KANA:
      return script == UScript.HIRAGANA ? WORD_HIRAGANA : WORD_KATAKANA;
    case RuleBasedBreakIterator.WORD_LETTER:
      return script == UScript.HANGUL ? WORD_HANGUL : WORD_LETTER;
    case RuleBasedBreakIterator.WORD_NUMBER:
      return WORD_NUMBER;
    case EMOJI_SEQUENCE_STATUS:
      return WORD_EMOJI;
    default: /* some other custom code */
      return "<OTHER>";
  }
}
 

private LineBreakIterator maybeLoad(Reporter reporter) {
    LineBreakIterator iterator = this.iterator;
    if (iterator != null)
        return iterator;
    else {
        BreakIterator bi = null;
        InputStream is = null;
        try {
            URL rulesLocator = getRulesLocator(name, RULES_BINARY_EXT);
            if (rulesLocator != null) {
                is = rulesLocator.openStream();
                bi = RuleBasedBreakIterator.getInstanceFromCompiledRules(is);
                reporter.logInfo(reporter.message("*KEY*", "Loaded rules based break iterator from ''{0}''.", rulesLocator.toString()));
            } else
                bi = BreakIterator.getCharacterInstance();
        } catch (IOException e) {
        } finally {
            IOUtil.closeSafely(is);
        }
        if (bi != null) {
            return this.iterator = new LineBreakIterator(bi);
        } else
            return null;
    }
}
 

private int calcStatus(int current, int next) {
    if (current == BreakIterator.DONE || next == BreakIterator.DONE) {
        return RuleBasedBreakIterator.WORD_NONE;
    }
    int begin = start + current;
    int end = start + next;
    int codepoint;
    for (int i = begin; i < end; i += UTF16.getCharCount(codepoint)) {
        codepoint = UTF16.charAt(text, 0, end, begin);
        if (UCharacter.isDigit(codepoint)) {
            return RuleBasedBreakIterator.WORD_NUMBER;
        } else if (UCharacter.isLetter(codepoint)) {
            return RuleBasedBreakIterator.WORD_LETTER;
        }
    }
    return RuleBasedBreakIterator.WORD_NONE;
}
 

void setText(char text[], int start, int length) {
  this.text = text;
  this.start = start;
  textIterator.setText(text, start, length);
  rbbi.setText(textIterator);
  status = RuleBasedBreakIterator.WORD_NONE;
}
 

private BreakIterator parseRules(String filename, ResourceLoader loader) throws IOException {
  StringBuilder rules = new StringBuilder();
  InputStream rulesStream = loader.openResource(filename);
  BufferedReader reader = new BufferedReader
      (IOUtils.getDecodingReader(rulesStream, StandardCharsets.UTF_8));
  String line = null;
  while ((line = reader.readLine()) != null) {
    if ( ! line.startsWith("#"))
      rules.append(line);
    rules.append('\n');
  }
  reader.close();
  return new RuleBasedBreakIterator(rules.toString());
}
 

@Override
public RuleBasedBreakIterator getBreakIterator(int script) {
  switch(script) {
    case UScript.JAPANESE: return (RuleBasedBreakIterator)cjkBreakIterator.clone();
    case UScript.MYANMAR: 
      if (myanmarAsWords) {
        return (RuleBasedBreakIterator)defaultBreakIterator.clone();
      } else {
        return (RuleBasedBreakIterator)myanmarSyllableIterator.clone();
      }
    default: return (RuleBasedBreakIterator)defaultBreakIterator.clone();
  }
}
 

private static RuleBasedBreakIterator readBreakIterator(String filename) {
  InputStream is = 
    DefaultICUTokenizerConfig.class.getResourceAsStream(filename);
  try {
    RuleBasedBreakIterator bi = 
      RuleBasedBreakIterator.getInstanceFromCompiledRules(is);
    is.close();
    return bi;
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 

static void compile(File srcDir, File destDir) throws Exception {
  File files[] = srcDir.listFiles(new FilenameFilter() {
    public boolean accept(File dir, String name) {
      return name.endsWith("rbbi");
    }});
  if (files == null) throw new IOException("Path does not exist: " + srcDir);
  for (int i = 0; i < files.length; i++) {
    File file = files[i];
    File outputFile = new File(destDir, 
        file.getName().replaceAll("rbbi$", "brk"));
    String rules = getRules(file);
    System.err.print("Compiling " + file.getName() + " to "
        + outputFile.getName() + ": ");
    /*
     * if there is a syntax error, compileRules() may succeed. the way to
     * check is to try to instantiate from the string. additionally if the
     * rules are invalid, you can get a useful syntax error.
     */
    try {
      new RuleBasedBreakIterator(rules);
    } catch (IllegalArgumentException e) {
      /*
       * do this intentionally, so you don't get a massive stack trace
       * instead, get a useful syntax error!
       */
      System.err.println(e.getMessage());
      System.exit(1);
    }
    FileOutputStream os = new FileOutputStream(outputFile);
    RuleBasedBreakIterator.compileRules(rules, os);
    os.close();
    System.err.println(outputFile.length() + " bytes.");
  }
}
 

private static RuleBasedBreakIterator readBreakIterator(String resourceName) {
    try (InputStream inputStream = DefaultIcuTokenizerConfig.class.getResourceAsStream(resourceName)) {
        return RuleBasedBreakIterator.getInstanceFromCompiledRules(inputStream);
    } catch (IOException e) {
        throw new UncheckedIOException("unable to load resource " + resourceName + " " + e.getMessage(), e);
    }
}
 

@Override
public String getType(int script, int ruleStatus) {
    switch (ruleStatus) {
        case RuleBasedBreakIterator.WORD_IDEO:
            return WORD_IDEO;
        case RuleBasedBreakIterator.WORD_KANA:
            return script == UScript.HIRAGANA ? WORD_HIRAGANA : WORD_KATAKANA;
        case RuleBasedBreakIterator.WORD_LETTER:
            return script == UScript.HANGUL ? WORD_HANGUL : WORD_LETTER;
        case RuleBasedBreakIterator.WORD_NUMBER:
            return WORD_NUMBER;
        default: /* some other custom code */
            return "<OTHER>";
    }
}
 

/**
 * If its a RuleBasedBreakIterator, the rule status can be used for token type. If its
 * any other BreakIterator, the rulestatus method is not available, so treat
 * it like a generic BreakIterator.
 */
static BreakIteratorWrapper wrap(BreakIterator breakIterator) {
    if (breakIterator instanceof RuleBasedBreakIterator) {
        return new RBBIWrapper((RuleBasedBreakIterator) breakIterator);
    } else {
        return new BIWrapper(breakIterator);
    }
}
 

public void compile(InputStream inputStream, OutputStream outputStream) throws IOException {
    String rules = getRules(inputStream);
    try (OutputStream os = outputStream) {
        new RuleBasedBreakIterator(rules);
        RuleBasedBreakIterator.compileRules(rules, os);
    } catch (IllegalArgumentException e) {
        logger.error(e.getMessage(), e);
    }
}
 

BreakIteratorWrapper(RuleBasedBreakIterator rbbi) {
  this.rbbi = rbbi;
}
 

/** Return a breakiterator capable of processing a given script. */
public abstract RuleBasedBreakIterator getBreakIterator(int script);
 

RBBIWrapper(RuleBasedBreakIterator rbbi) {
    this.rbbi = rbbi;
}
 

@Override
void setText(CharacterIterator text) {
    bi.setText(text);
    status = RuleBasedBreakIterator.WORD_NONE;
}