com.optimaize.langdetect.profiles.LanguageProfile Java Examples

static private void initOptimaize() {
    synchronized (initGuard) {
        if ((textObjectFactory != null) && (languageDetector != null)) return;

        // origin: https://github.com/optimaize/language-detector
        // load all languages:
        List<LanguageProfile> languageProfiles;
        try {
            languageProfiles = new LanguageProfileReader().readAllBuiltIn();
        } catch (IOException e) {
            throw new UncheckedIOException(e);
        }

        //build language detector:
        languageDetector = LanguageDetectorBuilder.create(NgramExtractors.standard())
                                                  .withProfiles(languageProfiles)
                                                  .build();

        //create a text object factory
        textObjectFactory = CommonTextObjectFactories.forDetectingOnLargeText();
    }
}
 

private static LanguageDetector getLanguageDetector(){
    if (languageDetector == null){
        try {
            List<String> languages = new ArrayList<>();
            for(Language lg : Language.values())
                languages.add(lg.getAbrev().toLowerCase());

            List<LanguageProfile> languageProfiles = new LanguageProfileReader().read(languages);
            languageDetector = LanguageDetectorBuilder.create(NgramExtractors.standard())
                            .withProfiles(languageProfiles).build();
        }
        catch (IOException e) {
            LOG.error("Translator.getLanguageDetector", e);
        }
    }
    return languageDetector;
}
 

private LanguageDetectionSingleton() throws IOException {
	// load all languages
	List<LanguageProfile> languageProfiles = new LanguageProfileReader().readAllBuiltIn();

	// build language detector
	languageDetector = LanguageDetectorBuilder.create(NgramExtractors.standard())
			.withProfiles(languageProfiles)
			.build();

	// create a text object factory
	textObjectFactory = CommonTextObjectFactories.forDetectingShortCleanText();
}
 

private LanguageDetector makeNewDetector() throws IOException {
    LanguageDetectorBuilder builder = LanguageDetectorBuilder.create(NgramExtractors.standard())
        .shortTextAlgorithm(50)
        .prefixFactor(1.5)
        .suffixFactor(2.0);

    LangProfileReader langProfileReader = new LangProfileReader();
    for (String language : ImmutableList.of("en", "fr", "nl", "de")) {
        LangProfile langProfile = langProfileReader.read(LanguageDetectorImplTest.class.getResourceAsStream("/languages/" + language));
        LanguageProfile languageProfile = OldLangProfileConverter.convert(langProfile);
        builder.withProfile(languageProfile);
    }

    return builder.build();
}
 

public DataLanguageDetectorImplTest() throws IOException {
    List<LanguageProfile> languageProfiles = new LanguageProfileReader().readAllBuiltIn();

    shortDetector = LanguageDetectorBuilder.create(NgramExtractors.standard())
            .shortTextAlgorithm(100)
            .withProfiles(languageProfiles)
            .build();

    longDetector = LanguageDetectorBuilder.create(NgramExtractors.standard())
            .shortTextAlgorithm(0)
            .withProfiles(new LanguageProfileReader().readAllBuiltIn())
            .build();
}
 

/**
 * Using all language profiles from the given directory.
 */
private LanguageDetector makeDetector() throws IOException {
    double alpha = getParamDouble("alpha", DEFAULT_ALPHA);
    String profileDirectory = requireParamString("directory") + "/";
    Optional<Long> seed = Optional.fromNullable(getParamLongOrNull("seed"));

    List<LanguageProfile> languageProfiles = new LanguageProfileReader().readAll(new File(profileDirectory));

    return LanguageDetectorBuilder.create(NgramExtractors.standard())
            .alpha(alpha)
            .seed(seed)
            .shortTextAlgorithm(50)
            .withProfiles(languageProfiles)
            .build();
}
 

/**
 * @param gramLengths for example [1,2,3]
 * @throws java.lang.IllegalArgumentException if languageProfiles or gramLengths is empty, or if one of the
 *         languageProfiles does not have the grams of the required sizes.
 */
@NotNull
public static NgramFrequencyData create(@NotNull Collection<LanguageProfile> languageProfiles, @NotNull Collection<Integer> gramLengths) throws IllegalArgumentException {
    if (languageProfiles.isEmpty()) throw new IllegalArgumentException("No languageProfiles provided!");
    if (gramLengths.isEmpty()) throw new IllegalArgumentException("No gramLengths provided!");

    Map<String, double[]> wordLangProbMap = new HashMap<>();
    List<LdLocale> langlist = new ArrayList<>();
    int langsize = languageProfiles.size();

    int index = -1;
    for (LanguageProfile profile : languageProfiles) {
        index++;

        langlist.add( profile.getLocale() );

        for (Integer gramLength : gramLengths) {
            if (!profile.getGramLengths().contains(gramLength)) {
                throw new IllegalArgumentException("The language profile for "+profile.getLocale()+" does not contain "+gramLength+"-grams!");
            }
            for (Map.Entry<String, Integer> ngramEntry : profile.iterateGrams(gramLength)) {
                String ngram      = ngramEntry.getKey();
                Integer frequency = ngramEntry.getValue();
                if (!wordLangProbMap.containsKey(ngram)) {
                    wordLangProbMap.put(ngram, new double[langsize]);
                }
                double prob = frequency.doubleValue() / profile.getNumGramOccurrences(ngram.length());
                wordLangProbMap.get(ngram)[index] = prob;
            }
        }
    }

    return new NgramFrequencyData(wordLangProbMap, langlist);
}
 

/**
 * Remove potential LanguageProfiles, e.g. in combination with {@link #loadAllBuiltInLanguageProfiles()}.
 * @param isoString the ISO string of the LanguageProfile to be removed.
 */
public LanguageProfileValidator removeLanguageProfile(final String isoString) {
    Iterables.removeIf(this.languageProfiles, new Predicate<LanguageProfile>() {
        @Override
        public boolean apply(LanguageProfile languageProfile) {
            return languageProfile.getLocale().getLanguage().equals(isoString);
        }
    });
    return this;
}
 

/**
 * @throws IllegalStateException if a profile for the same language was added already (must be a userland bug).
 */
public LanguageDetectorBuilder withProfiles(Iterable<LanguageProfile> languageProfiles) throws IllegalStateException {
    for (LanguageProfile languageProfile : languageProfiles) {
        withProfile(languageProfile);
    }
    return this;
}
 

/**
 * @throws IllegalStateException if a profile for the same language was added already (must be a userland bug).
 */
public LanguageDetectorBuilder withProfile(LanguageProfile languageProfile) throws IllegalStateException {
    if (langsAdded.contains(languageProfile.getLocale())) {
        throw new IllegalStateException("A language profile for language "+languageProfile.getLocale()+" was added already!");
    }
    for (Integer gramLength : ngramExtractor.getGramLengths()) {
        if (!languageProfile.getGramLengths().contains(gramLength)) {
            throw new IllegalArgumentException("The NgramExtractor is set to handle "+gramLength+"-grams but the given language profile for "+languageProfile.getLocale()+" does not support this!");
        }
    }
    langsAdded.add(languageProfile.getLocale());
    languageProfiles.add(languageProfile);
    return this;
}
 

@Override
public void doInitialize(UimaContext aContext) throws ResourceInitializationException {
  try {
    List<LanguageProfile> languageProfiles = new LanguageProfileReader().readAllBuiltIn();
    languageDetector =
        LanguageDetectorBuilder.create(NgramExtractors.standard())
            .withProfiles(languageProfiles)
            .build();

    textObjectFactory = CommonTextObjectFactories.forDetectingOnLargeText();
  } catch (IOException ioe) {
    throw new ResourceInitializationException(ioe);
  }
}
 

/**
 * @throws IllegalStateException if a profile for the same language was added already (must be a userland bug).
 */
public LanguageDetectorBuilder withProfile(LanguageProfile languageProfile) throws IllegalStateException {
    if (langsAdded.contains(languageProfile.getLocale())) {
        throw new IllegalStateException("A language profile for language " + languageProfile.getLocale() + " was added already!");
    }
    for (Integer gramLength : ngramExtractor.getGramLengths()) {
        if (!languageProfile.getGramLengths().contains(gramLength)) {
            throw new IllegalArgumentException("The NgramExtractor is set to handle " + gramLength + "-grams but the given language profile for " + languageProfile.getLocale() + " does not support this!");
        }
    }
    langsAdded.add(languageProfile.getLocale());
    languageProfiles.add(languageProfile);
    return this;
}
 

private LanguageDetector makeNewDetector() throws IOException {
    LanguageDetectorBuilder builder = LanguageDetectorBuilder.create(NgramExtractors.standard()).shortTextAlgorithm(50).prefixFactor(1.5).suffixFactor(2.0);

    LangProfileReader langProfileReader = new LangProfileReader();
    for (String language : ImmutableList.of("en", "fr", "nl", "de")) {
        LangProfile langProfile = langProfileReader.read(LanguageDetectorImplTest.class.getResourceAsStream("/languages/" + language));
        LanguageProfile languageProfile = OldLangProfileConverter.convert(langProfile);
        builder.withProfile(languageProfile);
    }

    return builder.build();
}
 

/**
 * Using all language profiles from the given directory.
 */
private LanguageDetector makeDetector() throws IOException {
    double alpha = getParamDouble("alpha", DEFAULT_ALPHA);
    String profileDirectory = requireParamString("directory") + "/";
    Optional<Long> seed = Optional.fromNullable(getParamLongOrNull("seed"));

    List<LanguageProfile> languageProfiles = new LanguageProfileReader().readAll(new File(profileDirectory));

    return LanguageDetectorBuilder.create(NgramExtractors.standard()).alpha(alpha).seed(seed).shortTextAlgorithm(50).withProfiles(languageProfiles).build();
}
 

/**
 * @param gramLengths for example [1,2,3]
 * @throws java.lang.IllegalArgumentException if languageProfiles or gramLengths is empty, or if one of the languageProfiles does not have the grams of the required sizes.
 */
@NotNull
public static NgramFrequencyData create(@NotNull Collection<LanguageProfile> languageProfiles, @NotNull Collection<Integer> gramLengths) throws IllegalArgumentException {
    if (languageProfiles.isEmpty())
        throw new IllegalArgumentException("No languageProfiles provided!");
    if (gramLengths.isEmpty())
        throw new IllegalArgumentException("No gramLengths provided!");

    Map<String, double[]> wordLangProbMap = new HashMap<>();
    List<Locale> langlist = new ArrayList<>();
    int langsize = languageProfiles.size();

    int index = -1;
    for (LanguageProfile profile : languageProfiles) {
        index++;

        langlist.add(profile.getLocale());

        for (Integer gramLength : gramLengths) {
            if (!profile.getGramLengths().contains(gramLength)) {
                throw new IllegalArgumentException("The language profile for " + profile.getLocale() + " does not contain " + gramLength + "-grams!");
            }
            for (Map.Entry<String, Integer> ngramEntry : profile.iterateGrams(gramLength)) {
                String ngram = ngramEntry.getKey();
                Integer frequency = ngramEntry.getValue();
                if (!wordLangProbMap.containsKey(ngram)) {
                    wordLangProbMap.put(ngram, new double[langsize]);
                }
                double prob = frequency.doubleValue() / profile.getNumGramOccurrences(ngram.length());
                wordLangProbMap.get(ngram)[index] = prob;
            }
        }
    }

    return new NgramFrequencyData(wordLangProbMap, langlist);
}
 

/**
 * Remove potential LanguageProfiles, e.g. in combination with {@link #loadAllBuiltInLanguageProfiles()}.
 * 
 * @param isoString the ISO string of the LanguageProfile to be removed.
 */
public LanguageProfileValidator removeLanguageProfile(final String isoString) {
    Iterables.removeIf(this.languageProfiles, new Predicate<LanguageProfile>() {
        @Override
        public boolean apply(LanguageProfile languageProfile) {
            return languageProfile.getLocale().getLanguage().equals(isoString);
        }
    });
    return this;
}
 

/**
 * @throws IllegalStateException if a profile for the same language was added already (must be a userland bug).
 */
public LanguageDetectorBuilder withProfiles(Iterable<LanguageProfile> languageProfiles) throws IllegalStateException {
    for (LanguageProfile languageProfile : languageProfiles) {
        withProfile(languageProfile);
    }
    return this;
}
 

private static NgramFrequencyData forAll(int gramSize) throws IOException {
    List<LanguageProfile> languageProfiles = new LanguageProfileReader().readAllBuiltIn();
    return NgramFrequencyData.create(languageProfiles, ImmutableSet.of(gramSize));
}
 

/**
 * Load the given {@link LanguageProfile}.
 */
public LanguageProfileValidator loadLanguageProfile(LanguageProfile languageProfile) {
    this.languageProfiles.add(languageProfile);
    return this;
}
 

/**
 * Load the given {@link LanguageProfile}s.
 */
public LanguageProfileValidator loadLanguageProfiles(Collection<LanguageProfile> languageProfiles) {
    this.languageProfiles.addAll(languageProfiles);
    return this;
}
 

/**
 * Load the given {@link LanguageProfile}s.
 */
public LanguageProfileValidator loadLanguageProfiles(Collection<LanguageProfile> languageProfiles) {
    this.languageProfiles.addAll(languageProfiles);
    return this;
}
 

private static NgramFrequencyData forAll(int gramSize) throws IOException {
    List<LanguageProfile> languageProfiles = new LanguageProfileReader().readAllBuiltIn();
    return NgramFrequencyData.create(languageProfiles, ImmutableSet.of(gramSize));
}
 

/**
 * Load the given {@link LanguageProfile}.
 */
public LanguageProfileValidator loadLanguageProfile(LanguageProfile languageProfile) {
    this.languageProfiles.add(languageProfile);
    return this;
}
 

public DataLanguageDetectorImplTest() throws IOException {
    List<LanguageProfile> languageProfiles = new LanguageProfileReader().readAllBuiltIn();

    shortDetector = LanguageDetectorBuilder.create(NgramExtractors.standard()).shortTextAlgorithm(100).withProfiles(languageProfiles).build();

    longDetector = LanguageDetectorBuilder.create(NgramExtractors.standard()).shortTextAlgorithm(0).withProfiles(new LanguageProfileReader().readAllBuiltIn()).build();
}