org.apache.solr.client.solrj.embedded.EmbeddedSolrServer Java Examples

public static void validateSolrServerDocumentCount(File solrHomeDir, FileSystem fs, Path outDir, int expectedDocs, int expectedShards)
    throws IOException, SolrServerException {
  
  long actualDocs = 0;
  int actualShards = 0;
  for (FileStatus dir : fs.listStatus(outDir)) { // for each shard
    if (dir.getPath().getName().startsWith("part") && dir.isDirectory()) {
      actualShards++;
      EmbeddedSolrServer solr = createEmbeddedSolrServer(
          solrHomeDir, fs, dir.getPath());
      
      try {
        SolrQuery query = new SolrQuery();
        query.setQuery("*:*");
        QueryResponse resp = solr.query(query);
        long numDocs = resp.getResults().getNumFound();
        actualDocs += numDocs;
      } finally {
        solr.close();
      }
    }
  }
  assertEquals(expectedShards, actualShards);
  assertEquals(expectedDocs, actualDocs);
}
 

@Override
public void activateSolr() throws Exception
{
   ClassLoader oldCl = Thread.currentThread().getContextClassLoader();
   Thread.currentThread().setContextClassLoader( getClass().getClassLoader() );

   try
   {
      File directory = new File( fileConfig.dataDirectory(), descriptor.identity().toString() );
      directory.mkdirs();

      System.setProperty( "solr.solr.home", directory.getAbsolutePath() );

      CoreContainer.Initializer initializer = new CoreContainer.Initializer();
      coreContainer = initializer.initialize();
      server = new EmbeddedSolrServer( coreContainer, "" );
      core = coreContainer.getCore( "" );
   } finally
   {
      Thread.currentThread().setContextClassLoader( oldCl );
   }
}
 

public BatchWriter(EmbeddedSolrServer solr, int batchSize, TaskID tid,
    int writerThreads, int queueSize) {
  this.solr = solr;
  this.writerThreads = writerThreads;
  this.queueSize = queueSize;
  taskId = tid;

  // we need to obtain the settings before the constructor
  if (writerThreads != 0) {
    batchPool = new ThreadPoolExecutor(writerThreads, writerThreads, 5,
        TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(queueSize),
        new ThreadPoolExecutor.CallerRunsPolicy());
  } else { // single threaded case
    batchPool = null;
  }
}
 

/**
 *
 * @param solrHome
 *              path to directory where solr.xml lives
 * @param coreName
 *              the name of the core to load
 * @param dataDir
 *              the data dir for the core
 *
 * @return an EmbeddedSolrServer for the given core
 */
public static SolrClient create(String solrHome, String coreName, String dataDir)
        throws IOException {

    Map<String,String> props = new HashMap<>();
    if (dataDir != null) {
        File coreDataDir = new File(dataDir + "/" + coreName);
        if (coreDataDir.exists()) {
            FileUtils.deleteDirectory(coreDataDir);
        }
        props.put("dataDir", dataDir + "/" + coreName);
    }

    final CoreContainer coreContainer = new CoreContainer(solrHome);
    coreContainer.load();

    return new EmbeddedSolrServer(coreContainer, coreName);
}
 

public static void main(String[] args) throws IOException, JATEException, SolrServerException {
	Logger logger = Logger.getLogger(Indexing.class.getName());
	JATEProperties prop = new JATEProperties(args[0]);
	boolean deletePrevious = Boolean.valueOf(args[2]);
	SolrClient solrClient = new EmbeddedSolrServer(Paths.get(args[3]), args[4]);
	try {
		if (deletePrevious) {
			logger.info("DELETING PREVIOUS INDEX");
			solrClient.deleteByQuery("*:*");
			solrClient.commit();
		}
		logger.info("INDEXING BEGINS");
		IndexingHandler m = new IndexingHandler();
		List<String> files = new ArrayList<>();
		for (File f : new File(args[1]).listFiles())
			files.add(f.toString());

		m.index(files, prop.getIndexerMaxUnitsToCommit(), new TikaSimpleDocumentCreator(), solrClient, prop);
		logger.info("INDEXING COMPLETE");
		System.exit(0);
	} finally {
		solrClient.close();
	}
}
 

protected void indexJATEDocuments(Path file, EmbeddedSolrServer solrServer, JATEProperties jateProp, boolean commit) throws JATEException {
    if (file == null) {
        return;
    }

    try {
        JATEDocument jateDocument = JATEUtil.loadJATEDocument(file);

        if (isNotEmpty(jateDocument))
            JATEUtil.addNewDoc(solrServer, jateDocument.getId(),
                    jateDocument.getId(), jateDocument.getContent(), jateProp, commit);
    } catch (FileNotFoundException ffe) {
        throw new JATEException(ffe.toString());
    } catch (IOException ioe) {
        throw new JATEException(String.format("failed to index [%s]", file.toString()) + ioe.toString());
    } catch (SolrServerException sse) {
        throw new JATEException(String.format("failed to index [%s] ", file.toString()) + sse.toString());
    }
}
 

public static void addNewDoc(EmbeddedSolrServer server, String docId, String docTitle,
                             String text, JATEProperties jateProperties, boolean commit)
        throws IOException, SolrServerException, JATEException {
    SolrInputDocument newDoc = new SolrInputDocument();
    newDoc.addField("id", docId);
    newDoc.addField("title_s", docTitle);
    newDoc.addField("text", text);

    newDoc.addField(jateProperties.getSolrFieldNameJATENGramInfo(), text);
    newDoc.addField(jateProperties.getSolrFieldNameJATECTerms(), text);

    server.add(newDoc);

    if (commit) {
        server.commit();
    }
}
 

public static void main(String[] args) throws IOException, ParseException, JATEException {
    /*mergeGS("/home/zqz/Work/data/semeval2017-scienceie/scienceie2017_test/scienceie2017_test_gs",
            "/home/zqz/Work/data/semeval2017-scienceie/scienceie2017_test/all_key_phrases.txt");*/
    Lemmatiser lem = new Lemmatiser(new EngLemmatiser(args[0],
            false, false));
    String solrHomePath = args[4];
    String solrCoreName = args[5];
    final EmbeddedSolrServer solrServer = new EmbeddedSolrServer(Paths.get(solrHomePath), solrCoreName);
    JATEProperties jateProp = App.getJateProperties(args[6]);
    for (File f : new File(args[1]).listFiles()) {
        File outFolder = new File(args[2] + "/" + f.getName() + "/");
        outFolder.mkdirs();
        System.out.println(outFolder);

        transformToKEAOutput(f.toString(), 0, args[3], outFolder.toString(), lem,
                solrServer.getCoreContainer().getCore(solrCoreName), jateProp);
    }
    solrServer.close();
    System.exit(0);
}
 

@Override
List<JATETerm> rankAndFilter(EmbeddedSolrServer server, String solrCoreName, JATEProperties jateProp)
        throws JATEException {
    LOG.info("AppATTF ranking and filtering ... ");
    List<JATETerm> terms = new ArrayList<>();
    Map<String, String> initParam = new HashMap<>();

    initParam.put(AppParams.PREFILTER_MIN_TERM_TOTAL_FREQUENCY.getParamKey(), "2");
    initParam.put(AppParams.CUTOFF_TOP_K_PERCENT.getParamKey(), "0.99999");

    AppATTF appATTF = new AppATTF(initParam);
    long startTime = System.currentTimeMillis();
    terms = appATTF.extract(server.getCoreContainer().getCore(solrCoreName), jateProp);
    long endTime = System.currentTimeMillis();
    LOG.info(String.format("AppATTF ranking took [%s] milliseconds", (endTime - startTime)));
    LOG.info("complete ranking and filtering.");

    LOG.info("Export results for evaluation ...");
    try {
        appATTF.outputFile = "attf_acltdtec.json";
        appATTF.write(terms);
    } catch (IOException e) {
        throw new JATEException("Fail to export results.");
    }
    return terms;
}
 

@Override
List<JATETerm> rankAndFilter(EmbeddedSolrServer server, String solrCoreName, JATEProperties jateProp)
        throws JATEException {
    LOG.info("AppRAKE ranking and filtering ... ");
    List<JATETerm> terms = new ArrayList<>();
    Map<String, String> initParam = new HashMap<>();
    initParam.put(AppParams.PREFILTER_MIN_TERM_TOTAL_FREQUENCY.getParamKey(), "2");
    initParam.put(AppParams.CUTOFF_TOP_K_PERCENT.getParamKey(), "0.99999");

    AppRAKE appRAKE = new AppRAKE(initParam);
    long startTime = System.currentTimeMillis();
    terms = appRAKE.extract(server.getCoreContainer().getCore(solrCoreName), jateProp);
    long endTime = System.currentTimeMillis();
    LOG.info(String.format("appRAKE ranking took [%s] milliseconds", (endTime - startTime)));

    LOG.info("complete ranking and filtering.");

    LOG.info("Export results for evaluation ...");
    try {
        appRAKE.outputFile = "rake_acltdtec.json";
        appRAKE.write(terms);
    } catch (IOException e) {
        throw new JATEException("Fail to export results.");
    }
    return terms;
}
 

@Test
public void testSelectsEmbeddedSolrServerAndAddDocument() throws Exception {
  //Solr locator should select EmbeddedSolrServer only solrHome is specified
  SolrLocator solrLocator = new SolrLocator(new SolrMorphlineContext.Builder().build());
  solrLocator.setSolrHomeDir(RESOURCES_DIR + "/solr");
  solrLocator.setCollectionName("collection1");

  SolrServerDocumentLoader documentLoader = (SolrServerDocumentLoader)solrLocator.getLoader();
  SolrClient solrServer = documentLoader.getSolrServer();
  
  assertTrue(solrServer instanceof EmbeddedSolrServer);
  
  SolrInputDocument doc = new SolrInputDocument();
  doc.addField("id", "myId");
  doc.addField("text", "myValue");
  solrServer.add(doc);
  solrServer.commit();
  
  SolrDocument resultDoc = solrServer.getById("myId");
  assertTrue(resultDoc.getFieldValues("text").contains("myValue"));
  
  UpdateResponse deleteResponse = solrServer.deleteById("myId");
  assertEquals(0, deleteResponse.getStatus());
  solrServer.commit();
  solrServer.close();
}
 

/**
 *
 * @param solrHome
 *              path to directory where solr.xml lives
 * @param coreName
 *              the name of the core to load
 * @param dataDir
 *              the data dir for the core
 *
 * @return an EmbeddedSolrServer for the given core
 */
public static SolrClient create(String solrHome, String coreName, String dataDir)
        throws IOException {

    Map<String,String> props = new HashMap<>();
    if (dataDir != null) {
        File coreDataDir = new File(dataDir + "/" + coreName);
        if (coreDataDir.exists()) {
            FileUtils.deleteDirectory(coreDataDir);
        }
        props.put("dataDir", dataDir + "/" + coreName);
    }

    final CoreContainer coreContainer = new CoreContainer(solrHome);
    coreContainer.load();

    return new EmbeddedSolrServer(coreContainer, coreName);
}
 

@Override
List<JATETerm> rankAndFilter(EmbeddedSolrServer server, String solrCoreName, JATEProperties jateProp)
        throws JATEException {
    LOG.info("AppCValue ranking and filtering ... ");

    List<JATETerm> terms = new ArrayList<>();
    Map<String, String> initParam = new HashMap<>();
    initParam.put(AppParams.PREFILTER_MIN_TERM_TOTAL_FREQUENCY.getParamKey(), "2");
    initParam.put(AppParams.CUTOFF_TOP_K_PERCENT.getParamKey(), "0.99999");

    AppCValue appCValue = new AppCValue(initParam);
    long startTime = System.currentTimeMillis();
    terms = appCValue.extract(server.getCoreContainer().getCore(solrCoreName), jateProp);
    long endTime = System.currentTimeMillis();
    LOG.info(String.format("AppCValue ranking took [%s] milliseconds", (endTime - startTime)));

    LOG.info("complete ranking and filtering.");

    LOG.info("Export results for evaluation ...");
    try {
        appCValue.outputFile = "cvalue_acltdtec.json";
        appCValue.write(terms);
    } catch (IOException e) {
        throw new JATEException("Fail to export results.");
    }
    return terms;
}
 

@Override
List<JATETerm> rankAndFilter(EmbeddedSolrServer server, String solrCoreName, JATEProperties jateProp)
        throws JATEException {
    LOG.info("AppChiSquare ranking and filtering ... ");

    List<JATETerm> terms = new ArrayList<>();
    Map<String, String> initParam = new HashMap<>();
    initParam.put(AppParams.PREFILTER_MIN_TERM_TOTAL_FREQUENCY.getParamKey(), "2");
    initParam.put(AppParams.CUTOFF_TOP_K_PERCENT.getParamKey(), "0.99999");
    initParam.put(AppParams.PREFILTER_MIN_TERM_CONTEXT_FREQUENCY.getParamKey(), "2");

    initParam.put(AppParams.CHISQUERE_FREQ_TERM_CUTOFF_PERCENTAGE.getParamKey(), "0.1");

    AppChiSquare appChiSquare = new AppChiSquare(initParam);
    long startTime = System.currentTimeMillis();
    terms = appChiSquare.extract(server.getCoreContainer().getCore(solrCoreName), jateProp);
    long endTime = System.currentTimeMillis();
    LOG.info(String.format("AppChiSquare ranking took [%s] milliseconds", (endTime - startTime)));

    LOG.info("complete ranking and filtering.");

    LOG.info("Export results for evaluation ...");
    try {
        appChiSquare.outputFile = "chi_square_acltdtec.json";
        appChiSquare.write(terms);
    } catch (IOException e) {
        throw new JATEException("Fail to export results.");
    }
    return terms;
}
 

@Override
List<JATETerm> rankAndFilter(EmbeddedSolrServer server, String solrCoreName, JATEProperties jateProp)
        throws JATEException {
    LOG.info("AppGlossEx ranking and filtering ... ");

    List<JATETerm> terms = new ArrayList<>();

    Map<String, String> initParam = new HashMap<>();
    initParam.put(AppParams.PREFILTER_MIN_TERM_TOTAL_FREQUENCY.getParamKey(), "2");
    initParam.put(AppParams.CUTOFF_TOP_K_PERCENT.getParamKey(), "0.99999");

    initParam.put(AppParams.REFERENCE_FREQUENCY_FILE.getParamKey(), FREQ_GENIC_FILE.toString());
    AppGlossEx appGlossEx = new AppGlossEx(initParam);

    long startTime = System.currentTimeMillis();
    terms = appGlossEx.extract(server.getCoreContainer().getCore(solrCoreName), jateProp);
    long endTime = System.currentTimeMillis();
    LOG.info(String.format("appGlossEx ranking took [%s] milliseconds", (endTime - startTime)));

    LOG.info("complete ranking and filtering.");

    LOG.info("Export results for evaluation ...");
    try {
        appGlossEx.outputFile = "glossEx_acltdtec.json";
        appGlossEx.write(terms);
    } catch (IOException e) {
        throw new JATEException("Fail to export results.");
    }

    return terms;
}
 

@Override
List<JATETerm> rankAndFilter(EmbeddedSolrServer server, String solrCoreName, JATEProperties jateProp)
        throws JATEException {
    LOG.info("AppWeirdness ranking and filtering ... ");

    List<JATETerm> terms = new ArrayList<>();
    Map<String, String> initParam = new HashMap<>();
    initParam.put(AppParams.PREFILTER_MIN_TERM_TOTAL_FREQUENCY.getParamKey(), "2");
    initParam.put(AppParams.CUTOFF_TOP_K_PERCENT.getParamKey(), "0.99999");

    initParam.put(AppParams.REFERENCE_FREQUENCY_FILE.getParamKey(), FREQ_GENIC_FILE.toString());

    AppWeirdness appWeirdness = new AppWeirdness(initParam);
    long startTime = System.currentTimeMillis();
    terms = appWeirdness.extract(server.getCoreContainer().getCore(solrCoreName), jateProp);
    long endTime = System.currentTimeMillis();
    LOG.info(String.format("AppWeirdness ranking took [%s] milliseconds", (endTime - startTime)));

    LOG.info("complete ranking and filtering.");

    LOG.info("Export results for evaluation ...");
    try {
        appWeirdness.outputFile = "weirdness_acltdtec.json";
        appWeirdness.write(terms);
    } catch (IOException e) {
        throw new JATEException("Fail to export results.");
    }
    return terms;
}
 

public SolrEmbedded(final Config config) throws IOException {
    // Copy all required files
    final var solrHome = config.solrHome;
    solrHome.mkdirs();

    copyResourceToFile("/solrhome/solr.xml", new File(solrHome, "solr.xml"));

    final var core1 = new File(solrHome, "core1");
    final var core1conf = new File(core1, "conf");
    final var core1data = new File(core1, "data");
    final var core1lang = new File(core1, "lang");

    core1conf.mkdirs();
    core1data.mkdirs();
    core1lang.mkdirs();

    // Core1
    copyResourceToFile("/solrhome/core1/core.properties", new File(core1, "core.properties"));
    copyResourceToFile("/solrhome/core1/currency.xml", new File(core1, "currency.xml"));
    copyResourceToFile("/solrhome/core1/protwords.txt", new File(core1, "protwords.txt"));
    copyResourceToFile("/solrhome/core1/solrconfig.xml", new File(core1, "solrconfig.xml"));
    copyResourceToFile("/solrhome/core1/stopwords.txt", new File(core1, "stopwords.txt"));
    copyResourceToFile("/solrhome/core1/synonyms.txt", new File(core1, "synonyms.txt"));
    copyResourceToFile("/solrhome/core1/update-script.js", new File(core1, "update-script.js"));

    // Core1 Config
    copyResourceToFile("/solrhome/core1/conf/elevate.xml", new File(core1conf, "elevate.xml"));
    copyResourceToFile("/solrhome/core1/conf/managed-schema", new File(core1conf, "managed-schema"));

    // Core1 Language
    copyResourceToFile("/solrhome/core1/lang/stopwords_en.txt", new File(core1lang, "stopwords_en.txt"));

    // Bootstrap
    coreContainer = new CoreContainer(solrHome.toString());
    coreContainer.load();

    embeddedSolrServer = new EmbeddedSolrServer(coreContainer, "core1");
}
 

public void setup() throws Exception {
        setSolrCoreName();
        setReindex();
//        if(reindex)
//            cleanIndexDirectory(solrHome.toString(), solrCoreName);
        CoreContainer testBedContainer = new CoreContainer(solrHome.toString());
        testBedContainer.load();
        server = new EmbeddedSolrServer(testBedContainer, solrCoreName);
    }
 

public static void main(String[] args) throws JATEException, IOException, ParseException {
    String outFolder=args[0];
    String solrHomePath = args[1];
    String solrCoreName = args[2];
    final EmbeddedSolrServer solrServer = new EmbeddedSolrServer(Paths.get(solrHomePath), solrCoreName);
    JATEProperties jateProp = App.getJateProperties(args[3]);
    List<String> predictions=null;
    if(args.length>4){
        predictions=ATEResultLoader.loadFromJSON(args[4]);
    }

    output(outFolder, solrServer.getCoreContainer().getCore(solrCoreName),
            jateProp, predictions);
}
 

@Override
List<JATETerm> rankAndFilter(EmbeddedSolrServer server, String solrCoreName, JATEProperties jateProp)
        throws JATEException {
    LOG.info("AppRIDF ranking and filtering ... ");

    List<JATETerm> terms = new ArrayList<>();
    Map<String, String> initParam = new HashMap<>();
    initParam.put(AppParams.PREFILTER_MIN_TERM_TOTAL_FREQUENCY.getParamKey(), "2");
    initParam.put(AppParams.CUTOFF_TOP_K_PERCENT.getParamKey(), "0.99999");

    AppRIDF appRIDF = new AppRIDF(initParam);
    long startTime = System.currentTimeMillis();
    terms = appRIDF.extract(server.getCoreContainer().getCore(solrCoreName), jateProp);
    long endTime = System.currentTimeMillis();
    LOG.info(String.format("AppRIDF ranking took [%s] milliseconds", (endTime - startTime)));

    LOG.info("complete ranking and filtering.");

    LOG.info("Export results for evaluation ...");
    try {
        appRIDF.outputFile = "ridf_acltdtec.json";
        appRIDF.write(terms);
    } catch (IOException e) {
        throw new JATEException("Fail to export results.");
    }

    return terms;
}
 

@Override
List<JATETerm> rankAndFilter(EmbeddedSolrServer server, String solrCoreName, JATEProperties jateProp)
        throws JATEException {
    LOG.info("AppTermEx ranking and filtering ... ");

    List<JATETerm> terms = new ArrayList<>();
    Map<String, String> initParam = new HashMap<>();
    initParam.put(AppParams.PREFILTER_MIN_TERM_TOTAL_FREQUENCY.getParamKey(), "2");
    initParam.put(AppParams.CUTOFF_TOP_K_PERCENT.getParamKey(), "0.99999");

    initParam.put(AppParams.REFERENCE_FREQUENCY_FILE.getParamKey(), FREQ_GENIC_FILE.toString());

    AppTermEx appTermEx = new AppTermEx(initParam);
    long startTime = System.currentTimeMillis();
    terms = appTermEx.extract(server.getCoreContainer().getCore(solrCoreName), jateProp);
    long endTime = System.currentTimeMillis();
    LOG.info(String.format("AppTermEx ranking took [%s] milliseconds", (endTime - startTime)));
    LOG.info("complete ranking and filtering.");

    LOG.info("Export results for evaluation ...");
    try {
        appTermEx.outputFile = "termEx_acltdtec.json";
        appTermEx.write(terms);
    } catch (IOException e) {
        throw new JATEException("Fail to export results.");
    }
    return terms;
}
 

@Override
List<JATETerm> rankAndFilter(EmbeddedSolrServer server, String solrCoreName, JATEProperties jateProp)
        throws JATEException {
    LOG.info("AppTFIDF ranking and filtering ... ");

    List<JATETerm> terms = new ArrayList<>();
    Map<String, String> initParam = new HashMap<>();
    initParam.put(AppParams.PREFILTER_MIN_TERM_TOTAL_FREQUENCY.getParamKey(), "2");
    initParam.put(AppParams.CUTOFF_TOP_K_PERCENT.getParamKey(), "0.99999");

    AppTFIDF appTFIDF = new AppTFIDF(initParam);
    long startTime = System.currentTimeMillis();
    terms = appTFIDF.extract(server.getCoreContainer().getCore(solrCoreName), jateProp);
    long endTime = System.currentTimeMillis();
    LOG.info(String.format("AppTFIDF ranking took [%s] milliseconds", (endTime - startTime)));

    LOG.info("complete ranking and filtering.");

    LOG.info("Export results for evaluation ...");
    try {
        appTFIDF.outputFile = "tfidf_acltdtec.json";
        appTFIDF.write(terms);
    } catch (IOException e) {
        throw new JATEException("Fail to export results.");
    }
    return terms;
}
 

/**
 * Uses the {@link EmbeddedSolrServer} to index some data to Solr.
 * 
 * @throws Exception in case of I/O or index failure.
 */
@Test
public void embeddedSolrServer() throws Exception {
	// 1. Create a new (local) container using "solr.solr.home" and "solr.data.dir" system property.
	// Note that we need to define a dedicated solr.data.dir for this test method because
	// otherwise we would end in a lock conflict (the embedded Solr instance is running) 
	System.setProperty("solr.solr.home", "src/solr/solr-home");
	System.setProperty("solr.data.dir", new File("target/solr-embedded-solr").getAbsolutePath());
	CoreContainer container = new CoreContainer();
	container.load();
	
	System.out.println(container.getAllCoreNames());
	
	// 2. Create a new instance of (Embedded)SolrServer 
	solr = new EmbeddedSolrServer(container, "example");
	
	// 3. Create some data
	final List<SolrInputDocument> albums = sampleData();
	
	// 4. Add those data
	solr.add(albums);
	
	// 5. Commit
	solr.commit();

	// 6. Verify
	verify();
}
 

public SolrRecordWriter(TaskAttemptContext context, Path outputShardDir, int batchSize) {
  this.batchSize = batchSize;
  this.batch = new ArrayList(batchSize);
  Configuration conf = context.getConfiguration();

  // setLogLevel("org.apache.solr.core", "WARN");
  // setLogLevel("org.apache.solr.update", "WARN");

  heartBeater = new HeartBeater(context);
  try {
    heartBeater.needHeartBeat();

    Path solrHomeDir = SolrRecordWriter.findSolrConfig(conf);
    FileSystem fs = outputShardDir.getFileSystem(conf);
    EmbeddedSolrServer solr = createEmbeddedSolrServer(solrHomeDir, fs, outputShardDir);
    batchWriter = new BatchWriter(solr, batchSize,
        context.getTaskAttemptID().getTaskID(),
        SolrOutputFormat.getSolrWriterThreadCount(conf),
        SolrOutputFormat.getSolrWriterQueueSize(conf));

  } catch (Exception e) {
    throw new IllegalStateException(String.format(Locale.ENGLISH,
        "Failed to initialize record writer for %s, %s", context.getJobName(), conf
            .get("mapred.task.id")), e);
  } finally {
    heartBeater.cancelHeartBeat();
  }
}
 

@Override
List<JATETerm> rankAndFilter(EmbeddedSolrServer server, String solrCoreName, JATEProperties jateProp)
        throws JATEException {
    LOG.info("AppTTF ranking and filtering ... ");

    List<JATETerm> terms = new ArrayList<>();
    Map<String, String> initParam = new HashMap<>();
    initParam.put(AppParams.PREFILTER_MIN_TERM_TOTAL_FREQUENCY.getParamKey(), "2");
    initParam.put(AppParams.CUTOFF_TOP_K_PERCENT.getParamKey(), "0.99999");

    AppTTF appTTF = new AppTTF(initParam);
    long startTime = System.currentTimeMillis();
    terms = appTTF.extract(server.getCoreContainer().getCore(solrCoreName), jateProp);
    long endTime = System.currentTimeMillis();
    LOG.info(String.format("AppTTF ranking took [%s] milliseconds", (endTime - startTime)));

    LOG.info("complete ranking and filtering.");

    LOG.info("Export results for evaluation ...");
    try {
        appTTF.outputFile = "ttf_acltdtec.json";
        appTTF.write(terms);
    } catch (IOException e) {
        throw new JATEException("Fail to export results.");
    }
    return terms;
}
 

/**
 * Performs Solr restoration.
 *
 * @param properties properties containing arguments to execute the restoration.
 */
private static void restoreSolr5Index (Properties properties) throws
      IOException, SolrServerException
{
   String solrHome = properties.getProperty ("dhus.solr.home");
   String coreName = properties.getProperty ("dhus.solr.core.name");
   final String name = properties.getProperty ("dhus.solr.backup.name");
   final String location = properties.getProperty (
         "dhus.solr.backup.location");

   if (solrHome == null || coreName == null || name == null ||
       location == null)
   {
      throw new UnsupportedOperationException ();
   }

   System.setProperty ("solr.solr.home", solrHome);
   CoreContainer core = new CoreContainer (solrHome);
   EmbeddedSolrServer server = new EmbeddedSolrServer (core, coreName);
   try
   {
      server.getCoreContainer ().load ();

      SolrQuery query = new SolrQuery();
      query.setRequestHandler("/replication");
      query.set("command", "restore");
      query.set("name", name);
      query.set("location", location);

      server.query(query);
      LOGGER.info("SolR indexes restored.");
   }
   finally
   {
      server.close();
   }
   
}
 

@Override
public EmbeddedSolrServer getSolrClient() {
	if (this.solrServer == null) {
		initSolrServer();
	}
	return this.solrServer;
}
 

/**
 * @param path
 *            Any Path expression valid for use with
 *            {@link org.springframework.util.ResourceUtils}
 * @return new instance of
 *         {@link org.apache.solr.client.solrj.embedded.EmbeddedSolrServer}
 * @throws javax.xml.parsers.ParserConfigurationException
 * @throws java.io.IOException
 * @throws org.xml.sax.SAXException
 */
public final EmbeddedSolrServer createPathConfiguredSolrServer(String path)
		throws ParserConfigurationException, IOException, SAXException {
	String solrHomeDirectory = System.getProperty(SOLR_HOME_SYSTEM_PROPERTY);

	if (StringUtils.isBlank(solrHomeDirectory)) {
		solrHomeDirectory = ResourceUtils.getURL(path).getPath();
	}

	solrHomeDirectory = URLDecoder.decode(solrHomeDirectory, "utf-8");
	return new EmbeddedSolrServer(createCoreContainer(solrHomeDirectory), "collection1");
}
 

public SolrClient getSolrServer() {
  if (zkHost != null && zkHost.length() > 0) {
    if (collectionName == null || collectionName.length() == 0) {
      throw new MorphlineCompilationException("Parameter 'zkHost' requires that you also pass parameter 'collection'", config);
    }
    CloudSolrClient cloudSolrClient = new Builder()
        .withZkHost(zkHost)
        .build();
    cloudSolrClient.setDefaultCollection(collectionName);
    cloudSolrClient.setZkClientTimeout(zkClientSessionTimeout); 
    cloudSolrClient.setZkConnectTimeout(zkClientConnectTimeout); 
    return cloudSolrClient;
  } else {
    if (solrUrl == null && solrHomeDir != null) {
      CoreContainer coreContainer = new CoreContainer(solrHomeDir);
      coreContainer.load();
      EmbeddedSolrServer embeddedSolrServer = new EmbeddedSolrServer(coreContainer, collectionName);
      return embeddedSolrServer;
    }
    if (solrUrl == null || solrUrl.length() == 0) {
      throw new MorphlineCompilationException("Missing parameter 'solrUrl'", config);
    }
    int solrServerNumThreads = 2;
    int solrServerQueueLength = solrServerNumThreads;
    SolrClient server = new SafeConcurrentUpdateSolrServer(solrUrl, solrServerQueueLength, solrServerNumThreads);
    return server;
  }
}
 

@SuppressWarnings({ "unchecked", "rawtypes" })
private static SolrClient cloneEmbeddedSolrServer(SolrClient solrClient, String core) {

	CoreContainer coreContainer = ((EmbeddedSolrServer) solrClient).getCoreContainer();
	try {
		Constructor constructor = ClassUtils.getConstructorIfAvailable(solrClient.getClass(), CoreContainer.class,
				String.class);
		return (SolrClient) BeanUtils.instantiateClass(constructor, coreContainer, core);
	} catch (Exception e) {
		throw new BeanInstantiationException(solrClient.getClass(),
				"Cannot create instace of " + solrClient.getClass() + ".", e);
	}
}