Java Code Examples for org.apache.solr.core.CoreContainer#load()

/**
 *
 * @param solrHome
 *              path to directory where solr.xml lives
 * @param coreName
 *              the name of the core to load
 * @param dataDir
 *              the data dir for the core
 *
 * @return an EmbeddedSolrServer for the given core
 */
public static SolrClient create(String solrHome, String coreName, String dataDir)
        throws IOException {

    Map<String,String> props = new HashMap<>();
    if (dataDir != null) {
        File coreDataDir = new File(dataDir + "/" + coreName);
        if (coreDataDir.exists()) {
            FileUtils.deleteDirectory(coreDataDir);
        }
        props.put("dataDir", dataDir + "/" + coreName);
    }

    final CoreContainer coreContainer = new CoreContainer(solrHome);
    coreContainer.load();

    return new EmbeddedSolrServer(coreContainer, coreName);
}
 

/**
 *
 * @param solrHome
 *              path to directory where solr.xml lives
 * @param coreName
 *              the name of the core to load
 * @param dataDir
 *              the data dir for the core
 *
 * @return an EmbeddedSolrServer for the given core
 */
public static SolrClient create(String solrHome, String coreName, String dataDir)
        throws IOException {

    Map<String,String> props = new HashMap<>();
    if (dataDir != null) {
        File coreDataDir = new File(dataDir + "/" + coreName);
        if (coreDataDir.exists()) {
            FileUtils.deleteDirectory(coreDataDir);
        }
        props.put("dataDir", dataDir + "/" + coreName);
    }

    final CoreContainer coreContainer = new CoreContainer(solrHome);
    coreContainer.load();

    return new EmbeddedSolrServer(coreContainer, coreName);
}
 

/**
 * Create a TestHarness using a specific config
 * @param config the ConfigSolr to use
 */
public TestHarness(NodeConfig config, CoresLocator coresLocator) {
  container = new CoreContainer(config, coresLocator);
  container.load();
  updater = new UpdateRequestHandler();
  updater.init(null);
}
 

/**
 * Override this to change CoreContainer initialization
 * @return a CoreContainer to hold this server's cores
 */
protected CoreContainer createCoreContainer(Path solrHome, Properties extraProperties) {
  NodeConfig nodeConfig = loadNodeConfig(solrHome, extraProperties);
  final CoreContainer coreContainer = new CoreContainer(nodeConfig, true);
  coreContainer.load();
  return coreContainer;
}
 

private CoreContainer init() throws Exception {
  File changed = new File(solrHomeDirectory, "changed");
  copyMinConf(changed, "name=changed");
  // Overlay with my local schema
  schemaFile = new File(new File(changed, "conf"), "schema.xml");
  FileUtils.writeStringToFile(schemaFile, withWhich, StandardCharsets.UTF_8);

  String discoveryXml = "<solr></solr>";
  File solrXml = new File(solrHomeDirectory, "solr.xml");
  FileUtils.write(solrXml, discoveryXml, StandardCharsets.UTF_8);

  final CoreContainer cores = new CoreContainer(solrHomeDirectory.toPath(), new Properties());
  cores.load();
  return cores;
}
 

/**
 * Setup.
 */
@org.junit.BeforeClass
public static void setup() {
  try {
    Path dataPath = Paths.get("src" + File.separator + "test"
        + File.separator + "resources" + File.separator + "data");
    // data
    Map<Integer, SolrInputDocument> solrDocuments = MtasSolrBase
        .createDocuments(true);

    // create
    ArrayList<String> collections = new ArrayList<>(
        Arrays.asList("collection1", "collection2", "collection3"));
    initializeDirectory(dataPath, collections);
    CoreContainer container = new CoreContainer(
        solrPath.toAbsolutePath().toString());
    container.load();
    server = new EmbeddedSolrServer(container, collections.get(0));

    // add
    server.add("collection1", solrDocuments.get(1));
    server.commit("collection1");
    server.add("collection1", solrDocuments.get(2));
    server.add("collection1", solrDocuments.get(3));
    server.commit("collection1");

    server.add("collection2", solrDocuments.get(1));
    server.commit("collection2");

    server.add("collection3", solrDocuments.get(3));
    server.add("collection3", solrDocuments.get(2));
    server.commit("collection3");

  } catch (IOException | SolrServerException e) {
    log.error(e);
  }
}
 

public void setup() throws Exception {
        setSolrCoreName();
        setReindex();
//        if(reindex)
//            cleanIndexDirectory(solrHome.toString(), solrCoreName);
        CoreContainer testBedContainer = new CoreContainer(solrHome.toString());
        testBedContainer.load();
        server = new EmbeddedSolrServer(testBedContainer, solrCoreName);
    }
 

public SolrEmbedded(final Config config) throws IOException {
    // Copy all required files
    final var solrHome = config.solrHome;
    solrHome.mkdirs();

    copyResourceToFile("/solrhome/solr.xml", new File(solrHome, "solr.xml"));

    final var core1 = new File(solrHome, "core1");
    final var core1conf = new File(core1, "conf");
    final var core1data = new File(core1, "data");
    final var core1lang = new File(core1, "lang");

    core1conf.mkdirs();
    core1data.mkdirs();
    core1lang.mkdirs();

    // Core1
    copyResourceToFile("/solrhome/core1/core.properties", new File(core1, "core.properties"));
    copyResourceToFile("/solrhome/core1/currency.xml", new File(core1, "currency.xml"));
    copyResourceToFile("/solrhome/core1/protwords.txt", new File(core1, "protwords.txt"));
    copyResourceToFile("/solrhome/core1/solrconfig.xml", new File(core1, "solrconfig.xml"));
    copyResourceToFile("/solrhome/core1/stopwords.txt", new File(core1, "stopwords.txt"));
    copyResourceToFile("/solrhome/core1/synonyms.txt", new File(core1, "synonyms.txt"));
    copyResourceToFile("/solrhome/core1/update-script.js", new File(core1, "update-script.js"));

    // Core1 Config
    copyResourceToFile("/solrhome/core1/conf/elevate.xml", new File(core1conf, "elevate.xml"));
    copyResourceToFile("/solrhome/core1/conf/managed-schema", new File(core1conf, "managed-schema"));

    // Core1 Language
    copyResourceToFile("/solrhome/core1/lang/stopwords_en.txt", new File(core1lang, "stopwords_en.txt"));

    // Bootstrap
    coreContainer = new CoreContainer(solrHome.toString());
    coreContainer.load();

    embeddedSolrServer = new EmbeddedSolrServer(coreContainer, "core1");
}
 

/**
 * Uses the {@link EmbeddedSolrServer} to index some data to Solr.
 * 
 * @throws Exception in case of I/O or index failure.
 */
@Test
public void embeddedSolrServer() throws Exception {
	// 1. Create a new (local) container using "solr.solr.home" and "solr.data.dir" system property.
	// Note that we need to define a dedicated solr.data.dir for this test method because
	// otherwise we would end in a lock conflict (the embedded Solr instance is running) 
	System.setProperty("solr.solr.home", "src/solr/solr-home");
	System.setProperty("solr.data.dir", new File("target/solr-embedded-solr").getAbsolutePath());
	CoreContainer container = new CoreContainer();
	container.load();
	
	System.out.println(container.getAllCoreNames());
	
	// 2. Create a new instance of (Embedded)SolrServer 
	solr = new EmbeddedSolrServer(container, "example");
	
	// 3. Create some data
	final List<SolrInputDocument> albums = sampleData();
	
	// 4. Add those data
	solr.add(albums);
	
	// 5. Commit
	solr.commit();

	// 6. Verify
	verify();
}
 

public SolrClient getSolrServer() {
  if (zkHost != null && zkHost.length() > 0) {
    if (collectionName == null || collectionName.length() == 0) {
      throw new MorphlineCompilationException("Parameter 'zkHost' requires that you also pass parameter 'collection'", config);
    }
    CloudSolrClient cloudSolrClient = new Builder()
        .withZkHost(zkHost)
        .build();
    cloudSolrClient.setDefaultCollection(collectionName);
    cloudSolrClient.setZkClientTimeout(zkClientSessionTimeout); 
    cloudSolrClient.setZkConnectTimeout(zkClientConnectTimeout); 
    return cloudSolrClient;
  } else {
    if (solrUrl == null && solrHomeDir != null) {
      CoreContainer coreContainer = new CoreContainer(solrHomeDir);
      coreContainer.load();
      EmbeddedSolrServer embeddedSolrServer = new EmbeddedSolrServer(coreContainer, collectionName);
      return embeddedSolrServer;
    }
    if (solrUrl == null || solrUrl.length() == 0) {
      throw new MorphlineCompilationException("Missing parameter 'solrUrl'", config);
    }
    int solrServerNumThreads = 2;
    int solrServerQueueLength = solrServerNumThreads;
    SolrClient server = new SafeConcurrentUpdateSolrServer(solrUrl, solrServerQueueLength, solrServerNumThreads);
    return server;
  }
}
 

protected void setupCoreContainer() {
  cores = new CoreContainer(getSolrHome(), new Properties());
  cores.load();
  //cores = CoreContainer.createAndLoad(getSolrHome(), new File(TEMP_DIR, "solr.xml"));
}
 

private static CoreContainer load(CoreContainer cc) {
  cc.load();
  return cc;
}
 

public static EmbeddedSolrServer createEmbeddedSolrServer(Path solrHomeDir, FileSystem fs, Path outputShardDir)
    throws IOException {

  if (solrHomeDir == null) {
    throw new IOException("Unable to find solr home setting");
  }
  LOG.info("Creating embedded Solr server with solrHomeDir: " + solrHomeDir + ", fs: " + fs + ", outputShardDir: " + outputShardDir);

  Path solrDataDir = new Path(outputShardDir, "data");

  String dataDirStr = solrDataDir.toUri().toString();

  SolrResourceLoader loader = new SolrResourceLoader(solrHomeDir.toString(), null, null);

  LOG.info(String
      .format(Locale.ENGLISH, 
          "Constructed instance information solr.home %s (%s), instance dir %s, conf dir %s, writing index to solr.data.dir %s, with permdir %s",
          solrHomeDir, solrHomeDir.toUri(), loader.getInstanceDir(),
          loader.getConfigDir(), dataDirStr, outputShardDir));

  // TODO: This is fragile and should be well documented
  System.setProperty("solr.directoryFactory", HdfsDirectoryFactory.class.getName()); 
  System.setProperty("solr.lock.type", "hdfs"); 
  System.setProperty("solr.hdfs.nrtcachingdirectory", "false");
  System.setProperty("solr.hdfs.blockcache.enabled", "false");
  System.setProperty("solr.autoCommit.maxTime", "600000");
  System.setProperty("solr.autoSoftCommit.maxTime", "-1");
  
  CoreContainer container = new CoreContainer(loader);
  container.load();
  
  Properties props = new Properties();
  props.setProperty(CoreDescriptor.CORE_DATADIR, dataDirStr);
  
  CoreDescriptor descr = new CoreDescriptor(container, "core1", solrHomeDir.toString(), props);
  
  SolrCore core = container.create(descr);
  
  if (!(core.getDirectoryFactory() instanceof HdfsDirectoryFactory)) {
    throw new UnsupportedOperationException(
        "Invalid configuration. Currently, the only DirectoryFactory supported is "
            + HdfsDirectoryFactory.class.getSimpleName());
  }

  EmbeddedSolrServer solr = new EmbeddedSolrServer(container, "core1");
  return solr;
}
 

public static EmbeddedSolrServer createEmbeddedSolrServer(Path solrHomeDir, FileSystem fs, Path outputShardDir)
      throws IOException {

    LOG.info("Creating embedded Solr server with solrHomeDir: " + solrHomeDir + ", fs: " + fs + ", outputShardDir: " + outputShardDir);
    
    LOG.info("Using custom SolrRecordWriter class for HBaseMapReduceIndexer");
    if (LOG.isDebugEnabled()) {
      LOG.debug("Listing files contained in solrHomeDir {} ...", solrHomeDir);
      int i = 0;
      for (File file : FileUtils.listFiles(new File(solrHomeDir.toString()), null, true)) {
        // strip off common path prefix for better human readability
        String relPath = file.toString();
        relPath = relPath.substring(solrHomeDir.toString().length() + 1, relPath.length());
        LOG.debug("solrHomeDirFile[{}]: {}", i++, relPath);
      }
    }

    Path solrDataDir = new Path(outputShardDir, "data");

    String dataDirStr = solrDataDir.toUri().toString();

    SolrResourceLoader loader = new SolrResourceLoader(Paths.get(solrHomeDir.toString()), null, null);

    LOG.info(String
        .format(Locale.ENGLISH, 
            "Constructed instance information solr.home %s (%s), instance dir %s, conf dir %s, writing index to solr.data.dir %s, with permdir %s",
            solrHomeDir, solrHomeDir.toUri(), loader.getInstancePath(),
            loader.getConfigDir(), dataDirStr, outputShardDir));

    // TODO: This is fragile and should be well documented
    System.setProperty("solr.directoryFactory", HdfsDirectoryFactory.class.getName()); 
    System.setProperty("solr.lock.type", DirectoryFactory.LOCK_TYPE_HDFS);
    System.setProperty("solr.hdfs.nrtcachingdirectory", "false");
    System.setProperty("solr.hdfs.blockcache.enabled", "false");
    System.setProperty("solr.autoCommit.maxTime", "600000");
    System.setProperty("solr.autoSoftCommit.maxTime", "-1");
    
    CoreContainer container = new CoreContainer(loader);
    container.load();

    SolrCore core = container.create("core1", Paths.get(solrHomeDir.toString()), ImmutableMap.of(CoreDescriptor.CORE_DATADIR, dataDirStr), false);
//    SolrCore core = container.create("", ImmutableMap.of(CoreDescriptor.CORE_DATADIR, dataDirStr));
    
    if (!(core.getDirectoryFactory() instanceof HdfsDirectoryFactory)) {
      throw new UnsupportedOperationException(
          "Invalid configuration. Currently, the only DirectoryFactory supported is "
              + HdfsDirectoryFactory.class.getSimpleName());
    }

    EmbeddedSolrServer solr = new EmbeddedSolrServer(container, "core1");
//    EmbeddedSolrServer solr = new EmbeddedSolrServer(container, "");
    return solr;
  }
 

private static EmbeddedSolrServer createEmbeddedSolrServer(File solrHomeDir, FileSystem fs, Path outputShardDir)
        throws IOException {

  LOG.info("Creating embedded Solr server with solrHomeDir: " + solrHomeDir + ", fs: " + fs + ", outputShardDir: " + outputShardDir);

  // copy solrHomeDir to ensure it isn't modified across multiple unit tests or multiple EmbeddedSolrServer instances
  File tmpDir = Files.createTempDir();
  tmpDir.deleteOnExit();
  FileUtils.copyDirectory(solrHomeDir, tmpDir);
  solrHomeDir = tmpDir;

  Path solrDataDir = new Path(outputShardDir, "data");

  String dataDirStr = solrDataDir.toUri().toString();

  SolrResourceLoader loader = new SolrResourceLoader(Paths.get(solrHomeDir.toString()), null, null);

  LOG.info(String
          .format(Locale.ENGLISH,
                  "Constructed instance information solr.home %s (%s), instance dir %s, conf dir %s, writing index to solr.data.dir %s, with permdir %s",
                  solrHomeDir, solrHomeDir.toURI(), loader.getInstancePath(),
                  loader.getConfigDir(), dataDirStr, outputShardDir));

  // TODO: This is fragile and should be well documented
  System.setProperty("solr.directoryFactory", HdfsDirectoryFactory.class.getName());
  System.setProperty("solr.lock.type", DirectoryFactory.LOCK_TYPE_HDFS);
  System.setProperty("solr.hdfs.nrtcachingdirectory", "false");
  System.setProperty("solr.hdfs.blockcache.enabled", "false");
  System.setProperty("solr.autoCommit.maxTime", "600000");
  System.setProperty("solr.autoSoftCommit.maxTime", "-1");

  CoreContainer container = new CoreContainer(loader);
  container.load();

  SolrCore core = container.create("core1", Paths.get(solrHomeDir.toString()), ImmutableMap.of(CoreDescriptor.CORE_DATADIR, dataDirStr), false);

  if (!(core.getDirectoryFactory() instanceof HdfsDirectoryFactory)) {
    throw new UnsupportedOperationException(
            "Invalid configuration. Currently, the only DirectoryFactory supported is "
                    + HdfsDirectoryFactory.class.getSimpleName());
  }

  EmbeddedSolrServer solr = new EmbeddedSolrServer(container, "core1");
  return solr;
}