Java Code Examples for gate.util.Files#fileFromURL()

/**
 * Create a corpus exporter instance for the given Exporter.
 * 
 * This method will create the proper corpus representation for the 
 * algorithm and the feature info. 
 * 
 * @param exporter exporter 
 * @param parms parameters
 * @param featureInfo feature info
 * @param instanceType instance type
 * @param datadir data directory
 * @return CorpusExporter instance
 */
public static CorpusExporter create(Exporter exporter, String parms, 
        FeatureInfo featureInfo, String instanceType, URL datadir) {
  CorpusExporter ce = null;
  try {
    @SuppressWarnings("unchecked")             
    Constructor<?> constr = exporter.getCorpusExporterClass().getDeclaredConstructor();
    ce = (CorpusExporter)constr.newInstance();
  } catch (IllegalAccessException | IllegalArgumentException | InstantiationException | NoSuchMethodException | SecurityException | InvocationTargetException ex) {
    throw new GateRuntimeException("Error creating CorpusExporter instance for "+exporter.getCorpusExporterClass(),ex);
  }
  ce.datadir = datadir;
  ce.exporter = exporter;
  ce.targetType = exporter.getTargetType();
  ce.featureInfo = featureInfo;
  ce.parms = parms;
  ce.instanceType = instanceType;
  ce.dataDirFile = Files.fileFromURL(datadir);
  ce.initWhenCreating();    
  return ce;
}
 

/**
 * Write out the (possibly modified) GAPP file to its new location.
 * 
 * @throws IOException if an I/O error occurs.
 */
public void write() throws IOException {
  finish();
  File newGappFile = Files.fileFromURL(gappFileURL);
  FileOutputStream fos = new FileOutputStream(newGappFile);
  BufferedOutputStream out = new BufferedOutputStream(fos);

  XMLOutputter outputter = new XMLOutputter(Format.getRawFormat());
  outputter.output(gappDocument, out);
}
 

@Override
protected void loadModel(URL directoryURL, String parms) {
  ArrayList<String> finalCommand = new ArrayList<>();
  // Instead of loading a model, this establishes a connection with the 
  // external sklearn process. 
  if(!"file".equals(directoryURL.getProtocol())) {
    throw new GateRuntimeException("The dataDirectory URL must be a file: URL for sklearn");
  }
  File directory = Files.fileFromURL(directoryURL);
  File commandFile = findWrapperCommand(directory, true);
  String modelFileName = new File(directory,MODEL_BASENAME).getAbsolutePath();
  finalCommand.add(commandFile.getAbsolutePath());
  finalCommand.add(modelFileName);
  // if we have a shell command prepend that, and if we have shell parms too, include them
  if(shellcmd != null) {
    finalCommand.add(0,shellcmd);
    if(shellparms != null) {
      String[] sps = shellparms.trim().split("\\s+");
      int i=0; for(String sp : sps) { finalCommand.add(++i,sp); }
    }
  }
  //System.err.println("Running: "+finalCommand);
  // Create a fake Model jsut to make LF_Apply... happy which checks if this is null
  model = MODEL_INSTANCE;
  Map<String,String> env = new HashMap<>();
  env.put(ENV_WRAPPER_HOME, wrapperhome);
  process = Process4JsonStream.create(directory,env,finalCommand);
}
 

@Override
protected void initWhenCreating(URL directory, Algorithm algorithm, 
        String parms, FeatureInfo featureInfo, TargetType targetType) {
  dataDir = Files.fileFromURL(directory);
  this.featureInfo = featureInfo;
  corpusRepresentation = new CorpusRepresentationVolatileDense2JsonStream(dataDir, featureInfo);
  corpusRepresentation.startAdding();
  this.featureInfo = featureInfo;
  // NOTE: we are copying the wrapper code only when starting training, not
  // here. This allows the user to copy their own code while the PR is running
  // but creating the corpus has not yet finished.
}
 

@Override
protected void loadAndSetCorpusRepresentation(URL directory) {
  //System.err.println("DEBUG EngineDVFileJson: running loadAndSetCorpusRepresentation "+directory);
  
  // this does not actually need to load anything but the featureInfo ... 
  // this is needed to convert our instance data to JSON, which is then sent
  // off to the script or server which is responsible to use any other saved
  // model info (the model itself, scaling info, vocab info, embeddings etc)
  dataDir = Files.fileFromURL(directory);
  featureInfo = FeatureInfo.load(directory);
  corpusRepresentation = new CorpusRepresentationVolatileDense2JsonStream(dataDir, featureInfo);
}
 

@Override
public void loadModel(URL directory, String parms) {
  if(!"file".equals(directory.getProtocol())) {
    throw new GateRuntimeException("The dataDirectory URL must be a file: URL for LibSVM");
  }
  try {      
    File directoryFile = Files.fileFromURL(directory);
    svm_model svmModel = svm.svm_load_model(new File(directoryFile, FILENAME_MODEL).getAbsolutePath());
    // System.out.println("Loaded LIBSVM model, nrclasses=" + svmModel.nr_class);
    model = svmModel;
  } catch (IOException | IllegalArgumentException ex) {
    throw new GateRuntimeException("Error loading the LIBSVM model from directory "+directory, ex);
  }
}
 

/** Open a connection to the data store. */
@Override
public void open() throws PersistenceException {
  super.open();
  /*
   * check if the storage directory is a valid serial datastore if we
   * want to support old style: String versionInVersionFile = "1.0";
   * (but this means it will open *any* directory)
   */
  try (BufferedReader isr = new BufferedReader(new FileReader(getVersionFile()))) {
    currentProtocolVersion = isr.readLine();
    String indexDirRelativePath = isr.readLine();

    if(indexDirRelativePath != null
            && indexDirRelativePath.trim().length() > 1) {
      URL storageDirURL = storageDir.toURI().toURL();
      URL theIndexURL = new URL(storageDirURL, indexDirRelativePath);
      // check if index directory exists
      File indexDir = Files.fileFromURL(theIndexURL);
      if(!indexDir.exists()) {
        throw new PersistenceException("Index directory "
                + indexDirRelativePath
                + " could not be found for datastore at " + storageDirURL);
      }

      indexURL = theIndexURL;
      this.indexer = new LuceneIndexer(indexURL);
      this.searcher = new LuceneSearcher();
      ((LuceneSearcher)this.searcher).setLuceneDatastore(this);
    }
  } catch(IOException e) {
    throw new PersistenceException("Invalid storage directory: " + e);
  }
  
  if(!isValidProtocolVersion(currentProtocolVersion))
    throw new PersistenceException("Invalid protocol version number: "
            + currentProtocolVersion);

  // Lets create a separate indexer thread which keeps running in the
  // background
  executor =
          new ScheduledThreadPoolExecutor(1, Executors.defaultThreadFactory());
  // set up the executor so it does not execute delayed indexing tasks
  // that are still waiting when it is shut down. We run these tasks
  // immediately at shutdown time rather than waiting.
  executor.setContinueExistingPeriodicTasksAfterShutdownPolicy(false);
  executor.setExecuteExistingDelayedTasksAfterShutdownPolicy(false);
  
  // start listening to Creole events
  Gate.getCreoleRegister().addCreoleListener(this);
}
 

/**
 * Fills the provided corpus with documents created on the fly from
 * selected files in a directory. Uses a {@link FileFilter} to select
 * which files will be used and which will be ignored. A simple file
 * filter based on extensions is provided in the Gate distribution (
 * {@link gate.util.ExtensionFileFilter}).
 * 
 * @param corpus the corpus to be populated
 * @param directory the directory from which the files will be picked.
 *          This parameter is an URL for uniformity. It needs to be a
 *          URL of type file otherwise an InvalidArgumentException
 *          will be thrown.
 * @param filter the file filter used to select files from the target
 *          directory. If the filter is <tt>null</tt> all the files
 *          will be accepted.
 * @param encoding the encoding to be used for reading the documents
 * @param recurseDirectories should the directory be parsed
 *          recursively?. If <tt>true</tt> all the files from the
 *          provided directory and all its children directories (on as
 *          many levels as necessary) will be picked if accepted by
 *          the filter otherwise the children directories will be
 *          ignored.
 * @throws java.io.IOException if a file doesn't exist
 */
public static void populate(Corpus corpus, URL directory, FileFilter filter,
        String encoding, String mimeType, boolean recurseDirectories)
        throws IOException {

  // check input
  if(!directory.getProtocol().equalsIgnoreCase("file"))
    throw new IllegalArgumentException(
            "The URL provided is not of type \"file:\"!");

  File dir = Files.fileFromURL(directory);
  if(!dir.exists()) throw new FileNotFoundException(dir.toString());

  if(!dir.isDirectory())
    throw new IllegalArgumentException(dir.getAbsolutePath()
            + " is not a directory!");

  File[] files;
  // populate the corpus
  if(recurseDirectories) {
    files = Files.listFilesRecursively(dir, filter);
  }
  else {
    files = dir.listFiles(filter);
  }

  if(files == null) {
    return;
  }

  // sort the files alphabetically regardless of their paths
  Arrays.sort(files, new Comparator<File>() {
    @Override
    public int compare(File f1, File f2) {
      return f1.getName().compareTo(f2.getName());
    }
  });

  // create the GATE documents
  for(File file : files) {
    if(file.isDirectory()) {
      continue;
    }
    StatusListener sListener = (StatusListener)Gate.getListeners().get(
            "gate.event.StatusListener");
    if(sListener != null)
      sListener.statusChanged("Reading: " + file.getName());
    String docName = file.getName() + "_" + Gate.genSym();
    FeatureMap params = Factory.newFeatureMap();
    params.put(Document.DOCUMENT_URL_PARAMETER_NAME, file.toURI().toURL());
    if(encoding != null)
      params.put(Document.DOCUMENT_ENCODING_PARAMETER_NAME, encoding);
    if(mimeType != null)
      params.put(Document.DOCUMENT_MIME_TYPE_PARAMETER_NAME, mimeType);

    try {
      Document doc = (Document)Factory.createResource(DocumentImpl.class
              .getName(), params, null, docName);
      corpus.add(doc);
      if(corpus.getLRPersistenceId() != null) {
        // persistent corpus -> unload the document
        corpus.unloadDocument(doc);
        Factory.deleteResource(doc);
      }
    }
    catch(Throwable t) {
      String nl = Strings.getNl();
      Err.prln("WARNING: Corpus.populate could not instantiate document" + nl
              + "  Document name was: " + docName + nl + "  Exception was: "
              + t + nl + nl);
      t.printStackTrace();
    }
    if(sListener != null) sListener.statusChanged(file.getName() + " read");
  }

}
 

@Override
protected void loadModel(URL directoryURL, String parms) {
  ArrayList<String> finalCommand = new ArrayList<>();
  // TODO: for now, we only allow URLs which are file: URLs here.
  // This is because the script wrapping Weka is currently not able to access 
  // the model from any other location. Also, we need to export the 
  // data and currently this is done into the directoryURL.
  // At some later point, we may be able to e.g. copy the model into 
  // a temporary directory and use the demporary directory also to store 
  // the data! 
  File directoryFile = null;
  if("file".equals(directoryURL.getProtocol())) {
    directoryFile = Files.fileFromURL(directoryURL);
  } else {
    throw new GateRuntimeException("The dataDirectory for WekaWrapper must be a file: URL");
  }
  
  // Instead of loading a model, this establishes a connection with the 
  // external weka process. For this, we expect an additional file in the 
  // directory, weka.yaml, which describes how to run the weka wrapper
  File commandFile = findWrapperCommand(directoryFile, true);
  // If the directoryURL 
  String modelFileName = new File(directoryFile,FILENAME_MODEL).getAbsolutePath();
  if(!new File(modelFileName).exists()) {
    throw new GateRuntimeException("File not found: "+modelFileName);
  }
  String header = new File(directoryFile,"header.arff").getAbsolutePath();
  if(!new File(header).exists()) {
    throw new GateRuntimeException("File not found: "+header);
  }
  
  if(shellcmd != null) {
    finalCommand.add(shellcmd);
    if(shellparms != null) {
      String[] sps = shellparms.trim().split("\\s+");
      for(String sp : sps) { finalCommand.add(sp); }
    }
  }
  
  finalCommand.add(commandFile.getAbsolutePath());
  finalCommand.add(modelFileName);
  finalCommand.add(header);
  
  //System.err.println("Running: "+finalCommand);
  // Create a fake Model jsut to make LF_Apply... happy which checks if this is null
  model = "ExternalWekaWrapperModel";
  Map<String,String> env = new HashMap<>();
  env.put(ENV_WRAPPER_HOME,wrapperhome);
  // NOTE: if the directoryFile is null, the current Java process' directory is used
  process = Process4ObjectStream.create(directoryFile,env,finalCommand);
}
 

@Override
protected void loadModel(URL directoryURL, String parms) {
  File directory = null;
  if("file".equals(directoryURL.getProtocol())) {
    directory = Files.fileFromURL(directoryURL);
  } else {
    throw new GateRuntimeException("The dataDirectory for WekaWrapper must be a file: URL not "+directoryURL);
  }
  ArrayList<String> finalCommand = new ArrayList<>();
  // we need the corpus representation here! Normally this is done from loadEngine and after
  // load model, but we do it here. The load crm method only loads anything if it is still
  // null, so we will do this only once anyway.
  loadAndSetCorpusRepresentation(directoryURL);
  CorpusRepresentationMalletTarget data = (CorpusRepresentationMalletTarget)corpusRepresentation;
  SimpleEntry<String,Integer> modeAndNrC = findOutMode(data);
  String mode = modeAndNrC.getKey();
  Integer nrClasses = modeAndNrC.getValue();
  // Instead of loading a model, this establishes a connection with the 
  // external wrapper process. 
  
  File commandFile = findWrapperCommand(directory, true);
  String modelFileName = new File(directory,MODEL_BASENAME).getAbsolutePath();
  finalCommand.add(commandFile.getAbsolutePath());
  finalCommand.add(modelFileName);
  finalCommand.add(mode);
  finalCommand.add(nrClasses.toString());
  // if we have a shell command prepend that, and if we have shell parms too, include them
  if(shellcmd != null) {
    finalCommand.add(0,shellcmd);
    if(shellparms != null) {
      String[] sps = shellparms.trim().split("\\s+");
      int i=0; for(String sp : sps) { finalCommand.add(++i,sp); }
    }
  }
  //System.err.println("Running: "+finalCommand);
  // Create a fake Model jsut to make LF_Apply... happy which checks if this is null
  model = MODEL_INSTANCE;
  Map<String,String> env = new HashMap<>();
  env.put(ENV_WRAPPER_HOME, wrapperhome);
  process = Process4JsonStream.create(directory,env,finalCommand);
}