be.tarsos.dsp.AudioDispatcher Java Examples

/**
 * Create a new AudioDispatcher connected to the default microphone.
 * 
 * @param sampleRate
 *            The requested sample rate.
 * @param audioBufferSize
 *            The size of the audio buffer (in samples).
 * 
 * @param bufferOverlap
 *            The size of the overlap (in samples).
 * @return A new AudioDispatcher
 */
public static AudioDispatcher fromDefaultMicrophone(final int sampleRate,
		final int audioBufferSize, final int bufferOverlap) {
	int minAudioBufferSize = AudioRecord.getMinBufferSize(sampleRate,
			android.media.AudioFormat.CHANNEL_IN_MONO,
			android.media.AudioFormat.ENCODING_PCM_16BIT);
	int minAudioBufferSizeInSamples =  minAudioBufferSize/2;
	if(minAudioBufferSizeInSamples <= audioBufferSize ){
	AudioRecord audioInputStream = new AudioRecord(
			MediaRecorder.AudioSource.MIC, sampleRate,
			android.media.AudioFormat.CHANNEL_IN_MONO,
			android.media.AudioFormat.ENCODING_PCM_16BIT,
			audioBufferSize * 2);

	TarsosDSPAudioFormat format = new TarsosDSPAudioFormat(sampleRate, 16,1, true, false);
	
	TarsosDSPAudioInputStream audioStream = new AndroidAudioInputStream(audioInputStream, format);
	//start recording ! Opens the stream.
	audioInputStream.startRecording();
	return new AudioDispatcher(audioStream,audioBufferSize,bufferOverlap);
	}else{
		throw new IllegalArgumentException("Buffer size too small should be at least " + (minAudioBufferSize *2));
	}
}
 

public BeatRootSpectralFluxOnsetDetector(AudioDispatcher d,int fftSize, int hopSize){
	
	this.hopSize = hopSize; 
	this.hopTime = hopSize/d.getFormat().getSampleRate();
	this.fftSize = fftSize;

	System.err.println("Please use the ComplexOnset detector: BeatRootSpectralFluxOnsetDetector does currenlty not support streaming");
	//no overlap
	//FIXME:		
	int durationInFrames = -1000; 
	totalFrames = (int)(durationInFrames / hopSize) + 4;
	energy = new double[totalFrames*energyOversampleFactor];
	spectralFlux = new double[totalFrames];
	
	reBuffer = new float[fftSize/2];
	imBuffer = new float[fftSize/2];
	prevFrame = new float[fftSize/2];
	
	makeFreqMap(fftSize, d.getFormat().getSampleRate());
	
	newFrame = new double[freqMapSize];
	frames = new double[totalFrames][freqMapSize];
	handler = new PrintOnsetHandler();
	fft = new FFT(fftSize,new ScaledHammingWindow());
}
 

private void processMonitorQuery(float[] audioData,QueryResultHandler handler, double timeStamp,Set<Integer> avoid){
	int samplerate = Config.getInt(Key.RAFS_SAMPLE_RATE);
	int size = Config.getInt(Key.RAFS_FFT_SIZE);
	int overlap = size - Config.getInt(Key.RAFS_FFT_STEP_SIZE);
	
	AudioDispatcher d;
	try {
		d = AudioDispatcherFactory.fromFloatArray(audioData, samplerate, size, overlap);
		d.setZeroPadFirstBuffer(true);
		final RafsExtractor processor = new RafsExtractor(null,true);
		d.addAudioProcessor(processor);
		d.run();
		queryForMonitor(processor.fingerprints, processor.fingerprintProbabilities, 10 , avoid, handler);
	} catch (UnsupportedAudioFileException e) {
		LOG.severe("Unsupported audio");
	}
}
 

private static RafsPacker extractPacker(File f, int fileIndex, boolean trackProbabilities){
	final int sampleRate = Config.getInt(Key.RAFS_SAMPLE_RATE);//2250Hz Nyquist frequency
	final int size = Config.getInt(Key.RAFS_FFT_SIZE);
	final int overlap =  size - Config.getInt(Key.RAFS_FFT_STEP_SIZE); //about an fft every 11.6ms (64/5500)
	String file = f.getAbsolutePath();
	AudioDispatcher d = AudioDispatcherFactory.fromPipe(file, sampleRate, size, overlap);
	d.setZeroPadFirstBuffer(true);
	RafsExtractor ex = new RafsExtractor(file, trackProbabilities);
	RafsPacker packer = new RafsPacker(ex,trackProbabilities);
	//String baseName = f.getName();
	d.setZeroPadFirstBuffer(true);
	d.addAudioProcessor(ex);
	d.addAudioProcessor(packer);
	d.run();
	return packer;
}
 

private static List<BitSetWithID> extractPackedPrints(File f,int fileIndex){		
	final int sampleRate = Config.getInt(Key.RAFS_SAMPLE_RATE);//2250Hz Nyquist frequency
	final int size = Config.getInt(Key.RAFS_FFT_SIZE);
	final int overlap =  size - Config.getInt(Key.RAFS_FFT_STEP_SIZE); 
	String file = f.getAbsolutePath();
	AudioDispatcher d = AudioDispatcherFactory.fromPipe(file, sampleRate, size, overlap);
	RafsExtractor ex = new RafsExtractor(file, true);
	RafsPacker packer = new RafsPacker(ex,true);
	//String baseName = f.getName();
	d.setZeroPadFirstBuffer(true);
	d.addAudioProcessor(ex);
	d.addAudioProcessor(packer);
	d.run();
	List<BitSetWithID> prints = new ArrayList<>();
	
	for (Map.Entry<Float, BitSet> frameEntry : packer.packedFingerprints.entrySet()) {
		int offset = (int) (frameEntry.getKey() * 1000);
		prints.add(new BitSetWithID(fileIndex * (1L<<32)  + offset, frameEntry.getValue()));
	}
	return prints;		
}
 

@Override
public void monitor(String query, final int maxNumberOfReqults,Set<Integer> avoid,
		final QueryResultHandler handler) {
	
	int samplerate = Config.getInt(Key.NCTEQ_SAMPLE_RATE);
	int size = Config.getInt(Key.MONITOR_STEP_SIZE) * samplerate;
	int overlap = Config.getInt(Key.MONITOR_OVERLAP) * samplerate;
	final ConstantQ constanQ = createConstantQ();
	
	AudioDispatcher d = AudioDispatcherFactory.fromPipe(query, samplerate, size, overlap);
	d.addAudioProcessor(new AudioProcessor() {
		@Override
		public boolean process(AudioEvent audioEvent) {
			double timeStamp = audioEvent.getTimeStamp() - Config.getInt(Key.MONITOR_OVERLAP);
			processMonitorQuery(audioEvent.getFloatBuffer().clone(), maxNumberOfReqults, handler,timeStamp,constanQ);
			return true;
		}
		
		@Override
		public void processingFinished() {
		}
	});
	d.run();

}
 

private void processMonitorQueryToSerializeFingerprints(float[] audioBuffer,SerializedFingerprintsHandler handler,double queryOffset){
	int samplerate = Config.getInt(Key.NFFT_SAMPLE_RATE);
	int size = Config.getInt(Key.NFFT_SIZE);
	int overlap = size - Config.getInt(Key.NFFT_STEP_SIZE);
	
	AudioDispatcher d;
	try {
		d = AudioDispatcherFactory.fromFloatArray(audioBuffer, samplerate, size, overlap);
		final NFFTEventPointProcessor minMaxProcessor = new NFFTEventPointProcessor(size,overlap,samplerate);
		d.addAudioProcessor(minMaxProcessor);
		d.run();
		double queryDuration = d.secondsProcessed();
		List<NFFTFingerprint> fingerprints = new ArrayList<NFFTFingerprint>(minMaxProcessor.getFingerprints());
		handler.handleSerializedFingerprints(PanakoWebserviceClient.serializeFingerprintsToJson(fingerprints),queryDuration,queryOffset);
	} catch (UnsupportedAudioFileException e) {
		LOG.severe("Unsupported audio");
	}
}
 

@Override
public void run(String... args) {
	String inputResource = AudioResourceUtils.sanitizeResource(args[0]);
	AudioDispatcher d;
	try {
		d = AudioDispatcherFactory.fromPipe(inputResource, TARGET_SAMPLE_RATE, 2028, 0);
		d.addAudioProcessor(new AudioPlayer(JVMAudioInputStream.toAudioFormat(d.getFormat())));
		d.run();
	}  catch (LineUnavailableException e) {
		e.printStackTrace();
		System.err.print(e.getLocalizedMessage());
	}
}
 

public static float[] getAudioBuffer(File file,double start,double stop){

	double sampleRate = 44100;
	int sampleStart = (int) Math.round(sampleRate * start);
	int sampleStop = (int) Math.round(sampleRate * stop);
	int diff = sampleStop - sampleStart;
	final float[] audioBuffer = new float[diff];
	
	AudioDispatcher d;
	
	d = AudioDispatcherFactory.fromPipe(file.getAbsolutePath(), 44100,diff, 0);
	d.skip(start);
	d.addAudioProcessor(new AudioProcessor() {
		boolean filled = false;
		@Override
		public void processingFinished() {
		}

		@Override
		public boolean process(AudioEvent audioEvent) {
			if(!filled){
				for (int i = 0; i < audioEvent.getFloatBuffer().length; i++) {
					audioBuffer[i] = audioEvent.getFloatBuffer()[i];
				}
				filled = true;
			}
			return false;
		}
	});
	d.run();
	
	
	
	return audioBuffer;
}
 

@Test
public void testPipeDecoder(){
	File reference = TestUtilities.getResource("dataset/61198.wav");
	File referenceFile = TestUtilities.getResource("dataset/61198.wav");
	final float[] referenceBuffer = TestUtilities.getAudioBuffer(reference,1.0,1.5);
	
	AudioDispatcher d = AudioDispatcherFactory.fromPipe(referenceFile.getAbsolutePath(), 44100, 22050, 0,1.0,0.5);
	d.addAudioProcessor(new AudioProcessor() {
		boolean ran = false;
		@Override
		public void processingFinished() {
		}
		
		@Override
		public boolean process(AudioEvent audioEvent) {
			if(!ran){
				float[] otherBuffer = audioEvent.getFloatBuffer();
				assertEquals("Buffers should be equal in length", referenceBuffer.length, otherBuffer.length); 
				for(int i = 0 ; i < otherBuffer.length; i++){
					assertEquals("Buffers should have the same content", referenceBuffer[i], otherBuffer[i],0.0000001);
				}
			}
			ran = true;
			return true;
		}
	});
	d.run();		
}
 

public static List<QIFFTFingerprint> extractFingerprintsFromQuery(String query){
	int samplerate = Config.getInt(Key.NFFT_SAMPLE_RATE);
	int size = Config.getInt(Key.NFFT_SIZE);
	int overlap = size - Config.getInt(Key.NFFT_STEP_SIZE);
	AudioDispatcher d = AudioDispatcherFactory.fromPipe(query, samplerate, size, overlap);
	final QIFFTEventPointProcessor minMaxProcessor = new QIFFTEventPointProcessor(size,overlap,samplerate,4);
	d.addAudioProcessor(minMaxProcessor);
	d.run();
	List<QIFFTFingerprint> fingerprints = new ArrayList<QIFFTFingerprint>(minMaxProcessor.getFingerprints());
	return fingerprints;
}
 

@Override
public double store(String resource, String description) {
	
	ConstantQ constantQ = createConstantQ();
	NCteQMapDBStorage storage = NCteQMapDBStorage.getInstance();
	
	int sampleRate = Config.getInt(Key.NCTEQ_SAMPLE_RATE);
	int size = constantQ.getFFTlength();
	int overlap = size - Config.getInt(Key.NCTEQ_STEP_SIZE);
	NCteQEventPointProcessor eventPointProcessor = new NCteQEventPointProcessor(constantQ,sampleRate,Config.getInt(Key.NCTEQ_STEP_SIZE));
	
	AudioDispatcher d = AudioDispatcherFactory.fromPipe(resource, sampleRate, size , overlap);
	d.addAudioProcessor(eventPointProcessor);
	d.run();
	
	int identifier = FileUtils.getIdentifier(resource);
	int hashesAdded = 0;
	
	float bucketFillFactorSum = 0.0f;
	for(NCteQFingerprint fingerprint : eventPointProcessor.getFingerprints()){
		float fillFactor = storage.addFingerprint(identifier, fingerprint.t1, fingerprint.hash(),fingerprint.timeDelta(),fingerprint.f1);
		bucketFillFactorSum += fillFactor;
		if(fillFactor != 1.0){
			hashesAdded++;
		}
	}
	LOG.info(String.format("Average hash bucket fill factor for %d hashes %.2f %%", hashesAdded, 100 * bucketFillFactorSum / (float)hashesAdded ) );
	float secondsProcessed = d.secondsProcessed();
	
	storage.addAudio(identifier, description);
	storage.audioObjectAdded(hashesAdded, Math.round(secondsProcessed));
	
	return secondsProcessed;
}
 

private void processMonitorQuery(float[] audioBuffer,int maxNumberOfResults,
		QueryResultHandler handler,double queryOffset,Set<Integer> avoid){
	int samplerate = Config.getInt(Key.NFFT_SAMPLE_RATE);
	int size = Config.getInt(Key.NFFT_SIZE);
	int overlap = size - Config.getInt(Key.NFFT_STEP_SIZE);
	
	AudioDispatcher d;
	try {
		d = AudioDispatcherFactory.fromFloatArray(audioBuffer, samplerate, size, overlap);
		final NFFTEventPointProcessor minMaxProcessor = new NFFTEventPointProcessor(size,overlap,samplerate);
		d.addAudioProcessor(minMaxProcessor);
		d.run();
		List<NFFTFingerprint> fingerprints = new ArrayList<NFFTFingerprint>(minMaxProcessor.getFingerprints());
		
		final List<NFFTFingerprintQueryMatch> queryMatches = new ArrayList<NFFTFingerprintQueryMatch>();
		
		queryMatches.addAll(storage.getMatches(fingerprints, maxNumberOfResults));
		
		double queryDuration = d.secondsProcessed();
		
		if(queryMatches.isEmpty()){
			QueryResult result = QueryResult.emptyQueryResult(queryOffset,queryOffset+queryDuration);
			handler.handleEmptyResult(result);
		}else{
			for(NFFTFingerprintQueryMatch match : queryMatches){
				//avoid the results in the avoid hash set
				if(!avoid.contains(match.identifier)){
					String description = storage.getAudioDescription(match.identifier);
					handler.handleQueryResult(new QueryResult(queryOffset,queryOffset+queryDuration,String.valueOf(match.identifier), description, match.score, match.getStartTime(),100.0,100.0));
				}
			}
		}
		
	} catch (UnsupportedAudioFileException e) {
		LOG.severe("Unsupported audio");
	}
	
}
 

@Override
public void monitor(String query,final  int maxNumberOfResults,Set<Integer> avoid,
		final QueryResultHandler handler) {
	
	int samplerate = Config.getInt(Key.NFFT_SAMPLE_RATE);
	int size = Config.getInt(Key.MONITOR_STEP_SIZE) * samplerate;
	int overlap = Config.getInt(Key.MONITOR_OVERLAP) * samplerate;
	AudioDispatcher d ;
	if (query.equals(Panako.DEFAULT_MICROPHONE)){
		try {
			d = AudioDispatcherFactory.fromDefaultMicrophone(samplerate,size, overlap);
		} catch (LineUnavailableException e) {
			LOG.warning("Could not connect to default microphone!" + e.getMessage());
			e.printStackTrace();
			d = null;
		}
	}else{
		d = AudioDispatcherFactory.fromPipe(query, samplerate, size, overlap);
	}
	d.addAudioProcessor(new AudioProcessor() {
		@Override
		public boolean process(AudioEvent audioEvent) {
			double timeStamp = audioEvent.getTimeStamp() - Config.getInt(Key.MONITOR_OVERLAP);
			processMonitorQuery(audioEvent.getFloatBuffer().clone(), maxNumberOfResults, handler,timeStamp,avoid);
			return true;
		}
		
		@Override
		public void processingFinished() {
		}
	});
	d.run();
}
 

public void monitor(String query,final SerializedFingerprintsHandler handler){
	
	int samplerate = Config.getInt(Key.NFFT_SAMPLE_RATE);
	int size = Config.getInt(Key.MONITOR_STEP_SIZE) * samplerate;
	int overlap = Config.getInt(Key.MONITOR_OVERLAP) * samplerate;
	AudioDispatcher d ;
	if (query.equals(Panako.DEFAULT_MICROPHONE)){
		try {
			d = AudioDispatcherFactory.fromDefaultMicrophone(samplerate,size, overlap);
		} catch (LineUnavailableException e) {
			LOG.warning("Could not connect to default microphone!" + e.getMessage());
			e.printStackTrace();
			d = null;
		}
	}else{
		d = AudioDispatcherFactory.fromPipe(query, samplerate, size, overlap);
	}
	d.addAudioProcessor(new AudioProcessor() {
		@Override
		public boolean process(AudioEvent audioEvent) {
			double timeStamp = audioEvent.getTimeStamp() - Config.getInt(Key.MONITOR_OVERLAP);
			processMonitorQueryToSerializeFingerprints(audioEvent.getFloatBuffer().clone(), handler,timeStamp);
			return true;
		}
		
		@Override
		public void processingFinished() {
		}
	});
	d.run();
}
 

public List<NFFTFingerprint> extractFingerprintsFromQuery(String query){
	int samplerate = Config.getInt(Key.NFFT_SAMPLE_RATE);
	int size = Config.getInt(Key.NFFT_SIZE);
	int overlap = size - Config.getInt(Key.NFFT_STEP_SIZE);
	AudioDispatcher d = AudioDispatcherFactory.fromPipe(query, samplerate, size, overlap);
	final NFFTEventPointProcessor minMaxProcessor = new NFFTEventPointProcessor(size,overlap,samplerate);
	d.addAudioProcessor(minMaxProcessor);
	d.run();
	List<NFFTFingerprint> fingerprints = new ArrayList<NFFTFingerprint>(minMaxProcessor.getFingerprints());
	return fingerprints;
}
 

@Override
public double store(String resource, String description) {
	int samplerate = Config.getInt(Key.NFFT_SAMPLE_RATE);
	int size = Config.getInt(Key.NFFT_SIZE);
	int overlap = size - Config.getInt(Key.NFFT_STEP_SIZE);
	
	AudioDispatcher d = AudioDispatcherFactory.fromPipe(resource, samplerate, size, overlap);
	final NFFTEventPointProcessor minMaxProcessor = new NFFTEventPointProcessor(size,overlap,samplerate);
	d.addAudioProcessor(minMaxProcessor);
	d.run();
	Set<NFFTFingerprint> fingerprints = new HashSet<NFFTFingerprint>(minMaxProcessor.getFingerprints());
	
	int identifier = FileUtils.getIdentifier(resource);
	
	
	for(NFFTFingerprint fingerprint: fingerprints){
		storage.addFingerprint(identifier, fingerprint.t1, fingerprint.hash());
	}
	
	// Store the meta data.
	storage.addAudio(identifier, description);
	
	// Commit the changes to store the fingerprints
	double durationInSeconds = d.secondsProcessed();
	storage.audioObjectAdded((int) Math.round(durationInSeconds));
	
	LOG.info(String.format("Stored %d fingerprints bundeled from %d event points for %s.",fingerprints.size(),minMaxProcessor.getEventPoints().size(),resource));
	return durationInSeconds;
}
 

private List<NFFTFingerprint> extractFingerprints(String resource) {
	int samplerate = Config.getInt(Key.NFFT_SAMPLE_RATE);
	int size = Config.getInt(Key.NFFT_SIZE);
	int overlap = size - Config.getInt(Key.NFFT_STEP_SIZE);

	AudioDispatcher d = AudioDispatcherFactory.fromPipe(resource, samplerate, size, overlap);
	final NFFTEventPointProcessor minMaxProcessor = new NFFTEventPointProcessor(size, overlap, samplerate);
	d.addAudioProcessor(minMaxProcessor);
	d.run();
	return new ArrayList<NFFTFingerprint>(minMaxProcessor.getFingerprints());
}
 

private static TreeMap<Float, BitSet> extractPackedPrints(File f,boolean trackProbabilities){		
	final int sampleRate = Config.getInt(Key.RAFS_SAMPLE_RATE);//2250Hz Nyquist frequency
	final int size = Config.getInt(Key.RAFS_FFT_SIZE);
	final int overlap = size - Config.getInt(Key.RAFS_FFT_STEP_SIZE);
	String file = f.getAbsolutePath();
	AudioDispatcher d = AudioDispatcherFactory.fromPipe(file, sampleRate, size, overlap);
	d.setZeroPadFirstBuffer(true);
	RafsExtractor ex = new RafsExtractor(file, trackProbabilities);
	//String baseName = f.getName();
	d.addAudioProcessor(ex);
	d.run();
	return ex.fingerprints;
}
 

public void starExtraction(){
	AudioDispatcher d = AudioDispatcherFactory.fromPipe(file, sampleRate, size, overlap);
	//every buffer has the same length
	d.setZeroPadFirstBuffer(true);
	d.addAudioProcessor(this);
	d.run();
}
 

private static RafsExtractor extractExtractor(File f, int fileIndex, boolean trackProbabilities){
	final int sampleRate = Config.getInt(Key.RAFS_SAMPLE_RATE);//2250Hz Nyquist frequency
	final int size = Config.getInt(Key.RAFS_FFT_SIZE);
	final int overlap = size -  Config.getInt(Key.RAFS_FFT_STEP_SIZE); //about an fft every 11.6ms (64/5500)
	String file = f.getAbsolutePath();
	AudioDispatcher d = AudioDispatcherFactory.fromPipe(file, sampleRate, size, overlap);
	d.setZeroPadFirstBuffer(true);
	RafsExtractor ex = new RafsExtractor(file, trackProbabilities);
	//String baseName = f.getName();
	d.addAudioProcessor(ex);
	d.run();
	return ex;
}
 

@Override
public void monitor(String query, int maxNumberOfReqults, Set<Integer> avoid, QueryResultHandler handler) {
	int samplerate = Config.getInt(Key.RAFS_SAMPLE_RATE);
	int size = Config.getInt(Key.MONITOR_STEP_SIZE) * samplerate;
	int overlap = Config.getInt(Key.MONITOR_OVERLAP) * samplerate;
	AudioDispatcher d ;
	if (query.equals(Panako.DEFAULT_MICROPHONE)){
		try {
			d = AudioDispatcherFactory.fromDefaultMicrophone(samplerate,size, overlap);
		} catch (LineUnavailableException e) {
			LOG.warning("Could not connect to default microphone!" + e.getMessage());
			e.printStackTrace();
			d = null;
		}
	}else{
		d = AudioDispatcherFactory.fromPipe(query, samplerate, size, overlap);
	}
	d.setZeroPadFirstBuffer(true);
	d.addAudioProcessor(new AudioProcessor() {
		@Override
		public boolean process(AudioEvent audioEvent) {
			double timeStamp = audioEvent.getTimeStamp() - Config.getInt(Key.MONITOR_OVERLAP);
			processMonitorQuery(audioEvent.getFloatBuffer().clone(), handler,timeStamp,avoid);
			return true;
		}
		
		@Override
		public void processingFinished() {
		}
	});
	d.run();
}
 

public void starExtraction(){
	AudioDispatcher d = AudioDispatcherFactory.fromPipe(file, sampleRate, size, overlap);
	d.addAudioProcessor(this);
	d.run();
}
 

public void setDispatcher(AudioDispatcher newDispatcher){
	this.dispatcher = newDispatcher;
}
 

private AudioDispatcher getAudioDispatcher() {
    return this.mAudioDispatcher;
}
 

/**
 * Create a stream from a piped sub process and use that to create a new
 * {@link AudioDispatcher} The sub-process writes a WAV-header and
 * PCM-samples to standard out. The header is ignored and the PCM samples
 * are are captured and interpreted. Examples of executables that can
 * convert audio in any format and write to stdout are ffmpeg and avconv.
 *
 * @param source
 *            The file or stream to capture.
 * @param targetSampleRate
 *            The target sample rate.
 * @param audioBufferSize
 *            The number of samples used in the buffer.
 * @param bufferOverlap
 * 			  The number of samples to overlap the current and previous buffer.
 * @return A new audioprocessor.
 */
public static AudioDispatcher fromPipe(final String source,final int targetSampleRate, final int audioBufferSize,final int bufferOverlap){
	PipedAudioStream f = new PipedAudioStream(source);
	TarsosDSPAudioInputStream audioStream = f.getMonoStream(targetSampleRate,0);
	return new AudioDispatcher(audioStream, audioBufferSize, bufferOverlap);
}