2 Copyright (C) 2003-2009 Paul Brossier <piem@aubio.org>
4 This file is part of aubio.
6 aubio is free software: you can redistribute it and/or modify
7 it under the terms of the GNU General Public License as published by
8 the Free Software Foundation, either version 3 of the License, or
9 (at your option) any later version.
11 aubio is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 GNU General Public License for more details.
16 You should have received a copy of the GNU General Public License
17 along with aubio. If not, see <http://www.gnu.org/licenses/>.
21 #include "aubio_priv.h"
24 #include "spectral/fft.h"
25 #include "spectral/specdesc.h"
26 #include "mathutils.h"
27 #include "utils/hist.h"
29 /** Energy based onset detection function
31 This function calculates the local energy of the input spectral frame.
33 \param o onset detection object as returned by new_aubio_specdesc()
34 \param fftgrain input spectral frame
35 \param onset output onset detection function
38 void aubio_specdesc_energy(aubio_specdesc_t *o, cvec_t * fftgrain, fvec_t * onset);
39 /** High Frequency Content onset detection function
41 This method computes the High Frequency Content (HFC) of the input spectral
42 frame. The resulting function is efficient at detecting percussive onsets.
44 Paul Masri. Computer modeling of Sound for Transformation and Synthesis of
45 Musical Signal. PhD dissertation, University of Bristol, UK, 1996.
47 \param o onset detection object as returned by new_aubio_specdesc()
48 \param fftgrain input spectral frame
49 \param onset output onset detection function
52 void aubio_specdesc_hfc(aubio_specdesc_t *o, cvec_t * fftgrain, fvec_t * onset);
53 /** Complex Domain Method onset detection function
55 Christopher Duxbury, Mike E. Davies, and Mark B. Sandler. Complex domain
56 onset detection for musical signals. In Proceedings of the Digital Audio
57 Effects Conference, DAFx-03, pages 90-93, London, UK, 2003.
59 \param o onset detection object as returned by new_aubio_specdesc()
60 \param fftgrain input spectral frame
61 \param onset output onset detection function
64 void aubio_specdesc_complex(aubio_specdesc_t *o, cvec_t * fftgrain, fvec_t * onset);
65 /** Phase Based Method onset detection function
67 Juan-Pablo Bello, Mike P. Davies, and Mark B. Sandler. Phase-based note onset
68 detection for music signals. In Proceedings of the IEEE International
69 Conference on Acoustics Speech and Signal Processing, pages 441444,
72 \param o onset detection object as returned by new_aubio_specdesc()
73 \param fftgrain input spectral frame
74 \param onset output onset detection function
77 void aubio_specdesc_phase(aubio_specdesc_t *o, cvec_t * fftgrain, fvec_t * onset);
78 /** Spectral difference method onset detection function
80 Jonhatan Foote and Shingo Uchihashi. The beat spectrum: a new approach to
81 rhythm analysis. In IEEE International Conference on Multimedia and Expo
82 (ICME 2001), pages 881884, Tokyo, Japan, August 2001.
84 \param o onset detection object as returned by new_aubio_specdesc()
85 \param fftgrain input spectral frame
86 \param onset output onset detection function
89 void aubio_specdesc_specdiff(aubio_specdesc_t *o, cvec_t * fftgrain, fvec_t * onset);
90 /** Kullback-Liebler onset detection function
92 Stephen Hainsworth and Malcom Macleod. Onset detection in music audio
93 signals. In Proceedings of the International Computer Music Conference
94 (ICMC), Singapore, 2003.
96 \param o onset detection object as returned by new_aubio_specdesc()
97 \param fftgrain input spectral frame
98 \param onset output onset detection function
101 void aubio_specdesc_kl(aubio_specdesc_t *o, cvec_t * fftgrain, fvec_t * onset);
102 /** Modified Kullback-Liebler onset detection function
104 Paul Brossier, ``Automatic annotation of musical audio for interactive
105 systems'', Chapter 2, Temporal segmentation, PhD thesis, Centre for Digital
106 music, Queen Mary University of London, London, UK, 2006.
108 \param o onset detection object as returned by new_aubio_specdesc()
109 \param fftgrain input spectral frame
110 \param onset output onset detection function
113 void aubio_specdesc_mkl(aubio_specdesc_t *o, cvec_t * fftgrain, fvec_t * onset);
116 Simon Dixon, Onset Detection Revisited, in ``Proceedings of the 9th
117 International Conference on Digital Audio Effects'' (DAFx-06), Montreal,
120 \param o onset detection object as returned by new_aubio_specdesc()
121 \param fftgrain input spectral frame
122 \param onset output onset detection function
125 void aubio_specdesc_specflux(aubio_specdesc_t *o, cvec_t * fftgrain, fvec_t * onset);
127 /** onsetdetection types */
129 aubio_onset_energy, /**< energy based */
130 aubio_onset_specdiff, /**< spectral diff */
131 aubio_onset_hfc, /**< high frequency content */
132 aubio_onset_complex, /**< complex domain */
133 aubio_onset_phase, /**< phase fast */
134 aubio_onset_kl, /**< Kullback Liebler */
135 aubio_onset_mkl, /**< modified Kullback Liebler */
136 aubio_onset_specflux, /**< spectral flux */
137 aubio_onset_default = aubio_onset_hfc, /**< default mode, set to hfc */
138 } aubio_specdesc_type;
140 /** structure to store object state */
141 struct _aubio_specdesc_t {
142 aubio_specdesc_type onset_type; /**< onset detection type */
143 /** Pointer to aubio_specdesc_<type> function */
144 void (*funcpointer)(aubio_specdesc_t *o,
145 cvec_t * fftgrain, fvec_t * onset);
146 smpl_t threshold; /**< minimum norm threshold for phase and specdiff */
147 fvec_t *oldmag; /**< previous norm vector */
148 fvec_t *dev1 ; /**< current onset detection measure vector */
149 fvec_t *theta1; /**< previous phase vector, one frame behind */
150 fvec_t *theta2; /**< previous phase vector, two frames behind */
151 aubio_hist_t * histog; /**< histogram */
155 /* Energy based onset detection function */
156 void aubio_specdesc_energy (aubio_specdesc_t *o UNUSED,
157 cvec_t * fftgrain, fvec_t * onset) {
159 for (i=0;i<fftgrain->channels;i++) {
160 onset->data[i][0] = 0.;
161 for (j=0;j<fftgrain->length;j++) {
162 onset->data[i][0] += SQR(fftgrain->norm[i][j]);
167 /* High Frequency Content onset detection function */
168 void aubio_specdesc_hfc(aubio_specdesc_t *o UNUSED,
169 cvec_t * fftgrain, fvec_t * onset){
171 for (i=0;i<fftgrain->channels;i++) {
172 onset->data[i][0] = 0.;
173 for (j=0;j<fftgrain->length;j++) {
174 onset->data[i][0] += (j+1)*fftgrain->norm[i][j];
180 /* Complex Domain Method onset detection function */
181 void aubio_specdesc_complex (aubio_specdesc_t *o, cvec_t * fftgrain, fvec_t * onset) {
183 uint_t nbins = fftgrain->length;
184 for (i=0;i<fftgrain->channels; i++) {
185 onset->data[i][0] = 0.;
186 for (j=0;j<nbins; j++) {
187 // compute the predicted phase
188 o->dev1->data[i][j] = 2. * o->theta1->data[i][j] - o->theta2->data[i][j];
189 // compute the euclidean distance in the complex domain
190 // sqrt ( r_1^2 + r_2^2 - 2 * r_1 * r_2 * \cos ( \phi_1 - \phi_2 ) )
192 SQRT (ABS (SQR (o->oldmag->data[i][j]) + SQR (fftgrain->norm[i][j])
193 - 2. * o->oldmag->data[i][j] * fftgrain->norm[i][j]
194 * COS (o->dev1->data[i][j] - fftgrain->phas[i][j])));
195 /* swap old phase data (need to remember 2 frames behind)*/
196 o->theta2->data[i][j] = o->theta1->data[i][j];
197 o->theta1->data[i][j] = fftgrain->phas[i][j];
198 /* swap old magnitude data (1 frame is enough) */
199 o->oldmag->data[i][j] = fftgrain->norm[i][j];
205 /* Phase Based Method onset detection function */
206 void aubio_specdesc_phase(aubio_specdesc_t *o,
207 cvec_t * fftgrain, fvec_t * onset){
209 uint_t nbins = fftgrain->length;
210 for (i=0;i<fftgrain->channels; i++) {
211 onset->data[i][0] = 0.0;
212 o->dev1->data[i][0]=0.;
213 for ( j=0;j<nbins; j++ ) {
214 o->dev1->data[i][j] =
217 -2.0*o->theta1->data[i][j]
218 +o->theta2->data[i][j]);
219 if ( o->threshold < fftgrain->norm[i][j] )
220 o->dev1->data[i][j] = ABS(o->dev1->data[i][j]);
222 o->dev1->data[i][j] = 0.0;
223 /* keep a track of the past frames */
224 o->theta2->data[i][j] = o->theta1->data[i][j];
225 o->theta1->data[i][j] = fftgrain->phas[i][j];
227 /* apply o->histogram */
228 aubio_hist_dyn_notnull(o->histog,o->dev1);
230 aubio_hist_weight(o->histog);
231 /* its mean is the result */
232 onset->data[i][0] = aubio_hist_mean(o->histog);
233 //onset->data[i][0] = fvec_mean(o->dev1);
237 /* Spectral difference method onset detection function */
238 void aubio_specdesc_specdiff(aubio_specdesc_t *o,
239 cvec_t * fftgrain, fvec_t * onset){
241 uint_t nbins = fftgrain->length;
242 for (i=0;i<fftgrain->channels; i++) {
243 onset->data[i][0] = 0.0;
244 for (j=0;j<nbins; j++) {
245 o->dev1->data[i][j] = SQRT(
246 ABS(SQR( fftgrain->norm[i][j])
247 - SQR(o->oldmag->data[i][j])));
248 if (o->threshold < fftgrain->norm[i][j] )
249 o->dev1->data[i][j] = ABS(o->dev1->data[i][j]);
251 o->dev1->data[i][j] = 0.0;
252 o->oldmag->data[i][j] = fftgrain->norm[i][j];
255 /* apply o->histogram (act somewhat as a low pass on the
256 * overall function)*/
257 aubio_hist_dyn_notnull(o->histog,o->dev1);
259 aubio_hist_weight(o->histog);
260 /* its mean is the result */
261 onset->data[i][0] = aubio_hist_mean(o->histog);
266 /* Kullback Liebler onset detection function
267 * note we use ln(1+Xn/(Xn-1+0.0001)) to avoid
268 * negative (1.+) and infinite values (+1.e-10) */
269 void aubio_specdesc_kl(aubio_specdesc_t *o, cvec_t * fftgrain, fvec_t * onset){
271 for (i=0;i<fftgrain->channels;i++) {
272 onset->data[i][0] = 0.;
273 for (j=0;j<fftgrain->length;j++) {
274 onset->data[i][0] += fftgrain->norm[i][j]
275 *LOG(1.+fftgrain->norm[i][j]/(o->oldmag->data[i][j]+1.e-10));
276 o->oldmag->data[i][j] = fftgrain->norm[i][j];
278 if (isnan(onset->data[i][0])) onset->data[i][0] = 0.;
282 /* Modified Kullback Liebler onset detection function
283 * note we use ln(1+Xn/(Xn-1+0.0001)) to avoid
284 * negative (1.+) and infinite values (+1.e-10) */
285 void aubio_specdesc_mkl(aubio_specdesc_t *o, cvec_t * fftgrain, fvec_t * onset){
287 for (i=0;i<fftgrain->channels;i++) {
288 onset->data[i][0] = 0.;
289 for (j=0;j<fftgrain->length;j++) {
290 onset->data[i][0] += LOG(1.+fftgrain->norm[i][j]/(o->oldmag->data[i][j]+1.e-10));
291 o->oldmag->data[i][j] = fftgrain->norm[i][j];
293 if (isnan(onset->data[i][0])) onset->data[i][0] = 0.;
298 void aubio_specdesc_specflux(aubio_specdesc_t *o, cvec_t * fftgrain, fvec_t * onset){
300 for (i=0;i<fftgrain->channels;i++) {
301 onset->data[i][0] = 0.;
302 for (j=0;j<fftgrain->length;j++) {
303 if (fftgrain->norm[i][j] > o->oldmag->data[i][j])
304 onset->data[i][0] += fftgrain->norm[i][j] - o->oldmag->data[i][j];
305 o->oldmag->data[i][j] = fftgrain->norm[i][j];
310 /* Generic function pointing to the choosen one */
312 aubio_specdesc_do (aubio_specdesc_t *o, cvec_t * fftgrain,
314 o->funcpointer(o,fftgrain,onset);
317 /* Allocate memory for an onset detection
318 * depending on the choosen type, allocate memory as needed
321 new_aubio_specdesc (char_t * onset_mode,
322 uint_t size, uint_t channels){
323 aubio_specdesc_t * o = AUBIO_NEW(aubio_specdesc_t);
324 uint_t rsize = size/2+1;
325 aubio_specdesc_type onset_type;
326 if (strcmp (onset_mode, "energy") == 0)
327 onset_type = aubio_onset_energy;
328 else if (strcmp (onset_mode, "specdiff") == 0)
329 onset_type = aubio_onset_specdiff;
330 else if (strcmp (onset_mode, "hfc") == 0)
331 onset_type = aubio_onset_hfc;
332 else if (strcmp (onset_mode, "complexdomain") == 0)
333 onset_type = aubio_onset_complex;
334 else if (strcmp (onset_mode, "complex") == 0)
335 onset_type = aubio_onset_complex;
336 else if (strcmp (onset_mode, "phase") == 0)
337 onset_type = aubio_onset_phase;
338 else if (strcmp (onset_mode, "mkl") == 0)
339 onset_type = aubio_onset_mkl;
340 else if (strcmp (onset_mode, "kl") == 0)
341 onset_type = aubio_onset_kl;
342 else if (strcmp (onset_mode, "specflux") == 0)
343 onset_type = aubio_onset_specflux;
344 else if (strcmp (onset_mode, "default") == 0)
345 onset_type = aubio_onset_default;
347 AUBIO_ERR("unknown onset type.\n");
348 onset_type = aubio_onset_default;
351 /* for both energy and hfc, only fftgrain->norm is required */
352 case aubio_onset_energy:
354 case aubio_onset_hfc:
356 /* the other approaches will need some more memory spaces */
357 case aubio_onset_complex:
358 o->oldmag = new_fvec(rsize,channels);
359 o->dev1 = new_fvec(rsize,channels);
360 o->theta1 = new_fvec(rsize,channels);
361 o->theta2 = new_fvec(rsize,channels);
363 case aubio_onset_phase:
364 o->dev1 = new_fvec(rsize,channels);
365 o->theta1 = new_fvec(rsize,channels);
366 o->theta2 = new_fvec(rsize,channels);
367 o->histog = new_aubio_hist(0.0, PI, 10, channels);
370 case aubio_onset_specdiff:
371 o->oldmag = new_fvec(rsize,channels);
372 o->dev1 = new_fvec(rsize,channels);
373 o->histog = new_aubio_hist(0.0, PI, 10, channels);
377 case aubio_onset_mkl:
378 case aubio_onset_specflux:
379 o->oldmag = new_fvec(rsize,channels);
385 /* this switch could be in its own function to change between
386 * detections on the fly. this would need getting rid of the switch
387 * above and always allocate all the structure */
390 case aubio_onset_energy:
391 o->funcpointer = aubio_specdesc_energy;
393 case aubio_onset_hfc:
394 o->funcpointer = aubio_specdesc_hfc;
396 case aubio_onset_complex:
397 o->funcpointer = aubio_specdesc_complex;
399 case aubio_onset_phase:
400 o->funcpointer = aubio_specdesc_phase;
402 case aubio_onset_specdiff:
403 o->funcpointer = aubio_specdesc_specdiff;
406 o->funcpointer = aubio_specdesc_kl;
408 case aubio_onset_mkl:
409 o->funcpointer = aubio_specdesc_mkl;
411 case aubio_onset_specflux:
412 o->funcpointer = aubio_specdesc_specflux;
417 o->onset_type = onset_type;
421 void del_aubio_specdesc (aubio_specdesc_t *o){
422 switch(o->onset_type) {
423 /* for both energy and hfc, only fftgrain->norm is required */
424 case aubio_onset_energy:
426 case aubio_onset_hfc:
428 /* the other approaches will need some more memory spaces */
429 case aubio_onset_complex:
435 case aubio_onset_phase:
439 del_aubio_hist(o->histog);
441 case aubio_onset_specdiff:
444 del_aubio_hist(o->histog);
447 case aubio_onset_mkl:
448 case aubio_onset_specflux: