From: Paul Brossier Date: Fri, 8 Mar 2013 14:30:45 +0000 (-0500) Subject: python/: move generator.py to lib, improve README, remove env_ files, update MANIFEST.in X-Git-Tag: 0.4.0-beta1~216 X-Git-Url: https://git.aubio.org/?a=commitdiff_plain;h=92c83ccc46e127c20a4fbedde8c1ba02e6815251;p=aubio.git python/: move generator.py to lib, improve README, remove env_ files, update MANIFEST.in --- diff --git a/python/MANIFEST.in b/python/MANIFEST.in index 93ba3974..cffccebf 100644 --- a/python/MANIFEST.in +++ b/python/MANIFEST.in @@ -1,9 +1,8 @@ include README COPYING VERSION include ext/*.h +include lib/generator.py +include lib/gen_pyobject.py include gen/aubio-generated.h -include generator.py -include gen_pyobject.py -include aubio/*.py -include demos/*.py include tests/run_all_tests include tests/*.py +include demos/*.py diff --git a/python/README b/python/README index 4103c8bc..537f4848 100644 --- a/python/README +++ b/python/README @@ -12,10 +12,27 @@ Compiling python aubio You should be able to build the aubio python module out of the box: - $ ./build_osx + $ python setup.py build -This should work on linux based systems as well as recent versions of OS X -(10.8.x). Let me know if you have issues on your platforms. +To use the python module without installing it, set PYTHONPATH: + + $ export PYTHONPATH=$PYTHONPATH:$PWD/`ls -rtd build/lib.* | head -1`:$PWD/tests + +And LD_LIBRARY_PATH: + + $ export LD_LIBRARY_PATH=$PWD/../build/src + +Or on macosx systems: + + $ export DYLD_LIBRARY_PATH=$PWD/../build/src + +The you should be able to run the tests: + + $ ./tests/run_all_tests + +And to try out the demos: + + $ ./demos/demo_source.wav ~/test.wav Additional tools ---------------- diff --git a/python/env_linux b/python/env_linux deleted file mode 100755 index adffa76d..00000000 --- a/python/env_linux +++ /dev/null @@ -1,4 +0,0 @@ -#! /bin/sh - -export PYTHONPATH=$PWD/`ls -rtd build/lib.* | head -1`:$PWD/tests:$PYTHONPATH -export LD_LIBRARY_PATH=$PWD/../build/src diff --git a/python/env_osx b/python/env_osx deleted file mode 100755 index aa27f7e3..00000000 --- a/python/env_osx +++ /dev/null @@ -1,4 +0,0 @@ -#! /bin/sh - -export PYTHONPATH=$PWD/`ls -rtd build/lib.* | head -1`:$PWD/tests:$PYTHONPATH -export DYLD_LIBRARY_PATH=$PWD/../build/src diff --git a/python/gen_pyobject.py b/python/gen_pyobject.py deleted file mode 100644 index 91c427ac..00000000 --- a/python/gen_pyobject.py +++ /dev/null @@ -1,526 +0,0 @@ -#! /usr/bin/python - -""" This madness of code is used to generate the C code of the python interface -to aubio. Don't try this at home. - -The list of typedefs and functions is obtained from the command line 'cpp -aubio.h'. This list is then used to parse all the functions about this object. - -I hear the ones asking "why not use swig, or cython, or something like that?" - -The requirements for this extension are the following: - - - aubio vectors can be viewed as numpy arrays, and vice versa - - aubio 'object' should be python classes, not just a bunch of functions - -I haven't met any python interface generator that can meet both these -requirements. If you know of one, please let me know, it will spare me -maintaining this bizarre file. -""" - -param_numbers = { - 'source': [0, 2], - 'sink': [2, 0], -} - -# TODO -# do function: for now, only the following pattern is supported: -# void aubio__do (aubio_foo_t * o, -# [input1_t * input, [output1_t * output, ..., output3_t * output]]); -# There is no way of knowing that output1 is actually input2. In the future, -# const could be used for the inputs in the C prototypes. - -def write_msg(*args): - pass - # uncomment out for debugging - #print args - -def split_type(arg): - """ arg = 'foo *name' - return ['foo*', 'name'] """ - l = arg.split() - type_arg = {'type': l[0], 'name': l[1]} - # ['foo', '*name'] -> ['foo*', 'name'] - if l[-1].startswith('*'): - #return [l[0]+'*', l[1][1:]] - type_arg['type'] = l[0] + '*' - type_arg['name'] = l[1][1:] - # ['foo', '*', 'name'] -> ['foo*', 'name'] - if len(l) == 3: - #return [l[0]+l[1], l[2]] - type_arg['type'] = l[0]+l[1] - type_arg['name'] = l[2] - else: - #return l - pass - return type_arg - -def get_params(proto): - """ get the list of parameters from a function prototype - example: proto = "int main (int argc, char ** argv)" - returns: ['int argc', 'char ** argv'] - """ - import re - paramregex = re.compile('[\(, ](\w+ \*?\*? ?\w+)[, \)]') - return paramregex.findall(proto) - -def get_params_types_names(proto): - """ get the list of parameters from a function prototype - example: proto = "int main (int argc, char ** argv)" - returns: [['int', 'argc'], ['char **','argv']] - """ - return map(split_type, get_params(proto)) - -def get_return_type(proto): - import re - paramregex = re.compile('(\w+ ?\*?).*') - outputs = paramregex.findall(proto) - assert len(outputs) == 1 - return outputs[0].replace(' ', '') - -def get_name(proto): - name = proto.split()[1].split('(')[0] - return name.replace('*','') - -# the important bits: the size of the output for each objects. this data should -# move into the C library at some point. -defaultsizes = { - 'resampler': ['input->length * self->ratio'], - 'specdesc': ['1'], - 'onset': ['1'], - 'pitchyin': ['1'], - 'pitchyinfft': ['1'], - 'pitchschmitt': ['1'], - 'pitchmcomb': ['1'], - 'pitchfcomb': ['1'], - 'pitch': ['1'], - 'tss': ['self->buf_size', 'self->buf_size'], - 'mfcc': ['self->n_coeffs'], - 'beattracking': ['self->hop_size'], - 'tempo': ['1'], - 'peakpicker': ['1'], - 'source': ['self->hop_size', '1'], -} - -# default value for variables -aubioinitvalue = { - 'uint_t': 0, - 'smpl_t': 0, - 'lsmp_t': 0., - 'char_t*': 'NULL', - } - -aubiodefvalue = { - # we have some clean up to do - 'buf_size': 'Py_default_vector_length', - # and here too - 'hop_size': 'Py_default_vector_length / 2', - # these should be alright - 'samplerate': 'Py_aubio_default_samplerate', - # now for the non obvious ones - 'n_filters': '40', - 'n_coeffs': '13', - 'nelems': '10', - 'flow': '0.', - 'fhig': '1.', - 'ilow': '0.', - 'ihig': '1.', - 'thrs': '0.5', - 'ratio': '0.5', - 'method': '"default"', - 'uri': '"none"', - } - -# aubio to python -aubio2pytypes = { - 'uint_t': 'I', - 'smpl_t': 'f', - 'lsmp_t': 'd', - 'fvec_t*': 'O', - 'cvec_t*': 'O', - 'char_t*': 's', -} - -# python to aubio -aubiovecfrompyobj = { - 'fvec_t*': 'PyAubio_ArrayToCFvec', - 'cvec_t*': 'PyAubio_ArrayToCCvec', - 'uint_t': '(uint_t)PyInt_AsLong', -} - -# aubio to python -aubiovectopyobj = { - 'fvec_t*': 'PyAubio_CFvecToArray', - 'cvec_t*': 'PyAubio_CCvecToPyCvec', - 'smpl_t': 'PyFloat_FromDouble', - 'uint_t*': 'PyInt_FromLong', - 'uint_t': 'PyInt_FromLong', -} - -def gen_new_init(newfunc, name): - newparams = get_params_types_names(newfunc) - # self->param1, self->param2, self->param3 - if len(newparams): - selfparams = ', self->'+', self->'.join([p['name'] for p in newparams]) - else: - selfparams = '' - # "param1", "param2", "param3" - paramnames = ", ".join(["\""+p['name']+"\"" for p in newparams]) - pyparams = "".join(map(lambda p: aubio2pytypes[p['type']], newparams)) - paramrefs = ", ".join(["&" + p['name'] for p in newparams]) - s = """\ -// WARNING: this file is generated, DO NOT EDIT - -// WARNING: if you haven't read the first line yet, please do so -#include "aubiowraphell.h" - -typedef struct -{ - PyObject_HEAD - aubio_%(name)s_t * o; -""" % locals() - for p in newparams: - ptype = p['type'] - pname = p['name'] - s += """\ - %(ptype)s %(pname)s; -""" % locals() - s += """\ -} Py_%(name)s; - -static char Py_%(name)s_doc[] = "%(name)s object"; - -static PyObject * -Py_%(name)s_new (PyTypeObject * pytype, PyObject * args, PyObject * kwds) -{ - Py_%(name)s *self; -""" % locals() - for p in newparams: - ptype = p['type'] - pname = p['name'] - initval = aubioinitvalue[ptype] - s += """\ - %(ptype)s %(pname)s = %(initval)s; -""" % locals() - # now the actual PyArg_Parse - if len(paramnames): - s += """\ - static char *kwlist[] = { %(paramnames)s, NULL }; - - if (!PyArg_ParseTupleAndKeywords (args, kwds, "|%(pyparams)s", kwlist, - %(paramrefs)s)) { - return NULL; - } -""" % locals() - s += """\ - - self = (Py_%(name)s *) pytype->tp_alloc (pytype, 0); - - if (self == NULL) { - return NULL; - } -""" % locals() - for p in newparams: - ptype = p['type'] - pname = p['name'] - defval = aubiodefvalue[pname] - if ptype == 'char_t*': - s += """\ - - self->%(pname)s = %(defval)s; - if (%(pname)s != NULL) { - self->%(pname)s = %(pname)s; - } -""" % locals() - elif ptype == 'uint_t': - s += """\ - - self->%(pname)s = %(defval)s; - if (%(pname)s > 0) { - self->%(pname)s = %(pname)s; - } else if (%(pname)s < 0) { - PyErr_SetString (PyExc_ValueError, - "can not use negative value for %(pname)s"); - return NULL; - } -""" % locals() - elif ptype == 'smpl_t': - s += """\ - - self->%(pname)s = %(defval)s; - if (%(pname)s != %(defval)s) { - self->%(pname)s = %(pname)s; - } -""" % locals() - else: - write_msg ("ERROR, unknown type of parameter %s %s" % (ptype, pname) ) - s += """\ - - return (PyObject *) self; -} - -AUBIO_INIT(%(name)s %(selfparams)s) - -AUBIO_DEL(%(name)s) - -""" % locals() - return s - -def gen_do_input_params(inputparams): - inputdefs = '' - parseinput = '' - inputrefs = '' - inputvecs = '' - pytypes = '' - - if len(inputparams): - # build the parsing string for PyArg_ParseTuple - pytypes = "".join([aubio2pytypes[p['type']] for p in inputparams]) - - inputdefs = " /* input vectors python prototypes */\n" - for p in inputparams: - if p['type'] != 'uint_t': - inputdefs += " PyObject * " + p['name'] + "_obj;\n" - - inputvecs = " /* input vectors prototypes */\n " - inputvecs += "\n ".join(map(lambda p: p['type'] + ' ' + p['name'] + ";", inputparams)) - - parseinput = " /* input vectors parsing */\n " - for p in inputparams: - inputvec = p['name'] - if p['type'] != 'uint_t': - inputdef = p['name'] + "_obj" - else: - inputdef = p['name'] - converter = aubiovecfrompyobj[p['type']] - if p['type'] != 'uint_t': - parseinput += """%(inputvec)s = %(converter)s (%(inputdef)s); - - if (%(inputvec)s == NULL) { - return NULL; - } - - """ % locals() - - # build the string for the input objects references - inputreflist = [] - for p in inputparams: - if p['type'] != 'uint_t': - inputreflist += [ "&" + p['name'] + "_obj" ] - else: - inputreflist += [ "&" + p['name'] ] - inputrefs = ", ".join(inputreflist) - # end of inputs strings - return inputdefs, parseinput, inputrefs, inputvecs, pytypes - -def gen_do_output_params(outputparams, name): - outputvecs = "" - outputcreate = "" - if len(outputparams): - outputvecs = " /* output vectors prototypes */\n" - for p in outputparams: - params = { - 'name': p['name'], 'pytype': p['type'], 'autype': p['type'][:-3], - 'length': defaultsizes[name].pop(0) } - if (p['type'] == 'uint_t*'): - outputvecs += ' uint_t' + ' ' + p['name'] + ";\n" - outputcreate += " %(name)s = 0;\n" % params - else: - outputvecs += " " + p['type'] + ' ' + p['name'] + ";\n" - outputcreate += " /* creating output %(name)s as a new_%(autype)s of length %(length)s */\n" % params - outputcreate += " %(name)s = new_%(autype)s (%(length)s);\n" % params - - returnval = ""; - if len(outputparams) > 1: - returnval += " PyObject *outputs = PyList_New(0);\n" - for p in outputparams: - returnval += " PyList_Append( outputs, (PyObject *)" + aubiovectopyobj[p['type']] + " (" + p['name'] + ")" +");\n" - returnval += " return outputs;" - elif len(outputparams) == 1: - if defaultsizes[name] == '1': - returnval += " return (PyObject *)PyFloat_FromDouble(" + p['name'] + "->data[0])" - else: - returnval += " return (PyObject *)" + aubiovectopyobj[p['type']] + " (" + p['name'] + ")" - else: - returnval += " Py_RETURN_NONE" - # end of output strings - return outputvecs, outputcreate, returnval - -def gen_do(dofunc, name): - funcname = dofunc.split()[1].split('(')[0] - doparams = get_params_types_names(dofunc) - # make sure the first parameter is the object - assert doparams[0]['type'] == "aubio_"+name+"_t*", \ - "method is not in 'aubio__t" - # and remove it - doparams = doparams[1:] - - n_param = len(doparams) - - if name in param_numbers.keys(): - n_input_param, n_output_param = param_numbers[name] - else: - n_input_param, n_output_param = 1, n_param - 1 - - assert n_output_param + n_input_param == n_param, "n_output_param + n_input_param != n_param for %s" % name - - inputparams = doparams[:n_input_param] - outputparams = doparams[n_input_param:n_input_param + n_output_param] - - inputdefs, parseinput, inputrefs, inputvecs, pytypes = gen_do_input_params(inputparams); - outputvecs, outputcreate, returnval = gen_do_output_params(outputparams, name) - - # build strings for outputs - # build the parameters for the _do() call - doparams_string = "self->o" - for p in doparams: - if p['type'] == 'uint_t*': - doparams_string += ", &" + p['name'] - else: - doparams_string += ", " + p['name'] - - if n_input_param: - arg_parse_tuple = """\ - if (!PyArg_ParseTuple (args, "%(pytypes)s", %(inputrefs)s)) { - return NULL; - } -""" % locals() - else: - arg_parse_tuple = "" - # put it all together - s = """\ -/* function Py_%(name)s_do */ -static PyObject * -Py_%(name)s_do(Py_%(name)s * self, PyObject * args) -{ -%(inputdefs)s -%(inputvecs)s -%(outputvecs)s - -%(arg_parse_tuple)s - -%(parseinput)s - -%(outputcreate)s - - /* compute _do function */ - %(funcname)s (%(doparams_string)s); - -%(returnval)s; -} -""" % locals() - return s - -def gen_members(new_method, name): - newparams = get_params_types_names(new_method) - s = """ -AUBIO_MEMBERS_START(%(name)s)""" % locals() - for param in newparams: - if param['type'] == 'char_t*': - s += """ - {"%(pname)s", T_STRING, offsetof (Py_%(name)s, %(pname)s), READONLY, ""},""" \ - % { 'pname': param['name'], 'ptype': param['type'], 'name': name} - elif param['type'] == 'uint_t': - s += """ - {"%(pname)s", T_INT, offsetof (Py_%(name)s, %(pname)s), READONLY, ""},""" \ - % { 'pname': param['name'], 'ptype': param['type'], 'name': name} - elif param['type'] == 'smpl_t': - s += """ - {"%(pname)s", T_FLOAT, offsetof (Py_%(name)s, %(pname)s), READONLY, ""},""" \ - % { 'pname': param['name'], 'ptype': param['type'], 'name': name} - else: - write_msg ("-- ERROR, unknown member type ", param ) - s += """ -AUBIO_MEMBERS_STOP(%(name)s) - -""" % locals() - return s - - -def gen_methods(get_methods, set_methods, name): - s = "" - method_defs = "" - for method in set_methods: - method_name = get_name(method) - params = get_params_types_names(method) - out_type = get_return_type(method) - assert params[0]['type'] == "aubio_"+name+"_t*", \ - "get method is not in 'aubio__t" - write_msg (method ) - write_msg (params[1:]) - setter_args = "self->o, " +",".join([p['name'] for p in params[1:]]) - parse_args = "" - for p in params[1:]: - parse_args += p['type'] + " " + p['name'] + ";\n" - argmap = "".join([aubio2pytypes[p['type']] for p in params[1:]]) - arglist = ", ".join(["&"+p['name'] for p in params[1:]]) - parse_args += """ - if (!PyArg_ParseTuple (args, "%(argmap)s", %(arglist)s)) { - return NULL; - } """ % locals() - s += """ -static PyObject * -Py%(funcname)s (Py_%(objname)s *self, PyObject *args) -{ - uint_t err = 0; - - %(parse_args)s - - err = %(funcname)s (%(setter_args)s); - - if (err > 0) { - PyErr_SetString (PyExc_ValueError, - "error running %(funcname)s"); - return NULL; - } - Py_RETURN_NONE; -} -""" % {'funcname': method_name, 'objname': name, - 'out_type': out_type, 'setter_args': setter_args, 'parse_args': parse_args } - shortname = method_name.split(name+'_')[-1] - method_defs += """\ - {"%(shortname)s", (PyCFunction) Py%(method_name)s, - METH_VARARGS, ""}, -""" % locals() - - for method in get_methods: - method_name = get_name(method) - params = get_params_types_names(method) - out_type = get_return_type(method) - assert params[0]['type'] == "aubio_"+name+"_t*", \ - "get method is not in 'aubio__t %s" % params[0]['type'] - assert len(params) == 1, \ - "get method has more than one parameter %s" % params - getter_args = "self->o" - returnval = "(PyObject *)" + aubiovectopyobj[out_type] + " (tmp)" - shortname = method_name.split(name+'_')[-1] - method_defs += """\ - {"%(shortname)s", (PyCFunction) Py%(method_name)s, - METH_NOARGS, ""}, -""" % locals() - s += """ -static PyObject * -Py%(funcname)s (Py_%(objname)s *self, PyObject *unused) -{ - %(out_type)s tmp = %(funcname)s (%(getter_args)s); - return %(returnval)s; -} -""" % {'funcname': method_name, 'objname': name, - 'out_type': out_type, 'getter_args': getter_args, 'returnval': returnval } - - s += """ -static PyMethodDef Py_%(name)s_methods[] = { -""" % locals() - s += method_defs - s += """\ - {NULL} /* sentinel */ -}; -""" % locals() - return s - -def gen_finish(name): - s = """\ - -AUBIO_TYPEOBJECT(%(name)s, "aubio.%(name)s") -""" % locals() - return s diff --git a/python/generator.py b/python/generator.py deleted file mode 100755 index ead37cf7..00000000 --- a/python/generator.py +++ /dev/null @@ -1,187 +0,0 @@ -#! /usr/bin/python - -""" This file generates a c file from a list of cpp prototypes. """ - -import os, sys, shutil -from gen_pyobject import write_msg, gen_new_init, gen_do, gen_members, gen_methods, gen_finish - -def get_cpp_objects(): - - cpp_output = [l.strip() for l in os.popen('cpp -DAUBIO_UNSTABLE=1 -I../build/src ../src/aubio.h').readlines()] - - cpp_output = filter(lambda y: len(y) > 1, cpp_output) - cpp_output = filter(lambda y: not y.startswith('#'), cpp_output) - - i = 1 - while 1: - if i >= len(cpp_output): break - if cpp_output[i-1].endswith(',') or cpp_output[i-1].endswith('{') or cpp_output[i].startswith('}'): - cpp_output[i] = cpp_output[i-1] + ' ' + cpp_output[i] - cpp_output.pop(i-1) - else: - i += 1 - - typedefs = filter(lambda y: y.startswith ('typedef struct _aubio'), cpp_output) - - cpp_objects = [a.split()[3][:-1] for a in typedefs] - - return cpp_output, cpp_objects - -def generate_object_files(output_path): - if os.path.isdir(output_path): shutil.rmtree(output_path) - os.mkdir(output_path) - - generated_objects = [] - cpp_output, cpp_objects = get_cpp_objects() - skip_objects = ['fft', - 'pvoc', - 'filter', - 'filterbank', - 'resampler', - 'sndfile', - 'sink_apple_audio', - 'sink_sndfile', - 'source_apple_audio', - 'source_sndfile'] - - write_msg("-- INFO: %d objects in total" % len(cpp_objects)) - - for this_object in cpp_objects: - lint = 0 - - if this_object[-2:] == '_t': - object_name = this_object[:-2] - else: - object_name = this_object - write_msg("-- WARNING: %s does not end in _t" % this_object) - - if object_name[:len('aubio_')] != 'aubio_': - write_msg("-- WARNING: %s does not start n aubio_" % this_object) - - write_msg("-- INFO: looking at", object_name) - object_methods = filter(lambda x: this_object in x, cpp_output) - object_methods = [a.strip() for a in object_methods] - object_methods = filter(lambda x: not x.startswith('typedef'), object_methods) - #for method in object_methods: - # write_msg(method) - new_methods = filter(lambda x: 'new_'+object_name in x, object_methods) - if len(new_methods) > 1: - write_msg("-- WARNING: more than one new method for", object_name) - for method in new_methods: - write_msg(method) - elif len(new_methods) < 1: - write_msg("-- WARNING: no new method for", object_name) - elif 0: - for method in new_methods: - write_msg(method) - - del_methods = filter(lambda x: 'del_'+object_name in x, object_methods) - if len(del_methods) > 1: - write_msg("-- WARNING: more than one del method for", object_name) - for method in del_methods: - write_msg(method) - elif len(del_methods) < 1: - write_msg("-- WARNING: no del method for", object_name) - - do_methods = filter(lambda x: object_name+'_do' in x, object_methods) - if len(do_methods) > 1: - pass - #write_msg("-- WARNING: more than one do method for", object_name) - #for method in do_methods: - # write_msg(method) - elif len(do_methods) < 1: - write_msg("-- WARNING: no do method for", object_name) - elif 0: - for method in do_methods: - write_msg(method) - - # check do methods return void - for method in do_methods: - if (method.split()[0] != 'void'): - write_msg("-- ERROR: _do method does not return void:", method ) - - get_methods = filter(lambda x: object_name+'_get_' in x, object_methods) - - set_methods = filter(lambda x: object_name+'_set_' in x, object_methods) - for method in set_methods: - if (method.split()[0] != 'uint_t'): - write_msg("-- ERROR: _set method does not return uint_t:", method ) - - other_methods = filter(lambda x: x not in new_methods, object_methods) - other_methods = filter(lambda x: x not in del_methods, other_methods) - other_methods = filter(lambda x: x not in do_methods, other_methods) - other_methods = filter(lambda x: x not in get_methods, other_methods) - other_methods = filter(lambda x: x not in set_methods, other_methods) - - if len(other_methods) > 0: - write_msg("-- WARNING: some methods for", object_name, "were unidentified") - for method in other_methods: - write_msg(method) - - - # generate this_object - short_name = object_name[len('aubio_'):] - if short_name in skip_objects: - write_msg("-- INFO: skipping object", short_name ) - continue - if 1: #try: - s = gen_new_init(new_methods[0], short_name) - s += gen_do(do_methods[0], short_name) - s += gen_members(new_methods[0], short_name) - s += gen_methods(get_methods, set_methods, short_name) - s += gen_finish(short_name) - generated_filepath = os.path.join(output_path,'gen-'+short_name+'.c') - fd = open(generated_filepath, 'w') - fd.write(s) - #except Exception, e: - # write_msg("-- ERROR:", type(e), str(e), "in", short_name) - # continue - generated_objects += [this_object] - - s = """// generated list of objects created with generator.py - -""" - - for each in generated_objects: - s += "extern PyTypeObject Py_%sType;\n" % \ - each.replace('aubio_','').replace('_t','') - - types_ready = [] - for each in generated_objects: - types_ready.append(" PyType_Ready (&Py_%sType) < 0" % \ - each.replace('aubio_','').replace('_t','') ) - - s += """ - int - generated_types_ready (void) - { - return ( - """ - s += ('\n ||').join(types_ready) - s += """); - } - """ - - s += """ - void - add_generated_objects ( PyObject *m ) - {""" - for each in generated_objects: - s += """ Py_INCREF (&Py_%(name)sType); - PyModule_AddObject (m, "%(name)s", (PyObject *) & Py_%(name)sType);""" % \ - { 'name': ( each.replace('aubio_','').replace('_t','') ) } - - s += """ - }""" - - fd = open(os.path.join(output_path,'aubio-generated.h'), 'w') - fd.write(s) - - from os import listdir - generated_files = listdir(output_path) - generated_files = filter(lambda x: x.endswith('.c'), generated_files) - generated_files = [output_path+'/'+f for f in generated_files] - return generated_files - -if __name__ == '__main__': - generate_object_files('gen') diff --git a/python/lib/__init__.py b/python/lib/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/lib/gen_pyobject.py b/python/lib/gen_pyobject.py new file mode 100644 index 00000000..91c427ac --- /dev/null +++ b/python/lib/gen_pyobject.py @@ -0,0 +1,526 @@ +#! /usr/bin/python + +""" This madness of code is used to generate the C code of the python interface +to aubio. Don't try this at home. + +The list of typedefs and functions is obtained from the command line 'cpp +aubio.h'. This list is then used to parse all the functions about this object. + +I hear the ones asking "why not use swig, or cython, or something like that?" + +The requirements for this extension are the following: + + - aubio vectors can be viewed as numpy arrays, and vice versa + - aubio 'object' should be python classes, not just a bunch of functions + +I haven't met any python interface generator that can meet both these +requirements. If you know of one, please let me know, it will spare me +maintaining this bizarre file. +""" + +param_numbers = { + 'source': [0, 2], + 'sink': [2, 0], +} + +# TODO +# do function: for now, only the following pattern is supported: +# void aubio__do (aubio_foo_t * o, +# [input1_t * input, [output1_t * output, ..., output3_t * output]]); +# There is no way of knowing that output1 is actually input2. In the future, +# const could be used for the inputs in the C prototypes. + +def write_msg(*args): + pass + # uncomment out for debugging + #print args + +def split_type(arg): + """ arg = 'foo *name' + return ['foo*', 'name'] """ + l = arg.split() + type_arg = {'type': l[0], 'name': l[1]} + # ['foo', '*name'] -> ['foo*', 'name'] + if l[-1].startswith('*'): + #return [l[0]+'*', l[1][1:]] + type_arg['type'] = l[0] + '*' + type_arg['name'] = l[1][1:] + # ['foo', '*', 'name'] -> ['foo*', 'name'] + if len(l) == 3: + #return [l[0]+l[1], l[2]] + type_arg['type'] = l[0]+l[1] + type_arg['name'] = l[2] + else: + #return l + pass + return type_arg + +def get_params(proto): + """ get the list of parameters from a function prototype + example: proto = "int main (int argc, char ** argv)" + returns: ['int argc', 'char ** argv'] + """ + import re + paramregex = re.compile('[\(, ](\w+ \*?\*? ?\w+)[, \)]') + return paramregex.findall(proto) + +def get_params_types_names(proto): + """ get the list of parameters from a function prototype + example: proto = "int main (int argc, char ** argv)" + returns: [['int', 'argc'], ['char **','argv']] + """ + return map(split_type, get_params(proto)) + +def get_return_type(proto): + import re + paramregex = re.compile('(\w+ ?\*?).*') + outputs = paramregex.findall(proto) + assert len(outputs) == 1 + return outputs[0].replace(' ', '') + +def get_name(proto): + name = proto.split()[1].split('(')[0] + return name.replace('*','') + +# the important bits: the size of the output for each objects. this data should +# move into the C library at some point. +defaultsizes = { + 'resampler': ['input->length * self->ratio'], + 'specdesc': ['1'], + 'onset': ['1'], + 'pitchyin': ['1'], + 'pitchyinfft': ['1'], + 'pitchschmitt': ['1'], + 'pitchmcomb': ['1'], + 'pitchfcomb': ['1'], + 'pitch': ['1'], + 'tss': ['self->buf_size', 'self->buf_size'], + 'mfcc': ['self->n_coeffs'], + 'beattracking': ['self->hop_size'], + 'tempo': ['1'], + 'peakpicker': ['1'], + 'source': ['self->hop_size', '1'], +} + +# default value for variables +aubioinitvalue = { + 'uint_t': 0, + 'smpl_t': 0, + 'lsmp_t': 0., + 'char_t*': 'NULL', + } + +aubiodefvalue = { + # we have some clean up to do + 'buf_size': 'Py_default_vector_length', + # and here too + 'hop_size': 'Py_default_vector_length / 2', + # these should be alright + 'samplerate': 'Py_aubio_default_samplerate', + # now for the non obvious ones + 'n_filters': '40', + 'n_coeffs': '13', + 'nelems': '10', + 'flow': '0.', + 'fhig': '1.', + 'ilow': '0.', + 'ihig': '1.', + 'thrs': '0.5', + 'ratio': '0.5', + 'method': '"default"', + 'uri': '"none"', + } + +# aubio to python +aubio2pytypes = { + 'uint_t': 'I', + 'smpl_t': 'f', + 'lsmp_t': 'd', + 'fvec_t*': 'O', + 'cvec_t*': 'O', + 'char_t*': 's', +} + +# python to aubio +aubiovecfrompyobj = { + 'fvec_t*': 'PyAubio_ArrayToCFvec', + 'cvec_t*': 'PyAubio_ArrayToCCvec', + 'uint_t': '(uint_t)PyInt_AsLong', +} + +# aubio to python +aubiovectopyobj = { + 'fvec_t*': 'PyAubio_CFvecToArray', + 'cvec_t*': 'PyAubio_CCvecToPyCvec', + 'smpl_t': 'PyFloat_FromDouble', + 'uint_t*': 'PyInt_FromLong', + 'uint_t': 'PyInt_FromLong', +} + +def gen_new_init(newfunc, name): + newparams = get_params_types_names(newfunc) + # self->param1, self->param2, self->param3 + if len(newparams): + selfparams = ', self->'+', self->'.join([p['name'] for p in newparams]) + else: + selfparams = '' + # "param1", "param2", "param3" + paramnames = ", ".join(["\""+p['name']+"\"" for p in newparams]) + pyparams = "".join(map(lambda p: aubio2pytypes[p['type']], newparams)) + paramrefs = ", ".join(["&" + p['name'] for p in newparams]) + s = """\ +// WARNING: this file is generated, DO NOT EDIT + +// WARNING: if you haven't read the first line yet, please do so +#include "aubiowraphell.h" + +typedef struct +{ + PyObject_HEAD + aubio_%(name)s_t * o; +""" % locals() + for p in newparams: + ptype = p['type'] + pname = p['name'] + s += """\ + %(ptype)s %(pname)s; +""" % locals() + s += """\ +} Py_%(name)s; + +static char Py_%(name)s_doc[] = "%(name)s object"; + +static PyObject * +Py_%(name)s_new (PyTypeObject * pytype, PyObject * args, PyObject * kwds) +{ + Py_%(name)s *self; +""" % locals() + for p in newparams: + ptype = p['type'] + pname = p['name'] + initval = aubioinitvalue[ptype] + s += """\ + %(ptype)s %(pname)s = %(initval)s; +""" % locals() + # now the actual PyArg_Parse + if len(paramnames): + s += """\ + static char *kwlist[] = { %(paramnames)s, NULL }; + + if (!PyArg_ParseTupleAndKeywords (args, kwds, "|%(pyparams)s", kwlist, + %(paramrefs)s)) { + return NULL; + } +""" % locals() + s += """\ + + self = (Py_%(name)s *) pytype->tp_alloc (pytype, 0); + + if (self == NULL) { + return NULL; + } +""" % locals() + for p in newparams: + ptype = p['type'] + pname = p['name'] + defval = aubiodefvalue[pname] + if ptype == 'char_t*': + s += """\ + + self->%(pname)s = %(defval)s; + if (%(pname)s != NULL) { + self->%(pname)s = %(pname)s; + } +""" % locals() + elif ptype == 'uint_t': + s += """\ + + self->%(pname)s = %(defval)s; + if (%(pname)s > 0) { + self->%(pname)s = %(pname)s; + } else if (%(pname)s < 0) { + PyErr_SetString (PyExc_ValueError, + "can not use negative value for %(pname)s"); + return NULL; + } +""" % locals() + elif ptype == 'smpl_t': + s += """\ + + self->%(pname)s = %(defval)s; + if (%(pname)s != %(defval)s) { + self->%(pname)s = %(pname)s; + } +""" % locals() + else: + write_msg ("ERROR, unknown type of parameter %s %s" % (ptype, pname) ) + s += """\ + + return (PyObject *) self; +} + +AUBIO_INIT(%(name)s %(selfparams)s) + +AUBIO_DEL(%(name)s) + +""" % locals() + return s + +def gen_do_input_params(inputparams): + inputdefs = '' + parseinput = '' + inputrefs = '' + inputvecs = '' + pytypes = '' + + if len(inputparams): + # build the parsing string for PyArg_ParseTuple + pytypes = "".join([aubio2pytypes[p['type']] for p in inputparams]) + + inputdefs = " /* input vectors python prototypes */\n" + for p in inputparams: + if p['type'] != 'uint_t': + inputdefs += " PyObject * " + p['name'] + "_obj;\n" + + inputvecs = " /* input vectors prototypes */\n " + inputvecs += "\n ".join(map(lambda p: p['type'] + ' ' + p['name'] + ";", inputparams)) + + parseinput = " /* input vectors parsing */\n " + for p in inputparams: + inputvec = p['name'] + if p['type'] != 'uint_t': + inputdef = p['name'] + "_obj" + else: + inputdef = p['name'] + converter = aubiovecfrompyobj[p['type']] + if p['type'] != 'uint_t': + parseinput += """%(inputvec)s = %(converter)s (%(inputdef)s); + + if (%(inputvec)s == NULL) { + return NULL; + } + + """ % locals() + + # build the string for the input objects references + inputreflist = [] + for p in inputparams: + if p['type'] != 'uint_t': + inputreflist += [ "&" + p['name'] + "_obj" ] + else: + inputreflist += [ "&" + p['name'] ] + inputrefs = ", ".join(inputreflist) + # end of inputs strings + return inputdefs, parseinput, inputrefs, inputvecs, pytypes + +def gen_do_output_params(outputparams, name): + outputvecs = "" + outputcreate = "" + if len(outputparams): + outputvecs = " /* output vectors prototypes */\n" + for p in outputparams: + params = { + 'name': p['name'], 'pytype': p['type'], 'autype': p['type'][:-3], + 'length': defaultsizes[name].pop(0) } + if (p['type'] == 'uint_t*'): + outputvecs += ' uint_t' + ' ' + p['name'] + ";\n" + outputcreate += " %(name)s = 0;\n" % params + else: + outputvecs += " " + p['type'] + ' ' + p['name'] + ";\n" + outputcreate += " /* creating output %(name)s as a new_%(autype)s of length %(length)s */\n" % params + outputcreate += " %(name)s = new_%(autype)s (%(length)s);\n" % params + + returnval = ""; + if len(outputparams) > 1: + returnval += " PyObject *outputs = PyList_New(0);\n" + for p in outputparams: + returnval += " PyList_Append( outputs, (PyObject *)" + aubiovectopyobj[p['type']] + " (" + p['name'] + ")" +");\n" + returnval += " return outputs;" + elif len(outputparams) == 1: + if defaultsizes[name] == '1': + returnval += " return (PyObject *)PyFloat_FromDouble(" + p['name'] + "->data[0])" + else: + returnval += " return (PyObject *)" + aubiovectopyobj[p['type']] + " (" + p['name'] + ")" + else: + returnval += " Py_RETURN_NONE" + # end of output strings + return outputvecs, outputcreate, returnval + +def gen_do(dofunc, name): + funcname = dofunc.split()[1].split('(')[0] + doparams = get_params_types_names(dofunc) + # make sure the first parameter is the object + assert doparams[0]['type'] == "aubio_"+name+"_t*", \ + "method is not in 'aubio__t" + # and remove it + doparams = doparams[1:] + + n_param = len(doparams) + + if name in param_numbers.keys(): + n_input_param, n_output_param = param_numbers[name] + else: + n_input_param, n_output_param = 1, n_param - 1 + + assert n_output_param + n_input_param == n_param, "n_output_param + n_input_param != n_param for %s" % name + + inputparams = doparams[:n_input_param] + outputparams = doparams[n_input_param:n_input_param + n_output_param] + + inputdefs, parseinput, inputrefs, inputvecs, pytypes = gen_do_input_params(inputparams); + outputvecs, outputcreate, returnval = gen_do_output_params(outputparams, name) + + # build strings for outputs + # build the parameters for the _do() call + doparams_string = "self->o" + for p in doparams: + if p['type'] == 'uint_t*': + doparams_string += ", &" + p['name'] + else: + doparams_string += ", " + p['name'] + + if n_input_param: + arg_parse_tuple = """\ + if (!PyArg_ParseTuple (args, "%(pytypes)s", %(inputrefs)s)) { + return NULL; + } +""" % locals() + else: + arg_parse_tuple = "" + # put it all together + s = """\ +/* function Py_%(name)s_do */ +static PyObject * +Py_%(name)s_do(Py_%(name)s * self, PyObject * args) +{ +%(inputdefs)s +%(inputvecs)s +%(outputvecs)s + +%(arg_parse_tuple)s + +%(parseinput)s + +%(outputcreate)s + + /* compute _do function */ + %(funcname)s (%(doparams_string)s); + +%(returnval)s; +} +""" % locals() + return s + +def gen_members(new_method, name): + newparams = get_params_types_names(new_method) + s = """ +AUBIO_MEMBERS_START(%(name)s)""" % locals() + for param in newparams: + if param['type'] == 'char_t*': + s += """ + {"%(pname)s", T_STRING, offsetof (Py_%(name)s, %(pname)s), READONLY, ""},""" \ + % { 'pname': param['name'], 'ptype': param['type'], 'name': name} + elif param['type'] == 'uint_t': + s += """ + {"%(pname)s", T_INT, offsetof (Py_%(name)s, %(pname)s), READONLY, ""},""" \ + % { 'pname': param['name'], 'ptype': param['type'], 'name': name} + elif param['type'] == 'smpl_t': + s += """ + {"%(pname)s", T_FLOAT, offsetof (Py_%(name)s, %(pname)s), READONLY, ""},""" \ + % { 'pname': param['name'], 'ptype': param['type'], 'name': name} + else: + write_msg ("-- ERROR, unknown member type ", param ) + s += """ +AUBIO_MEMBERS_STOP(%(name)s) + +""" % locals() + return s + + +def gen_methods(get_methods, set_methods, name): + s = "" + method_defs = "" + for method in set_methods: + method_name = get_name(method) + params = get_params_types_names(method) + out_type = get_return_type(method) + assert params[0]['type'] == "aubio_"+name+"_t*", \ + "get method is not in 'aubio__t" + write_msg (method ) + write_msg (params[1:]) + setter_args = "self->o, " +",".join([p['name'] for p in params[1:]]) + parse_args = "" + for p in params[1:]: + parse_args += p['type'] + " " + p['name'] + ";\n" + argmap = "".join([aubio2pytypes[p['type']] for p in params[1:]]) + arglist = ", ".join(["&"+p['name'] for p in params[1:]]) + parse_args += """ + if (!PyArg_ParseTuple (args, "%(argmap)s", %(arglist)s)) { + return NULL; + } """ % locals() + s += """ +static PyObject * +Py%(funcname)s (Py_%(objname)s *self, PyObject *args) +{ + uint_t err = 0; + + %(parse_args)s + + err = %(funcname)s (%(setter_args)s); + + if (err > 0) { + PyErr_SetString (PyExc_ValueError, + "error running %(funcname)s"); + return NULL; + } + Py_RETURN_NONE; +} +""" % {'funcname': method_name, 'objname': name, + 'out_type': out_type, 'setter_args': setter_args, 'parse_args': parse_args } + shortname = method_name.split(name+'_')[-1] + method_defs += """\ + {"%(shortname)s", (PyCFunction) Py%(method_name)s, + METH_VARARGS, ""}, +""" % locals() + + for method in get_methods: + method_name = get_name(method) + params = get_params_types_names(method) + out_type = get_return_type(method) + assert params[0]['type'] == "aubio_"+name+"_t*", \ + "get method is not in 'aubio__t %s" % params[0]['type'] + assert len(params) == 1, \ + "get method has more than one parameter %s" % params + getter_args = "self->o" + returnval = "(PyObject *)" + aubiovectopyobj[out_type] + " (tmp)" + shortname = method_name.split(name+'_')[-1] + method_defs += """\ + {"%(shortname)s", (PyCFunction) Py%(method_name)s, + METH_NOARGS, ""}, +""" % locals() + s += """ +static PyObject * +Py%(funcname)s (Py_%(objname)s *self, PyObject *unused) +{ + %(out_type)s tmp = %(funcname)s (%(getter_args)s); + return %(returnval)s; +} +""" % {'funcname': method_name, 'objname': name, + 'out_type': out_type, 'getter_args': getter_args, 'returnval': returnval } + + s += """ +static PyMethodDef Py_%(name)s_methods[] = { +""" % locals() + s += method_defs + s += """\ + {NULL} /* sentinel */ +}; +""" % locals() + return s + +def gen_finish(name): + s = """\ + +AUBIO_TYPEOBJECT(%(name)s, "aubio.%(name)s") +""" % locals() + return s diff --git a/python/lib/generator.py b/python/lib/generator.py new file mode 100755 index 00000000..ead37cf7 --- /dev/null +++ b/python/lib/generator.py @@ -0,0 +1,187 @@ +#! /usr/bin/python + +""" This file generates a c file from a list of cpp prototypes. """ + +import os, sys, shutil +from gen_pyobject import write_msg, gen_new_init, gen_do, gen_members, gen_methods, gen_finish + +def get_cpp_objects(): + + cpp_output = [l.strip() for l in os.popen('cpp -DAUBIO_UNSTABLE=1 -I../build/src ../src/aubio.h').readlines()] + + cpp_output = filter(lambda y: len(y) > 1, cpp_output) + cpp_output = filter(lambda y: not y.startswith('#'), cpp_output) + + i = 1 + while 1: + if i >= len(cpp_output): break + if cpp_output[i-1].endswith(',') or cpp_output[i-1].endswith('{') or cpp_output[i].startswith('}'): + cpp_output[i] = cpp_output[i-1] + ' ' + cpp_output[i] + cpp_output.pop(i-1) + else: + i += 1 + + typedefs = filter(lambda y: y.startswith ('typedef struct _aubio'), cpp_output) + + cpp_objects = [a.split()[3][:-1] for a in typedefs] + + return cpp_output, cpp_objects + +def generate_object_files(output_path): + if os.path.isdir(output_path): shutil.rmtree(output_path) + os.mkdir(output_path) + + generated_objects = [] + cpp_output, cpp_objects = get_cpp_objects() + skip_objects = ['fft', + 'pvoc', + 'filter', + 'filterbank', + 'resampler', + 'sndfile', + 'sink_apple_audio', + 'sink_sndfile', + 'source_apple_audio', + 'source_sndfile'] + + write_msg("-- INFO: %d objects in total" % len(cpp_objects)) + + for this_object in cpp_objects: + lint = 0 + + if this_object[-2:] == '_t': + object_name = this_object[:-2] + else: + object_name = this_object + write_msg("-- WARNING: %s does not end in _t" % this_object) + + if object_name[:len('aubio_')] != 'aubio_': + write_msg("-- WARNING: %s does not start n aubio_" % this_object) + + write_msg("-- INFO: looking at", object_name) + object_methods = filter(lambda x: this_object in x, cpp_output) + object_methods = [a.strip() for a in object_methods] + object_methods = filter(lambda x: not x.startswith('typedef'), object_methods) + #for method in object_methods: + # write_msg(method) + new_methods = filter(lambda x: 'new_'+object_name in x, object_methods) + if len(new_methods) > 1: + write_msg("-- WARNING: more than one new method for", object_name) + for method in new_methods: + write_msg(method) + elif len(new_methods) < 1: + write_msg("-- WARNING: no new method for", object_name) + elif 0: + for method in new_methods: + write_msg(method) + + del_methods = filter(lambda x: 'del_'+object_name in x, object_methods) + if len(del_methods) > 1: + write_msg("-- WARNING: more than one del method for", object_name) + for method in del_methods: + write_msg(method) + elif len(del_methods) < 1: + write_msg("-- WARNING: no del method for", object_name) + + do_methods = filter(lambda x: object_name+'_do' in x, object_methods) + if len(do_methods) > 1: + pass + #write_msg("-- WARNING: more than one do method for", object_name) + #for method in do_methods: + # write_msg(method) + elif len(do_methods) < 1: + write_msg("-- WARNING: no do method for", object_name) + elif 0: + for method in do_methods: + write_msg(method) + + # check do methods return void + for method in do_methods: + if (method.split()[0] != 'void'): + write_msg("-- ERROR: _do method does not return void:", method ) + + get_methods = filter(lambda x: object_name+'_get_' in x, object_methods) + + set_methods = filter(lambda x: object_name+'_set_' in x, object_methods) + for method in set_methods: + if (method.split()[0] != 'uint_t'): + write_msg("-- ERROR: _set method does not return uint_t:", method ) + + other_methods = filter(lambda x: x not in new_methods, object_methods) + other_methods = filter(lambda x: x not in del_methods, other_methods) + other_methods = filter(lambda x: x not in do_methods, other_methods) + other_methods = filter(lambda x: x not in get_methods, other_methods) + other_methods = filter(lambda x: x not in set_methods, other_methods) + + if len(other_methods) > 0: + write_msg("-- WARNING: some methods for", object_name, "were unidentified") + for method in other_methods: + write_msg(method) + + + # generate this_object + short_name = object_name[len('aubio_'):] + if short_name in skip_objects: + write_msg("-- INFO: skipping object", short_name ) + continue + if 1: #try: + s = gen_new_init(new_methods[0], short_name) + s += gen_do(do_methods[0], short_name) + s += gen_members(new_methods[0], short_name) + s += gen_methods(get_methods, set_methods, short_name) + s += gen_finish(short_name) + generated_filepath = os.path.join(output_path,'gen-'+short_name+'.c') + fd = open(generated_filepath, 'w') + fd.write(s) + #except Exception, e: + # write_msg("-- ERROR:", type(e), str(e), "in", short_name) + # continue + generated_objects += [this_object] + + s = """// generated list of objects created with generator.py + +""" + + for each in generated_objects: + s += "extern PyTypeObject Py_%sType;\n" % \ + each.replace('aubio_','').replace('_t','') + + types_ready = [] + for each in generated_objects: + types_ready.append(" PyType_Ready (&Py_%sType) < 0" % \ + each.replace('aubio_','').replace('_t','') ) + + s += """ + int + generated_types_ready (void) + { + return ( + """ + s += ('\n ||').join(types_ready) + s += """); + } + """ + + s += """ + void + add_generated_objects ( PyObject *m ) + {""" + for each in generated_objects: + s += """ Py_INCREF (&Py_%(name)sType); + PyModule_AddObject (m, "%(name)s", (PyObject *) & Py_%(name)sType);""" % \ + { 'name': ( each.replace('aubio_','').replace('_t','') ) } + + s += """ + }""" + + fd = open(os.path.join(output_path,'aubio-generated.h'), 'w') + fd.write(s) + + from os import listdir + generated_files = listdir(output_path) + generated_files = filter(lambda x: x.endswith('.c'), generated_files) + generated_files = [output_path+'/'+f for f in generated_files] + return generated_files + +if __name__ == '__main__': + generate_object_files('gen') diff --git a/python/setup.py b/python/setup.py index c7cef374..23a4f4a2 100755 --- a/python/setup.py +++ b/python/setup.py @@ -28,7 +28,7 @@ output_path = 'gen' generated_object_files = [] if not os.path.isdir(output_path): - from generator import generate_object_files + from lib.generator import generate_object_files generated_object_files = generate_object_files(output_path) # define include dirs else: