__init__.py

00001 #
00002 # UAVCAN DSDL compiler for libuavcan
00003 #
00004 # Copyright (C) 2014 Pavel Kirienko <pavel.kirienko@gmail.com>
00005 #
00006 
00007 '''
00008 This module implements the core functionality of the UAVCAN DSDL compiler for libuavcan.
00009 Supported Python versions: 3.2+, 2.7.
00010 It accepts a list of root namespaces and produces the set of C++ header files for libuavcan.
00011 It is based on the DSDL parsing package from pyuavcan.
00012 '''
00013 
00014 from __future__ import division, absolute_import, print_function, unicode_literals
00015 import sys, os, logging, errno, re
00016 from .pyratemp import Template
00017 from uavcan import dsdl
00018 
00019 # Python 2.7 compatibility
00020 try:
00021     str = unicode
00022 except NameError:
00023     pass
00024 
00025 OUTPUT_FILE_EXTENSION = 'hpp'
00026 OUTPUT_FILE_PERMISSIONS = 0o444  # Read only for all
00027 TEMPLATE_FILENAME = os.path.join(os.path.dirname(__file__), 'data_type_template.tmpl')
00028 
00029 __all__ = ['run', 'logger', 'DsdlCompilerException']
00030 
00031 class DsdlCompilerException(Exception):
00032     pass
00033 
00034 logger = logging.getLogger(__name__)
00035 
00036 def run(source_dirs, include_dirs, output_dir):
00037     '''
00038     This function takes a list of root namespace directories (containing DSDL definition files to parse), a
00039     possibly empty list of search directories (containing DSDL definition files that can be referenced from the types
00040     that are going to be parsed), and the output directory path (possibly nonexistent) where the generated C++
00041     header files will be stored.
00042     
00043     Note that this module features lazy write, i.e. if an output file does already exist and its content is not going
00044     to change, it will not be overwritten. This feature allows to avoid unnecessary recompilation of dependent object
00045     files.
00046     
00047     Args:
00048         source_dirs    List of root namespace directories to parse.
00049         include_dirs   List of root namespace directories with referenced types (possibly empty). This list is
00050                        automaitcally extended with source_dirs.
00051         output_dir     Output directory path. Will be created if doesn't exist.
00052     '''
00053     assert isinstance(source_dirs, list)
00054     assert isinstance(include_dirs, list)
00055     output_dir = str(output_dir)
00056 
00057     types = run_parser(source_dirs, include_dirs + source_dirs)
00058     if not types:
00059         die('No type definitions were found')
00060 
00061     logger.info('%d types total', len(types))
00062     run_generator(types, output_dir)
00063 
00064 # -----------------
00065 
00066 def pretty_filename(filename):
00067     try:
00068         a = os.path.abspath(filename)
00069         r = os.path.relpath(filename)
00070         return a if '..' in r else r
00071     except ValueError:
00072         return filename
00073 
00074 def type_output_filename(t):
00075     assert t.category == t.CATEGORY_COMPOUND
00076     return t.full_name.replace('.', os.path.sep) + '.' + OUTPUT_FILE_EXTENSION
00077 
00078 def makedirs(path):
00079     try:
00080         try:
00081             os.makedirs(path, exist_ok=True)  # May throw "File exists" when executed as root, which is wrong
00082         except TypeError:
00083             os.makedirs(path)  # Python 2.7 compatibility
00084     except OSError as ex:
00085         if ex.errno != errno.EEXIST:  # http://stackoverflow.com/questions/12468022
00086             raise
00087 
00088 def die(text):
00089     raise DsdlCompilerException(str(text))
00090 
00091 def run_parser(source_dirs, search_dirs):
00092     try:
00093         types = dsdl.parse_namespaces(source_dirs, search_dirs)
00094     except dsdl.DsdlException as ex:
00095         logger.info('Parser failure', exc_info=True)
00096         die(ex)
00097     return types
00098 
00099 def run_generator(types, dest_dir):
00100     try:
00101         template_expander = make_template_expander(TEMPLATE_FILENAME)
00102         dest_dir = os.path.abspath(dest_dir)  # Removing '..'
00103         makedirs(dest_dir)
00104         for t in types:
00105             logger.info('Generating type %s', t.full_name)
00106             filename = os.path.join(dest_dir, type_output_filename(t))
00107             text = generate_one_type(template_expander, t)
00108             write_generated_data(filename, text)
00109     except Exception as ex:
00110         logger.info('Generator failure', exc_info=True)
00111         die(ex)
00112 
00113 def write_generated_data(filename, data):
00114     dirname = os.path.dirname(filename)
00115     makedirs(dirname)
00116 
00117     # Lazy update - file will not be rewritten if its content is not going to change
00118     if os.path.exists(filename):
00119         with open(filename) as f:
00120             existing_data = f.read()
00121         if data == existing_data:
00122             logger.info('Up to date [%s]', pretty_filename(filename))
00123             return
00124         logger.info('Rewriting [%s]', pretty_filename(filename))
00125         os.remove(filename)
00126     else:
00127         logger.info('Creating [%s]', pretty_filename(filename))
00128 
00129     # Full rewrite
00130     with open(filename, 'w') as f:
00131         f.write(data)
00132     try:
00133         os.chmod(filename, OUTPUT_FILE_PERMISSIONS)
00134     except (OSError, IOError) as ex:
00135         logger.warning('Failed to set permissions for %s: %s', pretty_filename(filename), ex)
00136 
00137 def type_to_cpp_type(t):
00138     if t.category == t.CATEGORY_PRIMITIVE:
00139         cast_mode = {
00140             t.CAST_MODE_SATURATED: '::uavcan::CastModeSaturate',
00141             t.CAST_MODE_TRUNCATED: '::uavcan::CastModeTruncate',
00142         }[t.cast_mode]
00143         if t.kind == t.KIND_FLOAT:
00144             return '::uavcan::FloatSpec< %d, %s >' % (t.bitlen, cast_mode)
00145         else:
00146             signedness = {
00147                 t.KIND_BOOLEAN: '::uavcan::SignednessUnsigned',
00148                 t.KIND_UNSIGNED_INT: '::uavcan::SignednessUnsigned',
00149                 t.KIND_SIGNED_INT: '::uavcan::SignednessSigned',
00150             }[t.kind]
00151             return '::uavcan::IntegerSpec< %d, %s, %s >' % (t.bitlen, signedness, cast_mode)
00152     elif t.category == t.CATEGORY_ARRAY:
00153         value_type = type_to_cpp_type(t.value_type)
00154         mode = {
00155             t.MODE_STATIC: '::uavcan::ArrayModeStatic',
00156             t.MODE_DYNAMIC: '::uavcan::ArrayModeDynamic',
00157         }[t.mode]
00158         return '::uavcan::Array< %s, %s, %d >' % (value_type, mode, t.max_size)
00159     elif t.category == t.CATEGORY_COMPOUND:
00160         return '::' + t.full_name.replace('.', '::')
00161     elif t.category == t.CATEGORY_VOID:
00162         return '::uavcan::IntegerSpec< %d, ::uavcan::SignednessUnsigned, ::uavcan::CastModeSaturate >' % t.bitlen
00163     else:
00164         raise DsdlCompilerException('Unknown type category: %s' % t.category)
00165 
00166 def generate_one_type(template_expander, t):
00167     t.short_name = t.full_name.split('.')[-1]
00168     t.cpp_type_name = t.short_name + '_'
00169     t.cpp_full_type_name = '::' + t.full_name.replace('.', '::')
00170     t.include_guard = t.full_name.replace('.', '_').upper() + '_HPP_INCLUDED'
00171 
00172     # Dependencies (no duplicates)
00173     def fields_includes(fields):
00174         def detect_include(t):
00175             if t.category == t.CATEGORY_COMPOUND:
00176                 return type_output_filename(t)
00177             if t.category == t.CATEGORY_ARRAY:
00178                 return detect_include(t.value_type)
00179         return list(sorted(set(filter(None, [detect_include(x.type) for x in fields]))))
00180 
00181     if t.kind == t.KIND_MESSAGE:
00182         t.cpp_includes = fields_includes(t.fields)
00183     else:
00184         t.cpp_includes = fields_includes(t.request_fields + t.response_fields)
00185 
00186     t.cpp_namespace_components = t.full_name.split('.')[:-1]
00187     t.has_default_dtid = t.default_dtid is not None
00188 
00189     # Attribute types
00190     def inject_cpp_types(attributes):
00191         void_index = 0
00192         for a in attributes:
00193             a.cpp_type = type_to_cpp_type(a.type)
00194             a.void = a.type.category == a.type.CATEGORY_VOID
00195             if a.void:
00196                 assert not a.name
00197                 a.name = '_void_%d' % void_index
00198                 void_index += 1
00199 
00200     if t.kind == t.KIND_MESSAGE:
00201         inject_cpp_types(t.fields)
00202         inject_cpp_types(t.constants)
00203         t.all_attributes = t.fields + t.constants
00204         t.union = t.union and len(t.fields)
00205     else:
00206         inject_cpp_types(t.request_fields)
00207         inject_cpp_types(t.request_constants)
00208         inject_cpp_types(t.response_fields)
00209         inject_cpp_types(t.response_constants)
00210         t.all_attributes = t.request_fields + t.request_constants + t.response_fields + t.response_constants
00211         t.request_union = t.request_union and len(t.request_fields)
00212         t.response_union = t.response_union and len(t.response_fields)
00213 
00214     # Constant properties
00215     def inject_constant_info(constants):
00216         for c in constants:
00217             if c.type.kind == c.type.KIND_FLOAT:
00218                 float(c.string_value)  # Making sure that this is a valid float literal
00219                 c.cpp_value = c.string_value
00220             else:
00221                 int(c.string_value)  # Making sure that this is a valid integer literal
00222                 c.cpp_value = c.string_value
00223                 if c.type.kind == c.type.KIND_UNSIGNED_INT:
00224                     c.cpp_value += 'U'
00225 
00226     if t.kind == t.KIND_MESSAGE:
00227         inject_constant_info(t.constants)
00228     else:
00229         inject_constant_info(t.request_constants)
00230         inject_constant_info(t.response_constants)
00231 
00232     # Data type kind
00233     t.cpp_kind = {
00234         t.KIND_MESSAGE: '::uavcan::DataTypeKindMessage',
00235         t.KIND_SERVICE: '::uavcan::DataTypeKindService',
00236     }[t.kind]
00237 
00238     # Generation
00239     text = template_expander(t=t)  # t for Type
00240     text = '\n'.join(x.rstrip() for x in text.splitlines())
00241     text = text.replace('\n\n\n\n\n', '\n\n').replace('\n\n\n\n', '\n\n').replace('\n\n\n', '\n\n')
00242     text = text.replace('{\n\n ', '{\n ')
00243     return text
00244 
00245 def make_template_expander(filename):
00246     '''
00247     Templating is based on pyratemp (http://www.simple-is-better.org/template/pyratemp.html).
00248     The pyratemp's syntax is rather verbose and not so human friendly, so we define some
00249     custom extensions to make it easier to read and write.
00250     The resulting syntax somewhat resembles Mako (which was used earlier instead of pyratemp):
00251         Substitution:
00252             ${expression}
00253         Line joining through backslash (replaced with a single space):
00254             ${foo(bar(very_long_arument=42, \
00255                       second_line=72))}
00256         Blocks:
00257             % for a in range(10):
00258                 % if a == 5:
00259                     ${foo()}
00260                 % endif
00261             % endfor
00262     The extended syntax is converted into pyratemp's through regexp substitution.
00263     '''
00264     with open(filename) as f:
00265         template_text = f.read()
00266 
00267     # Backslash-newline elimination
00268     template_text = re.sub(r'\\\r{0,1}\n\ *', r' ', template_text)
00269 
00270     # Substitution syntax transformation: ${foo} ==> $!foo!$
00271     template_text = re.sub(r'([^\$]{0,1})\$\{([^\}]+)\}', r'\1$!\2!$', template_text)
00272 
00273     # Flow control expression transformation: % foo: ==> <!--(foo)-->
00274     template_text = re.sub(r'(?m)^(\ *)\%\ *(.+?):{0,1}$', r'\1<!--(\2)-->', template_text)
00275 
00276     # Block termination transformation: <!--(endfoo)--> ==> <!--(end)-->
00277     template_text = re.sub(r'<\!--\(end[a-z]+\)-->', r'<!--(end)-->', template_text)
00278 
00279     # Pyratemp workaround.
00280     # The problem is that if there's no empty line after a macro declaration, first line will be doubly indented.
00281     # Workaround:
00282     #  1. Remove trailing comments
00283     #  2. Add a newline after each macro declaration
00284     template_text = re.sub(r'\ *\#\!.*', '', template_text)
00285     template_text = re.sub(r'(<\!--\(macro\ [a-zA-Z0-9_]+\)-->.*?)', r'\1\n', template_text)
00286 
00287     # Preprocessed text output for debugging
00288 #   with open(filename + '.d', 'w') as f:
00289 #       f.write(template_text)
00290 
00291     template = Template(template_text)
00292 
00293     def expand(**args):
00294         # This function adds one indentation level (4 spaces); it will be used from the template
00295         args['indent'] = lambda text, idnt = '    ': idnt + text.replace('\n', '\n' + idnt)
00296         # This function works like enumerate(), telling you whether the current item is the last one
00297         def enum_last_value(iterable, start=0):
00298             it = iter(iterable)
00299             count = start
00300             last = next(it)
00301             for val in it:
00302                 yield count, False, last
00303                 last = val
00304                 count += 1
00305             yield count, True, last
00306         args['enum_last_value'] = enum_last_value
00307         return template(**args)
00308 
00309     return expand