memap.py

00001 #!/usr/bin/env python
00002 
00003 """Memory Map File Analyser for ARM mbed"""
00004 
00005 import sys
00006 import os
00007 import re
00008 import csv
00009 import json
00010 import argparse
00011 from prettytable import PrettyTable
00012 
00013 from utils import argparse_filestring_type, \
00014     argparse_lowercase_hyphen_type, argparse_uppercase_type
00015 
00016 DEBUG = False
00017 
00018 RE_ARMCC = re.compile(
00019     r'^\s+0x(\w{8})\s+0x(\w{8})\s+(\w+)\s+(\w+)\s+(\d+)\s+[*]?.+\s+(.+)$')
00020 RE_IAR = re.compile(
00021     r'^\s+(.+)\s+(zero|const|ro code|inited|uninit)\s'
00022     r'+0x(\w{8})\s+0x(\w+)\s+(.+)\s.+$')
00023 
00024 class MemapParser (object):
00025     """An object that represents parsed results, parses the memory map files,
00026     and writes out different file types of memory results
00027     """
00028 
00029     print_sections = ('.text', '.data', '.bss')
00030 
00031     misc_flash_sections = ('.interrupts', '.flash_config')
00032 
00033     other_sections = ('.interrupts_ram', '.init', '.ARM.extab',
00034                       '.ARM.exidx', '.ARM.attributes', '.eh_frame',
00035                       '.init_array', '.fini_array', '.jcr', '.stab',
00036                       '.stabstr', '.ARM.exidx', '.ARM')
00037 
00038     # sections to print info (generic for all toolchains)
00039     sections = ('.text', '.data', '.bss', '.heap', '.stack')
00040 
00041     def __init__ (self, detailed_misc=False):
00042         """ General initialization
00043         """
00044         # 
00045         self.detailed_misc  = detailed_misc
00046         
00047         # list of all modules and their sections
00048         self.modules  = dict()
00049 
00050         # sections must be defined in this order to take irrelevant out
00051         self.all_sections  = self.sections  + self.other_sections  + \
00052                             self.misc_flash_sections  + ('unknown', 'OUTPUT')
00053 
00054         # list of all object files and mappting to module names
00055         self.object_to_module  = dict()
00056 
00057         # Memory report (sections + summary)
00058         self.mem_report  = []
00059 
00060         # Just the memory summary section
00061         self.mem_summary  = dict()
00062 
00063         self.subtotal  = dict()
00064 
00065     def module_add (self, module_name, size, section):
00066         """ Adds a module / section to the list
00067 
00068         Positional arguments:
00069         module_name - name of the module to add
00070         size - the size of the module being added
00071         section - the section the module contributes to
00072         """
00073 
00074         if module_name in self.modules :
00075             self.modules [module_name][section] += size
00076         else:
00077             temp_dic = dict()
00078             for section_idx in self.all_sections :
00079                 temp_dic[section_idx] = 0
00080             temp_dic[section] = size
00081             self.modules [module_name] = temp_dic
00082 
00083     def check_new_section_gcc (self, line):
00084         """ Check whether a new section in a map file has been detected (only
00085         applies to gcc)
00086 
00087         Positional arguments:
00088         line - the line to check for a new section
00089         """
00090 
00091         for i in self.all_sections :
00092             if line.startswith(i):
00093                 # should name of the section (assuming it's a known one)
00094                 return i
00095 
00096         if line.startswith('.'):
00097             return 'unknown'     # all others are classified are unknown
00098         else:
00099             return False         # everything else, means no change in section
00100 
00101     
00102     def path_object_to_module_name (self, txt):
00103         """ Parse a path to object file to extract it's module and object data
00104 
00105         Positional arguments:
00106         txt - the path to parse the object and module name from
00107         """
00108 
00109         txt = txt.replace('\\', '/')
00110         rex_mbed_os_name = r'^.+mbed-os\/(.+)\/(.+\.o)$'
00111         test_rex_mbed_os_name = re.match(rex_mbed_os_name, txt)
00112 
00113         if test_rex_mbed_os_name:
00114 
00115             object_name = test_rex_mbed_os_name.group(2)
00116             data = test_rex_mbed_os_name.group(1).split('/')
00117             ndata = len(data)
00118 
00119             if ndata == 1:
00120                 module_name = data[0]
00121             else:
00122                 module_name = data[0] + '/' + data[1]
00123 
00124             return [module_name, object_name]
00125             
00126         elif self.detailed_misc :           
00127             rex_obj_name = r'^.+\/(.+\.o\)*)$'
00128             test_rex_obj_name = re.match(rex_obj_name, txt)
00129             if test_rex_obj_name:
00130                 object_name = test_rex_obj_name.group(1)
00131                 return ['Misc/' + object_name, ""]        
00132                 
00133             return ['Misc', ""]
00134         else: 
00135             return ['Misc', ""]
00136 
00137     def parse_section_gcc (self, line):
00138         """ Parse data from a section of gcc map file
00139 
00140         examples:
00141                         0x00004308       0x7c ./BUILD/K64F/GCC_ARM/mbed-os/hal/targets/hal/TARGET_Freescale/TARGET_KPSDK_MCUS/spi_api.o
00142          .text          0x00000608      0x198 ./BUILD/K64F/GCC_ARM/mbed-os/core/mbed-rtos/rtx/TARGET_CORTEX_M/TARGET_RTOS_M4_M7/TOOLCHAIN_GCC/HAL_CM4.o
00143 
00144         Positional arguments:
00145         line - the line to parse a section from
00146         """
00147         rex_address_len_name = re.compile(
00148             r'^\s+.*0x(\w{8,16})\s+0x(\w+)\s(.+)$')
00149 
00150         test_address_len_name = re.match(rex_address_len_name, line)
00151 
00152         if test_address_len_name:
00153 
00154             if int(test_address_len_name.group(2), 16) == 0: # size == 0
00155                 return ["", 0] # no valid entry
00156             else:
00157                 m_name, _ = self.path_object_to_module_name (
00158                     test_address_len_name.group(3))
00159                 m_size = int(test_address_len_name.group(2), 16)
00160                 return [m_name, m_size]
00161 
00162         else: # special corner case for *fill* sections
00163             #  example
00164             # *fill*         0x0000abe4        0x4
00165             rex_address_len = r'^\s+\*fill\*\s+0x(\w{8,16})\s+0x(\w+).*$'
00166             test_address_len = re.match(rex_address_len, line)
00167 
00168             if test_address_len:
00169                 if int(test_address_len.group(2), 16) == 0: # size == 0
00170                     return ["", 0] # no valid entry
00171                 else:
00172                     m_name = 'Fill'
00173                     m_size = int(test_address_len.group(2), 16)
00174                     return [m_name, m_size]
00175             else:
00176                 return ["", 0] # no valid entry
00177 
00178     def parse_map_file_gcc (self, file_desc):
00179         """ Main logic to decode gcc map files
00180 
00181         Positional arguments:
00182         file_desc - a stream object to parse as a gcc map file
00183         """
00184 
00185         current_section = 'unknown'
00186 
00187         with file_desc as infile:
00188 
00189             # Search area to parse
00190             for line in infile:
00191                 if line.startswith('Linker script and memory map'):
00192                     current_section = "unknown"
00193                     break
00194 
00195             # Start decoding the map file
00196             for line in infile:
00197 
00198                 change_section = self.check_new_section_gcc (line)
00199 
00200                 if change_section == "OUTPUT": # finish parsing file: exit
00201                     break
00202                 elif change_section != False:
00203                     current_section = change_section
00204 
00205                 [module_name, module_size] = self.parse_section_gcc (line)
00206 
00207                 if module_size == 0 or module_name == "":
00208                     pass
00209                 else:
00210                     self.module_add (module_name, module_size, current_section)
00211 
00212                 if DEBUG:
00213                     print "Line: %s" % line,
00214                     print "Module: %s\tSection: %s\tSize: %s" % \
00215                         (module_name, current_section, module_size)
00216                     raw_input("----------")
00217 
00218     def parse_section_armcc (self, line):
00219         """ Parse data from an armcc map file
00220 
00221         Examples of armcc map file:
00222             Base_Addr    Size         Type   Attr      Idx    E Section Name        Object
00223             0x00000000   0x00000400   Data   RO        11222    RESET               startup_MK64F12.o
00224             0x00000410   0x00000008   Code   RO        49364  * !!!main             c_w.l(__main.o)
00225 
00226         Positional arguments:
00227         line - the line to parse the section data from
00228         """
00229 
00230         test_rex_armcc = re.match(RE_ARMCC, line)
00231 
00232         if test_rex_armcc:
00233 
00234             size = int(test_rex_armcc.group(2), 16)
00235 
00236             if test_rex_armcc.group(4) == 'RO':
00237                 section = '.text'
00238             else:
00239                 if test_rex_armcc.group(3) == 'Data':
00240                     section = '.data'
00241                 elif test_rex_armcc.group(3) == 'Zero':
00242                     section = '.bss'
00243                 else:
00244                     print "BUG armcc map parser"
00245                     raw_input()
00246 
00247             # lookup object in dictionary and return module name
00248             object_name = test_rex_armcc.group(6)
00249             if object_name in self.object_to_module :
00250                 module_name = self.object_to_module [object_name]
00251             else:
00252                 module_name = 'Misc'
00253 
00254             return [module_name, size, section]
00255 
00256         else:
00257             return ["", 0, ""] # no valid entry
00258 
00259     def parse_section_iar (self, line):
00260         """ Parse data from an IAR map file
00261 
00262         Examples of IAR map file:
00263          Section             Kind        Address     Size  Object
00264          .intvec             ro code  0x00000000    0x198  startup_MK64F12.o [15]
00265          .rodata             const    0x00000198      0x0  zero_init3.o [133]
00266          .iar.init_table     const    0x00008384     0x2c  - Linker created -
00267          Initializer bytes   const    0x00000198     0xb2  <for P3 s0>
00268          .data               inited   0x20000000     0xd4  driverAtmelRFInterface.o [70]
00269          .bss                zero     0x20000598    0x318  RTX_Conf_CM.o [4]
00270          .iar.dynexit        uninit   0x20001448    0x204  <Block tail>
00271            HEAP              uninit   0x20001650  0x10000  <Block tail>
00272 
00273         Positional_arguments:
00274         line - the line to parse section data from
00275         """
00276 
00277         test_rex_iar = re.match(RE_IAR, line)
00278 
00279         if test_rex_iar:
00280 
00281             size = int(test_rex_iar.group(4), 16)
00282 
00283             if test_rex_iar.group(2) == 'const' or \
00284                test_rex_iar.group(2) == 'ro code':
00285                 section = '.text'
00286             elif test_rex_iar.group(2) == 'zero' or \
00287             test_rex_iar.group(2) == 'uninit':
00288                 if test_rex_iar.group(1)[0:4] == 'HEAP':
00289                     section = '.heap'
00290                 elif test_rex_iar.group(1)[0:6] == 'CSTACK':
00291                     section = '.stack'
00292                 else:
00293                     section = '.bss' #  default section
00294 
00295             elif test_rex_iar.group(2) == 'inited':
00296                 section = '.data'
00297             else:
00298                 print "BUG IAR map parser"
00299                 raw_input()
00300 
00301             # lookup object in dictionary and return module name
00302             object_name = test_rex_iar.group(5)
00303             if object_name in self.object_to_module :
00304                 module_name = self.object_to_module [object_name]
00305             else:
00306                 module_name = 'Misc'
00307 
00308             return [module_name, size, section]
00309 
00310         else:
00311             return ["", 0, ""] # no valid entry
00312 
00313     def parse_map_file_armcc (self, file_desc):
00314         """ Main logic to decode armc5 map files
00315 
00316         Positional arguments:
00317         file_desc - a file like object to parse as an armc5 map file
00318         """
00319 
00320         with file_desc as infile:
00321 
00322             # Search area to parse
00323             for line in infile:
00324                 if line.startswith('    Base Addr    Size'):
00325                     break
00326 
00327             # Start decoding the map file
00328             for line in infile:
00329 
00330                 [name, size, section] = self.parse_section_armcc (line)
00331 
00332                 if size == 0 or name == "" or section == "":
00333                     pass
00334                 else:
00335                     self.module_add (name, size, section)
00336 
00337     def parse_map_file_iar (self, file_desc):
00338         """ Main logic to decode IAR map files
00339 
00340         Positional arguments:
00341         file_desc - a file like object to parse as an IAR map file
00342         """
00343 
00344         with file_desc as infile:
00345 
00346             # Search area to parse
00347             for line in infile:
00348                 if line.startswith('  Section  '):
00349                     break
00350 
00351             # Start decoding the map file
00352             for line in infile:
00353 
00354                 [name, size, section] = self.parse_section_iar (line)
00355 
00356                 if size == 0 or name == "" or section == "":
00357                     pass
00358                 else:
00359                     self.module_add (name, size, section)
00360 
00361     def search_objects (self, path):
00362         """ Searches for object files and creates mapping: object --> module
00363 
00364         Positional arguments:
00365         path - the path to an object file
00366         """
00367 
00368         path = path.replace('\\', '/')
00369 
00370         # check location of map file
00371         rex = r'^(.+)' + r'\/(.+\.map)$'
00372         test_rex = re.match(rex, path)
00373 
00374         if test_rex:
00375             search_path = test_rex.group(1) + '/mbed-os/'
00376         else:
00377             print "Warning: this doesn't look like an mbed project"
00378             return
00379 
00380         for root, _, obj_files in os.walk(search_path):
00381             for obj_file in obj_files:
00382                 if obj_file.endswith(".o"):
00383                     module_name, object_name = self.path_object_to_module_name (
00384                         os.path.join(root, obj_file))
00385 
00386                     if object_name in self.object_to_module :
00387                         if DEBUG:
00388                             print "WARNING: multiple usages of object file: %s"\
00389                                 % object_name
00390                             print "    Current: %s" % \
00391                                 self.object_to_module [object_name]
00392                             print "    New:     %s" % module_name
00393                             print " "
00394                     else:
00395                         self.object_to_module .update({object_name:module_name})
00396 
00397     export_formats = ["json", "csv-ci", "table"]
00398 
00399     def generate_output (self, export_format, file_output=None):
00400         """ Generates summary of memory map data
00401 
00402         Positional arguments:
00403         export_format - the format to dump
00404 
00405         Keyword arguments:
00406         file_desc - descriptor (either stdout or file)
00407 
00408         Returns: generated string for the 'table' format, otherwise None
00409         """
00410 
00411         try:
00412             if file_output:
00413                 file_desc = open(file_output, 'wb')
00414             else:
00415                 file_desc = sys.stdout
00416         except IOError as error:
00417             print "I/O error({0}): {1}".format(error.errno, error.strerror)
00418             return False
00419 
00420         to_call = {'json': self.generate_json ,
00421                    'csv-ci': self.generate_csv ,
00422                    'table': self.generate_table }[export_format]
00423         output = to_call(file_desc)
00424 
00425         if file_desc is not sys.stdout:
00426             file_desc.close()
00427 
00428         return output
00429 
00430     def generate_json (self, file_desc):
00431         """Generate a json file from a memory map
00432 
00433         Positional arguments:
00434         file_desc - the file to write out the final report to
00435         """
00436         file_desc.write(json.dumps(self.mem_report , indent=4))
00437         file_desc.write('\n')
00438 
00439         return None
00440 
00441     def generate_csv (self, file_desc):
00442         """Generate a CSV file from a memoy map
00443 
00444         Positional arguments:
00445         file_desc - the file to write out the final report to
00446         """
00447         csv_writer = csv.writer(file_desc, delimiter=',',
00448                                 quoting=csv.QUOTE_MINIMAL)
00449 
00450         csv_module_section = []
00451         csv_sizes = []
00452         for i in sorted(self.modules ):
00453             for k in self.print_sections :
00454                 csv_module_section += [i+k]
00455                 csv_sizes += [self.modules [i][k]]
00456 
00457         csv_module_section += ['static_ram']
00458         csv_sizes += [self.mem_summary ['static_ram']]
00459 
00460         csv_module_section += ['heap']
00461         if self.mem_summary ['heap'] == 0:
00462             csv_sizes += ['unknown']
00463         else:
00464             csv_sizes += [self.mem_summary ['heap']]
00465 
00466         csv_module_section += ['stack']
00467         if self.mem_summary ['stack'] == 0:
00468             csv_sizes += ['unknown']
00469         else:
00470             csv_sizes += [self.mem_summary ['stack']]
00471 
00472         csv_module_section += ['total_ram']
00473         csv_sizes += [self.mem_summary ['total_ram']]
00474 
00475         csv_module_section += ['total_flash']
00476         csv_sizes += [self.mem_summary ['total_flash']]
00477 
00478         csv_writer.writerow(csv_module_section)
00479         csv_writer.writerow(csv_sizes)
00480 
00481         return None
00482 
00483     def generate_table (self, file_desc):
00484         """Generate a table from a memoy map
00485 
00486         Positional arguments:
00487         file_desc - the file to write out the final report to
00488 
00489         Returns: string of the generated table
00490         """
00491         # Create table
00492         columns = ['Module']
00493         columns.extend(self.print_sections )
00494 
00495         table = PrettyTable(columns)
00496         table.align["Module"] = "l"
00497         for col in self.print_sections :
00498             table.align[col] = 'r'
00499 
00500         for i in list(self.print_sections ):
00501             table.align[i] = 'r'
00502 
00503         for i in sorted(self.modules ):
00504             row = [i]
00505 
00506             for k in self.print_sections :
00507                 row.append(self.modules [i][k])
00508 
00509             table.add_row(row)
00510 
00511         subtotal_row = ['Subtotals']
00512         for k in self.print_sections :
00513             subtotal_row.append(self.subtotal [k])
00514 
00515         table.add_row(subtotal_row)
00516 
00517         output = table.get_string()
00518         output += '\n'
00519 
00520         if self.mem_summary ['heap'] == 0:
00521             output += "Allocated Heap: unknown\n"
00522         else:
00523             output += "Allocated Heap: %s bytes\n" % \
00524                         str(self.mem_summary ['heap'])
00525 
00526         if self.mem_summary ['stack'] == 0:
00527             output += "Allocated Stack: unknown\n"
00528         else:
00529             output += "Allocated Stack: %s bytes\n" % \
00530                         str(self.mem_summary ['stack'])
00531 
00532         output += "Total Static RAM memory (data + bss): %s bytes\n" % \
00533                         str(self.mem_summary ['static_ram'])
00534         output += "Total RAM memory (data + bss + heap + stack): %s bytes\n" % \
00535                         str(self.mem_summary ['total_ram'])
00536         output += "Total Flash memory (text + data + misc): %s bytes\n" % \
00537                         str(self.mem_summary ['total_flash'])
00538 
00539         return output
00540 
00541     toolchains = ["ARM", "ARM_STD", "ARM_MICRO", "GCC_ARM", "IAR"]
00542 
00543     def compute_report(self):
00544         for k in self.sections :
00545             self.subtotal [k] = 0
00546 
00547         for i in sorted(self.modules ):
00548             for k in self.sections :
00549                 self.subtotal [k] += self.modules [i][k]
00550 
00551         # Calculate misc flash sections
00552         self.misc_flash_mem  = 0
00553         for i in self.modules :
00554             for k in self.misc_flash_sections :
00555                 if self.modules [i][k]:
00556                     self.misc_flash_mem  += self.modules [i][k]
00557 
00558         self.mem_summary  = {
00559             'static_ram': (self.subtotal ['.data'] + self.subtotal ['.bss']),
00560             'heap': (self.subtotal ['.heap']),
00561             'stack': (self.subtotal ['.stack']),
00562             'total_ram': (self.subtotal ['.data'] + self.subtotal ['.bss'] +
00563                           self.subtotal ['.heap']+self.subtotal ['.stack']),
00564             'total_flash': (self.subtotal ['.text'] + self.subtotal ['.data'] +
00565                             self.misc_flash_mem ),
00566         }
00567 
00568         self.mem_report  = []
00569         for i in sorted(self.modules ):
00570             self.mem_report .append({
00571                 "module":i,
00572                 "size":{
00573                     k:self.modules [i][k] for k in self.print_sections 
00574                 }
00575             })
00576 
00577         self.mem_report .append({
00578             'summary': self.mem_summary 
00579         })
00580 
00581     def parse (self, mapfile, toolchain):
00582         """ Parse and decode map file depending on the toolchain
00583 
00584         Positional arguments:
00585         mapfile - the file name of the memory map file
00586         toolchain - the toolchain used to create the file
00587         """
00588 
00589         result = True
00590         try:
00591             with open(mapfile, 'r') as file_input:
00592                 if toolchain == "ARM" or toolchain == "ARM_STD" or\
00593                    toolchain == "ARM_MICRO":
00594                     self.search_objects (os.path.abspath(mapfile))
00595                     self.parse_map_file_armcc (file_input)
00596                 elif toolchain == "GCC_ARM":
00597                     self.parse_map_file_gcc (file_input)
00598                 elif toolchain == "IAR":
00599                     self.search_objects (os.path.abspath(mapfile))
00600                     self.parse_map_file_iar (file_input)
00601                 else:
00602                     result = False
00603             
00604             self.compute_report ()
00605         
00606         except IOError as error:
00607             print "I/O error({0}): {1}".format(error.errno, error.strerror)
00608             result = False
00609         return result
00610 
00611 def main():
00612     """Entry Point"""
00613 
00614     version = '0.3.12'
00615 
00616     # Parser handling
00617     parser = argparse.ArgumentParser(
00618         description="Memory Map File Analyser for ARM mbed\nversion %s" %
00619         version)
00620 
00621     parser.add_argument(
00622         'file', type=argparse_filestring_type, help='memory map file')
00623 
00624     parser.add_argument(
00625         '-t', '--toolchain', dest='toolchain',
00626         help='select a toolchain used to build the memory map file (%s)' %
00627         ", ".join(MemapParser.toolchains),
00628         required=True,
00629         type=argparse_uppercase_type(MemapParser.toolchains, "toolchain"))
00630 
00631     parser.add_argument(
00632         '-o', '--output', help='output file name', required=False)
00633 
00634     parser.add_argument(
00635         '-e', '--export', dest='export', required=False, default='table',
00636         type=argparse_lowercase_hyphen_type(MemapParser.export_formats,
00637                                             'export format'),
00638         help="export format (examples: %s: default)" %
00639         ", ".join(MemapParser.export_formats))
00640 
00641     parser.add_argument('-v', '--version', action='version', version=version)
00642     
00643     parser.add_argument('-d', '--detailed', action='store_true', help='Displays the elements in "Misc" in a detailed fashion', required=False)
00644 
00645     # Parse/run command
00646     if len(sys.argv) <= 1:
00647         parser.print_help()
00648         sys.exit(1)
00649 
00650 
00651     args = parser.parse_args()
00652 
00653     # Create memap object
00654     memap = MemapParser(detailed_misc=args.detailed)
00655 
00656     # Parse and decode a map file
00657     if args.file and args.toolchain:
00658         if memap.parse(args.file, args.toolchain) is False:
00659             sys.exit(0)
00660 
00661     returned_string = None
00662     # Write output in file
00663     if args.output != None:
00664         returned_string = memap.generate_output(args.export, args.output)
00665     else: # Write output in screen
00666         returned_string = memap.generate_output(args.export)
00667 
00668     if args.export == 'table' and returned_string:
00669         print returned_string
00670 
00671     sys.exit(0)
00672 
00673 if __name__ == "__main__":
00674     main()