memap.py

00001 #!/usr/bin/env python
00002 
00003 """Memory Map File Analyser for ARM mbed"""
00004 
00005 import sys
00006 import os
00007 import re
00008 import csv
00009 import json
00010 import argparse
00011 from prettytable import PrettyTable
00012 
00013 from tools.utils import argparse_filestring_type, \
00014     argparse_lowercase_hyphen_type, argparse_uppercase_type
00015 
00016 DEBUG = False
00017 RE_ARMCC = re.compile(
00018     r'^\s+0x(\w{8})\s+0x(\w{8})\s+(\w+)\s+(\w+)\s+(\d+)\s+[*]?.+\s+(.+)$')
00019 RE_IAR = re.compile(
00020     r'^\s+(.+)\s+(zero|const|ro code|inited|uninit)\s'
00021     r'+0x(\w{8})\s+0x(\w+)\s+(.+)\s.+$')
00022 
00023 class MemapParser (object):
00024     """An object that represents parsed results, parses the memory map files,
00025     and writes out different file types of memory results
00026     """
00027 
00028     print_sections = ('.text', '.data', '.bss')
00029 
00030     misc_flash_sections = ('.interrupts', '.flash_config')
00031 
00032     other_sections = ('.interrupts_ram', '.init', '.ARM.extab',
00033                       '.ARM.exidx', '.ARM.attributes', '.eh_frame',
00034                       '.init_array', '.fini_array', '.jcr', '.stab',
00035                       '.stabstr', '.ARM.exidx', '.ARM')
00036 
00037     # sections to print info (generic for all toolchains)
00038     sections = ('.text', '.data', '.bss', '.heap', '.stack')
00039 
00040     def __init__ (self):
00041         """ General initialization
00042         """
00043 
00044         # list of all modules and their sections
00045         self.modules  = dict()
00046 
00047         # sections must be defined in this order to take irrelevant out
00048         self.all_sections  = self.sections  + self.other_sections  + \
00049                             self.misc_flash_sections  + ('unknown', 'OUTPUT')
00050 
00051         # list of all object files and mappting to module names
00052         self.object_to_module  = dict()
00053 
00054         # Memory usage summary structure
00055         self.mem_summary  = dict()
00056 
00057     def module_add (self, module_name, size, section):
00058         """ Adds a module / section to the list
00059 
00060         Positional arguments:
00061         module_name - name of the module to add
00062         size - the size of the module being added
00063         section - the section the module contributes to
00064         """
00065 
00066         if module_name in self.modules :
00067             self.modules [module_name][section] += size
00068         else:
00069             temp_dic = dict()
00070             for section_idx in self.all_sections :
00071                 temp_dic[section_idx] = 0
00072             temp_dic[section] = size
00073             self.modules [module_name] = temp_dic
00074 
00075     def check_new_section_gcc (self, line):
00076         """ Check whether a new section in a map file has been detected (only
00077         applies to gcc)
00078 
00079         Positional arguments:
00080         line - the line to check for a new section
00081         """
00082 
00083         for i in self.all_sections :
00084             if line.startswith(i):
00085                 # should name of the section (assuming it's a known one)
00086                 return i
00087 
00088         if line.startswith('.'):
00089             return 'unknown'     # all others are classified are unknown
00090         else:
00091             return False         # everything else, means no change in section
00092 
00093     @staticmethod
00094     def path_object_to_module_name (txt):
00095         """ Parse a path to object file to extract it's module and object data
00096 
00097         Positional arguments:
00098         txt - the path to parse the object and module name from
00099         """
00100 
00101         txt = txt.replace('\\', '/')
00102         rex_mbed_os_name = r'^.+mbed-os\/(.+)\/(.+\.o)$'
00103         test_rex_mbed_os_name = re.match(rex_mbed_os_name, txt)
00104 
00105         if test_rex_mbed_os_name:
00106 
00107             object_name = test_rex_mbed_os_name.group(2)
00108             data = test_rex_mbed_os_name.group(1).split('/')
00109             ndata = len(data)
00110 
00111             if ndata == 1:
00112                 module_name = data[0]
00113             else:
00114                 module_name = data[0] + '/' + data[1]
00115 
00116             return [module_name, object_name]
00117         else:
00118             return ['Misc', ""]
00119 
00120 
00121     def parse_section_gcc (self, line):
00122         """ Parse data from a section of gcc map file
00123 
00124         examples:
00125                         0x00004308       0x7c ./.build/K64F/GCC_ARM/mbed-os/hal/targets/hal/TARGET_Freescale/TARGET_KPSDK_MCUS/spi_api.o
00126          .text          0x00000608      0x198 ./.build/K64F/GCC_ARM/mbed-os/core/mbed-rtos/rtx/TARGET_CORTEX_M/TARGET_RTOS_M4_M7/TOOLCHAIN_GCC/HAL_CM4.o
00127 
00128         Positional arguments:
00129         line - the line to parse a section from
00130         """
00131         rex_address_len_name = re.compile(
00132             r'^\s+.*0x(\w{8,16})\s+0x(\w+)\s(.+)$')
00133 
00134         test_address_len_name = re.match(rex_address_len_name, line)
00135 
00136         if test_address_len_name:
00137 
00138             if int(test_address_len_name.group(2), 16) == 0: # size == 0
00139                 return ["", 0] # no valid entry
00140             else:
00141                 m_name, _ = self.path_object_to_module_name (
00142                     test_address_len_name.group(3))
00143                 m_size = int(test_address_len_name.group(2), 16)
00144                 return [m_name, m_size]
00145 
00146         else: # special corner case for *fill* sections
00147             #  example
00148             # *fill*         0x0000abe4        0x4
00149             rex_address_len = r'^\s+\*fill\*\s+0x(\w{8,16})\s+0x(\w+).*$'
00150             test_address_len = re.match(rex_address_len, line)
00151 
00152             if test_address_len:
00153                 if int(test_address_len.group(2), 16) == 0: # size == 0
00154                     return ["", 0] # no valid entry
00155                 else:
00156                     m_name = 'Fill'
00157                     m_size = int(test_address_len.group(2), 16)
00158                     return [m_name, m_size]
00159             else:
00160                 return ["", 0] # no valid entry
00161 
00162     def parse_map_file_gcc (self, file_desc):
00163         """ Main logic to decode gcc map files
00164 
00165         Positional arguments:
00166         file_desc - a stream object to parse as a gcc map file
00167         """
00168 
00169         current_section = 'unknown'
00170 
00171         with file_desc as infile:
00172 
00173             # Search area to parse
00174             for line in infile:
00175                 if line.startswith('Linker script and memory map'):
00176                     current_section = "unknown"
00177                     break
00178 
00179             # Start decoding the map file
00180             for line in infile:
00181 
00182                 change_section = self.check_new_section_gcc (line)
00183 
00184                 if change_section == "OUTPUT": # finish parsing file: exit
00185                     break
00186                 elif change_section != False:
00187                     current_section = change_section
00188 
00189                 [module_name, module_size] = self.parse_section_gcc (line)
00190 
00191                 if module_size == 0 or module_name == "":
00192                     pass
00193                 else:
00194                     self.module_add (module_name, module_size, current_section)
00195 
00196                 if DEBUG:
00197                     print "Line: %s" % line,
00198                     print "Module: %s\tSection: %s\tSize: %s" % \
00199                         (module_name, current_section, module_size)
00200                     raw_input("----------")
00201 
00202     def parse_section_armcc (self, line):
00203         """ Parse data from an armcc map file
00204 
00205         Examples of armcc map file:
00206             Base_Addr    Size         Type   Attr      Idx    E Section Name        Object
00207             0x00000000   0x00000400   Data   RO        11222    RESET               startup_MK64F12.o
00208             0x00000410   0x00000008   Code   RO        49364  * !!!main             c_w.l(__main.o)
00209 
00210         Positional arguments:
00211         line - the line to parse the section data from
00212         """
00213 
00214         test_rex_armcc = re.match(RE_ARMCC, line)
00215 
00216         if test_rex_armcc:
00217 
00218             size = int(test_rex_armcc.group(2), 16)
00219 
00220             if test_rex_armcc.group(4) == 'RO':
00221                 section = '.text'
00222             else:
00223                 if test_rex_armcc.group(3) == 'Data':
00224                     section = '.data'
00225                 elif test_rex_armcc.group(3) == 'Zero':
00226                     section = '.bss'
00227                 else:
00228                     print "BUG armcc map parser"
00229                     raw_input()
00230 
00231             # lookup object in dictionary and return module name
00232             object_name = test_rex_armcc.group(6)
00233             if object_name in self.object_to_module :
00234                 module_name = self.object_to_module [object_name]
00235             else:
00236                 module_name = 'Misc'
00237 
00238             return [module_name, size, section]
00239 
00240         else:
00241             return ["", 0, ""] # no valid entry
00242 
00243     def parse_section_iar (self, line):
00244         """ Parse data from an IAR map file
00245 
00246         Examples of IAR map file:
00247          Section             Kind        Address     Size  Object
00248          .intvec             ro code  0x00000000    0x198  startup_MK64F12.o [15]
00249          .rodata             const    0x00000198      0x0  zero_init3.o [133]
00250          .iar.init_table     const    0x00008384     0x2c  - Linker created -
00251          Initializer bytes   const    0x00000198     0xb2  <for P3 s0>
00252          .data               inited   0x20000000     0xd4  driverAtmelRFInterface.o [70]
00253          .bss                zero     0x20000598    0x318  RTX_Conf_CM.o [4]
00254          .iar.dynexit        uninit   0x20001448    0x204  <Block tail>
00255            HEAP              uninit   0x20001650  0x10000  <Block tail>
00256 
00257         Positional_arguments:
00258         line - the line to parse section data from
00259         """
00260 
00261         test_rex_iar = re.match(RE_IAR, line)
00262 
00263         if test_rex_iar:
00264 
00265             size = int(test_rex_iar.group(4), 16)
00266 
00267             if test_rex_iar.group(2) == 'const' or \
00268                test_rex_iar.group(2) == 'ro code':
00269                 section = '.text'
00270             elif test_rex_iar.group(2) == 'zero' or \
00271             test_rex_iar.group(2) == 'uninit':
00272                 if test_rex_iar.group(1)[0:4] == 'HEAP':
00273                     section = '.heap'
00274                 elif test_rex_iar.group(1)[0:6] == 'CSTACK':
00275                     section = '.stack'
00276                 else:
00277                     section = '.bss' #  default section
00278 
00279             elif test_rex_iar.group(2) == 'inited':
00280                 section = '.data'
00281             else:
00282                 print "BUG IAR map parser"
00283                 raw_input()
00284 
00285             # lookup object in dictionary and return module name
00286             object_name = test_rex_iar.group(5)
00287             if object_name in self.object_to_module :
00288                 module_name = self.object_to_module [object_name]
00289             else:
00290                 module_name = 'Misc'
00291 
00292             return [module_name, size, section]
00293 
00294         else:
00295             return ["", 0, ""] # no valid entry
00296 
00297     def parse_map_file_armcc (self, file_desc):
00298         """ Main logic to decode armc5 map files
00299 
00300         Positional arguments:
00301         file_desc - a file like object to parse as an armc5 map file
00302         """
00303 
00304         with file_desc as infile:
00305 
00306             # Search area to parse
00307             for line in infile:
00308                 if line.startswith('    Base Addr    Size'):
00309                     break
00310 
00311             # Start decoding the map file
00312             for line in infile:
00313 
00314                 [name, size, section] = self.parse_section_armcc (line)
00315 
00316                 if size == 0 or name == "" or section == "":
00317                     pass
00318                 else:
00319                     self.module_add (name, size, section)
00320 
00321     def parse_map_file_iar (self, file_desc):
00322         """ Main logic to decode IAR map files
00323 
00324         Positional arguments:
00325         file_desc - a file like object to parse as an IAR map file
00326         """
00327 
00328         with file_desc as infile:
00329 
00330             # Search area to parse
00331             for line in infile:
00332                 if line.startswith('  Section  '):
00333                     break
00334 
00335             # Start decoding the map file
00336             for line in infile:
00337 
00338                 [name, size, section] = self.parse_section_iar (line)
00339 
00340                 if size == 0 or name == "" or section == "":
00341                     pass
00342                 else:
00343                     self.module_add (name, size, section)
00344 
00345     def search_objects (self, path, toolchain):
00346         """ Check whether the specified map file matches with the toolchain.
00347         Searches for object files and creates mapping: object --> module
00348 
00349         Positional arguments:
00350         path - the path to an object file
00351         toolchain - the toolchain used to build the object file
00352         """
00353 
00354         path = path.replace('\\', '/')
00355 
00356         # check location of map file
00357         rex = r'^(.+\/)' + re.escape(toolchain) + r'\/(.+\.map)$'
00358         test_rex = re.match(rex, path)
00359 
00360         if test_rex:
00361             search_path = test_rex.group(1) + toolchain + '/mbed-os/'
00362         else:
00363             # It looks this is not an mbed project
00364             # object-to-module mapping cannot be generated
00365             print "Warning: specified toolchain doesn't match with"\
00366                 " path to the memory map file."
00367             return
00368 
00369         for root, _, obj_files in os.walk(search_path):
00370             for obj_file in obj_files:
00371                 if obj_file.endswith(".o"):
00372                     module_name, object_name = self.path_object_to_module_name (
00373                         os.path.join(root, obj_file))
00374 
00375                     if object_name in self.object_to_module :
00376                         if DEBUG:
00377                             print "WARNING: multiple usages of object file: %s"\
00378                                 % object_name
00379                             print "    Current: %s" % \
00380                                 self.object_to_module [object_name]
00381                             print "    New:     %s" % module_name
00382                             print " "
00383                     else:
00384                         self.object_to_module .update({object_name:module_name})
00385 
00386     export_formats = ["json", "csv-ci", "table"]
00387 
00388     def generate_output (self, export_format, file_output=None):
00389         """ Generates summary of memory map data
00390 
00391         Positional arguments:
00392         export_format - the format to dump
00393 
00394         Keyword arguments:
00395         file_desc - descriptor (either stdout or file)
00396         """
00397 
00398         try:
00399             if file_output:
00400                 file_desc = open(file_output, 'wb')
00401             else:
00402                 file_desc = sys.stdout
00403         except IOError as error:
00404             print "I/O error({0}): {1}".format(error.errno, error.strerror)
00405             return False
00406 
00407         subtotal = dict()
00408         for k in self.sections :
00409             subtotal[k] = 0
00410 
00411         # Calculate misc flash sections
00412         misc_flash_mem = 0
00413         for i in self.modules :
00414             for k in self.misc_flash_sections :
00415                 if self.modules [i][k]:
00416                     misc_flash_mem += self.modules [i][k]
00417 
00418         json_obj = []
00419         for i in sorted(self.modules ):
00420 
00421             row = []
00422 
00423             json_obj.append({
00424                 "module":i,
00425                 "size":{
00426                     k:self.modules [i][k] for k in self.print_sections 
00427                 }
00428             })
00429 
00430         summary = {
00431             'summary':{
00432                 'static_ram': (subtotal['.data'] + subtotal['.bss']),
00433                 'heap': (subtotal['.heap']),
00434                 'stack': (subtotal['.stack']),
00435                 'total_ram': (subtotal['.data'] + subtotal['.bss'] +
00436                               subtotal['.heap']+subtotal['.stack']),
00437                 'total_flash': (subtotal['.text'] + subtotal['.data'] +
00438                                 misc_flash_mem),
00439             }
00440         }
00441 
00442         self.mem_summary  = json_obj + [summary]
00443 
00444         to_call = {'json': self.generate_json ,
00445                    'csv-ci': self.generate_csv ,
00446                    'table': self.generate_table }[export_format]
00447         to_call(subtotal, misc_flash_mem, file_desc)
00448 
00449         if file_desc is not sys.stdout:
00450             file_desc.close()
00451 
00452     def generate_json (self, _, dummy, file_desc):
00453         """Generate a json file from a memory map
00454 
00455         Positional arguments:
00456         subtotal - total sizes for each module
00457         misc_flash_mem - size of misc flash sections
00458         file_desc - the file to write out the final report to
00459         """
00460         file_desc.write(json.dumps(self.mem_summary , indent=4))
00461         file_desc.write('\n')
00462 
00463     def generate_csv (self, subtotal, misc_flash_mem, file_desc):
00464         """Generate a CSV file from a memoy map
00465 
00466         Positional arguments:
00467         subtotal - total sizes for each module
00468         misc_flash_mem - size of misc flash sections
00469         file_desc - the file to write out the final report to
00470         """
00471         csv_writer = csv.writer(file_desc, delimiter=',',
00472                                 quoting=csv.QUOTE_NONE)
00473 
00474         csv_module_section = []
00475         csv_sizes = []
00476         for i in sorted(self.modules ):
00477             for k in self.print_sections :
00478                 csv_module_section += [i+k]
00479                 csv_sizes += [self.modules [i][k]]
00480 
00481         csv_module_section += ['static_ram']
00482         csv_sizes += [subtotal['.data']+subtotal['.bss']]
00483 
00484         csv_module_section += ['heap']
00485         if subtotal['.heap'] == 0:
00486             csv_sizes += ['unknown']
00487         else:
00488             csv_sizes += [subtotal['.heap']]
00489 
00490         csv_module_section += ['stack']
00491         if subtotal['.stack'] == 0:
00492             csv_sizes += ['unknown']
00493         else:
00494             csv_sizes += [subtotal['.stack']]
00495 
00496         csv_module_section += ['total_ram']
00497         csv_sizes += [subtotal['.data'] + subtotal['.bss'] +
00498                       subtotal['.heap'] + subtotal['.stack']]
00499 
00500         csv_module_section += ['total_flash']
00501         csv_sizes += [subtotal['.text']+subtotal['.data']+misc_flash_mem]
00502 
00503         csv_writer.writerow(csv_module_section)
00504         csv_writer.writerow(csv_sizes)
00505 
00506     def generate_table (self, subtotal, misc_flash_mem, file_desc):
00507         """Generate a table from a memoy map
00508 
00509         Positional arguments:
00510         subtotal - total sizes for each module
00511         misc_flash_mem - size of misc flash sections
00512         file_desc - the file to write out the final report to
00513         """
00514         # Create table
00515         columns = ['Module']
00516         columns.extend(self.print_sections )
00517 
00518         table = PrettyTable(columns)
00519         table.align["Module"] = "l"
00520         for col in self.print_sections :
00521             table.align[col] = 'r'
00522 
00523         for i in list(self.print_sections ):
00524             table.align[i] = 'r'
00525 
00526         for i in sorted(self.modules ):
00527             row = [i]
00528 
00529             for k in self.sections :
00530                 subtotal[k] += self.modules [i][k]
00531 
00532             for k in self.print_sections :
00533                 row.append(self.modules [i][k])
00534 
00535             table.add_row(row)
00536 
00537         subtotal_row = ['Subtotals']
00538         for k in self.print_sections :
00539             subtotal_row.append(subtotal[k])
00540 
00541         table.add_row(subtotal_row)
00542 
00543         file_desc.write(table.get_string())
00544         file_desc.write('\n')
00545 
00546         if subtotal['.heap'] == 0:
00547             file_desc.write("Allocated Heap: unknown\n")
00548         else:
00549             file_desc.write("Allocated Heap: %s bytes\n" %
00550                             str(subtotal['.heap']))
00551 
00552         if subtotal['.stack'] == 0:
00553             file_desc.write("Allocated Stack: unknown\n")
00554         else:
00555             file_desc.write("Allocated Stack: %s bytes\n" %
00556                             str(subtotal['.stack']))
00557 
00558         file_desc.write("Total Static RAM memory (data + bss): %s bytes\n" %
00559                         (str(subtotal['.data'] + subtotal['.bss'])))
00560         file_desc.write(
00561             "Total RAM memory (data + bss + heap + stack): %s bytes\n"
00562             % (str(subtotal['.data'] + subtotal['.bss'] + subtotal['.heap'] +
00563                    subtotal['.stack'])))
00564         file_desc.write("Total Flash memory (text + data + misc): %s bytes\n" %
00565                         (str(subtotal['.text'] + subtotal['.data'] +
00566                              misc_flash_mem)))
00567 
00568     toolchains = ["ARM", "ARM_STD", "ARM_MICRO", "GCC_ARM", "IAR"]
00569 
00570     def parse (self, mapfile, toolchain):
00571         """ Parse and decode map file depending on the toolchain
00572 
00573         Positional arguments:
00574         mapfile - the file name of the memory map file
00575         toolchain - the toolchain used to create the file
00576         """
00577 
00578         result = True
00579         try:
00580             with open(mapfile, 'r') as file_input:
00581                 if toolchain == "ARM" or toolchain == "ARM_STD" or\
00582                    toolchain == "ARM_MICRO":
00583                     self.search_objects (os.path.abspath(mapfile), "ARM")
00584                     self.parse_map_file_armcc (file_input)
00585                 elif toolchain == "GCC_ARM":
00586                     self.parse_map_file_gcc (file_input)
00587                 elif toolchain == "IAR":
00588                     self.search_objects (os.path.abspath(mapfile), toolchain)
00589                     self.parse_map_file_iar (file_input)
00590                 else:
00591                     result = False
00592         except IOError as error:
00593             print "I/O error({0}): {1}".format(error.errno, error.strerror)
00594             result = False
00595         return result
00596 
00597 def main():
00598     """Entry Point"""
00599 
00600     version = '0.3.11'
00601 
00602     # Parser handling
00603     parser = argparse.ArgumentParser(
00604         description="Memory Map File Analyser for ARM mbed\nversion %s" %
00605         version)
00606 
00607     parser.add_argument(
00608         'file', type=argparse_filestring_type, help='memory map file')
00609 
00610     parser.add_argument(
00611         '-t', '--toolchain', dest='toolchain',
00612         help='select a toolchain used to build the memory map file (%s)' %
00613         ", ".join(MemapParser.toolchains),
00614         required=True,
00615         type=argparse_uppercase_type(MemapParser.toolchains, "toolchain"))
00616 
00617     parser.add_argument(
00618         '-o', '--output', help='output file name', required=False)
00619 
00620     parser.add_argument(
00621         '-e', '--export', dest='export', required=False, default='table',
00622         type=argparse_lowercase_hyphen_type(MemapParser.export_formats,
00623                                             'export format'),
00624         help="export format (examples: %s: default)" %
00625         ", ".join(MemapParser.export_formats))
00626 
00627     parser.add_argument('-v', '--version', action='version', version=version)
00628 
00629     # Parse/run command
00630     if len(sys.argv) <= 1:
00631         parser.print_help()
00632         sys.exit(1)
00633 
00634 
00635     args = parser.parse_args()
00636 
00637     # Create memap object
00638     memap = MemapParser()
00639 
00640     # Parse and decode a map file
00641     if args.file and args.toolchain:
00642         if memap.parse(args.file, args.toolchain) is False:
00643             sys.exit(0)
00644 
00645     # Write output in file
00646     if args.output != None:
00647         memap.generate_output(args.export, args.output)
00648     else: # Write output in screen
00649         memap.generate_output(args.export)
00650 
00651     sys.exit(0)
00652 
00653 if __name__ == "__main__":
00654     main()