memap.py

00001 #!/usr/bin/env python
00002 
00003 """Memory Map File Analyser for ARM mbed"""
00004 
00005 import sys
00006 import os
00007 import re
00008 import csv
00009 import json
00010 import argparse
00011 from prettytable import PrettyTable
00012 
00013 from utils import argparse_filestring_type, \
00014     argparse_lowercase_hyphen_type, argparse_uppercase_type
00015 
00016 DEBUG = False
00017 
00018 RE_ARMCC = re.compile(
00019     r'^\s+0x(\w{8})\s+0x(\w{8})\s+(\w+)\s+(\w+)\s+(\d+)\s+[*]?.+\s+(.+)$')
00020 RE_IAR = re.compile(
00021     r'^\s+(.+)\s+(zero|const|ro code|inited|uninit)\s'
00022     r'+0x(\w{8})\s+0x(\w+)\s+(.+)\s.+$')
00023 
00024 class MemapParser (object):
00025     """An object that represents parsed results, parses the memory map files,
00026     and writes out different file types of memory results
00027     """
00028 
00029     print_sections = ('.text', '.data', '.bss')
00030 
00031     misc_flash_sections = ('.interrupts', '.flash_config')
00032 
00033     other_sections = ('.interrupts_ram', '.init', '.ARM.extab',
00034                       '.ARM.exidx', '.ARM.attributes', '.eh_frame',
00035                       '.init_array', '.fini_array', '.jcr', '.stab',
00036                       '.stabstr', '.ARM.exidx', '.ARM')
00037 
00038     # sections to print info (generic for all toolchains)
00039     sections = ('.text', '.data', '.bss', '.heap', '.stack')
00040 
00041     def __init__ (self, detailed_misc=False):
00042         """ General initialization
00043         """
00044         # 
00045         self.detailed_misc  = detailed_misc
00046         
00047         # list of all modules and their sections
00048         self.modules  = dict()
00049 
00050         # sections must be defined in this order to take irrelevant out
00051         self.all_sections  = self.sections  + self.other_sections  + \
00052                             self.misc_flash_sections  + ('unknown', 'OUTPUT')
00053 
00054         # list of all object files and mappting to module names
00055         self.object_to_module  = dict()
00056 
00057         # Memory report (sections + summary)
00058         self.mem_report  = []
00059 
00060         # Just the memory summary section
00061         self.mem_summary  = dict()
00062 
00063         self.subtotal  = dict()
00064 
00065     def module_add (self, module_name, size, section):
00066         """ Adds a module / section to the list
00067 
00068         Positional arguments:
00069         module_name - name of the module to add
00070         size - the size of the module being added
00071         section - the section the module contributes to
00072         """
00073 
00074         if module_name in self.modules :
00075             self.modules [module_name][section] += size
00076         else:
00077             temp_dic = dict()
00078             for section_idx in self.all_sections :
00079                 temp_dic[section_idx] = 0
00080             temp_dic[section] = size
00081             self.modules [module_name] = temp_dic
00082 
00083     def check_new_section_gcc (self, line):
00084         """ Check whether a new section in a map file has been detected (only
00085         applies to gcc)
00086 
00087         Positional arguments:
00088         line - the line to check for a new section
00089         """
00090 
00091         for i in self.all_sections :
00092             if line.startswith(i):
00093                 # should name of the section (assuming it's a known one)
00094                 return i
00095 
00096         if line.startswith('.'):
00097             return 'unknown'     # all others are classified are unknown
00098         else:
00099             return False         # everything else, means no change in section
00100 
00101     
00102     def path_object_to_module_name (self, txt):
00103         """ Parse a path to object file to extract it's module and object data
00104 
00105         Positional arguments:
00106         txt - the path to parse the object and module name from
00107         """
00108 
00109         txt = txt.replace('\\', '/')
00110         rex_mbed_os_name = r'^.+mbed-os\/(.+)\/(.+\.o)$'
00111         test_rex_mbed_os_name = re.match(rex_mbed_os_name, txt)
00112 
00113         if test_rex_mbed_os_name:
00114 
00115             object_name = test_rex_mbed_os_name.group(2)
00116             data = test_rex_mbed_os_name.group(1).split('/')
00117             ndata = len(data)
00118 
00119             if ndata == 1:
00120                 module_name = data[0]
00121             else:
00122                 module_name = data[0] + '/' + data[1]
00123 
00124             if self.detailed_misc :
00125                 return [module_name + '/' + object_name, object_name]
00126             else:
00127                 return [module_name, object_name]
00128             
00129         elif self.detailed_misc :           
00130             rex_obj_name = r'^.+\/(.+\.o\)*)$'
00131             test_rex_obj_name = re.match(rex_obj_name, txt)
00132             if test_rex_obj_name:
00133                 object_name = test_rex_obj_name.group(1)
00134                 return ['Misc/' + object_name, ""]        
00135                 
00136             return ['Misc', ""]
00137         else: 
00138             return ['Misc', ""]
00139 
00140     def parse_section_gcc (self, line):
00141         """ Parse data from a section of gcc map file
00142 
00143         examples:
00144                         0x00004308       0x7c ./BUILD/K64F/GCC_ARM/mbed-os/hal/targets/hal/TARGET_Freescale/TARGET_KPSDK_MCUS/spi_api.o
00145          .text          0x00000608      0x198 ./BUILD/K64F/GCC_ARM/mbed-os/core/mbed-rtos/rtx/TARGET_CORTEX_M/TARGET_RTOS_M4_M7/TOOLCHAIN_GCC/HAL_CM4.o
00146 
00147         Positional arguments:
00148         line - the line to parse a section from
00149         """
00150         rex_address_len_name = re.compile(
00151             r'^\s+.*0x(\w{8,16})\s+0x(\w+)\s(.+)$')
00152 
00153         test_address_len_name = re.match(rex_address_len_name, line)
00154 
00155         if test_address_len_name:
00156 
00157             if int(test_address_len_name.group(2), 16) == 0: # size == 0
00158                 return ["", 0] # no valid entry
00159             else:
00160                 m_name, _ = self.path_object_to_module_name (
00161                     test_address_len_name.group(3))
00162                 m_size = int(test_address_len_name.group(2), 16)
00163                 return [m_name, m_size]
00164 
00165         else: # special corner case for *fill* sections
00166             #  example
00167             # *fill*         0x0000abe4        0x4
00168             rex_address_len = r'^\s+\*fill\*\s+0x(\w{8,16})\s+0x(\w+).*$'
00169             test_address_len = re.match(rex_address_len, line)
00170 
00171             if test_address_len:
00172                 if int(test_address_len.group(2), 16) == 0: # size == 0
00173                     return ["", 0] # no valid entry
00174                 else:
00175                     m_name = 'Fill'
00176                     m_size = int(test_address_len.group(2), 16)
00177                     return [m_name, m_size]
00178             else:
00179                 return ["", 0] # no valid entry
00180 
00181     def parse_map_file_gcc (self, file_desc):
00182         """ Main logic to decode gcc map files
00183 
00184         Positional arguments:
00185         file_desc - a stream object to parse as a gcc map file
00186         """
00187 
00188         current_section = 'unknown'
00189 
00190         with file_desc as infile:
00191 
00192             # Search area to parse
00193             for line in infile:
00194                 if line.startswith('Linker script and memory map'):
00195                     current_section = "unknown"
00196                     break
00197 
00198             # Start decoding the map file
00199             for line in infile:
00200 
00201                 change_section = self.check_new_section_gcc (line)
00202 
00203                 if change_section == "OUTPUT": # finish parsing file: exit
00204                     break
00205                 elif change_section != False:
00206                     current_section = change_section
00207 
00208                 [module_name, module_size] = self.parse_section_gcc (line)
00209 
00210                 if module_size == 0 or module_name == "":
00211                     pass
00212                 else:
00213                     self.module_add (module_name, module_size, current_section)
00214 
00215                 if DEBUG:
00216                     print "Line: %s" % line,
00217                     print "Module: %s\tSection: %s\tSize: %s" % \
00218                         (module_name, current_section, module_size)
00219                     raw_input("----------")
00220 
00221     def parse_section_armcc (self, line):
00222         """ Parse data from an armcc map file
00223 
00224         Examples of armcc map file:
00225             Base_Addr    Size         Type   Attr      Idx    E Section Name        Object
00226             0x00000000   0x00000400   Data   RO        11222    RESET               startup_MK64F12.o
00227             0x00000410   0x00000008   Code   RO        49364  * !!!main             c_w.l(__main.o)
00228 
00229         Positional arguments:
00230         line - the line to parse the section data from
00231         """
00232 
00233         test_rex_armcc = re.match(RE_ARMCC, line)
00234 
00235         if test_rex_armcc:
00236 
00237             size = int(test_rex_armcc.group(2), 16)
00238 
00239             if test_rex_armcc.group(4) == 'RO':
00240                 section = '.text'
00241             else:
00242                 if test_rex_armcc.group(3) == 'Data':
00243                     section = '.data'
00244                 elif test_rex_armcc.group(3) == 'Zero':
00245                     section = '.bss'
00246                 else:
00247                     print "BUG armcc map parser"
00248                     raw_input()
00249 
00250             # lookup object in dictionary and return module name
00251             object_name = test_rex_armcc.group(6)
00252             if object_name in self.object_to_module :
00253                 module_name = self.object_to_module [object_name]
00254             else:
00255                 module_name = 'Misc'
00256 
00257             return [module_name, size, section]
00258 
00259         else:
00260             return ["", 0, ""] # no valid entry
00261 
00262     def parse_section_iar (self, line):
00263         """ Parse data from an IAR map file
00264 
00265         Examples of IAR map file:
00266          Section             Kind        Address     Size  Object
00267          .intvec             ro code  0x00000000    0x198  startup_MK64F12.o [15]
00268          .rodata             const    0x00000198      0x0  zero_init3.o [133]
00269          .iar.init_table     const    0x00008384     0x2c  - Linker created -
00270          Initializer bytes   const    0x00000198     0xb2  <for P3 s0>
00271          .data               inited   0x20000000     0xd4  driverAtmelRFInterface.o [70]
00272          .bss                zero     0x20000598    0x318  RTX_Conf_CM.o [4]
00273          .iar.dynexit        uninit   0x20001448    0x204  <Block tail>
00274            HEAP              uninit   0x20001650  0x10000  <Block tail>
00275 
00276         Positional_arguments:
00277         line - the line to parse section data from
00278         """
00279 
00280         test_rex_iar = re.match(RE_IAR, line)
00281 
00282         if test_rex_iar:
00283 
00284             size = int(test_rex_iar.group(4), 16)
00285 
00286             if test_rex_iar.group(2) == 'const' or \
00287                test_rex_iar.group(2) == 'ro code':
00288                 section = '.text'
00289             elif test_rex_iar.group(2) == 'zero' or \
00290             test_rex_iar.group(2) == 'uninit':
00291                 if test_rex_iar.group(1)[0:4] == 'HEAP':
00292                     section = '.heap'
00293                 elif test_rex_iar.group(1)[0:6] == 'CSTACK':
00294                     section = '.stack'
00295                 else:
00296                     section = '.bss' #  default section
00297 
00298             elif test_rex_iar.group(2) == 'inited':
00299                 section = '.data'
00300             else:
00301                 print "BUG IAR map parser"
00302                 raw_input()
00303 
00304             # lookup object in dictionary and return module name
00305             object_name = test_rex_iar.group(5)
00306             if object_name in self.object_to_module :
00307                 module_name = self.object_to_module [object_name]
00308             else:
00309                 module_name = 'Misc'
00310 
00311             return [module_name, size, section]
00312 
00313         else:
00314             return ["", 0, ""] # no valid entry
00315 
00316     def parse_map_file_armcc (self, file_desc):
00317         """ Main logic to decode armc5 map files
00318 
00319         Positional arguments:
00320         file_desc - a file like object to parse as an armc5 map file
00321         """
00322 
00323         with file_desc as infile:
00324 
00325             # Search area to parse
00326             for line in infile:
00327                 if line.startswith('    Base Addr    Size'):
00328                     break
00329 
00330             # Start decoding the map file
00331             for line in infile:
00332 
00333                 [name, size, section] = self.parse_section_armcc (line)
00334 
00335                 if size == 0 or name == "" or section == "":
00336                     pass
00337                 else:
00338                     self.module_add (name, size, section)
00339 
00340     def parse_map_file_iar (self, file_desc):
00341         """ Main logic to decode IAR map files
00342 
00343         Positional arguments:
00344         file_desc - a file like object to parse as an IAR map file
00345         """
00346 
00347         with file_desc as infile:
00348 
00349             # Search area to parse
00350             for line in infile:
00351                 if line.startswith('  Section  '):
00352                     break
00353 
00354             # Start decoding the map file
00355             for line in infile:
00356 
00357                 [name, size, section] = self.parse_section_iar (line)
00358 
00359                 if size == 0 or name == "" or section == "":
00360                     pass
00361                 else:
00362                     self.module_add (name, size, section)
00363 
00364     def search_objects (self, path):
00365         """ Searches for object files and creates mapping: object --> module
00366 
00367         Positional arguments:
00368         path - the path to an object file
00369         """
00370 
00371         path = path.replace('\\', '/')
00372 
00373         # check location of map file
00374         rex = r'^(.+)' + r'\/(.+\.map)$'
00375         test_rex = re.match(rex, path)
00376 
00377         if test_rex:
00378             search_path = test_rex.group(1) + '/mbed-os/'
00379         else:
00380             print "Warning: this doesn't look like an mbed project"
00381             return
00382 
00383         for root, _, obj_files in os.walk(search_path):
00384             for obj_file in obj_files:
00385                 if obj_file.endswith(".o"):
00386                     module_name, object_name = self.path_object_to_module_name (
00387                         os.path.join(root, obj_file))
00388 
00389                     if object_name in self.object_to_module :
00390                         if DEBUG:
00391                             print "WARNING: multiple usages of object file: %s"\
00392                                 % object_name
00393                             print "    Current: %s" % \
00394                                 self.object_to_module [object_name]
00395                             print "    New:     %s" % module_name
00396                             print " "
00397                     else:
00398                         self.object_to_module .update({object_name:module_name})
00399 
00400     export_formats = ["json", "csv-ci", "table"]
00401 
00402     def generate_output (self, export_format, file_output=None):
00403         """ Generates summary of memory map data
00404 
00405         Positional arguments:
00406         export_format - the format to dump
00407 
00408         Keyword arguments:
00409         file_desc - descriptor (either stdout or file)
00410 
00411         Returns: generated string for the 'table' format, otherwise None
00412         """
00413 
00414         try:
00415             if file_output:
00416                 file_desc = open(file_output, 'wb')
00417             else:
00418                 file_desc = sys.stdout
00419         except IOError as error:
00420             print "I/O error({0}): {1}".format(error.errno, error.strerror)
00421             return False
00422 
00423         to_call = {'json': self.generate_json ,
00424                    'csv-ci': self.generate_csv ,
00425                    'table': self.generate_table }[export_format]
00426         output = to_call(file_desc)
00427 
00428         if file_desc is not sys.stdout:
00429             file_desc.close()
00430 
00431         return output
00432 
00433     def generate_json (self, file_desc):
00434         """Generate a json file from a memory map
00435 
00436         Positional arguments:
00437         file_desc - the file to write out the final report to
00438         """
00439         file_desc.write(json.dumps(self.mem_report , indent=4))
00440         file_desc.write('\n')
00441 
00442         return None
00443 
00444     def generate_csv (self, file_desc):
00445         """Generate a CSV file from a memoy map
00446 
00447         Positional arguments:
00448         file_desc - the file to write out the final report to
00449         """
00450         csv_writer = csv.writer(file_desc, delimiter=',',
00451                                 quoting=csv.QUOTE_MINIMAL)
00452 
00453         csv_module_section = []
00454         csv_sizes = []
00455         for i in sorted(self.modules ):
00456             for k in self.print_sections :
00457                 csv_module_section += [i+k]
00458                 csv_sizes += [self.modules [i][k]]
00459 
00460         csv_module_section += ['static_ram']
00461         csv_sizes += [self.mem_summary ['static_ram']]
00462 
00463         csv_module_section += ['heap']
00464         if self.mem_summary ['heap'] == 0:
00465             csv_sizes += ['unknown']
00466         else:
00467             csv_sizes += [self.mem_summary ['heap']]
00468 
00469         csv_module_section += ['stack']
00470         if self.mem_summary ['stack'] == 0:
00471             csv_sizes += ['unknown']
00472         else:
00473             csv_sizes += [self.mem_summary ['stack']]
00474 
00475         csv_module_section += ['total_ram']
00476         csv_sizes += [self.mem_summary ['total_ram']]
00477 
00478         csv_module_section += ['total_flash']
00479         csv_sizes += [self.mem_summary ['total_flash']]
00480 
00481         csv_writer.writerow(csv_module_section)
00482         csv_writer.writerow(csv_sizes)
00483 
00484         return None
00485 
00486     def generate_table (self, file_desc):
00487         """Generate a table from a memoy map
00488 
00489         Positional arguments:
00490         file_desc - the file to write out the final report to
00491 
00492         Returns: string of the generated table
00493         """
00494         # Create table
00495         columns = ['Module']
00496         columns.extend(self.print_sections )
00497 
00498         table = PrettyTable(columns)
00499         table.align["Module"] = "l"
00500         for col in self.print_sections :
00501             table.align[col] = 'r'
00502 
00503         for i in list(self.print_sections ):
00504             table.align[i] = 'r'
00505 
00506         for i in sorted(self.modules ):
00507             row = [i]
00508 
00509             for k in self.print_sections :
00510                 row.append(self.modules [i][k])
00511 
00512             table.add_row(row)
00513 
00514         subtotal_row = ['Subtotals']
00515         for k in self.print_sections :
00516             subtotal_row.append(self.subtotal [k])
00517 
00518         table.add_row(subtotal_row)
00519 
00520         output = table.get_string()
00521         output += '\n'
00522 
00523         if self.mem_summary ['heap'] == 0:
00524             output += "Allocated Heap: unknown\n"
00525         else:
00526             output += "Allocated Heap: %s bytes\n" % \
00527                         str(self.mem_summary ['heap'])
00528 
00529         if self.mem_summary ['stack'] == 0:
00530             output += "Allocated Stack: unknown\n"
00531         else:
00532             output += "Allocated Stack: %s bytes\n" % \
00533                         str(self.mem_summary ['stack'])
00534 
00535         output += "Total Static RAM memory (data + bss): %s bytes\n" % \
00536                         str(self.mem_summary ['static_ram'])
00537         output += "Total RAM memory (data + bss + heap + stack): %s bytes\n" % \
00538                         str(self.mem_summary ['total_ram'])
00539         output += "Total Flash memory (text + data + misc): %s bytes\n" % \
00540                         str(self.mem_summary ['total_flash'])
00541 
00542         return output
00543 
00544     toolchains = ["ARM", "ARM_STD", "ARM_MICRO", "GCC_ARM", "GCC_CR", "IAR"]
00545 
00546     def compute_report(self):
00547         for k in self.sections :
00548             self.subtotal [k] = 0
00549 
00550         for i in sorted(self.modules ):
00551             for k in self.sections :
00552                 self.subtotal [k] += self.modules [i][k]
00553 
00554         # Calculate misc flash sections
00555         self.misc_flash_mem  = 0
00556         for i in self.modules :
00557             for k in self.misc_flash_sections :
00558                 if self.modules [i][k]:
00559                     self.misc_flash_mem  += self.modules [i][k]
00560 
00561         self.mem_summary  = {
00562             'static_ram': (self.subtotal ['.data'] + self.subtotal ['.bss']),
00563             'heap': (self.subtotal ['.heap']),
00564             'stack': (self.subtotal ['.stack']),
00565             'total_ram': (self.subtotal ['.data'] + self.subtotal ['.bss'] +
00566                           self.subtotal ['.heap']+self.subtotal ['.stack']),
00567             'total_flash': (self.subtotal ['.text'] + self.subtotal ['.data'] +
00568                             self.misc_flash_mem ),
00569         }
00570 
00571         self.mem_report  = []
00572         for i in sorted(self.modules ):
00573             self.mem_report .append({
00574                 "module":i,
00575                 "size":{
00576                     k:self.modules [i][k] for k in self.print_sections 
00577                 }
00578             })
00579 
00580         self.mem_report .append({
00581             'summary': self.mem_summary 
00582         })
00583 
00584     def parse (self, mapfile, toolchain):
00585         """ Parse and decode map file depending on the toolchain
00586 
00587         Positional arguments:
00588         mapfile - the file name of the memory map file
00589         toolchain - the toolchain used to create the file
00590         """
00591 
00592         result = True
00593         try:
00594             with open(mapfile, 'r') as file_input:
00595                 if toolchain == "ARM" or toolchain == "ARM_STD" or\
00596                    toolchain == "ARM_MICRO":
00597                     self.search_objects (os.path.abspath(mapfile))
00598                     self.parse_map_file_armcc (file_input)
00599                 elif toolchain == "GCC_ARM" or toolchain == "GCC_CR":
00600                     self.parse_map_file_gcc (file_input)
00601                 elif toolchain == "IAR":
00602                     self.search_objects (os.path.abspath(mapfile))
00603                     self.parse_map_file_iar (file_input)
00604                 else:
00605                     result = False
00606             
00607             self.compute_report ()
00608         
00609         except IOError as error:
00610             print "I/O error({0}): {1}".format(error.errno, error.strerror)
00611             result = False
00612         return result
00613 
00614 def main():
00615     """Entry Point"""
00616 
00617     version = '0.3.12'
00618 
00619     # Parser handling
00620     parser = argparse.ArgumentParser(
00621         description="Memory Map File Analyser for ARM mbed\nversion %s" %
00622         version)
00623 
00624     parser.add_argument(
00625         'file', type=argparse_filestring_type, help='memory map file')
00626 
00627     parser.add_argument(
00628         '-t', '--toolchain', dest='toolchain',
00629         help='select a toolchain used to build the memory map file (%s)' %
00630         ", ".join(MemapParser.toolchains),
00631         required=True,
00632         type=argparse_uppercase_type(MemapParser.toolchains, "toolchain"))
00633 
00634     parser.add_argument(
00635         '-o', '--output', help='output file name', required=False)
00636 
00637     parser.add_argument(
00638         '-e', '--export', dest='export', required=False, default='table',
00639         type=argparse_lowercase_hyphen_type(MemapParser.export_formats,
00640                                             'export format'),
00641         help="export format (examples: %s: default)" %
00642         ", ".join(MemapParser.export_formats))
00643 
00644     parser.add_argument('-v', '--version', action='version', version=version)
00645     
00646     parser.add_argument('-d', '--detailed', action='store_true', help='Displays the elements in "Misc" in a detailed fashion', required=False)
00647 
00648     # Parse/run command
00649     if len(sys.argv) <= 1:
00650         parser.print_help()
00651         sys.exit(1)
00652 
00653 
00654     args = parser.parse_args()
00655 
00656     # Create memap object
00657     memap = MemapParser(detailed_misc=args.detailed)
00658 
00659     # Parse and decode a map file
00660     if args.file and args.toolchain:
00661         if memap.parse(args.file, args.toolchain) is False:
00662             sys.exit(0)
00663 
00664     returned_string = None
00665     # Write output in file
00666     if args.output != None:
00667         returned_string = memap.generate_output(args.export, args.output)
00668     else: # Write output in screen
00669         returned_string = memap.generate_output(args.export)
00670 
00671     if args.export == 'table' and returned_string:
00672         print returned_string
00673 
00674     sys.exit(0)
00675 
00676 if __name__ == "__main__":
00677     main()