Diff: memap.py

Revision:

9:2d27d77ada5c

Parent:

8:a8ac6ed29081

Child:

13:ab47a20b66f0

diff -r a8ac6ed29081 -r 2d27d77ada5c memap.py
--- a/memap.py	Tue Jun 07 11:35:02 2016 +0100
+++ b/memap.py	Tue Jun 14 11:07:30 2016 +0100
@@ -1,21 +1,20 @@
-#! /usr/bin/env python
+#!/usr/bin/env python
+# pylint: disable=too-many-arguments, too-many-locals, too-many-branches, too-many-lines, line-too-long, too-many-nested-blocks, too-many-public-methods, too-many-instance-attributes
+# pylint: disable=invalid-name, missing-docstring
 
 # Memory Map File Analyser for ARM mbed OS
 
-import argparse
 import sys
-import string
 import os
 import re
+import csv
 import json
-import time
-import string
-import StringIO 
+import argparse
 from prettytable import PrettyTable
 
 debug = False
 
-class MemmapParser(object):
+class MemapParser(object):
 
     def __init__(self):
         """
@@ -27,92 +26,22 @@
 
         self.misc_flash_sections = ('.interrupts', '.flash_config')
 
-        self.other_sections = ('.interrupts_ram', '.init', '.ARM.extab', '.ARM.exidx', '.ARM.attributes', \
-                 '.eh_frame', '.init_array', '.fini_array', '.jcr', '.stab', '.stabstr', \
-                 '.ARM.exidx','.ARM' )
+        self.other_sections = ('.interrupts_ram', '.init', '.ARM.extab', \
+                               '.ARM.exidx', '.ARM.attributes', '.eh_frame', \
+                               '.init_array', '.fini_array', '.jcr', '.stab', \
+                               '.stabstr', '.ARM.exidx', '.ARM')
 
         # sections to print info (generic for all toolchains)
-        self.sections = ('.text', '.data', '.bss', '.heap', '.stack',) 
+        self.sections = ('.text', '.data', '.bss', '.heap', '.stack')
 
-        # need to have sections merged in this order ()
+        # sections must be defined in this order to take irrelevant out
         self.all_sections = self.sections + self.other_sections + \
                             self.misc_flash_sections + ('unknown', 'OUTPUT')
 
         self.print_sections = ('.text', '.data', '.bss')
 
         # list of all object files and mappting to module names
-        self.object_to_module = dict() 
-
-
-    def generate_output(self, file, json_mode):
-        """
-        Generates summary of memory map data
-
-        Parameters
-            file: descriptor (either stdout or file)
-            json_mode: generates output in json formal (True/False)
-        """
-
-        buf = StringIO.StringIO()
-
-        # Calculate misc flash sections
-        misc_flash_mem = 0
-        for i in self.modules:
-            for k in self.misc_flash_sections:
-                if self.modules[i][k]:
-                    misc_flash_mem += self.modules[i][k]
-
-        # Create table
-        colums = ['Module']
-        for i in list(self.print_sections):
-            colums.append(i) 
-
-        table = PrettyTable(colums)
-        table.align["Module"] = "l"
-
-        subtotal = dict()
-        for k in self.sections:
-            subtotal[k] = 0
-
-        json_obj = []
-        for i in sorted(self.modules):
-            
-            row = []
-            row.append(i)
-
-            for k in self.sections: 
-                subtotal[k] += self.modules[i][k]
-
-            for k in self.print_sections:
-                row.append(self.modules[i][k])
-            
-            json_obj.append({ "module":i, "size":{k:self.modules[i][k] for k in self.print_sections}})
-            table.add_row(row)
-
-        subtotal_row = ['Subtotals']
-        for k in self.print_sections:
-            subtotal_row.append(subtotal[k]) 
-
-        table.add_row(subtotal_row)
-
-        if json_mode:
-            json_obj.append({ "summary":{'static_ram':(subtotal['.data']+subtotal['.bss']),
-                                               'heap':(subtotal['.heap']),
-                                              'stack':(subtotal['.stack']),
-                                          'total_ram':(subtotal['.data']+subtotal['.bss']+subtotal['.heap']+subtotal['.stack']),
-                                        'total_flash':(subtotal['.text']+subtotal['.data']+misc_flash_mem),}})
-
-            file.write(json.dumps(json_obj, indent=4))
-            file.write('\n')
-        else:
-            file.write(table.get_string())
-            file.write('\n')
-            file.write("Static RAM memory (data + bss): %s\n" % (str(subtotal['.data']+subtotal['.bss'])))
-            file.write("Heap: %s\n" % str(subtotal['.heap']))
-            file.write("Stack: %s\n" % str(subtotal['.stack']))
-            file.write("Total RAM memory (data + bss + heap + stack): %s\n" % (str(subtotal['.data']+subtotal['.bss']+subtotal['.heap']+subtotal['.stack'])))
-            file.write("Total Flash memory (text + data + misc): %s\n" % (str(subtotal['.text']+subtotal['.data']+misc_flash_mem)))
-        return
+        self.object_to_module = dict()
 
     def module_add(self, module_name, size, section):
         """
@@ -120,42 +49,15 @@
         """
 
         if module_name in self.modules:
-                self.modules[module_name][section] += size
+            self.modules[module_name][section] += size
         else:
             temp_dic = dict()
-            for x in self.all_sections:
-                temp_dic[x] = 0
+            for section_idx in self.all_sections:
+                temp_dic[section_idx] = 0
             temp_dic[section] = size
             self.modules[module_name] = temp_dic
 
-    def find_start_gcc(self,line):
-        """
-        Checks location of gcc map file to start parsing map file
-        """
-        if line.startswith('Linker script and memory map'):
-            return True
-        else:
-            return False
-
-    def find_start_armcc(self,line):
-        """
-        Checks location of armcc map file to start parsing map file
-        """
-        if line.startswith('    Base Addr    Size'):
-            return True
-        else:
-            return False
-
-    def find_start_iar(self,line):
-        """
-        Checks location of armcc map file to start parsing map file
-        """
-        if line.startswith('  Section  '):
-            return True
-        else:
-            return False
-
-    def check_new_section_gcc(self,line):
+    def check_new_section_gcc(self, line):
         """
         Check whether a new section in a map file has been detected (only applies to gcc)
         """
@@ -169,15 +71,15 @@
         else:
             return False         # everything else, means no change in section
 
-    def path_object_to_module_name(self,txt):
+    def path_object_to_module_name(self, txt):
         """
-        Parses path to object file and extracts module / object data 
+        Parses path to object file and extracts module / object data
         """
 
-        txt = txt.replace('\\','/')
+        txt = txt.replace('\\', '/')
         rex_mbed_os_name = r'^.+mbed-os\/(.+)\/(.+\.o)$'
-        test_rex_mbed_os_name = re.match(rex_mbed_os_name,txt)
- 
+        test_rex_mbed_os_name = re.match(rex_mbed_os_name, txt)
+
         if test_rex_mbed_os_name:
 
             object_name = test_rex_mbed_os_name.group(2)
@@ -192,9 +94,9 @@
             return [module_name, object_name]
         else:
             return ['Misc', ""]
-               
+
 
-    def parse_section_gcc(self,line):
+    def parse_section_gcc(self, line):
         """
         Parse data from a section of gcc map file
         """
@@ -203,45 +105,45 @@
         #  .text          0x00000608      0x198 ./.build/K64F/GCC_ARM/mbed-os/core/mbed-rtos/rtx/TARGET_CORTEX_M/TARGET_RTOS_M4_M7/TOOLCHAIN_GCC/HAL_CM4.o
         rex_address_len_name = r'^\s+.*0x(\w{8,16})\s+0x(\w+)\s(.+)$'
 
-        test_address_len_name = re.match(rex_address_len_name,line)
+        test_address_len_name = re.match(rex_address_len_name, line)
 
         if test_address_len_name:
 
-            if int(test_address_len_name.group(2),16) == 0: # size == 0
-                return ["",0] # no valid entry
+            if int(test_address_len_name.group(2), 16) == 0: # size == 0
+                return ["", 0] # no valid entry
             else:
                 m_name, m_object = self.path_object_to_module_name(test_address_len_name.group(3))
-                m_size = int(test_address_len_name.group(2),16) 
-                return [m_name,m_size]
+                m_size = int(test_address_len_name.group(2), 16)
+                return [m_name, m_size]
 
-        else: # special cortner case for *fill* sections
+        else: # special corner case for *fill* sections
             #  example
             # *fill*         0x0000abe4        0x4
             rex_address_len = r'^\s+\*fill\*\s+0x(\w{8,16})\s+0x(\w+).*$'
-            test_address_len = re.match(rex_address_len,line)
+            test_address_len = re.match(rex_address_len, line)
 
             if test_address_len:
-                if int(test_address_len.group(2),16) == 0: # size == 0
-                    return ["",0] # no valid entry
+                if int(test_address_len.group(2), 16) == 0: # size == 0
+                    return ["", 0] # no valid entry
                 else:
-                    m_name = 'Misc'
-                    m_size = int(test_address_len.group(2),16)
-                    return [m_name,m_size]
+                    m_name = 'Fill'
+                    m_size = int(test_address_len.group(2), 16)
+                    return [m_name, m_size]
             else:
-                return ["",0] # no valid entry
+                return ["", 0] # no valid entry
 
-    def parse_map_file_gcc(self, file):
+    def parse_map_file_gcc(self, file_desc):
         """
         Main logic to decode gcc map files
         """
 
         current_section = 'unknown'
 
-        with file as infile:
+        with file_desc as infile:
 
             # Search area to parse
             for line in infile:
-                if self.find_start_gcc(line) == True:
+                if line.startswith('Linker script and memory map'):
                     current_section = "unknown"
                     break
 
@@ -252,7 +154,7 @@
 
                 if change_section == "OUTPUT": # finish parsing file: exit
                     break
-                elif change_section != False:  
+                elif change_section != False:
                     current_section = change_section
 
                 [module_name, module_size] = self.parse_section_gcc(line)
@@ -264,10 +166,10 @@
 
                 if debug:
                     print "Line: %s" % line,
-                    print "Module: %s\tSection: %s\tSize: %s" % (module_name,current_section,module_size)
+                    print "Module: %s\tSection: %s\tSize: %s" % (module_name, current_section, module_size)
                     raw_input("----------")
 
-    def parse_section_armcc(self,line):
+    def parse_section_armcc(self, line):
         """
         Parse data from an armcc map file
         """
@@ -277,11 +179,11 @@
         #     0x00000410   0x00000008   Code   RO        49364  * !!!main             c_w.l(__main.o)
         rex_armcc = r'^\s+0x(\w{8})\s+0x(\w{8})\s+(\w+)\s+(\w+)\s+(\d+)\s+[*]?.+\s+(.+)$'
 
-        test_rex_armcc = re.match(rex_armcc,line)
+        test_rex_armcc = re.match(rex_armcc, line)
 
         if test_rex_armcc:
 
-            size = int(test_rex_armcc.group(2),16)
+            size = int(test_rex_armcc.group(2), 16)
 
             if test_rex_armcc.group(4) == 'RO':
                 section = '.text'
@@ -301,12 +203,12 @@
             else:
                 module_name = 'Misc'
 
-            return [module_name,size,section]
+            return [module_name, size, section]
 
         else:
-            return ["",0,""] # no valid entry
+            return ["", 0, ""] # no valid entry
 
-    def parse_section_iar(self,line):
+    def parse_section_iar(self, line):
         """
         Parse data from an IAR map file
         """
@@ -322,16 +224,15 @@
         #    HEAP              uninit   0x20001650  0x10000  <Block tail>
         rex_iar = r'^\s+(.+)\s+(zero|const|ro code|inited|uninit)\s+0x(\w{8})\s+0x(\w+)\s+(.+)\s.+$'
 
-        test_rex_iar = re.match(rex_iar,line)
+        test_rex_iar = re.match(rex_iar, line)
 
         if test_rex_iar:
 
-            size = int(test_rex_iar.group(4),16)
+            size = int(test_rex_iar.group(4), 16)
 
             if test_rex_iar.group(2) == 'const' or test_rex_iar.group(2) == 'ro code':
                 section = '.text'
             elif test_rex_iar.group(2) == 'zero' or test_rex_iar.group(2) == 'uninit':
-               
                 if test_rex_iar.group(1)[0:4] == 'HEAP':
                     section = '.heap'
                 elif test_rex_iar.group(1)[0:6] == 'CSTACK':
@@ -352,21 +253,21 @@
             else:
                 module_name = 'Misc'
 
-            return [module_name,size,section]
+            return [module_name, size, section]
 
         else:
-            return ["",0,""] # no valid entry
+            return ["", 0, ""] # no valid entry
 
-    def parse_map_file_armcc(self, file):
+    def parse_map_file_armcc(self, file_desc):
         """
         Main logic to decode armcc map files
         """
 
-        with file as infile:
+        with file_desc as infile:
 
             # Search area to parse
             for line in infile:
-                if self.find_start_armcc(line) == True:
+                if line.startswith('    Base Addr    Size'):
                     break
 
             # Start decoding the map file
@@ -379,16 +280,16 @@
                 else:
                     self.module_add(name, size, section)
 
-    def parse_map_file_iar(self, file):
+    def parse_map_file_iar(self, file_desc):
         """
         Main logic to decode armcc map files
         """
 
-        with file as infile:
+        with file_desc as infile:
 
             # Search area to parse
             for line in infile:
-                if self.find_start_iar(line) == True:
+                if line.startswith('  Section  '):
                     break
 
             # Start decoding the map file
@@ -401,18 +302,18 @@
                 else:
                     self.module_add(name, size, section)
 
-    def search_objects(self,path,toolchain):
+    def search_objects(self, path, toolchain):
         """
         Check whether the specified map file matches with the toolchain.
         Searches for object files and creates mapping: object --> module
         """
 
-        path = path.replace('\\','/')
+        path = path.replace('\\', '/')
 
         # check location of map file
         rex = r'^(.+\/)' + re.escape(toolchain) + r'\/(.+\.map)$'
-        test_rex = re.match(rex,path)
-        
+        test_rex = re.match(rex, path)
+
         if test_rex:
             search_path = test_rex.group(1) + toolchain + '/mbed-os/'
         else:
@@ -421,37 +322,177 @@
             print "Warning: specified toolchain doesn't match with path to the memory map file."
             return
 
-        for root, dirs, files in os.walk(search_path):
-            for file in files:
-                if file.endswith(".o"):
-                    module_name, object_name = self.path_object_to_module_name(os.path.join(root, file))
+        for root, dir, obj_files in os.walk(search_path):
+            for obj_file in obj_files:
+                if obj_file.endswith(".o"):
+                    module_name, object_name = self.path_object_to_module_name(os.path.join(root, obj_file))
 
                     if object_name in self.object_to_module:
-                        print "WARNING: multiple usages of object file: %s" % object_name
-                        print "    Current: %s" % self.object_to_module[object_name]
-                        print "    New:     %s" % module_name
-                        print " "
-                        
+                        if debug:
+                            print "WARNING: multiple usages of object file: %s" % object_name
+                            print "    Current: %s" % self.object_to_module[object_name]
+                            print "    New:     %s" % module_name
+                            print " "
                     else:
                         self.object_to_module.update({object_name:module_name})
 
+    def generate_output(self, export_format, file_output=None):
+        """
+        Generates summary of memory map data
+
+        Parameters
+            json_mode: generates output in json formal (True/False)
+            file_desc: descriptor (either stdout or file)
+        """
+
+        try:
+            if file_output:
+                file_desc = open(file_output, 'wb')
+            else:
+                file_desc = sys.stdout
+        except IOError as error:
+            print "I/O error({0}): {1}".format(error.errno, error.strerror)
+            return False
+
+        # Calculate misc flash sections
+        misc_flash_mem = 0
+        for i in self.modules:
+            for k in self.misc_flash_sections:
+                if self.modules[i][k]:
+                    misc_flash_mem += self.modules[i][k]
+
+        # Create table
+        columns = ['Module']
+        for i in list(self.print_sections):
+            columns.append(i)
+
+        table = PrettyTable(columns)
+        table.align["Module"] = "l"
+
+        subtotal = dict()
+        for k in self.sections:
+            subtotal[k] = 0
+
+        json_obj = []
+        for i in sorted(self.modules):
+
+            row = []
+            row.append(i)
+
+            for k in self.sections:
+                subtotal[k] += self.modules[i][k]
+
+            for k in self.print_sections:
+                row.append(self.modules[i][k])
+
+            json_obj.append({"module":i, "size":{\
+                k:self.modules[i][k] for k in self.print_sections}})
+
+            table.add_row(row)
+
+        subtotal_row = ['Subtotals']
+        for k in self.print_sections:
+            subtotal_row.append(subtotal[k])
+
+        table.add_row(subtotal_row)
+
+        if export_format == 'json':
+            json_obj.append({\
+                  'summary':{\
+                  'static_ram':(subtotal['.data']+subtotal['.bss']),\
+                  'heap':(subtotal['.heap']),\
+                  'stack':(subtotal['.stack']),\
+                  'total_ram':(subtotal['.data']+subtotal['.bss']+subtotal['.heap']+subtotal['.stack']),\
+                  'total_flash':(subtotal['.text']+subtotal['.data']+misc_flash_mem),}})
+
+            file_desc.write(json.dumps(json_obj, indent=4))
+            file_desc.write('\n')
+
+        elif export_format == 'csv-ci': # CSV format for the CI system
+
+            csv_writer = csv.writer(file_desc, delimiter=',', quoting=csv.QUOTE_NONE)
+
+            csv_module_section = []
+            csv_sizes = []
+            for i in sorted(self.modules):
+                for k in self.print_sections:
+                    csv_module_section += [i+k]
+                    csv_sizes += [self.modules[i][k]]
+
+            csv_module_section += ['static_ram']
+            csv_sizes += [subtotal['.data']+subtotal['.bss']]
+
+            csv_module_section += ['heap']
+            csv_sizes += [subtotal['.heap']]
+
+            csv_module_section += ['stack']
+            csv_sizes += [subtotal['.stack']]
+
+            csv_module_section += ['total_ram']
+            csv_sizes += [subtotal['.data']+subtotal['.bss']+subtotal['.heap']+subtotal['.stack']]
+
+            csv_module_section += ['total_flash']
+            csv_sizes += [subtotal['.text']+subtotal['.data']+misc_flash_mem]
+
+            csv_writer.writerow(csv_module_section)
+            csv_writer.writerow(csv_sizes)
+
+        else: # default format is 'table'
+            file_desc.write(table.get_string())
+            file_desc.write('\n')
+            file_desc.write("Static RAM memory (data + bss): %s\n" % (str(subtotal['.data']+subtotal['.bss'])))
+            file_desc.write("Heap: %s\n" % str(subtotal['.heap']))
+            file_desc.write("Stack: %s\n" % str(subtotal['.stack']))
+            file_desc.write("Total RAM memory (data + bss + heap + stack): %s\n" % (str(subtotal['.data']+subtotal['.bss']+subtotal['.heap']+subtotal['.stack'])))
+            file_desc.write("Total Flash memory (text + data + misc): %s\n" % (str(subtotal['.text']+subtotal['.data']+misc_flash_mem)))
+
+        if file_desc is not sys.stdout:
+            file_desc.close()
+
+        return True
+
+    def parse(self, mapfile, toolchain):
+        """
+        Parse and decode map file depending on the toolchain
+        """
+
+        try:
+            file_input = open(mapfile, 'rt')
+        except IOError as error:
+            print "I/O error({0}): {1}".format(error.errno, error.strerror)
+            return False
+
+        if toolchain == "ARM" or toolchain == "ARM_STD" or toolchain == "ARM_MICRO":
+            self.search_objects(os.path.abspath(mapfile), "ARM")
+            self.parse_map_file_armcc(file_input)
+        elif toolchain == "GCC_ARM":
+            self.parse_map_file_gcc(file_input)
+        elif toolchain == "IAR":
+            self.search_objects(os.path.abspath(mapfile), toolchain)
+            self.parse_map_file_iar(file_input)
+        else:
+            return False
+
+        file_input.close()
+
+        return True
+
 def main():
 
-    version = '0.3.7'
-    time_start = time.clock()
+    version = '0.3.10'
 
     # Parser handling
     parser = argparse.ArgumentParser(description="Memory Map File Analyser for ARM mbed OS\nversion %s" % version)
 
     parser.add_argument('file', help='memory map file')
 
-    parser.add_argument('-t','--toolchain', dest='toolchain', help='select a toolchain that corresponds to the memory map file (ARM, GCC_ARM, IAR)',
+    parser.add_argument('-t', '--toolchain', dest='toolchain', help='select a toolchain used to build the memory map file (ARM, GCC_ARM, IAR)',\
                         required=True)
 
-    parser.add_argument('-o','--output',help='output file name', required=False)
+    parser.add_argument('-o', '--output', help='output file name', required=False)
 
-    parser.add_argument('-j', '--json', dest='json', required=False, action="store_true",
-                      help='output in JSON formatted list')
+    parser.add_argument('-e', '--export', dest='export', required=False,\
+                        help="export format (examples: 'json', 'csv-ci', 'table': default)")
 
     parser.add_argument('-v', '--version', action='version', version=version)
 
@@ -460,49 +501,29 @@
         parser.print_help()
         sys.exit(1)
 
+
     args, remainder = parser.parse_known_args()
 
-    try:
-        file_input = open(args.file,'rt')
-    except IOError as e:
-        print "I/O error({0}): {1}".format(e.errno, e.strerror)
-        sys.exit(0)
+    # Create memap object
+    memap = MemapParser()
 
-    # Creates parser object
-    t = MemmapParser()
-  
-    # Decode map file depending on the toolchain
-    if args.toolchain == "ARM":
-        t.search_objects(os.path.abspath(args.file),args.toolchain)
-        t.parse_map_file_armcc(file_input)
-    elif args.toolchain == "GCC_ARM":
-        t.parse_map_file_gcc(file_input)
-    elif args.toolchain == "IAR":
-        print "WARNING: IAR Compiler not fully supported (yet)"
-        print " "
-        t.search_objects(os.path.abspath(args.file),args.toolchain)
-        t.parse_map_file_iar(file_input)
-    else:
-        print "Invalid toolchain. Options are: ARM, GCC_ARM, IAR"
-        sys.exit(0)
+    # Parse and decode a map file
+    if args.file and args.toolchain:
+        if memap.parse(args.file, args.toolchain) is False:
+            print "Unknown toolchain for memory statistics %s" %  args.toolchain
+            sys.exit(0)
+
+    # default export format is table
+    if not args.export:
+        args.export = 'table'
 
     # Write output in file
     if args.output != None:
-        try:
-            file_output = open(args.output,'w')
-            t.generate_output(file_output,args.json)
-            file_output.close()
-        except IOError as e:
-            print "I/O error({0}): {1}".format(e.errno, e.strerror)
-            sys.exit(0)
+        memap.generate_output(args.export, args.output)
     else: # Write output in screen
-        t.generate_output(sys.stdout,args.json)
-    
-    file_input.close()
+        memap.generate_output(args.export)
 
-    print "Elapsed time: %smS" %int(round((time.clock()-time_start)*1000))
-    
     sys.exit(0)
 
 if __name__ == "__main__":
-    main()
\ No newline at end of file
+    main()