Clone of official tools
arm_pack_manager/__init__.py@47:21ae3e5a7128, 2021-02-04 (annotated)
- Committer:
- Anders Blomdell
- Date:
- Thu Feb 04 17:17:13 2021 +0100
- Revision:
- 47:21ae3e5a7128
- Parent:
- 43:2a7da56ebd24
Add a few normpath calls
Who changed what in which revision?
User | Revision | Line number | New contents of line |
---|---|---|---|
theotherjimmy |
43:2a7da56ebd24 | 1 | try: |
theotherjimmy |
43:2a7da56ebd24 | 2 | from urllib2 import urlopen, URLError |
theotherjimmy |
43:2a7da56ebd24 | 3 | except ImportError: |
theotherjimmy |
43:2a7da56ebd24 | 4 | from urllib.request import urlopen, URLError |
The Other Jimmy |
31:8ea194f6145b | 5 | from bs4 import BeautifulSoup |
The Other Jimmy |
31:8ea194f6145b | 6 | from os.path import join, dirname, basename |
The Other Jimmy |
31:8ea194f6145b | 7 | from os import makedirs |
The Other Jimmy |
31:8ea194f6145b | 8 | from errno import EEXIST |
The Other Jimmy |
31:8ea194f6145b | 9 | from threading import Thread |
theotherjimmy |
43:2a7da56ebd24 | 10 | try: |
theotherjimmy |
43:2a7da56ebd24 | 11 | from Queue import Queue |
theotherjimmy |
43:2a7da56ebd24 | 12 | except ImportError: |
theotherjimmy |
43:2a7da56ebd24 | 13 | from queue import Queue |
The Other Jimmy |
31:8ea194f6145b | 14 | from re import compile, sub |
The Other Jimmy |
31:8ea194f6145b | 15 | from sys import stderr, stdout |
The Other Jimmy |
31:8ea194f6145b | 16 | from itertools import takewhile |
The Other Jimmy |
31:8ea194f6145b | 17 | import argparse |
The Other Jimmy |
31:8ea194f6145b | 18 | from json import dump, load |
The Other Jimmy |
31:8ea194f6145b | 19 | from zipfile import ZipFile |
The Other Jimmy |
31:8ea194f6145b | 20 | from tempfile import gettempdir |
The Other Jimmy |
36:96847d42f010 | 21 | import warnings |
The Other Jimmy |
36:96847d42f010 | 22 | from distutils.version import LooseVersion |
The Other Jimmy |
36:96847d42f010 | 23 | |
theotherjimmy |
43:2a7da56ebd24 | 24 | from tools.flash_algo import PackFlashAlgo |
theotherjimmy |
43:2a7da56ebd24 | 25 | |
The Other Jimmy |
36:96847d42f010 | 26 | warnings.filterwarnings("ignore") |
The Other Jimmy |
36:96847d42f010 | 27 | |
The Other Jimmy |
36:96847d42f010 | 28 | from fuzzywuzzy import process |
The Other Jimmy |
31:8ea194f6145b | 29 | |
The Other Jimmy |
31:8ea194f6145b | 30 | RootPackURL = "http://www.keil.com/pack/index.idx" |
The Other Jimmy |
31:8ea194f6145b | 31 | |
The Other Jimmy |
31:8ea194f6145b | 32 | LocalPackDir = dirname(__file__) |
The Other Jimmy |
31:8ea194f6145b | 33 | LocalPackIndex = join(LocalPackDir, "index.json") |
The Other Jimmy |
31:8ea194f6145b | 34 | LocalPackAliases = join(LocalPackDir, "aliases.json") |
The Other Jimmy |
31:8ea194f6145b | 35 | |
The Other Jimmy |
31:8ea194f6145b | 36 | |
The Other Jimmy |
31:8ea194f6145b | 37 | protocol_matcher = compile("\w*://") |
The Other Jimmy |
31:8ea194f6145b | 38 | def strip_protocol(url) : |
The Other Jimmy |
31:8ea194f6145b | 39 | return protocol_matcher.sub("", str(url)) |
The Other Jimmy |
31:8ea194f6145b | 40 | |
The Other Jimmy |
31:8ea194f6145b | 41 | def largest_version(content) : |
The Other Jimmy |
35:da9c89f8be7d | 42 | return sorted([t['version'] for t in content.package.releases('release')], |
The Other Jimmy |
36:96847d42f010 | 43 | reverse=True, key=lambda v: LooseVersion(v))[0] |
The Other Jimmy |
31:8ea194f6145b | 44 | |
The Other Jimmy |
31:8ea194f6145b | 45 | def do_queue(Class, function, interable) : |
The Other Jimmy |
31:8ea194f6145b | 46 | q = Queue() |
The Other Jimmy |
31:8ea194f6145b | 47 | threads = [Class(q, function) for each in range(20)] |
The Other Jimmy |
31:8ea194f6145b | 48 | for each in threads : |
The Other Jimmy |
31:8ea194f6145b | 49 | each.setDaemon(True) |
The Other Jimmy |
31:8ea194f6145b | 50 | each.start() |
The Other Jimmy |
31:8ea194f6145b | 51 | for thing in interable : |
The Other Jimmy |
31:8ea194f6145b | 52 | q.put(thing) |
The Other Jimmy |
31:8ea194f6145b | 53 | q.join() |
The Other Jimmy |
31:8ea194f6145b | 54 | |
The Other Jimmy |
31:8ea194f6145b | 55 | class Reader (Thread) : |
The Other Jimmy |
31:8ea194f6145b | 56 | def __init__(self, queue, func) : |
The Other Jimmy |
31:8ea194f6145b | 57 | Thread.__init__(self) |
The Other Jimmy |
31:8ea194f6145b | 58 | self.queue = queue |
The Other Jimmy |
31:8ea194f6145b | 59 | self.func = func |
The Other Jimmy |
31:8ea194f6145b | 60 | def run(self) : |
The Other Jimmy |
31:8ea194f6145b | 61 | while True : |
The Other Jimmy |
31:8ea194f6145b | 62 | url = self.queue.get() |
The Other Jimmy |
31:8ea194f6145b | 63 | self.func(url) |
The Other Jimmy |
31:8ea194f6145b | 64 | self.queue.task_done() |
The Other Jimmy |
31:8ea194f6145b | 65 | |
The Other Jimmy |
31:8ea194f6145b | 66 | |
The Other Jimmy |
31:8ea194f6145b | 67 | class Cache () : |
The Other Jimmy |
31:8ea194f6145b | 68 | """ The Cache object is the only relevant API object at the moment |
The Other Jimmy |
31:8ea194f6145b | 69 | |
The Other Jimmy |
31:8ea194f6145b | 70 | Constructing the Cache object does not imply any caching. |
The Other Jimmy |
31:8ea194f6145b | 71 | A user of the API must explicitly call caching functions. |
The Other Jimmy |
31:8ea194f6145b | 72 | |
The Other Jimmy |
31:8ea194f6145b | 73 | :param silent: A boolean that, when True, significantly reduces the printing of this Object |
The Other Jimmy |
31:8ea194f6145b | 74 | :type silent: bool |
The Other Jimmy |
31:8ea194f6145b | 75 | :param no_timeouts: A boolean that, when True, disables the default connection timeout and low speed timeout for downloading things. |
The Other Jimmy |
31:8ea194f6145b | 76 | :type no_timeouts: bool |
The Other Jimmy |
31:8ea194f6145b | 77 | """ |
The Other Jimmy |
31:8ea194f6145b | 78 | def __init__ (self, silent, no_timeouts) : |
The Other Jimmy |
31:8ea194f6145b | 79 | self.silent = silent |
The Other Jimmy |
31:8ea194f6145b | 80 | self.counter = 0 |
The Other Jimmy |
31:8ea194f6145b | 81 | self.total = 1 |
The Other Jimmy |
31:8ea194f6145b | 82 | self._index = {} |
The Other Jimmy |
31:8ea194f6145b | 83 | self._aliases = {} |
The Other Jimmy |
31:8ea194f6145b | 84 | self.urls = None |
The Other Jimmy |
31:8ea194f6145b | 85 | self.no_timeouts = no_timeouts |
The Other Jimmy |
31:8ea194f6145b | 86 | self.data_path = gettempdir() |
The Other Jimmy |
31:8ea194f6145b | 87 | |
The Other Jimmy |
31:8ea194f6145b | 88 | def display_counter (self, message) : |
The Other Jimmy |
31:8ea194f6145b | 89 | stdout.write("{} {}/{}\r".format(message, self.counter, self.total)) |
The Other Jimmy |
31:8ea194f6145b | 90 | stdout.flush() |
The Other Jimmy |
31:8ea194f6145b | 91 | |
The Other Jimmy |
31:8ea194f6145b | 92 | def cache_file (self, url) : |
The Other Jimmy |
31:8ea194f6145b | 93 | """Low level interface to caching a single file. |
The Other Jimmy |
31:8ea194f6145b | 94 | |
The Other Jimmy |
31:8ea194f6145b | 95 | :param url: The URL to cache. |
The Other Jimmy |
31:8ea194f6145b | 96 | :type url: str |
The Other Jimmy |
31:8ea194f6145b | 97 | :rtype: None |
The Other Jimmy |
31:8ea194f6145b | 98 | """ |
The Other Jimmy |
31:8ea194f6145b | 99 | if not self.silent : print("Caching {}...".format(url)) |
The Other Jimmy |
31:8ea194f6145b | 100 | dest = join(self.data_path, strip_protocol(url)) |
The Other Jimmy |
31:8ea194f6145b | 101 | try : |
The Other Jimmy |
31:8ea194f6145b | 102 | makedirs(dirname(dest)) |
The Other Jimmy |
31:8ea194f6145b | 103 | except OSError as exc : |
The Other Jimmy |
31:8ea194f6145b | 104 | if exc.errno == EEXIST : pass |
The Other Jimmy |
31:8ea194f6145b | 105 | else : raise |
The Other Jimmy |
31:8ea194f6145b | 106 | try: |
The Other Jimmy |
31:8ea194f6145b | 107 | with open(dest, "wb+") as fd : |
The Other Jimmy |
31:8ea194f6145b | 108 | fd.write(urlopen(url).read()) |
The Other Jimmy |
31:8ea194f6145b | 109 | except URLError as e: |
The Other Jimmy |
31:8ea194f6145b | 110 | stderr.write(e.reason) |
The Other Jimmy |
31:8ea194f6145b | 111 | self.counter += 1 |
The Other Jimmy |
31:8ea194f6145b | 112 | self.display_counter("Caching Files") |
The Other Jimmy |
31:8ea194f6145b | 113 | |
The Other Jimmy |
31:8ea194f6145b | 114 | def pdsc_to_pack (self, url) : |
The Other Jimmy |
31:8ea194f6145b | 115 | """Find the URL of the specified pack file described by a PDSC. |
The Other Jimmy |
31:8ea194f6145b | 116 | |
The Other Jimmy |
31:8ea194f6145b | 117 | The PDSC is assumed to be cached and is looked up in the cache by its URL. |
The Other Jimmy |
31:8ea194f6145b | 118 | |
The Other Jimmy |
31:8ea194f6145b | 119 | :param url: The url used to look up the PDSC. |
The Other Jimmy |
31:8ea194f6145b | 120 | :type url: str |
The Other Jimmy |
31:8ea194f6145b | 121 | :return: The url of the PACK file. |
The Other Jimmy |
31:8ea194f6145b | 122 | :rtype: str |
The Other Jimmy |
31:8ea194f6145b | 123 | """ |
The Other Jimmy |
31:8ea194f6145b | 124 | content = self.pdsc_from_cache(url) |
The Other Jimmy |
31:8ea194f6145b | 125 | new_url = content.package.url.get_text() |
The Other Jimmy |
31:8ea194f6145b | 126 | if not new_url.endswith("/") : |
The Other Jimmy |
31:8ea194f6145b | 127 | new_url = new_url + "/" |
The Other Jimmy |
31:8ea194f6145b | 128 | return (new_url + content.package.vendor.get_text() + "." + |
The Other Jimmy |
31:8ea194f6145b | 129 | content.package.find('name').get_text() + "." + |
The Other Jimmy |
31:8ea194f6145b | 130 | largest_version(content) + ".pack") |
The Other Jimmy |
31:8ea194f6145b | 131 | |
The Other Jimmy |
31:8ea194f6145b | 132 | def cache_pdsc_and_pack (self, url) : |
The Other Jimmy |
31:8ea194f6145b | 133 | self.cache_file(url) |
The Other Jimmy |
31:8ea194f6145b | 134 | try : |
The Other Jimmy |
31:8ea194f6145b | 135 | self.cache_file(self.pdsc_to_pack(url)) |
The Other Jimmy |
31:8ea194f6145b | 136 | except AttributeError : |
The Other Jimmy |
31:8ea194f6145b | 137 | stderr.write("[ ERROR ] {} does not appear to be a conforming .pdsc file\n".format(url)) |
The Other Jimmy |
31:8ea194f6145b | 138 | self.counter += 1 |
The Other Jimmy |
31:8ea194f6145b | 139 | |
The Other Jimmy |
31:8ea194f6145b | 140 | def get_urls(self): |
The Other Jimmy |
31:8ea194f6145b | 141 | """Extract the URLs of all know PDSC files. |
The Other Jimmy |
31:8ea194f6145b | 142 | |
The Other Jimmy |
31:8ea194f6145b | 143 | Will pull the index from the internet if it is not cached. |
The Other Jimmy |
31:8ea194f6145b | 144 | |
The Other Jimmy |
31:8ea194f6145b | 145 | :return: A list of all PDSC URLs |
The Other Jimmy |
31:8ea194f6145b | 146 | :rtype: [str] |
The Other Jimmy |
31:8ea194f6145b | 147 | """ |
The Other Jimmy |
31:8ea194f6145b | 148 | if not self.urls : |
The Other Jimmy |
31:8ea194f6145b | 149 | try : root_data = self.pdsc_from_cache(RootPackURL) |
The Other Jimmy |
31:8ea194f6145b | 150 | except IOError : root_data = self.cache_and_parse(RootPackURL) |
The Other Jimmy |
31:8ea194f6145b | 151 | self.urls = ["/".join([pdsc.get('url').strip("/"), |
The Other Jimmy |
31:8ea194f6145b | 152 | pdsc.get('name').strip("/")]) |
The Other Jimmy |
31:8ea194f6145b | 153 | for pdsc in root_data.find_all("pdsc")] |
The Other Jimmy |
31:8ea194f6145b | 154 | return self.urls |
The Other Jimmy |
31:8ea194f6145b | 155 | |
theotherjimmy |
43:2a7da56ebd24 | 156 | def _get_sectors(self, device): |
theotherjimmy |
43:2a7da56ebd24 | 157 | """Extract sector sizes from device FLM algorithm |
theotherjimmy |
43:2a7da56ebd24 | 158 | |
theotherjimmy |
43:2a7da56ebd24 | 159 | Will return None if there is no algorithm, pdsc URL formatted in correctly |
theotherjimmy |
43:2a7da56ebd24 | 160 | |
theotherjimmy |
43:2a7da56ebd24 | 161 | :return: A list tuples of sector start and size |
theotherjimmy |
43:2a7da56ebd24 | 162 | :rtype: [list] |
theotherjimmy |
43:2a7da56ebd24 | 163 | """ |
theotherjimmy |
43:2a7da56ebd24 | 164 | try: |
theotherjimmy |
43:2a7da56ebd24 | 165 | pack = self.pack_from_cache(device) |
theotherjimmy |
43:2a7da56ebd24 | 166 | ret = [] |
theotherjimmy |
43:2a7da56ebd24 | 167 | for filename in device['algorithm'].keys(): |
theotherjimmy |
43:2a7da56ebd24 | 168 | try: |
theotherjimmy |
43:2a7da56ebd24 | 169 | flm = pack.open(filename) |
theotherjimmy |
43:2a7da56ebd24 | 170 | flash_alg = PackFlashAlgo(flm.read()) |
theotherjimmy |
43:2a7da56ebd24 | 171 | sectors = [(flash_alg.flash_start + offset, size) |
theotherjimmy |
43:2a7da56ebd24 | 172 | for offset, size in flash_alg.sector_sizes] |
theotherjimmy |
43:2a7da56ebd24 | 173 | ret.extend(sectors) |
theotherjimmy |
43:2a7da56ebd24 | 174 | except Exception: |
theotherjimmy |
43:2a7da56ebd24 | 175 | pass |
theotherjimmy |
43:2a7da56ebd24 | 176 | ret.sort(key=lambda sector: sector[0]) |
theotherjimmy |
43:2a7da56ebd24 | 177 | return ret |
theotherjimmy |
43:2a7da56ebd24 | 178 | except Exception: |
theotherjimmy |
43:2a7da56ebd24 | 179 | return None |
theotherjimmy |
43:2a7da56ebd24 | 180 | |
The Other Jimmy |
31:8ea194f6145b | 181 | def _extract_dict(self, device, filename, pack) : |
The Other Jimmy |
31:8ea194f6145b | 182 | to_ret = dict(pdsc_file=filename, pack_file=pack) |
The Other Jimmy |
31:8ea194f6145b | 183 | try : to_ret["memory"] = dict([(m["id"], dict(start=m["start"], |
The Other Jimmy |
31:8ea194f6145b | 184 | size=m["size"])) |
The Other Jimmy |
31:8ea194f6145b | 185 | for m in device("memory")]) |
theotherjimmy |
43:2a7da56ebd24 | 186 | except (KeyError, TypeError, IndexError) as e: |
theotherjimmy |
43:2a7da56ebd24 | 187 | try : to_ret["memory"] = dict([(m["name"], dict(start=m["start"], |
theotherjimmy |
43:2a7da56ebd24 | 188 | size=m["size"])) |
theotherjimmy |
43:2a7da56ebd24 | 189 | for m in device("memory")]) |
theotherjimmy |
43:2a7da56ebd24 | 190 | except (KeyError, TypeError, IndexError) as e : pass |
The Other Jimmy |
31:8ea194f6145b | 191 | try: algorithms = device("algorithm") |
The Other Jimmy |
31:8ea194f6145b | 192 | except: |
The Other Jimmy |
31:8ea194f6145b | 193 | try: algorithms = device.parent("algorithm") |
The Other Jimmy |
31:8ea194f6145b | 194 | except: pass |
The Other Jimmy |
31:8ea194f6145b | 195 | else: |
The Other Jimmy |
31:8ea194f6145b | 196 | if not algorithms: |
The Other Jimmy |
31:8ea194f6145b | 197 | try: algorithms = device.parent("algorithm") |
The Other Jimmy |
31:8ea194f6145b | 198 | except: pass |
The Other Jimmy |
31:8ea194f6145b | 199 | try : to_ret["algorithm"] = dict([(algo.get("name").replace('\\','/'), |
The Other Jimmy |
31:8ea194f6145b | 200 | dict(start=algo["start"], |
The Other Jimmy |
31:8ea194f6145b | 201 | size=algo["size"], |
The Other Jimmy |
31:8ea194f6145b | 202 | ramstart=algo.get("ramstart",None), |
The Other Jimmy |
31:8ea194f6145b | 203 | ramsize=algo.get("ramsize",None), |
The Other Jimmy |
31:8ea194f6145b | 204 | default=algo.get("default",1))) |
The Other Jimmy |
31:8ea194f6145b | 205 | for algo in algorithms]) |
The Other Jimmy |
31:8ea194f6145b | 206 | except (KeyError, TypeError, IndexError) as e: pass |
The Other Jimmy |
31:8ea194f6145b | 207 | try: to_ret["debug"] = device.parent.parent.debug["svd"] |
The Other Jimmy |
31:8ea194f6145b | 208 | except (KeyError, TypeError, IndexError) as e : pass |
The Other Jimmy |
31:8ea194f6145b | 209 | try: to_ret["debug"] = device.parent.debug["svd"] |
The Other Jimmy |
31:8ea194f6145b | 210 | except (KeyError, TypeError, IndexError) as e : pass |
The Other Jimmy |
31:8ea194f6145b | 211 | try: to_ret["debug"] = device.debug["svd"] |
The Other Jimmy |
31:8ea194f6145b | 212 | except (KeyError, TypeError, IndexError) as e : pass |
The Other Jimmy |
31:8ea194f6145b | 213 | |
The Other Jimmy |
31:8ea194f6145b | 214 | to_ret["compile"] = {} |
The Other Jimmy |
31:8ea194f6145b | 215 | try: compile_l1 = device.parent("compile") |
The Other Jimmy |
31:8ea194f6145b | 216 | except (KeyError, TypeError, IndexError) as e : compile_l1 = [] |
The Other Jimmy |
31:8ea194f6145b | 217 | try: compile_l2 = device.parent.parent("compile") |
The Other Jimmy |
31:8ea194f6145b | 218 | except (KeyError, TypeError, IndexError) as e : compile_l2 = [] |
The Other Jimmy |
31:8ea194f6145b | 219 | compile = compile_l2 + compile_l1 |
The Other Jimmy |
31:8ea194f6145b | 220 | for c in compile: |
The Other Jimmy |
31:8ea194f6145b | 221 | try: to_ret["compile"]["header"] = c["header"] |
The Other Jimmy |
31:8ea194f6145b | 222 | except (KeyError, TypeError, IndexError) as e : pass |
The Other Jimmy |
31:8ea194f6145b | 223 | try: to_ret["compile"]["define"] = c["define"] |
The Other Jimmy |
31:8ea194f6145b | 224 | except (KeyError, TypeError, IndexError) as e : pass |
The Other Jimmy |
31:8ea194f6145b | 225 | |
The Other Jimmy |
31:8ea194f6145b | 226 | try: to_ret["core"] = device.parent.processor['dcore'] |
The Other Jimmy |
31:8ea194f6145b | 227 | except (KeyError, TypeError, IndexError) as e : pass |
The Other Jimmy |
31:8ea194f6145b | 228 | try: to_ret["core"] = device.parent.parent.processor['dcore'] |
The Other Jimmy |
31:8ea194f6145b | 229 | except (KeyError, TypeError, IndexError) as e : pass |
The Other Jimmy |
31:8ea194f6145b | 230 | |
The Other Jimmy |
31:8ea194f6145b | 231 | to_ret["processor"] = {} |
The Other Jimmy |
31:8ea194f6145b | 232 | try: proc_l1 = device("processor") |
The Other Jimmy |
31:8ea194f6145b | 233 | except (KeyError, TypeError, IndexError) as e: proc_l1 = [] |
The Other Jimmy |
31:8ea194f6145b | 234 | try: proc_l2 = device.parent("processor") |
The Other Jimmy |
31:8ea194f6145b | 235 | except (KeyError, TypeError, IndexError) as e: proc_l2 = [] |
The Other Jimmy |
31:8ea194f6145b | 236 | try: proc_l3 = device.parent.parent("processor") |
The Other Jimmy |
31:8ea194f6145b | 237 | except (KeyError, TypeError, IndexError) as e: proc_l3 = [] |
The Other Jimmy |
31:8ea194f6145b | 238 | proc = proc_l3 + proc_l2 + proc_l1 |
The Other Jimmy |
31:8ea194f6145b | 239 | for p in proc: |
The Other Jimmy |
31:8ea194f6145b | 240 | try: to_ret["processor"]["fpu"] = p['dfpu'] |
The Other Jimmy |
31:8ea194f6145b | 241 | except (KeyError, TypeError, IndexError) as e: pass |
The Other Jimmy |
31:8ea194f6145b | 242 | try: to_ret["processor"]["endianness"] = p['dendian'] |
The Other Jimmy |
31:8ea194f6145b | 243 | except (KeyError, TypeError, IndexError) as e: pass |
The Other Jimmy |
31:8ea194f6145b | 244 | try: to_ret["processor"]["clock"] = p['dclock'] |
The Other Jimmy |
31:8ea194f6145b | 245 | except (KeyError, TypeError, IndexError) as e: pass |
The Other Jimmy |
31:8ea194f6145b | 246 | |
The Other Jimmy |
31:8ea194f6145b | 247 | try: to_ret["vendor"] = device.parent['dvendor'] |
The Other Jimmy |
31:8ea194f6145b | 248 | except (KeyError, TypeError, IndexError) as e: pass |
The Other Jimmy |
31:8ea194f6145b | 249 | try: to_ret["vendor"] = device.parent.parent['dvendor'] |
The Other Jimmy |
31:8ea194f6145b | 250 | except (KeyError, TypeError, IndexError) as e: pass |
The Other Jimmy |
31:8ea194f6145b | 251 | |
The Other Jimmy |
31:8ea194f6145b | 252 | if not to_ret["processor"]: |
The Other Jimmy |
31:8ea194f6145b | 253 | del to_ret["processor"] |
The Other Jimmy |
31:8ea194f6145b | 254 | |
The Other Jimmy |
31:8ea194f6145b | 255 | if not to_ret["compile"]: |
The Other Jimmy |
31:8ea194f6145b | 256 | del to_ret["compile"] |
The Other Jimmy |
31:8ea194f6145b | 257 | |
The Other Jimmy |
31:8ea194f6145b | 258 | to_ret['debug-interface'] = [] |
theotherjimmy |
43:2a7da56ebd24 | 259 | to_ret['sectors'] = self._get_sectors(to_ret) |
The Other Jimmy |
31:8ea194f6145b | 260 | |
The Other Jimmy |
31:8ea194f6145b | 261 | return to_ret |
The Other Jimmy |
31:8ea194f6145b | 262 | |
The Other Jimmy |
31:8ea194f6145b | 263 | def _generate_index_helper(self, d) : |
The Other Jimmy |
31:8ea194f6145b | 264 | try : |
The Other Jimmy |
31:8ea194f6145b | 265 | pack = self.pdsc_to_pack(d) |
The Other Jimmy |
31:8ea194f6145b | 266 | self._index.update(dict([(dev['dname'], self._extract_dict(dev, d, pack)) for dev in |
The Other Jimmy |
31:8ea194f6145b | 267 | (self.pdsc_from_cache(d)("device"))])) |
The Other Jimmy |
31:8ea194f6145b | 268 | except AttributeError as e : |
The Other Jimmy |
31:8ea194f6145b | 269 | stderr.write("[ ERROR ] file {}\n".format(d)) |
The Other Jimmy |
31:8ea194f6145b | 270 | print(e) |
The Other Jimmy |
31:8ea194f6145b | 271 | self.counter += 1 |
The Other Jimmy |
31:8ea194f6145b | 272 | self.display_counter("Generating Index") |
The Other Jimmy |
31:8ea194f6145b | 273 | |
The Other Jimmy |
31:8ea194f6145b | 274 | def _generate_aliases_helper(self, d) : |
The Other Jimmy |
31:8ea194f6145b | 275 | try : |
The Other Jimmy |
31:8ea194f6145b | 276 | mydict = [] |
The Other Jimmy |
31:8ea194f6145b | 277 | for dev in self.pdsc_from_cache(d)("board"): |
The Other Jimmy |
31:8ea194f6145b | 278 | try : |
The Other Jimmy |
31:8ea194f6145b | 279 | mydict.append((dev['name'], dev.mounteddevice['dname'])) |
The Other Jimmy |
31:8ea194f6145b | 280 | except (KeyError, TypeError, IndexError) as e: |
The Other Jimmy |
31:8ea194f6145b | 281 | pass |
The Other Jimmy |
31:8ea194f6145b | 282 | self._aliases.update(dict(mydict)) |
The Other Jimmy |
31:8ea194f6145b | 283 | except (AttributeError, TypeError) as e : |
The Other Jimmy |
31:8ea194f6145b | 284 | pass |
The Other Jimmy |
31:8ea194f6145b | 285 | self.counter += 1 |
The Other Jimmy |
31:8ea194f6145b | 286 | self.display_counter("Scanning for Aliases") |
The Other Jimmy |
31:8ea194f6145b | 287 | |
The Other Jimmy |
36:96847d42f010 | 288 | def get_flash_algorthim_binary(self, device_name, all=False) : |
The Other Jimmy |
31:8ea194f6145b | 289 | """Retrieve the flash algorithm file for a particular part. |
The Other Jimmy |
31:8ea194f6145b | 290 | |
The Other Jimmy |
31:8ea194f6145b | 291 | Assumes that both the PDSC and the PACK file associated with that part are in the cache. |
The Other Jimmy |
31:8ea194f6145b | 292 | |
The Other Jimmy |
31:8ea194f6145b | 293 | :param device_name: The exact name of a device |
The Other Jimmy |
36:96847d42f010 | 294 | :param all: Return an iterator of all flash algos for this device |
The Other Jimmy |
31:8ea194f6145b | 295 | :type device_name: str |
The Other Jimmy |
31:8ea194f6145b | 296 | :return: A file-like object that, when read, is the ELF file that describes the flashing algorithm |
The Other Jimmy |
36:96847d42f010 | 297 | :return: A file-like object that, when read, is the ELF file that describes the flashing algorithm. |
The Other Jimmy |
36:96847d42f010 | 298 | When "all" is set to True then an iterator for file-like objects is returned |
The Other Jimmy |
36:96847d42f010 | 299 | :rtype: ZipExtFile or ZipExtFile iterator if all is True |
The Other Jimmy |
31:8ea194f6145b | 300 | """ |
The Other Jimmy |
35:da9c89f8be7d | 301 | device = self.index[device_name] |
The Other Jimmy |
35:da9c89f8be7d | 302 | pack = self.pack_from_cache(device) |
The Other Jimmy |
36:96847d42f010 | 303 | algo_itr = (pack.open(path) for path in device['algorithm'].keys()) |
The Other Jimmy |
36:96847d42f010 | 304 | return algo_itr if all else algo_itr.next() |
The Other Jimmy |
31:8ea194f6145b | 305 | |
The Other Jimmy |
31:8ea194f6145b | 306 | def get_svd_file(self, device_name) : |
The Other Jimmy |
31:8ea194f6145b | 307 | """Retrieve the flash algorithm file for a particular part. |
The Other Jimmy |
31:8ea194f6145b | 308 | |
The Other Jimmy |
31:8ea194f6145b | 309 | Assumes that both the PDSC and the PACK file associated with that part are in the cache. |
The Other Jimmy |
31:8ea194f6145b | 310 | |
The Other Jimmy |
31:8ea194f6145b | 311 | :param device_name: The exact name of a device |
The Other Jimmy |
31:8ea194f6145b | 312 | :type device_name: str |
The Other Jimmy |
31:8ea194f6145b | 313 | :return: A file-like object that, when read, is the ELF file that describes the flashing algorithm |
The Other Jimmy |
31:8ea194f6145b | 314 | :rtype: ZipExtFile |
The Other Jimmy |
31:8ea194f6145b | 315 | """ |
The Other Jimmy |
35:da9c89f8be7d | 316 | device = self.index[device_name] |
The Other Jimmy |
35:da9c89f8be7d | 317 | pack = self.pack_from_cache(device) |
The Other Jimmy |
31:8ea194f6145b | 318 | return pack.open(device['debug']) |
The Other Jimmy |
31:8ea194f6145b | 319 | |
The Other Jimmy |
31:8ea194f6145b | 320 | def generate_index(self) : |
The Other Jimmy |
31:8ea194f6145b | 321 | self._index = {} |
The Other Jimmy |
31:8ea194f6145b | 322 | self.counter = 0 |
The Other Jimmy |
31:8ea194f6145b | 323 | do_queue(Reader, self._generate_index_helper, self.get_urls()) |
The Other Jimmy |
31:8ea194f6145b | 324 | with open(LocalPackIndex, "wb+") as out: |
The Other Jimmy |
31:8ea194f6145b | 325 | self._index["version"] = "0.1.0" |
The Other Jimmy |
31:8ea194f6145b | 326 | dump(self._index, out) |
The Other Jimmy |
31:8ea194f6145b | 327 | stdout.write("\n") |
The Other Jimmy |
31:8ea194f6145b | 328 | |
The Other Jimmy |
31:8ea194f6145b | 329 | def generate_aliases(self) : |
The Other Jimmy |
31:8ea194f6145b | 330 | self._aliases = {} |
The Other Jimmy |
31:8ea194f6145b | 331 | self.counter = 0 |
The Other Jimmy |
31:8ea194f6145b | 332 | do_queue(Reader, self._generate_aliases_helper, self.get_urls()) |
The Other Jimmy |
31:8ea194f6145b | 333 | with open(LocalPackAliases, "wb+") as out: |
The Other Jimmy |
31:8ea194f6145b | 334 | dump(self._aliases, out) |
The Other Jimmy |
31:8ea194f6145b | 335 | stdout.write("\n") |
The Other Jimmy |
31:8ea194f6145b | 336 | |
The Other Jimmy |
31:8ea194f6145b | 337 | def find_device(self, match) : |
The Other Jimmy |
31:8ea194f6145b | 338 | choices = process.extract(match, self.index.keys(), limit=len(self.index)) |
The Other Jimmy |
31:8ea194f6145b | 339 | choices = sorted([(v, k) for k, v in choices], reverse=True) |
The Other Jimmy |
31:8ea194f6145b | 340 | if choices : choices = list(takewhile(lambda t: t[0] == choices[0][0], choices)) |
The Other Jimmy |
31:8ea194f6145b | 341 | return [(v, self.index[v]) for k,v in choices] |
The Other Jimmy |
31:8ea194f6145b | 342 | |
The Other Jimmy |
31:8ea194f6145b | 343 | def dump_index_to_file(self, file) : |
The Other Jimmy |
31:8ea194f6145b | 344 | with open(file, "wb+") as out: |
The Other Jimmy |
31:8ea194f6145b | 345 | dump(self.index, out) |
The Other Jimmy |
31:8ea194f6145b | 346 | |
The Other Jimmy |
31:8ea194f6145b | 347 | @property |
The Other Jimmy |
31:8ea194f6145b | 348 | def index(self) : |
The Other Jimmy |
31:8ea194f6145b | 349 | """An index of most of the important data in all cached PDSC files. |
The Other Jimmy |
31:8ea194f6145b | 350 | |
The Other Jimmy |
31:8ea194f6145b | 351 | :Example: |
The Other Jimmy |
31:8ea194f6145b | 352 | |
The Other Jimmy |
31:8ea194f6145b | 353 | >>> from ArmPackManager import Cache |
The Other Jimmy |
31:8ea194f6145b | 354 | >>> a = Cache() |
The Other Jimmy |
31:8ea194f6145b | 355 | >>> a.index["LPC1768"] |
The Other Jimmy |
31:8ea194f6145b | 356 | {u'algorithm': {u'RAMsize': u'0x0FE0', |
The Other Jimmy |
31:8ea194f6145b | 357 | u'RAMstart': u'0x10000000', |
The Other Jimmy |
31:8ea194f6145b | 358 | u'name': u'Flash/LPC_IAP_512.FLM', |
The Other Jimmy |
31:8ea194f6145b | 359 | u'size': u'0x80000', |
The Other Jimmy |
31:8ea194f6145b | 360 | u'start': u'0x00000000'}, |
The Other Jimmy |
31:8ea194f6145b | 361 | u'compile': [u'Device/Include/LPC17xx.h', u'LPC175x_6x'], |
The Other Jimmy |
31:8ea194f6145b | 362 | u'debug': u'SVD/LPC176x5x.svd', |
The Other Jimmy |
31:8ea194f6145b | 363 | u'pdsc_file': u'http://www.keil.com/pack/Keil.LPC1700_DFP.pdsc', |
The Other Jimmy |
31:8ea194f6145b | 364 | u'memory': {u'IRAM1': {u'size': u'0x8000', u'start': u'0x10000000'}, |
The Other Jimmy |
31:8ea194f6145b | 365 | u'IRAM2': {u'size': u'0x8000', u'start': u'0x2007C000'}, |
The Other Jimmy |
31:8ea194f6145b | 366 | u'IROM1': {u'size': u'0x80000', u'start': u'0x00000000'}}} |
The Other Jimmy |
31:8ea194f6145b | 367 | |
The Other Jimmy |
31:8ea194f6145b | 368 | |
The Other Jimmy |
31:8ea194f6145b | 369 | """ |
The Other Jimmy |
31:8ea194f6145b | 370 | if not self._index : |
The Other Jimmy |
31:8ea194f6145b | 371 | with open(LocalPackIndex) as i : |
The Other Jimmy |
31:8ea194f6145b | 372 | self._index = load(i) |
The Other Jimmy |
31:8ea194f6145b | 373 | return self._index |
The Other Jimmy |
31:8ea194f6145b | 374 | @property |
The Other Jimmy |
31:8ea194f6145b | 375 | def aliases(self) : |
The Other Jimmy |
31:8ea194f6145b | 376 | """An index of most of the important data in all cached PDSC files. |
The Other Jimmy |
31:8ea194f6145b | 377 | |
The Other Jimmy |
31:8ea194f6145b | 378 | :Example: |
The Other Jimmy |
31:8ea194f6145b | 379 | |
The Other Jimmy |
31:8ea194f6145b | 380 | >>> from ArmPackManager import Cache |
The Other Jimmy |
31:8ea194f6145b | 381 | >>> a = Cache() |
The Other Jimmy |
31:8ea194f6145b | 382 | >>> a.index["LPC1768"] |
The Other Jimmy |
31:8ea194f6145b | 383 | {u'algorithm': {u'RAMsize': u'0x0FE0', |
The Other Jimmy |
31:8ea194f6145b | 384 | u'RAMstart': u'0x10000000', |
The Other Jimmy |
31:8ea194f6145b | 385 | u'name': u'Flash/LPC_IAP_512.FLM', |
The Other Jimmy |
31:8ea194f6145b | 386 | u'size': u'0x80000', |
The Other Jimmy |
31:8ea194f6145b | 387 | u'start': u'0x00000000'}, |
The Other Jimmy |
31:8ea194f6145b | 388 | u'compile': [u'Device/Include/LPC17xx.h', u'LPC175x_6x'], |
The Other Jimmy |
31:8ea194f6145b | 389 | u'debug': u'SVD/LPC176x5x.svd', |
The Other Jimmy |
31:8ea194f6145b | 390 | u'pdsc_file': u'http://www.keil.com/pack/Keil.LPC1700_DFP.pdsc', |
The Other Jimmy |
31:8ea194f6145b | 391 | u'memory': {u'IRAM1': {u'size': u'0x8000', u'start': u'0x10000000'}, |
The Other Jimmy |
31:8ea194f6145b | 392 | u'IRAM2': {u'size': u'0x8000', u'start': u'0x2007C000'}, |
The Other Jimmy |
31:8ea194f6145b | 393 | u'IROM1': {u'size': u'0x80000', u'start': u'0x00000000'}}} |
The Other Jimmy |
31:8ea194f6145b | 394 | |
The Other Jimmy |
31:8ea194f6145b | 395 | |
The Other Jimmy |
31:8ea194f6145b | 396 | """ |
The Other Jimmy |
31:8ea194f6145b | 397 | if not self._aliases : |
The Other Jimmy |
36:96847d42f010 | 398 | with open(LocalPackAliases) as i : |
The Other Jimmy |
31:8ea194f6145b | 399 | self._aliases = load(i) |
The Other Jimmy |
31:8ea194f6145b | 400 | return self._aliases |
The Other Jimmy |
31:8ea194f6145b | 401 | |
The Other Jimmy |
31:8ea194f6145b | 402 | def cache_everything(self) : |
The Other Jimmy |
31:8ea194f6145b | 403 | """Cache every PACK and PDSC file known. |
The Other Jimmy |
31:8ea194f6145b | 404 | |
The Other Jimmy |
31:8ea194f6145b | 405 | Generates an index afterwards. |
The Other Jimmy |
31:8ea194f6145b | 406 | |
The Other Jimmy |
31:8ea194f6145b | 407 | .. note:: This process may use 4GB of drive space and take upwards of 10 minutes to complete. |
The Other Jimmy |
31:8ea194f6145b | 408 | """ |
The Other Jimmy |
31:8ea194f6145b | 409 | self.cache_pack_list(self.get_urls()) |
The Other Jimmy |
31:8ea194f6145b | 410 | self.generate_index() |
The Other Jimmy |
31:8ea194f6145b | 411 | self.generate_aliases() |
The Other Jimmy |
31:8ea194f6145b | 412 | |
The Other Jimmy |
31:8ea194f6145b | 413 | def cache_descriptors(self) : |
The Other Jimmy |
31:8ea194f6145b | 414 | """Cache every PDSC file known. |
The Other Jimmy |
31:8ea194f6145b | 415 | |
The Other Jimmy |
31:8ea194f6145b | 416 | Generates an index afterwards. |
The Other Jimmy |
31:8ea194f6145b | 417 | |
The Other Jimmy |
31:8ea194f6145b | 418 | .. note:: This process may use 11MB of drive space and take upwards of 1 minute. |
The Other Jimmy |
31:8ea194f6145b | 419 | """ |
The Other Jimmy |
31:8ea194f6145b | 420 | self.cache_descriptor_list(self.get_urls()) |
The Other Jimmy |
31:8ea194f6145b | 421 | self.generate_index() |
The Other Jimmy |
31:8ea194f6145b | 422 | self.generate_aliases() |
The Other Jimmy |
31:8ea194f6145b | 423 | |
The Other Jimmy |
31:8ea194f6145b | 424 | def cache_descriptor_list(self, list) : |
The Other Jimmy |
31:8ea194f6145b | 425 | """Cache a list of PDSC files. |
The Other Jimmy |
31:8ea194f6145b | 426 | |
The Other Jimmy |
31:8ea194f6145b | 427 | :param list: URLs of PDSC files to cache. |
The Other Jimmy |
31:8ea194f6145b | 428 | :type list: [str] |
The Other Jimmy |
31:8ea194f6145b | 429 | """ |
The Other Jimmy |
31:8ea194f6145b | 430 | self.total = len(list) |
The Other Jimmy |
31:8ea194f6145b | 431 | self.display_counter("Caching Files") |
The Other Jimmy |
31:8ea194f6145b | 432 | do_queue(Reader, self.cache_file, list) |
The Other Jimmy |
31:8ea194f6145b | 433 | stdout.write("\n") |
The Other Jimmy |
31:8ea194f6145b | 434 | |
The Other Jimmy |
31:8ea194f6145b | 435 | def cache_pack_list(self, list) : |
The Other Jimmy |
31:8ea194f6145b | 436 | """Cache a list of PACK files, referenced by their PDSC URL |
The Other Jimmy |
31:8ea194f6145b | 437 | |
The Other Jimmy |
31:8ea194f6145b | 438 | :param list: URLs of PDSC files to cache. |
The Other Jimmy |
31:8ea194f6145b | 439 | :type list: [str] |
The Other Jimmy |
31:8ea194f6145b | 440 | """ |
The Other Jimmy |
31:8ea194f6145b | 441 | self.total = len(list) * 2 |
The Other Jimmy |
31:8ea194f6145b | 442 | self.display_counter("Caching Files") |
The Other Jimmy |
31:8ea194f6145b | 443 | do_queue(Reader, self.cache_pdsc_and_pack, list) |
The Other Jimmy |
31:8ea194f6145b | 444 | stdout.write("\n") |
The Other Jimmy |
31:8ea194f6145b | 445 | |
The Other Jimmy |
31:8ea194f6145b | 446 | def pdsc_from_cache(self, url) : |
The Other Jimmy |
31:8ea194f6145b | 447 | """Low level inteface for extracting a PDSC file from the cache. |
The Other Jimmy |
31:8ea194f6145b | 448 | |
The Other Jimmy |
31:8ea194f6145b | 449 | Assumes that the file specified is a PDSC file and is in the cache. |
The Other Jimmy |
31:8ea194f6145b | 450 | |
The Other Jimmy |
31:8ea194f6145b | 451 | :param url: The URL of a PDSC file. |
The Other Jimmy |
31:8ea194f6145b | 452 | :type url: str |
The Other Jimmy |
31:8ea194f6145b | 453 | :return: A parsed representation of the PDSC file. |
The Other Jimmy |
31:8ea194f6145b | 454 | :rtype: BeautifulSoup |
The Other Jimmy |
31:8ea194f6145b | 455 | """ |
The Other Jimmy |
31:8ea194f6145b | 456 | dest = join(self.data_path, strip_protocol(url)) |
The Other Jimmy |
31:8ea194f6145b | 457 | with open(dest, "r") as fd : |
The Other Jimmy |
31:8ea194f6145b | 458 | return BeautifulSoup(fd, "html.parser") |
The Other Jimmy |
31:8ea194f6145b | 459 | |
The Other Jimmy |
35:da9c89f8be7d | 460 | def pack_from_cache(self, device) : |
The Other Jimmy |
31:8ea194f6145b | 461 | """Low level inteface for extracting a PACK file from the cache. |
The Other Jimmy |
31:8ea194f6145b | 462 | |
The Other Jimmy |
31:8ea194f6145b | 463 | Assumes that the file specified is a PACK file and is in the cache. |
The Other Jimmy |
31:8ea194f6145b | 464 | |
The Other Jimmy |
31:8ea194f6145b | 465 | :param url: The URL of a PACK file. |
The Other Jimmy |
31:8ea194f6145b | 466 | :type url: str |
The Other Jimmy |
31:8ea194f6145b | 467 | :return: A parsed representation of the PACK file. |
The Other Jimmy |
31:8ea194f6145b | 468 | :rtype: ZipFile |
The Other Jimmy |
31:8ea194f6145b | 469 | """ |
The Other Jimmy |
31:8ea194f6145b | 470 | return ZipFile(join(self.data_path, |
The Other Jimmy |
31:8ea194f6145b | 471 | strip_protocol(device['pack_file']))) |
The Other Jimmy |
31:8ea194f6145b | 472 | |
The Other Jimmy |
31:8ea194f6145b | 473 | def gen_dict_from_cache() : |
The Other Jimmy |
31:8ea194f6145b | 474 | pdsc_files = pdsc_from_cache(RootPackUrl) |
The Other Jimmy |
31:8ea194f6145b | 475 | |
The Other Jimmy |
31:8ea194f6145b | 476 | def cache_and_parse(self, url) : |
The Other Jimmy |
31:8ea194f6145b | 477 | """A low level shortcut that Caches and Parses a PDSC file. |
The Other Jimmy |
31:8ea194f6145b | 478 | |
The Other Jimmy |
31:8ea194f6145b | 479 | :param url: The URL of the PDSC file. |
The Other Jimmy |
31:8ea194f6145b | 480 | :type url: str |
The Other Jimmy |
31:8ea194f6145b | 481 | :return: A parsed representation of the PDSC file. |
The Other Jimmy |
31:8ea194f6145b | 482 | :rtype: BeautifulSoup |
The Other Jimmy |
31:8ea194f6145b | 483 | """ |
The Other Jimmy |
31:8ea194f6145b | 484 | self.cache_file(url) |
The Other Jimmy |
31:8ea194f6145b | 485 | return self.pdsc_from_cache(url) |