Clone of official tools

Committer:
Anders Blomdell
Date:
Thu Feb 04 17:17:13 2021 +0100
Revision:
47:21ae3e5a7128
Parent:
43:2a7da56ebd24
Add a few normpath calls

Who changed what in which revision?

UserRevisionLine numberNew contents of line
theotherjimmy 43:2a7da56ebd24 1 try:
theotherjimmy 43:2a7da56ebd24 2 from urllib2 import urlopen, URLError
theotherjimmy 43:2a7da56ebd24 3 except ImportError:
theotherjimmy 43:2a7da56ebd24 4 from urllib.request import urlopen, URLError
The Other Jimmy 31:8ea194f6145b 5 from bs4 import BeautifulSoup
The Other Jimmy 31:8ea194f6145b 6 from os.path import join, dirname, basename
The Other Jimmy 31:8ea194f6145b 7 from os import makedirs
The Other Jimmy 31:8ea194f6145b 8 from errno import EEXIST
The Other Jimmy 31:8ea194f6145b 9 from threading import Thread
theotherjimmy 43:2a7da56ebd24 10 try:
theotherjimmy 43:2a7da56ebd24 11 from Queue import Queue
theotherjimmy 43:2a7da56ebd24 12 except ImportError:
theotherjimmy 43:2a7da56ebd24 13 from queue import Queue
The Other Jimmy 31:8ea194f6145b 14 from re import compile, sub
The Other Jimmy 31:8ea194f6145b 15 from sys import stderr, stdout
The Other Jimmy 31:8ea194f6145b 16 from itertools import takewhile
The Other Jimmy 31:8ea194f6145b 17 import argparse
The Other Jimmy 31:8ea194f6145b 18 from json import dump, load
The Other Jimmy 31:8ea194f6145b 19 from zipfile import ZipFile
The Other Jimmy 31:8ea194f6145b 20 from tempfile import gettempdir
The Other Jimmy 36:96847d42f010 21 import warnings
The Other Jimmy 36:96847d42f010 22 from distutils.version import LooseVersion
The Other Jimmy 36:96847d42f010 23
theotherjimmy 43:2a7da56ebd24 24 from tools.flash_algo import PackFlashAlgo
theotherjimmy 43:2a7da56ebd24 25
The Other Jimmy 36:96847d42f010 26 warnings.filterwarnings("ignore")
The Other Jimmy 36:96847d42f010 27
The Other Jimmy 36:96847d42f010 28 from fuzzywuzzy import process
The Other Jimmy 31:8ea194f6145b 29
The Other Jimmy 31:8ea194f6145b 30 RootPackURL = "http://www.keil.com/pack/index.idx"
The Other Jimmy 31:8ea194f6145b 31
The Other Jimmy 31:8ea194f6145b 32 LocalPackDir = dirname(__file__)
The Other Jimmy 31:8ea194f6145b 33 LocalPackIndex = join(LocalPackDir, "index.json")
The Other Jimmy 31:8ea194f6145b 34 LocalPackAliases = join(LocalPackDir, "aliases.json")
The Other Jimmy 31:8ea194f6145b 35
The Other Jimmy 31:8ea194f6145b 36
The Other Jimmy 31:8ea194f6145b 37 protocol_matcher = compile("\w*://")
The Other Jimmy 31:8ea194f6145b 38 def strip_protocol(url) :
The Other Jimmy 31:8ea194f6145b 39 return protocol_matcher.sub("", str(url))
The Other Jimmy 31:8ea194f6145b 40
The Other Jimmy 31:8ea194f6145b 41 def largest_version(content) :
The Other Jimmy 35:da9c89f8be7d 42 return sorted([t['version'] for t in content.package.releases('release')],
The Other Jimmy 36:96847d42f010 43 reverse=True, key=lambda v: LooseVersion(v))[0]
The Other Jimmy 31:8ea194f6145b 44
The Other Jimmy 31:8ea194f6145b 45 def do_queue(Class, function, interable) :
The Other Jimmy 31:8ea194f6145b 46 q = Queue()
The Other Jimmy 31:8ea194f6145b 47 threads = [Class(q, function) for each in range(20)]
The Other Jimmy 31:8ea194f6145b 48 for each in threads :
The Other Jimmy 31:8ea194f6145b 49 each.setDaemon(True)
The Other Jimmy 31:8ea194f6145b 50 each.start()
The Other Jimmy 31:8ea194f6145b 51 for thing in interable :
The Other Jimmy 31:8ea194f6145b 52 q.put(thing)
The Other Jimmy 31:8ea194f6145b 53 q.join()
The Other Jimmy 31:8ea194f6145b 54
The Other Jimmy 31:8ea194f6145b 55 class Reader (Thread) :
The Other Jimmy 31:8ea194f6145b 56 def __init__(self, queue, func) :
The Other Jimmy 31:8ea194f6145b 57 Thread.__init__(self)
The Other Jimmy 31:8ea194f6145b 58 self.queue = queue
The Other Jimmy 31:8ea194f6145b 59 self.func = func
The Other Jimmy 31:8ea194f6145b 60 def run(self) :
The Other Jimmy 31:8ea194f6145b 61 while True :
The Other Jimmy 31:8ea194f6145b 62 url = self.queue.get()
The Other Jimmy 31:8ea194f6145b 63 self.func(url)
The Other Jimmy 31:8ea194f6145b 64 self.queue.task_done()
The Other Jimmy 31:8ea194f6145b 65
The Other Jimmy 31:8ea194f6145b 66
The Other Jimmy 31:8ea194f6145b 67 class Cache () :
The Other Jimmy 31:8ea194f6145b 68 """ The Cache object is the only relevant API object at the moment
The Other Jimmy 31:8ea194f6145b 69
The Other Jimmy 31:8ea194f6145b 70 Constructing the Cache object does not imply any caching.
The Other Jimmy 31:8ea194f6145b 71 A user of the API must explicitly call caching functions.
The Other Jimmy 31:8ea194f6145b 72
The Other Jimmy 31:8ea194f6145b 73 :param silent: A boolean that, when True, significantly reduces the printing of this Object
The Other Jimmy 31:8ea194f6145b 74 :type silent: bool
The Other Jimmy 31:8ea194f6145b 75 :param no_timeouts: A boolean that, when True, disables the default connection timeout and low speed timeout for downloading things.
The Other Jimmy 31:8ea194f6145b 76 :type no_timeouts: bool
The Other Jimmy 31:8ea194f6145b 77 """
The Other Jimmy 31:8ea194f6145b 78 def __init__ (self, silent, no_timeouts) :
The Other Jimmy 31:8ea194f6145b 79 self.silent = silent
The Other Jimmy 31:8ea194f6145b 80 self.counter = 0
The Other Jimmy 31:8ea194f6145b 81 self.total = 1
The Other Jimmy 31:8ea194f6145b 82 self._index = {}
The Other Jimmy 31:8ea194f6145b 83 self._aliases = {}
The Other Jimmy 31:8ea194f6145b 84 self.urls = None
The Other Jimmy 31:8ea194f6145b 85 self.no_timeouts = no_timeouts
The Other Jimmy 31:8ea194f6145b 86 self.data_path = gettempdir()
The Other Jimmy 31:8ea194f6145b 87
The Other Jimmy 31:8ea194f6145b 88 def display_counter (self, message) :
The Other Jimmy 31:8ea194f6145b 89 stdout.write("{} {}/{}\r".format(message, self.counter, self.total))
The Other Jimmy 31:8ea194f6145b 90 stdout.flush()
The Other Jimmy 31:8ea194f6145b 91
The Other Jimmy 31:8ea194f6145b 92 def cache_file (self, url) :
The Other Jimmy 31:8ea194f6145b 93 """Low level interface to caching a single file.
The Other Jimmy 31:8ea194f6145b 94
The Other Jimmy 31:8ea194f6145b 95 :param url: The URL to cache.
The Other Jimmy 31:8ea194f6145b 96 :type url: str
The Other Jimmy 31:8ea194f6145b 97 :rtype: None
The Other Jimmy 31:8ea194f6145b 98 """
The Other Jimmy 31:8ea194f6145b 99 if not self.silent : print("Caching {}...".format(url))
The Other Jimmy 31:8ea194f6145b 100 dest = join(self.data_path, strip_protocol(url))
The Other Jimmy 31:8ea194f6145b 101 try :
The Other Jimmy 31:8ea194f6145b 102 makedirs(dirname(dest))
The Other Jimmy 31:8ea194f6145b 103 except OSError as exc :
The Other Jimmy 31:8ea194f6145b 104 if exc.errno == EEXIST : pass
The Other Jimmy 31:8ea194f6145b 105 else : raise
The Other Jimmy 31:8ea194f6145b 106 try:
The Other Jimmy 31:8ea194f6145b 107 with open(dest, "wb+") as fd :
The Other Jimmy 31:8ea194f6145b 108 fd.write(urlopen(url).read())
The Other Jimmy 31:8ea194f6145b 109 except URLError as e:
The Other Jimmy 31:8ea194f6145b 110 stderr.write(e.reason)
The Other Jimmy 31:8ea194f6145b 111 self.counter += 1
The Other Jimmy 31:8ea194f6145b 112 self.display_counter("Caching Files")
The Other Jimmy 31:8ea194f6145b 113
The Other Jimmy 31:8ea194f6145b 114 def pdsc_to_pack (self, url) :
The Other Jimmy 31:8ea194f6145b 115 """Find the URL of the specified pack file described by a PDSC.
The Other Jimmy 31:8ea194f6145b 116
The Other Jimmy 31:8ea194f6145b 117 The PDSC is assumed to be cached and is looked up in the cache by its URL.
The Other Jimmy 31:8ea194f6145b 118
The Other Jimmy 31:8ea194f6145b 119 :param url: The url used to look up the PDSC.
The Other Jimmy 31:8ea194f6145b 120 :type url: str
The Other Jimmy 31:8ea194f6145b 121 :return: The url of the PACK file.
The Other Jimmy 31:8ea194f6145b 122 :rtype: str
The Other Jimmy 31:8ea194f6145b 123 """
The Other Jimmy 31:8ea194f6145b 124 content = self.pdsc_from_cache(url)
The Other Jimmy 31:8ea194f6145b 125 new_url = content.package.url.get_text()
The Other Jimmy 31:8ea194f6145b 126 if not new_url.endswith("/") :
The Other Jimmy 31:8ea194f6145b 127 new_url = new_url + "/"
The Other Jimmy 31:8ea194f6145b 128 return (new_url + content.package.vendor.get_text() + "." +
The Other Jimmy 31:8ea194f6145b 129 content.package.find('name').get_text() + "." +
The Other Jimmy 31:8ea194f6145b 130 largest_version(content) + ".pack")
The Other Jimmy 31:8ea194f6145b 131
The Other Jimmy 31:8ea194f6145b 132 def cache_pdsc_and_pack (self, url) :
The Other Jimmy 31:8ea194f6145b 133 self.cache_file(url)
The Other Jimmy 31:8ea194f6145b 134 try :
The Other Jimmy 31:8ea194f6145b 135 self.cache_file(self.pdsc_to_pack(url))
The Other Jimmy 31:8ea194f6145b 136 except AttributeError :
The Other Jimmy 31:8ea194f6145b 137 stderr.write("[ ERROR ] {} does not appear to be a conforming .pdsc file\n".format(url))
The Other Jimmy 31:8ea194f6145b 138 self.counter += 1
The Other Jimmy 31:8ea194f6145b 139
The Other Jimmy 31:8ea194f6145b 140 def get_urls(self):
The Other Jimmy 31:8ea194f6145b 141 """Extract the URLs of all know PDSC files.
The Other Jimmy 31:8ea194f6145b 142
The Other Jimmy 31:8ea194f6145b 143 Will pull the index from the internet if it is not cached.
The Other Jimmy 31:8ea194f6145b 144
The Other Jimmy 31:8ea194f6145b 145 :return: A list of all PDSC URLs
The Other Jimmy 31:8ea194f6145b 146 :rtype: [str]
The Other Jimmy 31:8ea194f6145b 147 """
The Other Jimmy 31:8ea194f6145b 148 if not self.urls :
The Other Jimmy 31:8ea194f6145b 149 try : root_data = self.pdsc_from_cache(RootPackURL)
The Other Jimmy 31:8ea194f6145b 150 except IOError : root_data = self.cache_and_parse(RootPackURL)
The Other Jimmy 31:8ea194f6145b 151 self.urls = ["/".join([pdsc.get('url').strip("/"),
The Other Jimmy 31:8ea194f6145b 152 pdsc.get('name').strip("/")])
The Other Jimmy 31:8ea194f6145b 153 for pdsc in root_data.find_all("pdsc")]
The Other Jimmy 31:8ea194f6145b 154 return self.urls
The Other Jimmy 31:8ea194f6145b 155
theotherjimmy 43:2a7da56ebd24 156 def _get_sectors(self, device):
theotherjimmy 43:2a7da56ebd24 157 """Extract sector sizes from device FLM algorithm
theotherjimmy 43:2a7da56ebd24 158
theotherjimmy 43:2a7da56ebd24 159 Will return None if there is no algorithm, pdsc URL formatted in correctly
theotherjimmy 43:2a7da56ebd24 160
theotherjimmy 43:2a7da56ebd24 161 :return: A list tuples of sector start and size
theotherjimmy 43:2a7da56ebd24 162 :rtype: [list]
theotherjimmy 43:2a7da56ebd24 163 """
theotherjimmy 43:2a7da56ebd24 164 try:
theotherjimmy 43:2a7da56ebd24 165 pack = self.pack_from_cache(device)
theotherjimmy 43:2a7da56ebd24 166 ret = []
theotherjimmy 43:2a7da56ebd24 167 for filename in device['algorithm'].keys():
theotherjimmy 43:2a7da56ebd24 168 try:
theotherjimmy 43:2a7da56ebd24 169 flm = pack.open(filename)
theotherjimmy 43:2a7da56ebd24 170 flash_alg = PackFlashAlgo(flm.read())
theotherjimmy 43:2a7da56ebd24 171 sectors = [(flash_alg.flash_start + offset, size)
theotherjimmy 43:2a7da56ebd24 172 for offset, size in flash_alg.sector_sizes]
theotherjimmy 43:2a7da56ebd24 173 ret.extend(sectors)
theotherjimmy 43:2a7da56ebd24 174 except Exception:
theotherjimmy 43:2a7da56ebd24 175 pass
theotherjimmy 43:2a7da56ebd24 176 ret.sort(key=lambda sector: sector[0])
theotherjimmy 43:2a7da56ebd24 177 return ret
theotherjimmy 43:2a7da56ebd24 178 except Exception:
theotherjimmy 43:2a7da56ebd24 179 return None
theotherjimmy 43:2a7da56ebd24 180
The Other Jimmy 31:8ea194f6145b 181 def _extract_dict(self, device, filename, pack) :
The Other Jimmy 31:8ea194f6145b 182 to_ret = dict(pdsc_file=filename, pack_file=pack)
The Other Jimmy 31:8ea194f6145b 183 try : to_ret["memory"] = dict([(m["id"], dict(start=m["start"],
The Other Jimmy 31:8ea194f6145b 184 size=m["size"]))
The Other Jimmy 31:8ea194f6145b 185 for m in device("memory")])
theotherjimmy 43:2a7da56ebd24 186 except (KeyError, TypeError, IndexError) as e:
theotherjimmy 43:2a7da56ebd24 187 try : to_ret["memory"] = dict([(m["name"], dict(start=m["start"],
theotherjimmy 43:2a7da56ebd24 188 size=m["size"]))
theotherjimmy 43:2a7da56ebd24 189 for m in device("memory")])
theotherjimmy 43:2a7da56ebd24 190 except (KeyError, TypeError, IndexError) as e : pass
The Other Jimmy 31:8ea194f6145b 191 try: algorithms = device("algorithm")
The Other Jimmy 31:8ea194f6145b 192 except:
The Other Jimmy 31:8ea194f6145b 193 try: algorithms = device.parent("algorithm")
The Other Jimmy 31:8ea194f6145b 194 except: pass
The Other Jimmy 31:8ea194f6145b 195 else:
The Other Jimmy 31:8ea194f6145b 196 if not algorithms:
The Other Jimmy 31:8ea194f6145b 197 try: algorithms = device.parent("algorithm")
The Other Jimmy 31:8ea194f6145b 198 except: pass
The Other Jimmy 31:8ea194f6145b 199 try : to_ret["algorithm"] = dict([(algo.get("name").replace('\\','/'),
The Other Jimmy 31:8ea194f6145b 200 dict(start=algo["start"],
The Other Jimmy 31:8ea194f6145b 201 size=algo["size"],
The Other Jimmy 31:8ea194f6145b 202 ramstart=algo.get("ramstart",None),
The Other Jimmy 31:8ea194f6145b 203 ramsize=algo.get("ramsize",None),
The Other Jimmy 31:8ea194f6145b 204 default=algo.get("default",1)))
The Other Jimmy 31:8ea194f6145b 205 for algo in algorithms])
The Other Jimmy 31:8ea194f6145b 206 except (KeyError, TypeError, IndexError) as e: pass
The Other Jimmy 31:8ea194f6145b 207 try: to_ret["debug"] = device.parent.parent.debug["svd"]
The Other Jimmy 31:8ea194f6145b 208 except (KeyError, TypeError, IndexError) as e : pass
The Other Jimmy 31:8ea194f6145b 209 try: to_ret["debug"] = device.parent.debug["svd"]
The Other Jimmy 31:8ea194f6145b 210 except (KeyError, TypeError, IndexError) as e : pass
The Other Jimmy 31:8ea194f6145b 211 try: to_ret["debug"] = device.debug["svd"]
The Other Jimmy 31:8ea194f6145b 212 except (KeyError, TypeError, IndexError) as e : pass
The Other Jimmy 31:8ea194f6145b 213
The Other Jimmy 31:8ea194f6145b 214 to_ret["compile"] = {}
The Other Jimmy 31:8ea194f6145b 215 try: compile_l1 = device.parent("compile")
The Other Jimmy 31:8ea194f6145b 216 except (KeyError, TypeError, IndexError) as e : compile_l1 = []
The Other Jimmy 31:8ea194f6145b 217 try: compile_l2 = device.parent.parent("compile")
The Other Jimmy 31:8ea194f6145b 218 except (KeyError, TypeError, IndexError) as e : compile_l2 = []
The Other Jimmy 31:8ea194f6145b 219 compile = compile_l2 + compile_l1
The Other Jimmy 31:8ea194f6145b 220 for c in compile:
The Other Jimmy 31:8ea194f6145b 221 try: to_ret["compile"]["header"] = c["header"]
The Other Jimmy 31:8ea194f6145b 222 except (KeyError, TypeError, IndexError) as e : pass
The Other Jimmy 31:8ea194f6145b 223 try: to_ret["compile"]["define"] = c["define"]
The Other Jimmy 31:8ea194f6145b 224 except (KeyError, TypeError, IndexError) as e : pass
The Other Jimmy 31:8ea194f6145b 225
The Other Jimmy 31:8ea194f6145b 226 try: to_ret["core"] = device.parent.processor['dcore']
The Other Jimmy 31:8ea194f6145b 227 except (KeyError, TypeError, IndexError) as e : pass
The Other Jimmy 31:8ea194f6145b 228 try: to_ret["core"] = device.parent.parent.processor['dcore']
The Other Jimmy 31:8ea194f6145b 229 except (KeyError, TypeError, IndexError) as e : pass
The Other Jimmy 31:8ea194f6145b 230
The Other Jimmy 31:8ea194f6145b 231 to_ret["processor"] = {}
The Other Jimmy 31:8ea194f6145b 232 try: proc_l1 = device("processor")
The Other Jimmy 31:8ea194f6145b 233 except (KeyError, TypeError, IndexError) as e: proc_l1 = []
The Other Jimmy 31:8ea194f6145b 234 try: proc_l2 = device.parent("processor")
The Other Jimmy 31:8ea194f6145b 235 except (KeyError, TypeError, IndexError) as e: proc_l2 = []
The Other Jimmy 31:8ea194f6145b 236 try: proc_l3 = device.parent.parent("processor")
The Other Jimmy 31:8ea194f6145b 237 except (KeyError, TypeError, IndexError) as e: proc_l3 = []
The Other Jimmy 31:8ea194f6145b 238 proc = proc_l3 + proc_l2 + proc_l1
The Other Jimmy 31:8ea194f6145b 239 for p in proc:
The Other Jimmy 31:8ea194f6145b 240 try: to_ret["processor"]["fpu"] = p['dfpu']
The Other Jimmy 31:8ea194f6145b 241 except (KeyError, TypeError, IndexError) as e: pass
The Other Jimmy 31:8ea194f6145b 242 try: to_ret["processor"]["endianness"] = p['dendian']
The Other Jimmy 31:8ea194f6145b 243 except (KeyError, TypeError, IndexError) as e: pass
The Other Jimmy 31:8ea194f6145b 244 try: to_ret["processor"]["clock"] = p['dclock']
The Other Jimmy 31:8ea194f6145b 245 except (KeyError, TypeError, IndexError) as e: pass
The Other Jimmy 31:8ea194f6145b 246
The Other Jimmy 31:8ea194f6145b 247 try: to_ret["vendor"] = device.parent['dvendor']
The Other Jimmy 31:8ea194f6145b 248 except (KeyError, TypeError, IndexError) as e: pass
The Other Jimmy 31:8ea194f6145b 249 try: to_ret["vendor"] = device.parent.parent['dvendor']
The Other Jimmy 31:8ea194f6145b 250 except (KeyError, TypeError, IndexError) as e: pass
The Other Jimmy 31:8ea194f6145b 251
The Other Jimmy 31:8ea194f6145b 252 if not to_ret["processor"]:
The Other Jimmy 31:8ea194f6145b 253 del to_ret["processor"]
The Other Jimmy 31:8ea194f6145b 254
The Other Jimmy 31:8ea194f6145b 255 if not to_ret["compile"]:
The Other Jimmy 31:8ea194f6145b 256 del to_ret["compile"]
The Other Jimmy 31:8ea194f6145b 257
The Other Jimmy 31:8ea194f6145b 258 to_ret['debug-interface'] = []
theotherjimmy 43:2a7da56ebd24 259 to_ret['sectors'] = self._get_sectors(to_ret)
The Other Jimmy 31:8ea194f6145b 260
The Other Jimmy 31:8ea194f6145b 261 return to_ret
The Other Jimmy 31:8ea194f6145b 262
The Other Jimmy 31:8ea194f6145b 263 def _generate_index_helper(self, d) :
The Other Jimmy 31:8ea194f6145b 264 try :
The Other Jimmy 31:8ea194f6145b 265 pack = self.pdsc_to_pack(d)
The Other Jimmy 31:8ea194f6145b 266 self._index.update(dict([(dev['dname'], self._extract_dict(dev, d, pack)) for dev in
The Other Jimmy 31:8ea194f6145b 267 (self.pdsc_from_cache(d)("device"))]))
The Other Jimmy 31:8ea194f6145b 268 except AttributeError as e :
The Other Jimmy 31:8ea194f6145b 269 stderr.write("[ ERROR ] file {}\n".format(d))
The Other Jimmy 31:8ea194f6145b 270 print(e)
The Other Jimmy 31:8ea194f6145b 271 self.counter += 1
The Other Jimmy 31:8ea194f6145b 272 self.display_counter("Generating Index")
The Other Jimmy 31:8ea194f6145b 273
The Other Jimmy 31:8ea194f6145b 274 def _generate_aliases_helper(self, d) :
The Other Jimmy 31:8ea194f6145b 275 try :
The Other Jimmy 31:8ea194f6145b 276 mydict = []
The Other Jimmy 31:8ea194f6145b 277 for dev in self.pdsc_from_cache(d)("board"):
The Other Jimmy 31:8ea194f6145b 278 try :
The Other Jimmy 31:8ea194f6145b 279 mydict.append((dev['name'], dev.mounteddevice['dname']))
The Other Jimmy 31:8ea194f6145b 280 except (KeyError, TypeError, IndexError) as e:
The Other Jimmy 31:8ea194f6145b 281 pass
The Other Jimmy 31:8ea194f6145b 282 self._aliases.update(dict(mydict))
The Other Jimmy 31:8ea194f6145b 283 except (AttributeError, TypeError) as e :
The Other Jimmy 31:8ea194f6145b 284 pass
The Other Jimmy 31:8ea194f6145b 285 self.counter += 1
The Other Jimmy 31:8ea194f6145b 286 self.display_counter("Scanning for Aliases")
The Other Jimmy 31:8ea194f6145b 287
The Other Jimmy 36:96847d42f010 288 def get_flash_algorthim_binary(self, device_name, all=False) :
The Other Jimmy 31:8ea194f6145b 289 """Retrieve the flash algorithm file for a particular part.
The Other Jimmy 31:8ea194f6145b 290
The Other Jimmy 31:8ea194f6145b 291 Assumes that both the PDSC and the PACK file associated with that part are in the cache.
The Other Jimmy 31:8ea194f6145b 292
The Other Jimmy 31:8ea194f6145b 293 :param device_name: The exact name of a device
The Other Jimmy 36:96847d42f010 294 :param all: Return an iterator of all flash algos for this device
The Other Jimmy 31:8ea194f6145b 295 :type device_name: str
The Other Jimmy 31:8ea194f6145b 296 :return: A file-like object that, when read, is the ELF file that describes the flashing algorithm
The Other Jimmy 36:96847d42f010 297 :return: A file-like object that, when read, is the ELF file that describes the flashing algorithm.
The Other Jimmy 36:96847d42f010 298 When "all" is set to True then an iterator for file-like objects is returned
The Other Jimmy 36:96847d42f010 299 :rtype: ZipExtFile or ZipExtFile iterator if all is True
The Other Jimmy 31:8ea194f6145b 300 """
The Other Jimmy 35:da9c89f8be7d 301 device = self.index[device_name]
The Other Jimmy 35:da9c89f8be7d 302 pack = self.pack_from_cache(device)
The Other Jimmy 36:96847d42f010 303 algo_itr = (pack.open(path) for path in device['algorithm'].keys())
The Other Jimmy 36:96847d42f010 304 return algo_itr if all else algo_itr.next()
The Other Jimmy 31:8ea194f6145b 305
The Other Jimmy 31:8ea194f6145b 306 def get_svd_file(self, device_name) :
The Other Jimmy 31:8ea194f6145b 307 """Retrieve the flash algorithm file for a particular part.
The Other Jimmy 31:8ea194f6145b 308
The Other Jimmy 31:8ea194f6145b 309 Assumes that both the PDSC and the PACK file associated with that part are in the cache.
The Other Jimmy 31:8ea194f6145b 310
The Other Jimmy 31:8ea194f6145b 311 :param device_name: The exact name of a device
The Other Jimmy 31:8ea194f6145b 312 :type device_name: str
The Other Jimmy 31:8ea194f6145b 313 :return: A file-like object that, when read, is the ELF file that describes the flashing algorithm
The Other Jimmy 31:8ea194f6145b 314 :rtype: ZipExtFile
The Other Jimmy 31:8ea194f6145b 315 """
The Other Jimmy 35:da9c89f8be7d 316 device = self.index[device_name]
The Other Jimmy 35:da9c89f8be7d 317 pack = self.pack_from_cache(device)
The Other Jimmy 31:8ea194f6145b 318 return pack.open(device['debug'])
The Other Jimmy 31:8ea194f6145b 319
The Other Jimmy 31:8ea194f6145b 320 def generate_index(self) :
The Other Jimmy 31:8ea194f6145b 321 self._index = {}
The Other Jimmy 31:8ea194f6145b 322 self.counter = 0
The Other Jimmy 31:8ea194f6145b 323 do_queue(Reader, self._generate_index_helper, self.get_urls())
The Other Jimmy 31:8ea194f6145b 324 with open(LocalPackIndex, "wb+") as out:
The Other Jimmy 31:8ea194f6145b 325 self._index["version"] = "0.1.0"
The Other Jimmy 31:8ea194f6145b 326 dump(self._index, out)
The Other Jimmy 31:8ea194f6145b 327 stdout.write("\n")
The Other Jimmy 31:8ea194f6145b 328
The Other Jimmy 31:8ea194f6145b 329 def generate_aliases(self) :
The Other Jimmy 31:8ea194f6145b 330 self._aliases = {}
The Other Jimmy 31:8ea194f6145b 331 self.counter = 0
The Other Jimmy 31:8ea194f6145b 332 do_queue(Reader, self._generate_aliases_helper, self.get_urls())
The Other Jimmy 31:8ea194f6145b 333 with open(LocalPackAliases, "wb+") as out:
The Other Jimmy 31:8ea194f6145b 334 dump(self._aliases, out)
The Other Jimmy 31:8ea194f6145b 335 stdout.write("\n")
The Other Jimmy 31:8ea194f6145b 336
The Other Jimmy 31:8ea194f6145b 337 def find_device(self, match) :
The Other Jimmy 31:8ea194f6145b 338 choices = process.extract(match, self.index.keys(), limit=len(self.index))
The Other Jimmy 31:8ea194f6145b 339 choices = sorted([(v, k) for k, v in choices], reverse=True)
The Other Jimmy 31:8ea194f6145b 340 if choices : choices = list(takewhile(lambda t: t[0] == choices[0][0], choices))
The Other Jimmy 31:8ea194f6145b 341 return [(v, self.index[v]) for k,v in choices]
The Other Jimmy 31:8ea194f6145b 342
The Other Jimmy 31:8ea194f6145b 343 def dump_index_to_file(self, file) :
The Other Jimmy 31:8ea194f6145b 344 with open(file, "wb+") as out:
The Other Jimmy 31:8ea194f6145b 345 dump(self.index, out)
The Other Jimmy 31:8ea194f6145b 346
The Other Jimmy 31:8ea194f6145b 347 @property
The Other Jimmy 31:8ea194f6145b 348 def index(self) :
The Other Jimmy 31:8ea194f6145b 349 """An index of most of the important data in all cached PDSC files.
The Other Jimmy 31:8ea194f6145b 350
The Other Jimmy 31:8ea194f6145b 351 :Example:
The Other Jimmy 31:8ea194f6145b 352
The Other Jimmy 31:8ea194f6145b 353 >>> from ArmPackManager import Cache
The Other Jimmy 31:8ea194f6145b 354 >>> a = Cache()
The Other Jimmy 31:8ea194f6145b 355 >>> a.index["LPC1768"]
The Other Jimmy 31:8ea194f6145b 356 {u'algorithm': {u'RAMsize': u'0x0FE0',
The Other Jimmy 31:8ea194f6145b 357 u'RAMstart': u'0x10000000',
The Other Jimmy 31:8ea194f6145b 358 u'name': u'Flash/LPC_IAP_512.FLM',
The Other Jimmy 31:8ea194f6145b 359 u'size': u'0x80000',
The Other Jimmy 31:8ea194f6145b 360 u'start': u'0x00000000'},
The Other Jimmy 31:8ea194f6145b 361 u'compile': [u'Device/Include/LPC17xx.h', u'LPC175x_6x'],
The Other Jimmy 31:8ea194f6145b 362 u'debug': u'SVD/LPC176x5x.svd',
The Other Jimmy 31:8ea194f6145b 363 u'pdsc_file': u'http://www.keil.com/pack/Keil.LPC1700_DFP.pdsc',
The Other Jimmy 31:8ea194f6145b 364 u'memory': {u'IRAM1': {u'size': u'0x8000', u'start': u'0x10000000'},
The Other Jimmy 31:8ea194f6145b 365 u'IRAM2': {u'size': u'0x8000', u'start': u'0x2007C000'},
The Other Jimmy 31:8ea194f6145b 366 u'IROM1': {u'size': u'0x80000', u'start': u'0x00000000'}}}
The Other Jimmy 31:8ea194f6145b 367
The Other Jimmy 31:8ea194f6145b 368
The Other Jimmy 31:8ea194f6145b 369 """
The Other Jimmy 31:8ea194f6145b 370 if not self._index :
The Other Jimmy 31:8ea194f6145b 371 with open(LocalPackIndex) as i :
The Other Jimmy 31:8ea194f6145b 372 self._index = load(i)
The Other Jimmy 31:8ea194f6145b 373 return self._index
The Other Jimmy 31:8ea194f6145b 374 @property
The Other Jimmy 31:8ea194f6145b 375 def aliases(self) :
The Other Jimmy 31:8ea194f6145b 376 """An index of most of the important data in all cached PDSC files.
The Other Jimmy 31:8ea194f6145b 377
The Other Jimmy 31:8ea194f6145b 378 :Example:
The Other Jimmy 31:8ea194f6145b 379
The Other Jimmy 31:8ea194f6145b 380 >>> from ArmPackManager import Cache
The Other Jimmy 31:8ea194f6145b 381 >>> a = Cache()
The Other Jimmy 31:8ea194f6145b 382 >>> a.index["LPC1768"]
The Other Jimmy 31:8ea194f6145b 383 {u'algorithm': {u'RAMsize': u'0x0FE0',
The Other Jimmy 31:8ea194f6145b 384 u'RAMstart': u'0x10000000',
The Other Jimmy 31:8ea194f6145b 385 u'name': u'Flash/LPC_IAP_512.FLM',
The Other Jimmy 31:8ea194f6145b 386 u'size': u'0x80000',
The Other Jimmy 31:8ea194f6145b 387 u'start': u'0x00000000'},
The Other Jimmy 31:8ea194f6145b 388 u'compile': [u'Device/Include/LPC17xx.h', u'LPC175x_6x'],
The Other Jimmy 31:8ea194f6145b 389 u'debug': u'SVD/LPC176x5x.svd',
The Other Jimmy 31:8ea194f6145b 390 u'pdsc_file': u'http://www.keil.com/pack/Keil.LPC1700_DFP.pdsc',
The Other Jimmy 31:8ea194f6145b 391 u'memory': {u'IRAM1': {u'size': u'0x8000', u'start': u'0x10000000'},
The Other Jimmy 31:8ea194f6145b 392 u'IRAM2': {u'size': u'0x8000', u'start': u'0x2007C000'},
The Other Jimmy 31:8ea194f6145b 393 u'IROM1': {u'size': u'0x80000', u'start': u'0x00000000'}}}
The Other Jimmy 31:8ea194f6145b 394
The Other Jimmy 31:8ea194f6145b 395
The Other Jimmy 31:8ea194f6145b 396 """
The Other Jimmy 31:8ea194f6145b 397 if not self._aliases :
The Other Jimmy 36:96847d42f010 398 with open(LocalPackAliases) as i :
The Other Jimmy 31:8ea194f6145b 399 self._aliases = load(i)
The Other Jimmy 31:8ea194f6145b 400 return self._aliases
The Other Jimmy 31:8ea194f6145b 401
The Other Jimmy 31:8ea194f6145b 402 def cache_everything(self) :
The Other Jimmy 31:8ea194f6145b 403 """Cache every PACK and PDSC file known.
The Other Jimmy 31:8ea194f6145b 404
The Other Jimmy 31:8ea194f6145b 405 Generates an index afterwards.
The Other Jimmy 31:8ea194f6145b 406
The Other Jimmy 31:8ea194f6145b 407 .. note:: This process may use 4GB of drive space and take upwards of 10 minutes to complete.
The Other Jimmy 31:8ea194f6145b 408 """
The Other Jimmy 31:8ea194f6145b 409 self.cache_pack_list(self.get_urls())
The Other Jimmy 31:8ea194f6145b 410 self.generate_index()
The Other Jimmy 31:8ea194f6145b 411 self.generate_aliases()
The Other Jimmy 31:8ea194f6145b 412
The Other Jimmy 31:8ea194f6145b 413 def cache_descriptors(self) :
The Other Jimmy 31:8ea194f6145b 414 """Cache every PDSC file known.
The Other Jimmy 31:8ea194f6145b 415
The Other Jimmy 31:8ea194f6145b 416 Generates an index afterwards.
The Other Jimmy 31:8ea194f6145b 417
The Other Jimmy 31:8ea194f6145b 418 .. note:: This process may use 11MB of drive space and take upwards of 1 minute.
The Other Jimmy 31:8ea194f6145b 419 """
The Other Jimmy 31:8ea194f6145b 420 self.cache_descriptor_list(self.get_urls())
The Other Jimmy 31:8ea194f6145b 421 self.generate_index()
The Other Jimmy 31:8ea194f6145b 422 self.generate_aliases()
The Other Jimmy 31:8ea194f6145b 423
The Other Jimmy 31:8ea194f6145b 424 def cache_descriptor_list(self, list) :
The Other Jimmy 31:8ea194f6145b 425 """Cache a list of PDSC files.
The Other Jimmy 31:8ea194f6145b 426
The Other Jimmy 31:8ea194f6145b 427 :param list: URLs of PDSC files to cache.
The Other Jimmy 31:8ea194f6145b 428 :type list: [str]
The Other Jimmy 31:8ea194f6145b 429 """
The Other Jimmy 31:8ea194f6145b 430 self.total = len(list)
The Other Jimmy 31:8ea194f6145b 431 self.display_counter("Caching Files")
The Other Jimmy 31:8ea194f6145b 432 do_queue(Reader, self.cache_file, list)
The Other Jimmy 31:8ea194f6145b 433 stdout.write("\n")
The Other Jimmy 31:8ea194f6145b 434
The Other Jimmy 31:8ea194f6145b 435 def cache_pack_list(self, list) :
The Other Jimmy 31:8ea194f6145b 436 """Cache a list of PACK files, referenced by their PDSC URL
The Other Jimmy 31:8ea194f6145b 437
The Other Jimmy 31:8ea194f6145b 438 :param list: URLs of PDSC files to cache.
The Other Jimmy 31:8ea194f6145b 439 :type list: [str]
The Other Jimmy 31:8ea194f6145b 440 """
The Other Jimmy 31:8ea194f6145b 441 self.total = len(list) * 2
The Other Jimmy 31:8ea194f6145b 442 self.display_counter("Caching Files")
The Other Jimmy 31:8ea194f6145b 443 do_queue(Reader, self.cache_pdsc_and_pack, list)
The Other Jimmy 31:8ea194f6145b 444 stdout.write("\n")
The Other Jimmy 31:8ea194f6145b 445
The Other Jimmy 31:8ea194f6145b 446 def pdsc_from_cache(self, url) :
The Other Jimmy 31:8ea194f6145b 447 """Low level inteface for extracting a PDSC file from the cache.
The Other Jimmy 31:8ea194f6145b 448
The Other Jimmy 31:8ea194f6145b 449 Assumes that the file specified is a PDSC file and is in the cache.
The Other Jimmy 31:8ea194f6145b 450
The Other Jimmy 31:8ea194f6145b 451 :param url: The URL of a PDSC file.
The Other Jimmy 31:8ea194f6145b 452 :type url: str
The Other Jimmy 31:8ea194f6145b 453 :return: A parsed representation of the PDSC file.
The Other Jimmy 31:8ea194f6145b 454 :rtype: BeautifulSoup
The Other Jimmy 31:8ea194f6145b 455 """
The Other Jimmy 31:8ea194f6145b 456 dest = join(self.data_path, strip_protocol(url))
The Other Jimmy 31:8ea194f6145b 457 with open(dest, "r") as fd :
The Other Jimmy 31:8ea194f6145b 458 return BeautifulSoup(fd, "html.parser")
The Other Jimmy 31:8ea194f6145b 459
The Other Jimmy 35:da9c89f8be7d 460 def pack_from_cache(self, device) :
The Other Jimmy 31:8ea194f6145b 461 """Low level inteface for extracting a PACK file from the cache.
The Other Jimmy 31:8ea194f6145b 462
The Other Jimmy 31:8ea194f6145b 463 Assumes that the file specified is a PACK file and is in the cache.
The Other Jimmy 31:8ea194f6145b 464
The Other Jimmy 31:8ea194f6145b 465 :param url: The URL of a PACK file.
The Other Jimmy 31:8ea194f6145b 466 :type url: str
The Other Jimmy 31:8ea194f6145b 467 :return: A parsed representation of the PACK file.
The Other Jimmy 31:8ea194f6145b 468 :rtype: ZipFile
The Other Jimmy 31:8ea194f6145b 469 """
The Other Jimmy 31:8ea194f6145b 470 return ZipFile(join(self.data_path,
The Other Jimmy 31:8ea194f6145b 471 strip_protocol(device['pack_file'])))
The Other Jimmy 31:8ea194f6145b 472
The Other Jimmy 31:8ea194f6145b 473 def gen_dict_from_cache() :
The Other Jimmy 31:8ea194f6145b 474 pdsc_files = pdsc_from_cache(RootPackUrl)
The Other Jimmy 31:8ea194f6145b 475
The Other Jimmy 31:8ea194f6145b 476 def cache_and_parse(self, url) :
The Other Jimmy 31:8ea194f6145b 477 """A low level shortcut that Caches and Parses a PDSC file.
The Other Jimmy 31:8ea194f6145b 478
The Other Jimmy 31:8ea194f6145b 479 :param url: The URL of the PDSC file.
The Other Jimmy 31:8ea194f6145b 480 :type url: str
The Other Jimmy 31:8ea194f6145b 481 :return: A parsed representation of the PDSC file.
The Other Jimmy 31:8ea194f6145b 482 :rtype: BeautifulSoup
The Other Jimmy 31:8ea194f6145b 483 """
The Other Jimmy 31:8ea194f6145b 484 self.cache_file(url)
The Other Jimmy 31:8ea194f6145b 485 return self.pdsc_from_cache(url)