EL4121 Embedded System / mbed-os

mbed-os

Dependents:   cobaLCDJoyMotor_Thread odometry_omni_3roda_v3 odometry_omni_3roda_v1 odometry_omni_3roda_v2 ... more

tools/arm_pack_manager/__init__.py@0:b74591d5ab33, 2017-12-11 (annotated)

Committer:
be_bryan
Date:
Mon Dec 11 17:54:04 2017 +0000
Revision:
0:b74591d5ab33
motor ++

Who changed what in which revision?

UserRevisionLine numberNew contents of line
be_bryan 0:b74591d5ab33 1 from urllib2 import urlopen, URLError
be_bryan 0:b74591d5ab33 2 from bs4 import BeautifulSoup
be_bryan 0:b74591d5ab33 3 from os.path import join, dirname, basename
be_bryan 0:b74591d5ab33 4 from os import makedirs
be_bryan 0:b74591d5ab33 5 from errno import EEXIST
be_bryan 0:b74591d5ab33 6 from threading import Thread
be_bryan 0:b74591d5ab33 7 from Queue import Queue
be_bryan 0:b74591d5ab33 8 from re import compile, sub
be_bryan 0:b74591d5ab33 9 from sys import stderr, stdout
be_bryan 0:b74591d5ab33 10 from itertools import takewhile
be_bryan 0:b74591d5ab33 11 import argparse
be_bryan 0:b74591d5ab33 12 from json import dump, load
be_bryan 0:b74591d5ab33 13 from zipfile import ZipFile
be_bryan 0:b74591d5ab33 14 from tempfile import gettempdir
be_bryan 0:b74591d5ab33 15 import warnings
be_bryan 0:b74591d5ab33 16 from distutils.version import LooseVersion
be_bryan 0:b74591d5ab33 17
be_bryan 0:b74591d5ab33 18 warnings.filterwarnings("ignore")
be_bryan 0:b74591d5ab33 19
be_bryan 0:b74591d5ab33 20 from fuzzywuzzy import process
be_bryan 0:b74591d5ab33 21
be_bryan 0:b74591d5ab33 22 RootPackURL = "http://www.keil.com/pack/index.idx"
be_bryan 0:b74591d5ab33 23
be_bryan 0:b74591d5ab33 24 LocalPackDir = dirname(__file__)
be_bryan 0:b74591d5ab33 25 LocalPackIndex = join(LocalPackDir, "index.json")
be_bryan 0:b74591d5ab33 26 LocalPackAliases = join(LocalPackDir, "aliases.json")
be_bryan 0:b74591d5ab33 27
be_bryan 0:b74591d5ab33 28
be_bryan 0:b74591d5ab33 29 protocol_matcher = compile("\w*://")
be_bryan 0:b74591d5ab33 30 def strip_protocol(url) :
be_bryan 0:b74591d5ab33 31 return protocol_matcher.sub("", str(url))
be_bryan 0:b74591d5ab33 32
be_bryan 0:b74591d5ab33 33 def largest_version(content) :
be_bryan 0:b74591d5ab33 34 return sorted([t['version'] for t in content.package.releases('release')],
be_bryan 0:b74591d5ab33 35 reverse=True, key=lambda v: LooseVersion(v))[0]
be_bryan 0:b74591d5ab33 36
be_bryan 0:b74591d5ab33 37 def do_queue(Class, function, interable) :
be_bryan 0:b74591d5ab33 38 q = Queue()
be_bryan 0:b74591d5ab33 39 threads = [Class(q, function) for each in range(20)]
be_bryan 0:b74591d5ab33 40 for each in threads :
be_bryan 0:b74591d5ab33 41 each.setDaemon(True)
be_bryan 0:b74591d5ab33 42 each.start()
be_bryan 0:b74591d5ab33 43 for thing in interable :
be_bryan 0:b74591d5ab33 44 q.put(thing)
be_bryan 0:b74591d5ab33 45 q.join()
be_bryan 0:b74591d5ab33 46
be_bryan 0:b74591d5ab33 47 class Reader (Thread) :
be_bryan 0:b74591d5ab33 48 def __init__(self, queue, func) :
be_bryan 0:b74591d5ab33 49 Thread.__init__(self)
be_bryan 0:b74591d5ab33 50 self.queue = queue
be_bryan 0:b74591d5ab33 51 self.func = func
be_bryan 0:b74591d5ab33 52 def run(self) :
be_bryan 0:b74591d5ab33 53 while True :
be_bryan 0:b74591d5ab33 54 url = self.queue.get()
be_bryan 0:b74591d5ab33 55 self.func(url)
be_bryan 0:b74591d5ab33 56 self.queue.task_done()
be_bryan 0:b74591d5ab33 57
be_bryan 0:b74591d5ab33 58
be_bryan 0:b74591d5ab33 59 class Cache () :
be_bryan 0:b74591d5ab33 60 """ The Cache object is the only relevant API object at the moment
be_bryan 0:b74591d5ab33 61
be_bryan 0:b74591d5ab33 62 Constructing the Cache object does not imply any caching.
be_bryan 0:b74591d5ab33 63 A user of the API must explicitly call caching functions.
be_bryan 0:b74591d5ab33 64
be_bryan 0:b74591d5ab33 65 :param silent: A boolean that, when True, significantly reduces the printing of this Object
be_bryan 0:b74591d5ab33 66 :type silent: bool
be_bryan 0:b74591d5ab33 67 :param no_timeouts: A boolean that, when True, disables the default connection timeout and low speed timeout for downloading things.
be_bryan 0:b74591d5ab33 68 :type no_timeouts: bool
be_bryan 0:b74591d5ab33 69 """
be_bryan 0:b74591d5ab33 70 def __init__ (self, silent, no_timeouts) :
be_bryan 0:b74591d5ab33 71 self.silent = silent
be_bryan 0:b74591d5ab33 72 self.counter = 0
be_bryan 0:b74591d5ab33 73 self.total = 1
be_bryan 0:b74591d5ab33 74 self._index = {}
be_bryan 0:b74591d5ab33 75 self._aliases = {}
be_bryan 0:b74591d5ab33 76 self.urls = None
be_bryan 0:b74591d5ab33 77 self.no_timeouts = no_timeouts
be_bryan 0:b74591d5ab33 78 self.data_path = gettempdir()
be_bryan 0:b74591d5ab33 79
be_bryan 0:b74591d5ab33 80 def display_counter (self, message) :
be_bryan 0:b74591d5ab33 81 stdout.write("{} {}/{}\r".format(message, self.counter, self.total))
be_bryan 0:b74591d5ab33 82 stdout.flush()
be_bryan 0:b74591d5ab33 83
be_bryan 0:b74591d5ab33 84 def cache_file (self, url) :
be_bryan 0:b74591d5ab33 85 """Low level interface to caching a single file.
be_bryan 0:b74591d5ab33 86
be_bryan 0:b74591d5ab33 87 :param url: The URL to cache.
be_bryan 0:b74591d5ab33 88 :type url: str
be_bryan 0:b74591d5ab33 89 :rtype: None
be_bryan 0:b74591d5ab33 90 """
be_bryan 0:b74591d5ab33 91 if not self.silent : print("Caching {}...".format(url))
be_bryan 0:b74591d5ab33 92 dest = join(self.data_path, strip_protocol(url))
be_bryan 0:b74591d5ab33 93 try :
be_bryan 0:b74591d5ab33 94 makedirs(dirname(dest))
be_bryan 0:b74591d5ab33 95 except OSError as exc :
be_bryan 0:b74591d5ab33 96 if exc.errno == EEXIST : pass
be_bryan 0:b74591d5ab33 97 else : raise
be_bryan 0:b74591d5ab33 98 try:
be_bryan 0:b74591d5ab33 99 with open(dest, "wb+") as fd :
be_bryan 0:b74591d5ab33 100 fd.write(urlopen(url).read())
be_bryan 0:b74591d5ab33 101 except URLError as e:
be_bryan 0:b74591d5ab33 102 stderr.write(e.reason)
be_bryan 0:b74591d5ab33 103 self.counter += 1
be_bryan 0:b74591d5ab33 104 self.display_counter("Caching Files")
be_bryan 0:b74591d5ab33 105
be_bryan 0:b74591d5ab33 106 def pdsc_to_pack (self, url) :
be_bryan 0:b74591d5ab33 107 """Find the URL of the specified pack file described by a PDSC.
be_bryan 0:b74591d5ab33 108
be_bryan 0:b74591d5ab33 109 The PDSC is assumed to be cached and is looked up in the cache by its URL.
be_bryan 0:b74591d5ab33 110
be_bryan 0:b74591d5ab33 111 :param url: The url used to look up the PDSC.
be_bryan 0:b74591d5ab33 112 :type url: str
be_bryan 0:b74591d5ab33 113 :return: The url of the PACK file.
be_bryan 0:b74591d5ab33 114 :rtype: str
be_bryan 0:b74591d5ab33 115 """
be_bryan 0:b74591d5ab33 116 content = self.pdsc_from_cache(url)
be_bryan 0:b74591d5ab33 117 new_url = content.package.url.get_text()
be_bryan 0:b74591d5ab33 118 if not new_url.endswith("/") :
be_bryan 0:b74591d5ab33 119 new_url = new_url + "/"
be_bryan 0:b74591d5ab33 120 return (new_url + content.package.vendor.get_text() + "." +
be_bryan 0:b74591d5ab33 121 content.package.find('name').get_text() + "." +
be_bryan 0:b74591d5ab33 122 largest_version(content) + ".pack")
be_bryan 0:b74591d5ab33 123
be_bryan 0:b74591d5ab33 124 def cache_pdsc_and_pack (self, url) :
be_bryan 0:b74591d5ab33 125 self.cache_file(url)
be_bryan 0:b74591d5ab33 126 try :
be_bryan 0:b74591d5ab33 127 self.cache_file(self.pdsc_to_pack(url))
be_bryan 0:b74591d5ab33 128 except AttributeError :
be_bryan 0:b74591d5ab33 129 stderr.write("[ ERROR ] {} does not appear to be a conforming .pdsc file\n".format(url))
be_bryan 0:b74591d5ab33 130 self.counter += 1
be_bryan 0:b74591d5ab33 131
be_bryan 0:b74591d5ab33 132 def get_urls(self):
be_bryan 0:b74591d5ab33 133 """Extract the URLs of all know PDSC files.
be_bryan 0:b74591d5ab33 134
be_bryan 0:b74591d5ab33 135 Will pull the index from the internet if it is not cached.
be_bryan 0:b74591d5ab33 136
be_bryan 0:b74591d5ab33 137 :return: A list of all PDSC URLs
be_bryan 0:b74591d5ab33 138 :rtype: [str]
be_bryan 0:b74591d5ab33 139 """
be_bryan 0:b74591d5ab33 140 if not self.urls :
be_bryan 0:b74591d5ab33 141 try : root_data = self.pdsc_from_cache(RootPackURL)
be_bryan 0:b74591d5ab33 142 except IOError : root_data = self.cache_and_parse(RootPackURL)
be_bryan 0:b74591d5ab33 143 self.urls = ["/".join([pdsc.get('url').strip("/"),
be_bryan 0:b74591d5ab33 144 pdsc.get('name').strip("/")])
be_bryan 0:b74591d5ab33 145 for pdsc in root_data.find_all("pdsc")]
be_bryan 0:b74591d5ab33 146 return self.urls
be_bryan 0:b74591d5ab33 147
be_bryan 0:b74591d5ab33 148 def _extract_dict(self, device, filename, pack) :
be_bryan 0:b74591d5ab33 149 to_ret = dict(pdsc_file=filename, pack_file=pack)
be_bryan 0:b74591d5ab33 150 try : to_ret["memory"] = dict([(m["id"], dict(start=m["start"],
be_bryan 0:b74591d5ab33 151 size=m["size"]))
be_bryan 0:b74591d5ab33 152 for m in device("memory")])
be_bryan 0:b74591d5ab33 153 except (KeyError, TypeError, IndexError) as e : pass
be_bryan 0:b74591d5ab33 154 try: algorithms = device("algorithm")
be_bryan 0:b74591d5ab33 155 except:
be_bryan 0:b74591d5ab33 156 try: algorithms = device.parent("algorithm")
be_bryan 0:b74591d5ab33 157 except: pass
be_bryan 0:b74591d5ab33 158 else:
be_bryan 0:b74591d5ab33 159 if not algorithms:
be_bryan 0:b74591d5ab33 160 try: algorithms = device.parent("algorithm")
be_bryan 0:b74591d5ab33 161 except: pass
be_bryan 0:b74591d5ab33 162 try : to_ret["algorithm"] = dict([(algo.get("name").replace('\\','/'),
be_bryan 0:b74591d5ab33 163 dict(start=algo["start"],
be_bryan 0:b74591d5ab33 164 size=algo["size"],
be_bryan 0:b74591d5ab33 165 ramstart=algo.get("ramstart",None),
be_bryan 0:b74591d5ab33 166 ramsize=algo.get("ramsize",None),
be_bryan 0:b74591d5ab33 167 default=algo.get("default",1)))
be_bryan 0:b74591d5ab33 168 for algo in algorithms])
be_bryan 0:b74591d5ab33 169 except (KeyError, TypeError, IndexError) as e: pass
be_bryan 0:b74591d5ab33 170 try: to_ret["debug"] = device.parent.parent.debug["svd"]
be_bryan 0:b74591d5ab33 171 except (KeyError, TypeError, IndexError) as e : pass
be_bryan 0:b74591d5ab33 172 try: to_ret["debug"] = device.parent.debug["svd"]
be_bryan 0:b74591d5ab33 173 except (KeyError, TypeError, IndexError) as e : pass
be_bryan 0:b74591d5ab33 174 try: to_ret["debug"] = device.debug["svd"]
be_bryan 0:b74591d5ab33 175 except (KeyError, TypeError, IndexError) as e : pass
be_bryan 0:b74591d5ab33 176
be_bryan 0:b74591d5ab33 177 to_ret["compile"] = {}
be_bryan 0:b74591d5ab33 178 try: compile_l1 = device.parent("compile")
be_bryan 0:b74591d5ab33 179 except (KeyError, TypeError, IndexError) as e : compile_l1 = []
be_bryan 0:b74591d5ab33 180 try: compile_l2 = device.parent.parent("compile")
be_bryan 0:b74591d5ab33 181 except (KeyError, TypeError, IndexError) as e : compile_l2 = []
be_bryan 0:b74591d5ab33 182 compile = compile_l2 + compile_l1
be_bryan 0:b74591d5ab33 183 for c in compile:
be_bryan 0:b74591d5ab33 184 try: to_ret["compile"]["header"] = c["header"]
be_bryan 0:b74591d5ab33 185 except (KeyError, TypeError, IndexError) as e : pass
be_bryan 0:b74591d5ab33 186 try: to_ret["compile"]["define"] = c["define"]
be_bryan 0:b74591d5ab33 187 except (KeyError, TypeError, IndexError) as e : pass
be_bryan 0:b74591d5ab33 188
be_bryan 0:b74591d5ab33 189 try: to_ret["core"] = device.parent.processor['dcore']
be_bryan 0:b74591d5ab33 190 except (KeyError, TypeError, IndexError) as e : pass
be_bryan 0:b74591d5ab33 191 try: to_ret["core"] = device.parent.parent.processor['dcore']
be_bryan 0:b74591d5ab33 192 except (KeyError, TypeError, IndexError) as e : pass
be_bryan 0:b74591d5ab33 193
be_bryan 0:b74591d5ab33 194 to_ret["processor"] = {}
be_bryan 0:b74591d5ab33 195 try: proc_l1 = device("processor")
be_bryan 0:b74591d5ab33 196 except (KeyError, TypeError, IndexError) as e: proc_l1 = []
be_bryan 0:b74591d5ab33 197 try: proc_l2 = device.parent("processor")
be_bryan 0:b74591d5ab33 198 except (KeyError, TypeError, IndexError) as e: proc_l2 = []
be_bryan 0:b74591d5ab33 199 try: proc_l3 = device.parent.parent("processor")
be_bryan 0:b74591d5ab33 200 except (KeyError, TypeError, IndexError) as e: proc_l3 = []
be_bryan 0:b74591d5ab33 201 proc = proc_l3 + proc_l2 + proc_l1
be_bryan 0:b74591d5ab33 202 for p in proc:
be_bryan 0:b74591d5ab33 203 try: to_ret["processor"]["fpu"] = p['dfpu']
be_bryan 0:b74591d5ab33 204 except (KeyError, TypeError, IndexError) as e: pass
be_bryan 0:b74591d5ab33 205 try: to_ret["processor"]["endianness"] = p['dendian']
be_bryan 0:b74591d5ab33 206 except (KeyError, TypeError, IndexError) as e: pass
be_bryan 0:b74591d5ab33 207 try: to_ret["processor"]["clock"] = p['dclock']
be_bryan 0:b74591d5ab33 208 except (KeyError, TypeError, IndexError) as e: pass
be_bryan 0:b74591d5ab33 209
be_bryan 0:b74591d5ab33 210 try: to_ret["vendor"] = device.parent['dvendor']
be_bryan 0:b74591d5ab33 211 except (KeyError, TypeError, IndexError) as e: pass
be_bryan 0:b74591d5ab33 212 try: to_ret["vendor"] = device.parent.parent['dvendor']
be_bryan 0:b74591d5ab33 213 except (KeyError, TypeError, IndexError) as e: pass
be_bryan 0:b74591d5ab33 214
be_bryan 0:b74591d5ab33 215 if not to_ret["processor"]:
be_bryan 0:b74591d5ab33 216 del to_ret["processor"]
be_bryan 0:b74591d5ab33 217
be_bryan 0:b74591d5ab33 218 if not to_ret["compile"]:
be_bryan 0:b74591d5ab33 219 del to_ret["compile"]
be_bryan 0:b74591d5ab33 220
be_bryan 0:b74591d5ab33 221 to_ret['debug-interface'] = []
be_bryan 0:b74591d5ab33 222
be_bryan 0:b74591d5ab33 223 return to_ret
be_bryan 0:b74591d5ab33 224
be_bryan 0:b74591d5ab33 225 def _generate_index_helper(self, d) :
be_bryan 0:b74591d5ab33 226 try :
be_bryan 0:b74591d5ab33 227 pack = self.pdsc_to_pack(d)
be_bryan 0:b74591d5ab33 228 self._index.update(dict([(dev['dname'], self._extract_dict(dev, d, pack)) for dev in
be_bryan 0:b74591d5ab33 229 (self.pdsc_from_cache(d)("device"))]))
be_bryan 0:b74591d5ab33 230 except AttributeError as e :
be_bryan 0:b74591d5ab33 231 stderr.write("[ ERROR ] file {}\n".format(d))
be_bryan 0:b74591d5ab33 232 print(e)
be_bryan 0:b74591d5ab33 233 self.counter += 1
be_bryan 0:b74591d5ab33 234 self.display_counter("Generating Index")
be_bryan 0:b74591d5ab33 235
be_bryan 0:b74591d5ab33 236 def _generate_aliases_helper(self, d) :
be_bryan 0:b74591d5ab33 237 try :
be_bryan 0:b74591d5ab33 238 mydict = []
be_bryan 0:b74591d5ab33 239 for dev in self.pdsc_from_cache(d)("board"):
be_bryan 0:b74591d5ab33 240 try :
be_bryan 0:b74591d5ab33 241 mydict.append((dev['name'], dev.mounteddevice['dname']))
be_bryan 0:b74591d5ab33 242 except (KeyError, TypeError, IndexError) as e:
be_bryan 0:b74591d5ab33 243 pass
be_bryan 0:b74591d5ab33 244 self._aliases.update(dict(mydict))
be_bryan 0:b74591d5ab33 245 except (AttributeError, TypeError) as e :
be_bryan 0:b74591d5ab33 246 pass
be_bryan 0:b74591d5ab33 247 self.counter += 1
be_bryan 0:b74591d5ab33 248 self.display_counter("Scanning for Aliases")
be_bryan 0:b74591d5ab33 249
be_bryan 0:b74591d5ab33 250 def get_flash_algorthim_binary(self, device_name, all=False) :
be_bryan 0:b74591d5ab33 251 """Retrieve the flash algorithm file for a particular part.
be_bryan 0:b74591d5ab33 252
be_bryan 0:b74591d5ab33 253 Assumes that both the PDSC and the PACK file associated with that part are in the cache.
be_bryan 0:b74591d5ab33 254
be_bryan 0:b74591d5ab33 255 :param device_name: The exact name of a device
be_bryan 0:b74591d5ab33 256 :param all: Return an iterator of all flash algos for this device
be_bryan 0:b74591d5ab33 257 :type device_name: str
be_bryan 0:b74591d5ab33 258 :return: A file-like object that, when read, is the ELF file that describes the flashing algorithm
be_bryan 0:b74591d5ab33 259 :return: A file-like object that, when read, is the ELF file that describes the flashing algorithm.
be_bryan 0:b74591d5ab33 260 When "all" is set to True then an iterator for file-like objects is returned
be_bryan 0:b74591d5ab33 261 :rtype: ZipExtFile or ZipExtFile iterator if all is True
be_bryan 0:b74591d5ab33 262 """
be_bryan 0:b74591d5ab33 263 device = self.index[device_name]
be_bryan 0:b74591d5ab33 264 pack = self.pack_from_cache(device)
be_bryan 0:b74591d5ab33 265 algo_itr = (pack.open(path) for path in device['algorithm'].keys())
be_bryan 0:b74591d5ab33 266 return algo_itr if all else algo_itr.next()
be_bryan 0:b74591d5ab33 267
be_bryan 0:b74591d5ab33 268 def get_svd_file(self, device_name) :
be_bryan 0:b74591d5ab33 269 """Retrieve the flash algorithm file for a particular part.
be_bryan 0:b74591d5ab33 270
be_bryan 0:b74591d5ab33 271 Assumes that both the PDSC and the PACK file associated with that part are in the cache.
be_bryan 0:b74591d5ab33 272
be_bryan 0:b74591d5ab33 273 :param device_name: The exact name of a device
be_bryan 0:b74591d5ab33 274 :type device_name: str
be_bryan 0:b74591d5ab33 275 :return: A file-like object that, when read, is the ELF file that describes the flashing algorithm
be_bryan 0:b74591d5ab33 276 :rtype: ZipExtFile
be_bryan 0:b74591d5ab33 277 """
be_bryan 0:b74591d5ab33 278 device = self.index[device_name]
be_bryan 0:b74591d5ab33 279 pack = self.pack_from_cache(device)
be_bryan 0:b74591d5ab33 280 return pack.open(device['debug'])
be_bryan 0:b74591d5ab33 281
be_bryan 0:b74591d5ab33 282 def generate_index(self) :
be_bryan 0:b74591d5ab33 283 self._index = {}
be_bryan 0:b74591d5ab33 284 self.counter = 0
be_bryan 0:b74591d5ab33 285 do_queue(Reader, self._generate_index_helper, self.get_urls())
be_bryan 0:b74591d5ab33 286 with open(LocalPackIndex, "wb+") as out:
be_bryan 0:b74591d5ab33 287 self._index["version"] = "0.1.0"
be_bryan 0:b74591d5ab33 288 dump(self._index, out)
be_bryan 0:b74591d5ab33 289 stdout.write("\n")
be_bryan 0:b74591d5ab33 290
be_bryan 0:b74591d5ab33 291 def generate_aliases(self) :
be_bryan 0:b74591d5ab33 292 self._aliases = {}
be_bryan 0:b74591d5ab33 293 self.counter = 0
be_bryan 0:b74591d5ab33 294 do_queue(Reader, self._generate_aliases_helper, self.get_urls())
be_bryan 0:b74591d5ab33 295 with open(LocalPackAliases, "wb+") as out:
be_bryan 0:b74591d5ab33 296 dump(self._aliases, out)
be_bryan 0:b74591d5ab33 297 stdout.write("\n")
be_bryan 0:b74591d5ab33 298
be_bryan 0:b74591d5ab33 299 def find_device(self, match) :
be_bryan 0:b74591d5ab33 300 choices = process.extract(match, self.index.keys(), limit=len(self.index))
be_bryan 0:b74591d5ab33 301 choices = sorted([(v, k) for k, v in choices], reverse=True)
be_bryan 0:b74591d5ab33 302 if choices : choices = list(takewhile(lambda t: t[0] == choices[0][0], choices))
be_bryan 0:b74591d5ab33 303 return [(v, self.index[v]) for k,v in choices]
be_bryan 0:b74591d5ab33 304
be_bryan 0:b74591d5ab33 305 def dump_index_to_file(self, file) :
be_bryan 0:b74591d5ab33 306 with open(file, "wb+") as out:
be_bryan 0:b74591d5ab33 307 dump(self.index, out)
be_bryan 0:b74591d5ab33 308
be_bryan 0:b74591d5ab33 309 @property
be_bryan 0:b74591d5ab33 310 def index(self) :
be_bryan 0:b74591d5ab33 311 """An index of most of the important data in all cached PDSC files.
be_bryan 0:b74591d5ab33 312
be_bryan 0:b74591d5ab33 313 :Example:
be_bryan 0:b74591d5ab33 314
be_bryan 0:b74591d5ab33 315 >>> from ArmPackManager import Cache
be_bryan 0:b74591d5ab33 316 >>> a = Cache()
be_bryan 0:b74591d5ab33 317 >>> a.index["LPC1768"]
be_bryan 0:b74591d5ab33 318 {u'algorithm': {u'RAMsize': u'0x0FE0',
be_bryan 0:b74591d5ab33 319 u'RAMstart': u'0x10000000',
be_bryan 0:b74591d5ab33 320 u'name': u'Flash/LPC_IAP_512.FLM',
be_bryan 0:b74591d5ab33 321 u'size': u'0x80000',
be_bryan 0:b74591d5ab33 322 u'start': u'0x00000000'},
be_bryan 0:b74591d5ab33 323 u'compile': [u'Device/Include/LPC17xx.h', u'LPC175x_6x'],
be_bryan 0:b74591d5ab33 324 u'debug': u'SVD/LPC176x5x.svd',
be_bryan 0:b74591d5ab33 325 u'pdsc_file': u'http://www.keil.com/pack/Keil.LPC1700_DFP.pdsc',
be_bryan 0:b74591d5ab33 326 u'memory': {u'IRAM1': {u'size': u'0x8000', u'start': u'0x10000000'},
be_bryan 0:b74591d5ab33 327 u'IRAM2': {u'size': u'0x8000', u'start': u'0x2007C000'},
be_bryan 0:b74591d5ab33 328 u'IROM1': {u'size': u'0x80000', u'start': u'0x00000000'}}}
be_bryan 0:b74591d5ab33 329
be_bryan 0:b74591d5ab33 330
be_bryan 0:b74591d5ab33 331 """
be_bryan 0:b74591d5ab33 332 if not self._index :
be_bryan 0:b74591d5ab33 333 with open(LocalPackIndex) as i :
be_bryan 0:b74591d5ab33 334 self._index = load(i)
be_bryan 0:b74591d5ab33 335 return self._index
be_bryan 0:b74591d5ab33 336 @property
be_bryan 0:b74591d5ab33 337 def aliases(self) :
be_bryan 0:b74591d5ab33 338 """An index of most of the important data in all cached PDSC files.
be_bryan 0:b74591d5ab33 339
be_bryan 0:b74591d5ab33 340 :Example:
be_bryan 0:b74591d5ab33 341
be_bryan 0:b74591d5ab33 342 >>> from ArmPackManager import Cache
be_bryan 0:b74591d5ab33 343 >>> a = Cache()
be_bryan 0:b74591d5ab33 344 >>> a.index["LPC1768"]
be_bryan 0:b74591d5ab33 345 {u'algorithm': {u'RAMsize': u'0x0FE0',
be_bryan 0:b74591d5ab33 346 u'RAMstart': u'0x10000000',
be_bryan 0:b74591d5ab33 347 u'name': u'Flash/LPC_IAP_512.FLM',
be_bryan 0:b74591d5ab33 348 u'size': u'0x80000',
be_bryan 0:b74591d5ab33 349 u'start': u'0x00000000'},
be_bryan 0:b74591d5ab33 350 u'compile': [u'Device/Include/LPC17xx.h', u'LPC175x_6x'],
be_bryan 0:b74591d5ab33 351 u'debug': u'SVD/LPC176x5x.svd',
be_bryan 0:b74591d5ab33 352 u'pdsc_file': u'http://www.keil.com/pack/Keil.LPC1700_DFP.pdsc',
be_bryan 0:b74591d5ab33 353 u'memory': {u'IRAM1': {u'size': u'0x8000', u'start': u'0x10000000'},
be_bryan 0:b74591d5ab33 354 u'IRAM2': {u'size': u'0x8000', u'start': u'0x2007C000'},
be_bryan 0:b74591d5ab33 355 u'IROM1': {u'size': u'0x80000', u'start': u'0x00000000'}}}
be_bryan 0:b74591d5ab33 356
be_bryan 0:b74591d5ab33 357
be_bryan 0:b74591d5ab33 358 """
be_bryan 0:b74591d5ab33 359 if not self._aliases :
be_bryan 0:b74591d5ab33 360 with open(LocalPackAliases) as i :
be_bryan 0:b74591d5ab33 361 self._aliases = load(i)
be_bryan 0:b74591d5ab33 362 return self._aliases
be_bryan 0:b74591d5ab33 363
be_bryan 0:b74591d5ab33 364 def cache_everything(self) :
be_bryan 0:b74591d5ab33 365 """Cache every PACK and PDSC file known.
be_bryan 0:b74591d5ab33 366
be_bryan 0:b74591d5ab33 367 Generates an index afterwards.
be_bryan 0:b74591d5ab33 368
be_bryan 0:b74591d5ab33 369 .. note:: This process may use 4GB of drive space and take upwards of 10 minutes to complete.
be_bryan 0:b74591d5ab33 370 """
be_bryan 0:b74591d5ab33 371 self.cache_pack_list(self.get_urls())
be_bryan 0:b74591d5ab33 372 self.generate_index()
be_bryan 0:b74591d5ab33 373 self.generate_aliases()
be_bryan 0:b74591d5ab33 374
be_bryan 0:b74591d5ab33 375 def cache_descriptors(self) :
be_bryan 0:b74591d5ab33 376 """Cache every PDSC file known.
be_bryan 0:b74591d5ab33 377
be_bryan 0:b74591d5ab33 378 Generates an index afterwards.
be_bryan 0:b74591d5ab33 379
be_bryan 0:b74591d5ab33 380 .. note:: This process may use 11MB of drive space and take upwards of 1 minute.
be_bryan 0:b74591d5ab33 381 """
be_bryan 0:b74591d5ab33 382 self.cache_descriptor_list(self.get_urls())
be_bryan 0:b74591d5ab33 383 self.generate_index()
be_bryan 0:b74591d5ab33 384 self.generate_aliases()
be_bryan 0:b74591d5ab33 385
be_bryan 0:b74591d5ab33 386 def cache_descriptor_list(self, list) :
be_bryan 0:b74591d5ab33 387 """Cache a list of PDSC files.
be_bryan 0:b74591d5ab33 388
be_bryan 0:b74591d5ab33 389 :param list: URLs of PDSC files to cache.
be_bryan 0:b74591d5ab33 390 :type list: [str]
be_bryan 0:b74591d5ab33 391 """
be_bryan 0:b74591d5ab33 392 self.total = len(list)
be_bryan 0:b74591d5ab33 393 self.display_counter("Caching Files")
be_bryan 0:b74591d5ab33 394 do_queue(Reader, self.cache_file, list)
be_bryan 0:b74591d5ab33 395 stdout.write("\n")
be_bryan 0:b74591d5ab33 396
be_bryan 0:b74591d5ab33 397 def cache_pack_list(self, list) :
be_bryan 0:b74591d5ab33 398 """Cache a list of PACK files, referenced by their PDSC URL
be_bryan 0:b74591d5ab33 399
be_bryan 0:b74591d5ab33 400 :param list: URLs of PDSC files to cache.
be_bryan 0:b74591d5ab33 401 :type list: [str]
be_bryan 0:b74591d5ab33 402 """
be_bryan 0:b74591d5ab33 403 self.total = len(list) * 2
be_bryan 0:b74591d5ab33 404 self.display_counter("Caching Files")
be_bryan 0:b74591d5ab33 405 do_queue(Reader, self.cache_pdsc_and_pack, list)
be_bryan 0:b74591d5ab33 406 stdout.write("\n")
be_bryan 0:b74591d5ab33 407
be_bryan 0:b74591d5ab33 408 def pdsc_from_cache(self, url) :
be_bryan 0:b74591d5ab33 409 """Low level inteface for extracting a PDSC file from the cache.
be_bryan 0:b74591d5ab33 410
be_bryan 0:b74591d5ab33 411 Assumes that the file specified is a PDSC file and is in the cache.
be_bryan 0:b74591d5ab33 412
be_bryan 0:b74591d5ab33 413 :param url: The URL of a PDSC file.
be_bryan 0:b74591d5ab33 414 :type url: str
be_bryan 0:b74591d5ab33 415 :return: A parsed representation of the PDSC file.
be_bryan 0:b74591d5ab33 416 :rtype: BeautifulSoup
be_bryan 0:b74591d5ab33 417 """
be_bryan 0:b74591d5ab33 418 dest = join(self.data_path, strip_protocol(url))
be_bryan 0:b74591d5ab33 419 with open(dest, "r") as fd :
be_bryan 0:b74591d5ab33 420 return BeautifulSoup(fd, "html.parser")
be_bryan 0:b74591d5ab33 421
be_bryan 0:b74591d5ab33 422 def pack_from_cache(self, device) :
be_bryan 0:b74591d5ab33 423 """Low level inteface for extracting a PACK file from the cache.
be_bryan 0:b74591d5ab33 424
be_bryan 0:b74591d5ab33 425 Assumes that the file specified is a PACK file and is in the cache.
be_bryan 0:b74591d5ab33 426
be_bryan 0:b74591d5ab33 427 :param url: The URL of a PACK file.
be_bryan 0:b74591d5ab33 428 :type url: str
be_bryan 0:b74591d5ab33 429 :return: A parsed representation of the PACK file.
be_bryan 0:b74591d5ab33 430 :rtype: ZipFile
be_bryan 0:b74591d5ab33 431 """
be_bryan 0:b74591d5ab33 432 return ZipFile(join(self.data_path,
be_bryan 0:b74591d5ab33 433 strip_protocol(device['pack_file'])))
be_bryan 0:b74591d5ab33 434
be_bryan 0:b74591d5ab33 435 def gen_dict_from_cache() :
be_bryan 0:b74591d5ab33 436 pdsc_files = pdsc_from_cache(RootPackUrl)
be_bryan 0:b74591d5ab33 437
be_bryan 0:b74591d5ab33 438 def cache_and_parse(self, url) :
be_bryan 0:b74591d5ab33 439 """A low level shortcut that Caches and Parses a PDSC file.
be_bryan 0:b74591d5ab33 440
be_bryan 0:b74591d5ab33 441 :param url: The URL of the PDSC file.
be_bryan 0:b74591d5ab33 442 :type url: str
be_bryan 0:b74591d5ab33 443 :return: A parsed representation of the PDSC file.
be_bryan 0:b74591d5ab33 444 :rtype: BeautifulSoup
be_bryan 0:b74591d5ab33 445 """
be_bryan 0:b74591d5ab33 446 self.cache_file(url)
be_bryan 0:b74591d5ab33 447 return self.pdsc_from_cache(url)
be_bryan 0:b74591d5ab33 448