| """ |
| A tool for extracting module port definitions from PDF specifications |
| """ |
| |
| # the following cases required specific adjustments to the base algorithm: |
| # CFGLUT5, IDDR, IDELAYCTRL, IDELAYE2, ISERDESE2, KEEPER, LUT6, LUT6_2, |
| # MMCME2_BASE, ODDR, OSERDESE2, PLLE2_BASE, RAM128X1D, RAM64M |
| # any changes to the algorithm should be checked against these entries |
| |
| # we use pdfminer to parse the PDF document and interpret the elements |
| # for python3 support you need to `pip install pdfminer.six` |
| from pdfminer.pdfparser import PDFParser |
| from pdfminer.pdfdocument import PDFDocument |
| from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter |
| from pdfminer.pdfpage import PDFPage |
| from pdfminer.converter import PDFLayoutAnalyzer |
| from pdfminer.layout import LAParams, LTContainer, LTTextLineHorizontal |
| |
| from collections import OrderedDict |
| import sys |
| import re |
| |
| from datetime import datetime |
| from lxml import objectify, etree |
| |
| PAGE_MARGIN = 60 # space ignored at the top and bottom of the page, for page header/footer |
| HEADER_MARGIN = 30 # space ignored at the start of the section due to the section header |
| COL_MARGIN = 5 # acceptable variation in x-position of entries in the same column |
| |
| |
| def rev_enumerate(it, last=None): |
| for i in range(len(it))[last::-1]: |
| yield i, it[i] |
| |
| |
| class PDFTableParser(PDFLayoutAnalyzer): |
| """Custom interpreter that hooks into pdfminer to process PDF text elements""" |
| |
| def __init__( |
| self, rsrcmgr, laparams=None, stop_at=None, top=None, bottom=None |
| ): |
| PDFLayoutAnalyzer.__init__(self, rsrcmgr, pageno=1, laparams=laparams) |
| self.stop_at = stop_at |
| self.reset(top, bottom) |
| |
| def process_text(self, bbox, txt, obj): |
| # there's no guarantee on the order in which text strings appear |
| # so store them all for post-processing after sorting them into order |
| if self.stop_at is not None and txt.startswith(self.stop_at): |
| self.done = True |
| self.bottom = bbox[3] + COL_MARGIN |
| return True |
| txt = txt.replace(u'\u2019', "'") # all HEX constants |
| txt = txt.replace(u'\u2022', '\n*') # bullet lists |
| txt = txt.replace( |
| u'\u201c', '"' |
| ) # e.g. FRAME_ECCE2 (FRAME_RBT_IN_FILENAME) |
| txt = txt.replace( |
| u'\u201d', '"' |
| ) # e.g. FRAME_ECCE2 (FRAME_RBT_IN_FILENAME) |
| self.items.append([int(-bbox[3]), int(bbox[0]), txt]) |
| return True |
| |
| def reset(self, top, bottom): |
| self.top = top |
| self.bottom = bottom |
| self.done = False |
| self.items = [] |
| |
| def receive_layout(self, ltpage): |
| if self.done: |
| return |
| |
| def render(item): |
| # don't process it if it's outside the FOV |
| if getattr(item, 'y1', self.bottom) <= self.bottom: |
| return True |
| if getattr(item, 'y0', self.top) >= self.top: |
| return True |
| # process individual lines of text |
| if isinstance(item, LTTextLineHorizontal): |
| if not self.process_text(item.bbox, item.get_text().strip(), |
| item): |
| return False |
| # process containers that (might) contain text (e.g. LTTextBoxHorizontal) |
| elif isinstance(item, LTContainer): |
| for child in item: |
| if not render(child): |
| return False |
| return True |
| |
| # default to processing the entire page |
| if self.top is None: |
| self.top = ltpage.mediabox[3] |
| if self.bottom is None: |
| self.bottom = ltpage.mediabox[1] |
| render(ltpage) |
| self.items[:] = sorted(self.items) |
| |
| # don't bother doing any image/line rendering |
| def render_image(self, name, stream): |
| return |
| |
| def paint_path(self, gstate, stroke, fill, evenodd, path): |
| return |
| |
| # data processing functions |
| def process_table(self): |
| # figure out the table header |
| top = -self.bottom |
| self.heads = [] |
| for y, x, t in self.items: |
| if y < top: |
| top = y |
| for i, (y, x, t) in rev_enumerate(self.items): |
| if y < top + COL_MARGIN: |
| self.heads.append((x - COL_MARGIN, t)) |
| self.items.pop(i) |
| self.heads = sorted(self.heads) |
| # figure out the rows |
| leftcol = self.heads[1][0] |
| self.rows = [] |
| self.items[:] = sorted(self.items) |
| for i, (y, x, t) in rev_enumerate(self.items): |
| if x < leftcol: |
| self.items.pop(i) |
| if t[0] == '<' and t[-1] == '>': |
| continue # IDELAYE2 |
| self.rows.append((y, t)) |
| if t.startswith('NOTE:') or not ( |
| t.isupper() |
| or t in ['/', '-', 'to']) or y > -self.bottom: |
| self.bottom = -y |
| self.done = True |
| self.rows = [(y, t) for y, t in self.rows[::-1] if y < -self.bottom] |
| # join rows together if required |
| for i, x in rev_enumerate(self.rows, -3): |
| if self.rows[i + 1][1] in ('to', '/', '-'): |
| self.rows[i] = ( |
| x[0], x[1] + self.rows[i + 1][1] + self.rows.pop(i + 2)[1] |
| ) |
| self.rows.pop(i + 1) |
| for i, x in rev_enumerate(self.rows, -2): |
| if x[1][-1] in ',_/' or x[1].endswith('to') or self.rows[ |
| i + 1][1][0] == '_' or x[0] == self.rows[i + 1][0]: |
| self.rows[i] = (x[0], x[1] + self.rows.pop(i + 1)[1]) |
| # correct y-positions |
| for i, (y, x, t) in rev_enumerate(self.items): |
| if y >= -self.bottom: |
| self.items.pop(i) |
| else: |
| self.items[i][0] = int(y) |
| # now sort items by corrected y-positions |
| self.items = sorted(self.items) |
| |
| def get_row(self, y): |
| for i, (y0, t) in enumerate(self.rows[1:]): |
| if y < y0 - COL_MARGIN: |
| return i |
| return len(self.rows) - 1 |
| |
| def arrange_items(self): |
| data = [] |
| for r in self.rows: |
| d = OrderedDict([(k[1], '') for k in self.heads]) |
| d[self.heads[0][1]] = r[1].replace(' ', '') |
| data.append(d) |
| for y, x, t in sorted(self.items): |
| for x0, head in self.heads[::-1]: |
| if x >= x0: |
| entry = data[self.get_row(y)] |
| if len(entry[head] |
| ) and entry[head][-1] != '_' and t[0] != '_': |
| t = ' ' + t |
| entry[head] += t |
| break |
| return data |
| |
| |
| def resolve_goto_action(doc, a): |
| """Resolves a "goto" action from the PDF outline into the associated page |
| object and position on that page""" |
| a = a.resolve() |
| assert a['S'].name == 'GoTo' |
| link = doc.get_dest(a['D']).resolve()['D'] |
| y = link[3] if link[1].name == 'XYZ' else 0 |
| return link[0], y |
| |
| |
| def find_pages(pgs, start, stop): |
| """Find all the pages between the resolved "start" and "stop" actions""" |
| extract = False |
| bottom = PAGE_MARGIN |
| for pg in pgs: |
| if pg.pageid == start[0].objid: |
| extract = True |
| top = start[1] - HEADER_MARGIN |
| else: |
| top = pg.mediabox[3] - PAGE_MARGIN |
| if pg.pageid == stop[0].objid: |
| bottom = stop[1] |
| if extract: |
| yield pg, top, bottom |
| if pg.pageid == stop[0].objid: |
| return |
| |
| |
| def parse_module_pages(doc, start_at): |
| """Deconstruct the PDF outline into a list of modules, with links to the |
| start and end of the associated "port descriptions" section""" |
| parts = OrderedDict() |
| module = None |
| start = None |
| process = False |
| pgs = PDFPage.create_pages(doc) |
| for (level, title, dest, a, se) in doc.get_outlines(): |
| if level == 2: # chapter titles |
| # modules are defined in chapter 4 |
| process = title.startswith('Ch. 4:') |
| elif process and level == 3: # module names are defined at level 3 of the TOC |
| module = title |
| elif level == 4 and module is not None and title.startswith(start_at): |
| # NB: possible incosistency in "title" name (e.g. LUT6) |
| start = resolve_goto_action(doc, a) |
| elif start is not None and module not in parts: # i.e. this is the FIRST following section |
| stop = resolve_goto_action(doc, a) |
| parts[module] = find_pages(pgs, start, stop) |
| start = None |
| return parts |
| |
| |
| def process_ports(tbl): |
| """Process the text elements corresponding to the "port descriptions" |
| table and return an ordered list of ports""" |
| if not len(tbl.items): |
| return [] |
| # fixes for MMCME2_BASE |
| for i, (y, x, t) in rev_enumerate(tbl.items): |
| if tbl.items[i][2] == 'Clock' and tbl.items[i + 1][2] == 'Inputs': |
| tbl.items[i][2] = 'CLKIN1' |
| tbl.items.pop(i + 1) |
| elif tbl.items[i][2] == 'Status' and tbl.items[i + 1][2] == 'Ports': |
| tbl.items[i][2] = 'LOCKED' |
| tbl.items.pop(i + 1) |
| elif t == 'Direction Width': |
| tbl.items[i][2] = 'Direction' |
| tbl.items.insert( |
| i + 1, (y, (x + tbl.items[i + 1][1]) / 2, 'Width') |
| ) |
| tbl.process_table() |
| # transform headers as necessary |
| for i, (x, name) in enumerate(tbl.heads): |
| if name.lower().startswith('direction'): |
| tbl.heads[i] = (x, 'Type') |
| # sort the items into categories |
| ports = tbl.arrange_items() |
| # process the rows one-by-one (in reverse, because we might insert new entries) |
| for i, x in rev_enumerate(ports): |
| # process the "width" entry |
| M = re.match( |
| r'([0-9]+)', x['Width'] |
| ) # remove any additional text (e.g. ODDR, KEEPER) |
| if M is None: |
| print( |
| '\tInvalid width %s on %s, skipping item' % |
| (repr(x['Width']), repr(x['Port'])) |
| ) |
| return [] |
| x['Width'] = wid = int(M.group(0)) |
| # process the "direction" entry |
| if x['Type'] == 'Input': |
| dir = 'input' |
| elif x['Type'] == 'Output': |
| dir = 'output' |
| elif x['Type'] == 'In/out': |
| dir = 'inout' |
| else: |
| assert False, 'Invalid pin type %s' % repr(x['Type']) |
| x['Type'] = dir |
| # process the port name |
| name = re.sub( |
| r'\s', '', x['Port'] |
| ) # remove any spaces from (multiline) name entries (e.g. MMCME2_BASE) |
| if '<' in name: # bus pins MIGHT be explicitly listed in name; perform sanity check |
| name, bits = name.split('<', 1) |
| assert bits == '%d:0>' % (wid - 1) |
| elif '-' in name: # entry is a range of pins (e.g. ISERDESE2) |
| n, start, stop = re.match( |
| r'([A-Z]+)([0-9]+)\s*-\s*[A-Z]+([0-9]+)', name |
| ).groups() |
| ports.pop(i) |
| for j in range(int(start), int(stop) + 1): |
| y = x.copy() |
| y['Port'] = n + '%d' % j |
| ports.insert(i + j - 1, y) |
| continue |
| # is the entry actually a LIST of pins? (e.g. CFGLUT5, OSERDESE2, ODDR) |
| M = re.split(r'[,/:]', name) |
| if len(M) > 1: |
| ports.pop(i) |
| for j, n in enumerate(M): |
| y = x.copy() |
| y['Port'] = n.strip() |
| ports.insert(i + j, y) |
| else: |
| x['Port'] = name |
| return ports |
| |
| |
| def process_attributes(tbl): |
| if not len(tbl.items): |
| return [] |
| tbl.process_table() |
| # transform headers as necessary |
| for i, (x, name) in enumerate(tbl.heads): |
| if name in ('Allowed Values', 'Allowed_Values'): |
| tbl.heads[i] = (x, 'Allowed') |
| if name == 'Descriptions': |
| tbl.heads[i] = (x, 'Description') |
| # transform text as necessary |
| for i, (y, x, t) in enumerate(tbl.items): |
| t = t.replace(u'\u2122', '(tm)') # IOBUF (DRIVE) |
| t = t.replace('""', '"') # RAM18E1 (SIM_DEVICE) |
| tbl.items[i][2] = t |
| # sort the items into categories |
| attribs = tbl.arrange_items() |
| # post-process the entries |
| for i, x in rev_enumerate(attribs): |
| if x['Type'] == 'STRING' and x[ |
| 'Default'] == 'None': # ICAPE2 (SIM_CFG_FILE_NAME) |
| x['Default'] = '""' |
| if x['Default'][0] == '"' and x['Default'][ |
| -1] != '"': # RAMB18E1 (WRITE_MODE_A) |
| s1, s2 = x['Default'].rsplit('"', 1) |
| x['Default'] = s1 + '"' |
| x['Description'] = s2 + ' ' + x['Description'] |
| if x['Default'].startswith('All'): |
| if 'one' in x['Default']: |
| val = 'F' |
| elif 'zero' in x['Default']: |
| val = '0' |
| else: |
| raise TypeError |
| assert x['Type'] == 'HEX' |
| M = re.search(r'(\d+)[-\s][Bb]it', x['Allowed']) |
| if M is None: |
| break |
| sz = int(M.group(1)) |
| pad = 1 if sz % 4 else 0 |
| x['Default'] = "%d'h%s" % (sz, val * ((sz // 4) + pad)) |
| elif x['Default'].startswith("0'h"): # ICAPE2 (DEVICE_ID) |
| x['Default'] = "32'h0" + x['Default'][3:] |
| if ',' in x['Attribute']: |
| attribs.pop(i) |
| for j, n in enumerate(x['Attribute'].split(',')): |
| n = n.strip() |
| if not len(n): |
| continue |
| y = x.copy() |
| y['Attribute'] = n |
| attribs.insert(i + j, y) |
| M = re.match( |
| r'([A-Z_]+)([0-9A-F]+)?(_[A-Z_]+)?to([A-Z_]+)([0-9A-F]+)(_[A-Z_]+)?', |
| x['Attribute'] |
| ) |
| if M is not None: |
| pre1, start, post1, pre2, stop, post2 = M.groups() |
| attribs.pop(i) |
| if start is None: |
| y = x.copy() |
| y['Attribute'] = pre1 |
| attribs.insert(i, y) |
| start = '0' |
| pre1 = pre2 |
| post1 = post2 |
| i += 1 |
| else: |
| assert pre1 == pre2 and post1 == post2 and len(start |
| ) == len(stop) |
| if post1 is None: |
| post1 = '' |
| nchar = len(stop) |
| if re.match(r'[0-9]+$', start) is not None and re.match( |
| r'[0-9]+$', stop) is not None: |
| start = int(start) |
| stop = int(stop) |
| fmt = '%s%0*d%s' |
| else: |
| start = int(start, 16) |
| stop = int(stop, 16) |
| fmt = '%s%0*X%s' |
| for j in range(start, stop + 1): |
| y = x.copy() |
| y['Attribute'] = fmt % (pre1, nchar, j, post1) |
| attribs.insert(i + j, y) |
| return attribs |
| |
| |
| def process_specs(infile, modules=None): |
| """Process the module specifications in the input PDF into an XML tree""" |
| # initialise the pdfminer interface -- |
| # we use a custom "render device" to receive the text objects in the PDF for further processing |
| resman = PDFResourceManager() |
| doc = PDFDocument(PDFParser(open(infile, 'rb'))) |
| laparams = LAParams( |
| line_margin=0.1, char_margin=0.7 |
| ) # parameters optimised to prevent incorrectly joining together words |
| |
| device = PDFTableParser(resman, laparams, stop_at='VHDL') |
| interpreter = PDFPageInterpreter(resman, device) |
| |
| # parse the PDF table of contents to figure out what modules exist and which pages to process |
| port_pages = parse_module_pages(doc, 'Port Desc') |
| if modules is None or len(modules) == 0: |
| modules = port_pages.keys() # default to processing ALL modules |
| attrib_list = parse_module_pages( |
| doc, 'Available Attrib' |
| ) # NB: not all modules have attributes |
| |
| # parse the specifications and generate an XML tree |
| E = objectify.ElementMaker(annotate=False) |
| root = E.xml(source=infile, processed=datetime.now().isoformat()) |
| |
| # run through the modules |
| for module in modules: |
| sys.stderr.write('Processing %s...\n' % module) |
| node = E.module(name=module) |
| # process the ports of this module |
| for pg, top, bottom in port_pages[module]: |
| device.reset(top, bottom) |
| interpreter.process_page(pg) |
| for P in process_ports(device): |
| node.append( |
| E.port( |
| name=P['Port'], type=P['Type'], width=str(P['Width']) |
| ) |
| ) |
| if device.done: |
| break |
| # process the attributes of this module |
| for pg, top, bottom in attrib_list.get(module, []): |
| device.reset(top, bottom) |
| interpreter.process_page(pg) |
| for A in process_attributes(device): |
| node.append( |
| E.attribute( |
| name=A['Attribute'], |
| type=A['Type'], |
| default=A['Default'].replace('"', ''), |
| values=A['Allowed'].replace('"', '') |
| ) |
| ) |
| if device.done: |
| break |
| # add it to the root object |
| root.append(node) |
| return root |
| |
| |
| if __name__ == '__main__': |
| import argparse |
| parser = argparse.ArgumentParser() |
| parser.add_argument( |
| '--input', |
| '-i', |
| nargs='?', |
| default='ug953-vivado-7series-libraries.pdf' |
| ) |
| parser.add_argument( |
| '--output', |
| '-o', |
| nargs='?', |
| type=argparse.FileType('w'), |
| default=sys.stdout |
| ) |
| parser.add_argument('--modules', '-m', nargs='*') |
| args = parser.parse_args() |
| |
| xml = process_specs(args.input, args.modules) |
| args.output.write(etree.tostring(xml, pretty_print=True).decode('ascii')) |
