| #!/usr/bin/env python3 |
| # -*- coding: utf-8 -*- |
| # |
| # Copyright (C) 2017-2020 The Project X-Ray Authors. |
| # |
| # Use of this source code is governed by a ISC-style |
| # license that can be found in the LICENSE file or at |
| # https://opensource.org/licenses/ISC |
| # |
| # SPDX-License-Identifier: ISC |
| |
| import os, re |
| from prjxray import util |
| |
| |
| def noprefix(tag): |
| n = tag.find('.') |
| assert n > 0 |
| return tag[n + 1:] |
| |
| |
| def getprefix(tag): |
| n = tag.find('.') |
| assert n > 0 |
| return tag[0:n] |
| |
| |
| def load_pipfile(pipfile, verbose=False): |
| '''Returns a set of tags containing real tile type prefixes (ex: INT_L)''' |
| todos = set() |
| tile_type = None |
| with open(pipfile, "r") as f: |
| # INT_L.WW2BEG0.SR1BEG_S0 |
| for line in f: |
| tag = line.strip().split(' ')[0] |
| prefix_line = getprefix(tag) |
| if tile_type is None: |
| tile_type = prefix_line |
| else: |
| assert tile_type == prefix_line |
| todos.add(tag) |
| return todos, tile_type |
| |
| |
| def balance_todo_list( |
| pipfile, |
| todos, |
| balance_wire_re, |
| balance_wire_direction, |
| balance_wire_cnt, |
| verbose=False): |
| """Balance the contents of the todo list |
| |
| Todo list balancing allows to specify the name, direction and minimal number of |
| occurrences of a PIP wire in the final todo list. |
| The mechanism should be used in cases where a fuzzer times out because of an |
| unsolvable todo list, i.e. the netlist and resulting segdata generated from an |
| iteration keep segmatch from properly disambiguating the bits for some features. |
| |
| When the balance wire name regexp is specified it's guaranteed that all PIPs |
| with matching wire name (whether we want to match a src or dst wire has to be |
| specified with the --balance-wire-direction switch) will have at least the number |
| of entries specified with the --balance-wire-cnt switch in the final todo list. |
| """ |
| orig_todos, tile_type = load_pipfile(pipfile, verbose=verbose) |
| if balance_wire_re is not None: |
| todo_wires = {} |
| verbose and print("Start balancing the TODO list") |
| for todo in todos: |
| tile_type, dst, src = todo.split(".") |
| wire = src |
| other_wire = dst |
| if balance_wire_direction not in "src": |
| wire = dst |
| other_wire = src |
| balance_wire_match = re.match(balance_wire_re, wire) |
| if balance_wire_match is None: |
| continue |
| if wire not in todo_wires: |
| todo_wires[wire] = set() |
| todo_wires[wire].add(other_wire) |
| for wire, other_wires in todo_wires.items(): |
| if len(other_wires) >= balance_wire_cnt: |
| continue |
| else: |
| for todo in orig_todos: |
| tile_type, dst, src = todo.split(".") |
| if balance_wire_direction in "src": |
| if wire in src and dst not in todo_wires[wire]: |
| todo_wires[wire].add(dst) |
| else: |
| if wire in dst and src not in todo_wires[wire]: |
| todo_wires[wire].add(src) |
| if len(todo_wires[wire]) == balance_wire_cnt: |
| break |
| for wire, other_wires in todo_wires.items(): |
| if len(other_wires) < balance_wire_cnt: |
| verbose and print( |
| "Warning: failed to balance the todo list for wire {}, there are only {} PIPs which meet the requirement: {}" |
| .format(wire, len(other_wires), other_wires)) |
| for other_wire in other_wires: |
| line = tile_type + "." |
| if balance_wire_direction in "src": |
| line += other_wire + "." + wire |
| else: |
| line += wire + "." + other_wire |
| verbose and print("Adding {}".format(line)) |
| todos.add(line) |
| verbose and print("Finished balancing the TODO list") |
| |
| |
| def maketodo( |
| pipfile, |
| dbfile, |
| intre, |
| exclude_re=None, |
| balance_wire_re=None, |
| balance_wire_direction=None, |
| balance_wire_cnt=None, |
| not_endswith=None, |
| verbose=False): |
| ''' |
| db files start with INT., but pipfile lines start with INT_L |
| Normalize by removing before the first dot |
| 050-intpips doesn't care about contents, but most fuzzers use the tile type prefix |
| ''' |
| |
| todos, tile_type = load_pipfile(pipfile, verbose=verbose) |
| verbose and print('%s: %u entries' % (pipfile, len(todos))) |
| if not todos: |
| verbose and print('%s: %u entries, done!' % (pipfile, len(todos))) |
| return |
| |
| verbose and print("pipfile todo sample: %s" % list(todos)[0]) |
| |
| if 0 and verbose: |
| print("TODOs") |
| for todo in sorted(list(todos)): |
| print(' %s' % todo) |
| |
| verbose and print('Pre db %s: %u entries' % (dbfile, len(todos))) |
| # Allow against empty db |
| if os.path.exists(dbfile): |
| verbose and print("Loading %s" % dbfile) |
| with open(dbfile, "r") as f: |
| # INT.BYP_ALT0.BYP_BOUNCE_N3_3 !22_07 !23_07 !25_07 21_07 24_07 |
| for line in f: |
| tag, _bits, mode, _ = util.parse_db_line(line.strip()) |
| # Only count resolved entries |
| if mode: |
| continue |
| # INT.BLAH => INT_L.BLAH |
| tag = tile_type + '.' + noprefix(tag) |
| |
| # bipips works on a subset |
| if tag in todos: |
| todos.remove(tag) |
| else: |
| verbose and print( |
| "WARNING: couldnt remove %s (line %s)" % |
| (tag, line.strip())) |
| else: |
| verbose and print("WARNING: dbfile doesnt exist: %s" % dbfile) |
| verbose and print('Post db %s: %u entries' % (dbfile, len(todos))) |
| |
| drops = 0 |
| lines = 0 |
| filtered_todos = set() |
| for line in todos: |
| include = re.match(intre, line) is not None |
| |
| if include and not_endswith is not None: |
| include = not line.endswith(not_endswith) |
| |
| if include and exclude_re is not None: |
| include = re.match(exclude_re, line) is None |
| |
| if include: |
| filtered_todos.add(line) |
| else: |
| drops += 1 |
| lines += 1 |
| verbose and print('Print %u entries w/ %u drops' % (lines, drops)) |
| |
| balance_todo_list( |
| pipfile, filtered_todos, balance_wire_re, balance_wire_direction, |
| balance_wire_cnt, verbose) |
| for todo in filtered_todos: |
| print(todo) |
| |
| |
| def run( |
| build_dir, |
| db_dir, |
| pip_dir, |
| intre, |
| sides, |
| l, |
| r, |
| pip_type, |
| seg_type, |
| exclude_re=None, |
| balance_wire_re=None, |
| balance_wire_direction=None, |
| balance_wire_cnt=None, |
| not_endswith=None, |
| verbose=False): |
| if db_dir is None: |
| db_dir = "%s/%s" % ( |
| os.getenv("XRAY_DATABASE_DIR"), os.getenv("XRAY_DATABASE")) |
| |
| if pip_dir is None: |
| pip_dir = "%s/piplist/build/%s" % ( |
| os.getenv("XRAY_FUZZERS_DIR"), pip_type) |
| |
| assert intre, "RE is required" |
| |
| for side in sides: |
| if side == "l" and not l: |
| continue |
| |
| if side == "r" and not r: |
| continue |
| |
| if side == "xl": |
| segfile = "l{}".format(seg_type) |
| pipfile = "l{}".format(pip_type) |
| elif side == "xr": |
| segfile = "r{}".format(seg_type) |
| pipfile = "r{}".format(pip_type) |
| elif side != "": |
| segfile = "{}_{}".format(seg_type, side) |
| pipfile = "{}_{}".format(pip_type, side) |
| else: |
| segfile = "{}".format(seg_type) |
| pipfile = "{}".format(pip_type) |
| |
| maketodo( |
| "%s/%s.txt" % (pip_dir, pipfile), |
| "%s/segbits_%s.db" % (db_dir, segfile), |
| intre, |
| exclude_re=exclude_re, |
| balance_wire_re=balance_wire_re, |
| balance_wire_direction=balance_wire_direction, |
| balance_wire_cnt=balance_wire_cnt, |
| not_endswith=not_endswith, |
| verbose=verbose) |
| |
| |
| def main(): |
| import argparse |
| |
| parser = argparse.ArgumentParser( |
| description="Print list of known but unsolved PIPs") |
| |
| parser.add_argument('--verbose', action='store_true', help='') |
| parser.add_argument('--build-dir', default="build", help='') |
| parser.add_argument('--db-dir', default=None, help='') |
| parser.add_argument('--pip-dir', default=None, help='') |
| parser.add_argument('--re', required=True, help='') |
| parser.add_argument('--exclude-re', required=False, default=None, help='') |
| parser.add_argument( |
| '--balance-wire-re', required=False, default=None, help='') |
| parser.add_argument( |
| '--balance-wire-direction', required=False, default="src", help='') |
| parser.add_argument( |
| '--balance-wire-cnt', required=False, default="1", help='') |
| parser.add_argument( |
| '--balance-re-wire', required=False, default="src", help='') |
| parser.add_argument('--pip-type', default="pips_int", help='') |
| parser.add_argument('--seg-type', default="int", help='') |
| parser.add_argument('--sides', default="l,r", help='') |
| util.add_bool_arg(parser, '--l', default=True, help='') |
| util.add_bool_arg(parser, '--r', default=True, help='') |
| parser.add_argument( |
| '--not-endswith', help='Drop lines if they end with this') |
| args = parser.parse_args() |
| |
| run( |
| build_dir=args.build_dir, |
| db_dir=args.db_dir, |
| pip_dir=args.pip_dir, |
| intre=args.re, |
| exclude_re=args.exclude_re, |
| balance_wire_re=args.balance_wire_re, |
| balance_wire_direction=args.balance_wire_direction, |
| balance_wire_cnt=int(args.balance_wire_cnt), |
| sides=args.sides.split(','), |
| l=args.l, |
| r=args.r, |
| pip_type=args.pip_type, |
| seg_type=args.seg_type, |
| not_endswith=args.not_endswith, |
| verbose=args.verbose) |
| |
| |
| if __name__ == '__main__': |
| main() |
