From 413d4f1aa3a7d80ca291f523880ab30ae39d2d97 Mon Sep 17 00:00:00 2001 From: Ad Schellevis Date: Mon, 13 May 2019 11:41:53 +0200 Subject: [PATCH] cleanup, remove some helpers that where useful back in 2015 to search dependencies in legacy files. Don't want to upgrade them to python3, since we don't use this anymore. --- Scripts/development/crawl_legacy_deps.py | 72 ----- Scripts/development/inspect_function_usage.py | 48 ---- Scripts/development/lib/__init__.py | 0 Scripts/development/lib/legacy_deps.py | 253 ------------------ 4 files changed, 373 deletions(-) delete mode 100755 Scripts/development/crawl_legacy_deps.py delete mode 100755 Scripts/development/inspect_function_usage.py delete mode 100644 Scripts/development/lib/__init__.py delete mode 100644 Scripts/development/lib/legacy_deps.py diff --git a/Scripts/development/crawl_legacy_deps.py b/Scripts/development/crawl_legacy_deps.py deleted file mode 100755 index c71c699bf..000000000 --- a/Scripts/development/crawl_legacy_deps.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/local/bin/python2.7 - -""" - Copyright (c) 2015 Ad Schellevis - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, - INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY - AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, - OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - - -------------------------------------------------------------------------------------- - - Generate dependency maps for legacy files. - To use this script, please install graphviz package ( pkg install graphviz ) -""" - -import time -import os -import os.path -from lib.legacy_deps import DependancyCrawler - -# set source and target directories -target_directory = '/tmp/legacy/' -src_root = '/usr/local/' - -# create target directory if not existing -if not os.path.exists(target_directory): - os.mkdir(target_directory) - -# start crawling -crawler = DependancyCrawler(src_root) -print '[%.2f] started ' % (time.time()) -crawler.crawl() -print '[%.2f] collected %d dependencies in %d files' % (time.time(), - crawler.get_total_dependencies(), - crawler.get_total_files()) - -# generate graphs -generated_files = list() -for filename in crawler.get_files(): - file_stats = crawler.file_info(filename) - if file_stats['levels'] > 1: - print '[%.2f] ... writing %s' % (time.time(), filename) - dot_filename = ('%s/%s.dot' % (target_directory, filename)).replace('//', '/') - target_filename = dot_filename.replace('.dot', '.png') - open(dot_filename, 'w').write(crawler.generate_dot(filename)) - os.system('/usr/local/bin/dot -Tpng %s -o %s ' % (dot_filename, target_filename)) - generated_files.append(os.path.basename(target_filename)) - else: - # not interested, item has no children. - print '[%.2f] ... skip %s' % (time.time(), filename) - -# write a simple index page for our generated files -open(('%s/index.html' % target_directory).replace('//', '/'), 'w').write(crawler.generate_index_html(generated_files)) -print '[%.2f] done (all results in %s)' % (time.time(), target_directory) diff --git a/Scripts/development/inspect_function_usage.py b/Scripts/development/inspect_function_usage.py deleted file mode 100755 index 971d71afc..000000000 --- a/Scripts/development/inspect_function_usage.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/local/bin/python2.7 - -""" - Copyright (c) 2015 Ad Schellevis - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, - INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY - AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, - OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - -""" -import sys -from lib.legacy_deps import DependancyCrawler - -src_root = '/usr/local/' -crawler = DependancyCrawler(src_root) -crawler.crawl() -if len(sys.argv) < 2: - print ('inspect legacy function usage of module, choose one of:') - for module_name in crawler.get_files(): - print (module_name) - sys.exit() - -chk_source = sys.argv[1] - -usage = crawler.where_used(chk_source) -for src_filename in usage: - module_name = crawler.get_dependency_by_src(src_filename) - trace_data = crawler.trace(module_name) - if chk_source not in trace_data: - print ('inspect : %s (%s)' % (src_filename, ',' .join(usage[src_filename] ) )) diff --git a/Scripts/development/lib/__init__.py b/Scripts/development/lib/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/Scripts/development/lib/legacy_deps.py b/Scripts/development/lib/legacy_deps.py deleted file mode 100644 index 52b594299..000000000 --- a/Scripts/development/lib/legacy_deps.py +++ /dev/null @@ -1,253 +0,0 @@ -""" - Copyright (c) 2015 Ad Schellevis - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, - INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY - AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, - OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - - -------------------------------------------------------------------------------------- - Crawler class to find module (require/include) dependencies -""" -import os -import os.path - -class DependancyCrawler(object): - """ Legacy dependency crawler and grapher - """ - def __init__(self, root): - """ init - :param root: start crawling at - :return: - """ - self._all_dependencies = {} - self._all_dependencies_src = {} - self._all_functions = {} - self._exclude_deps = ['/usr/local/opnsense/mvc/app/config/config.php'] - self.root = root - - def get_dependency_by_src(self, src_filename): - """ dependencies are stored by a single name, this method maps a filename back to its name - usually the basename of the file. - :param src_filename: - :return: - """ - if src_filename in self._all_dependencies_src: - return self._all_dependencies_src[src_filename] - else: - return None - - def fetch_php_modules(self, src_filename): - # create a new list for this base filename - base_filename = os.path.basename(src_filename) - if base_filename in self._all_dependencies: - base_filename = '%s__%s' % (src_filename.split('/')[-2], base_filename) - self._all_dependencies[base_filename] = [] - self._all_dependencies_src[src_filename] = base_filename - - source_data = open(src_filename).read() - # fetch all include, include_once, require, require_once statements and - # add dependencies to object dependency list. - for tag in ('include', 'require'): - data = source_data - while True: - startpos = data.find(tag) - if startpos == -1: - break - else: - strlen = data[startpos:].find(';') - if strlen > -1: - # parse (single) statement, check if this could be an include type command - dep_stmt = data[startpos-1:strlen+startpos] - if dep_stmt[0] in (' ', '\n'): - dep_stmt = dep_stmt[1:].replace("'", '"') - if dep_stmt.find('\n') == -1 and dep_stmt.count('"') == 2: - dep_filename = dep_stmt.split('"')[1] - if dep_filename not in self._all_dependencies[base_filename]: - if dep_filename not in self._exclude_deps: - self._all_dependencies[base_filename].append(dep_filename) - data = data[strlen+startpos:] - - def fetch_php_functions(self, src_filename): - """ find php functions - :param src_filename: - :return: - """ - base_filename = os.path.basename(src_filename) - if base_filename in self._all_functions: - base_filename = '%s__%s' % (src_filename.split('/')[-2], base_filename) - - function_list = [] - for line in open(src_filename,'r').read().split('\n'): - if line.find('function ') > -1 and line.find('(') > -1: - if line.find('*') > -1 and line.find('function') > line.find('*'): - continue - function_nm = line.split('(')[0].strip().split(' ')[-1].strip() - function_list.append(function_nm) - - self._all_functions[base_filename] = function_list - - def find_files(self, analyse_dirs=('etc','www', 'captiveportal', 'sbin')): - """ - :param analyse_dirs: directories to analyse - :return: - """ - for analyse_dir in analyse_dirs: - analyse_dir = ('%s/%s' % (self.root, analyse_dir)).replace('//', '/') - for wroot, wdirs, wfiles in os.walk(analyse_dir): - for src_filename in wfiles: - src_filename = '%s/%s' % (wroot, src_filename) - if src_filename.split('.')[-1] in ('php', 'inc','class') \ - or open(src_filename).read(1024).find('/bin/php') > -1: - yield src_filename - - def crawl(self): - """ Crawl through legacy code - :param analyse_dirs: only analyse these directories - :return: None - """ - for src_filename in self.find_files(): - self.fetch_php_modules(src_filename) - self.fetch_php_functions(src_filename) - - def where_used(self, src): - """ - :param src: source object name (base name) - :return: dictionary containing files and functions - """ - where_used_lst={} - for src_filename in self.find_files(): - data = open(src_filename,'r').read().replace('\n',' ').replace('\t',' ').replace('@',' ') - use_list = [] - for function in self._all_functions[src]: - if data.find(' %s(' % (function)) > -1 or \ - data.find('!%s ' % (function)) > -1 or \ - data.find('!%s(' % (function)) > -1 or \ - data.find('(%s(' % (function)) > -1 or \ - data.find('(%s ' % (function)) > -1 or \ - data.find(' %s ' % (function)) > -1: - use_list.append(function) - - if len(use_list) > 0: - where_used_lst[src_filename] = sorted(use_list) - - return where_used_lst - - def get_total_files(self): - """ get total number of analysed files - :return: int - """ - return len(self._all_dependencies) - - def get_total_dependencies(self): - """ get total number of dependencies - :return: int - """ - count = 0 - for src_filename in self._all_dependencies: - count += len(self._all_dependencies[src_filename]) - return count - - def get_files(self): - """ retrieve all analysed files as iterator (ordered by name) - :return: iterator - """ - for src_filename in sorted(self._all_dependencies): - yield src_filename - - def trace(self, src_filename, parent_filename=None, result=None, level=0): - """ trace dependencies (recursive) - :param src_filename: - :param parent_filename: - :param result: - :param level: - :return: - """ - if result is None: - result = {} - if src_filename not in result: - result[src_filename] = {'level': level, 'dup': list(), 'parent': parent_filename} - else: - result[src_filename]['dup'].append(parent_filename) - return - - if src_filename in self._all_dependencies: - for dependency in self._all_dependencies[src_filename]: - self.trace(dependency, src_filename, result, level=level+1) - - return result - - def file_info(self, src_filename): - """ retrieve file info, like maximum recursive depth and number of duplicate dependencies - :param src_filename: - :return: - """ - result = {'levels': 0,'dup_count':0} - if src_filename in self._all_dependencies: - data = self.trace(src_filename) - for dep_filename in data: - if data[dep_filename]['level'] > result['levels']: - result['levels'] = data[dep_filename]['level'] - result['dup_count'] += len(data[dep_filename]['dup']) - - return result - - def generate_dot(self, filename_to_inspect): - """ convert trace data to do graph - :param filename_to_inspect: source filename to generate graph for - :return: string (dot) data - """ - trace_data = self.trace(filename_to_inspect) - result = list() - result.append('digraph dependencies {') - result.append('\toverlap=scale;') - nodes = {} - for level in range(100): - for src_filename in trace_data: - if trace_data[src_filename]['level'] == level: - if trace_data[src_filename]['parent'] is not None: - result.append('\tedge [color=black style=filled];') - result.append('\t"%s" -> "%s" [weight=%d];' % (trace_data[src_filename]['parent'], - src_filename, trace_data[src_filename]['level'])) - if len(trace_data[src_filename]['dup']) > 0: - for target in trace_data[src_filename]['dup']: - result.append('\tedge [color=red style=dotted];') - result.append('\t"%s" -> "%s";' % (target, src_filename)) - - if trace_data[src_filename]['parent'] is None: - nodes[src_filename] = '[shape=Mdiamond]' - elif len(trace_data[src_filename]['dup']) > 0: - nodes[src_filename] = '[shape=box,style=filled,color=".7 .3 1.0"]' - else: - nodes[src_filename] = '[shape=box]' - - for node in nodes: - result.append('\t"%s" %s;' % (node, nodes[node])) - - result.append('}') - return '\n'.join(result) - - @staticmethod - def generate_index_html(filelist): - html_body = "\n%s" - html_row = '\n' - html = html_body % ('\n'.join(map(lambda x: html_row % (x, x), sorted(filelist)))) - - return html
Name
%s