cleanup, remove some helpers that where useful back in 2015 to search dependencies in legacy files. Don't want to upgrade them to python3, since we don't use this anymore.
This commit is contained in:
parent
404f796eb8
commit
413d4f1aa3
|
@ -1,72 +0,0 @@
|
|||
#!/usr/local/bin/python2.7
|
||||
|
||||
"""
|
||||
Copyright (c) 2015 Ad Schellevis <ad@opnsense.org>
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
1. Redistributions of source code must retain the above copyright notice,
|
||||
this list of conditions and the following disclaimer.
|
||||
|
||||
2. Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in the
|
||||
documentation and/or other materials provided with the distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
|
||||
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||
AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
|
||||
AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
|
||||
OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
--------------------------------------------------------------------------------------
|
||||
|
||||
Generate dependency maps for legacy files.
|
||||
To use this script, please install graphviz package ( pkg install graphviz )
|
||||
"""
|
||||
|
||||
import time
|
||||
import os
|
||||
import os.path
|
||||
from lib.legacy_deps import DependancyCrawler
|
||||
|
||||
# set source and target directories
|
||||
target_directory = '/tmp/legacy/'
|
||||
src_root = '/usr/local/'
|
||||
|
||||
# create target directory if not existing
|
||||
if not os.path.exists(target_directory):
|
||||
os.mkdir(target_directory)
|
||||
|
||||
# start crawling
|
||||
crawler = DependancyCrawler(src_root)
|
||||
print '[%.2f] started ' % (time.time())
|
||||
crawler.crawl()
|
||||
print '[%.2f] collected %d dependencies in %d files' % (time.time(),
|
||||
crawler.get_total_dependencies(),
|
||||
crawler.get_total_files())
|
||||
|
||||
# generate graphs
|
||||
generated_files = list()
|
||||
for filename in crawler.get_files():
|
||||
file_stats = crawler.file_info(filename)
|
||||
if file_stats['levels'] > 1:
|
||||
print '[%.2f] ... writing %s' % (time.time(), filename)
|
||||
dot_filename = ('%s/%s.dot' % (target_directory, filename)).replace('//', '/')
|
||||
target_filename = dot_filename.replace('.dot', '.png')
|
||||
open(dot_filename, 'w').write(crawler.generate_dot(filename))
|
||||
os.system('/usr/local/bin/dot -Tpng %s -o %s ' % (dot_filename, target_filename))
|
||||
generated_files.append(os.path.basename(target_filename))
|
||||
else:
|
||||
# not interested, item has no children.
|
||||
print '[%.2f] ... skip %s' % (time.time(), filename)
|
||||
|
||||
# write a simple index page for our generated files
|
||||
open(('%s/index.html' % target_directory).replace('//', '/'), 'w').write(crawler.generate_index_html(generated_files))
|
||||
print '[%.2f] done (all results in %s)' % (time.time(), target_directory)
|
|
@ -1,48 +0,0 @@
|
|||
#!/usr/local/bin/python2.7
|
||||
|
||||
"""
|
||||
Copyright (c) 2015 Ad Schellevis <ad@opnsense.org>
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
1. Redistributions of source code must retain the above copyright notice,
|
||||
this list of conditions and the following disclaimer.
|
||||
|
||||
2. Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in the
|
||||
documentation and/or other materials provided with the distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
|
||||
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||
AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
|
||||
AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
|
||||
OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
"""
|
||||
import sys
|
||||
from lib.legacy_deps import DependancyCrawler
|
||||
|
||||
src_root = '/usr/local/'
|
||||
crawler = DependancyCrawler(src_root)
|
||||
crawler.crawl()
|
||||
if len(sys.argv) < 2:
|
||||
print ('inspect legacy function usage of module, choose one of:')
|
||||
for module_name in crawler.get_files():
|
||||
print (module_name)
|
||||
sys.exit()
|
||||
|
||||
chk_source = sys.argv[1]
|
||||
|
||||
usage = crawler.where_used(chk_source)
|
||||
for src_filename in usage:
|
||||
module_name = crawler.get_dependency_by_src(src_filename)
|
||||
trace_data = crawler.trace(module_name)
|
||||
if chk_source not in trace_data:
|
||||
print ('inspect : %s (%s)' % (src_filename, ',' .join(usage[src_filename] ) ))
|
|
@ -1,253 +0,0 @@
|
|||
"""
|
||||
Copyright (c) 2015 Ad Schellevis <ad@opnsense.org>
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
1. Redistributions of source code must retain the above copyright notice,
|
||||
this list of conditions and the following disclaimer.
|
||||
|
||||
2. Redistributions in binary form must reproduce the above copyright
|
||||
notice, this list of conditions and the following disclaimer in the
|
||||
documentation and/or other materials provided with the distribution.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
|
||||
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
|
||||
AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
|
||||
AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
|
||||
OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
||||
SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
||||
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
||||
CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
||||
ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
--------------------------------------------------------------------------------------
|
||||
Crawler class to find module (require/include) dependencies
|
||||
"""
|
||||
import os
|
||||
import os.path
|
||||
|
||||
class DependancyCrawler(object):
|
||||
""" Legacy dependency crawler and grapher
|
||||
"""
|
||||
def __init__(self, root):
|
||||
""" init
|
||||
:param root: start crawling at
|
||||
:return:
|
||||
"""
|
||||
self._all_dependencies = {}
|
||||
self._all_dependencies_src = {}
|
||||
self._all_functions = {}
|
||||
self._exclude_deps = ['/usr/local/opnsense/mvc/app/config/config.php']
|
||||
self.root = root
|
||||
|
||||
def get_dependency_by_src(self, src_filename):
|
||||
""" dependencies are stored by a single name, this method maps a filename back to its name
|
||||
usually the basename of the file.
|
||||
:param src_filename:
|
||||
:return:
|
||||
"""
|
||||
if src_filename in self._all_dependencies_src:
|
||||
return self._all_dependencies_src[src_filename]
|
||||
else:
|
||||
return None
|
||||
|
||||
def fetch_php_modules(self, src_filename):
|
||||
# create a new list for this base filename
|
||||
base_filename = os.path.basename(src_filename)
|
||||
if base_filename in self._all_dependencies:
|
||||
base_filename = '%s__%s' % (src_filename.split('/')[-2], base_filename)
|
||||
self._all_dependencies[base_filename] = []
|
||||
self._all_dependencies_src[src_filename] = base_filename
|
||||
|
||||
source_data = open(src_filename).read()
|
||||
# fetch all include, include_once, require, require_once statements and
|
||||
# add dependencies to object dependency list.
|
||||
for tag in ('include', 'require'):
|
||||
data = source_data
|
||||
while True:
|
||||
startpos = data.find(tag)
|
||||
if startpos == -1:
|
||||
break
|
||||
else:
|
||||
strlen = data[startpos:].find(';')
|
||||
if strlen > -1:
|
||||
# parse (single) statement, check if this could be an include type command
|
||||
dep_stmt = data[startpos-1:strlen+startpos]
|
||||
if dep_stmt[0] in (' ', '\n'):
|
||||
dep_stmt = dep_stmt[1:].replace("'", '"')
|
||||
if dep_stmt.find('\n') == -1 and dep_stmt.count('"') == 2:
|
||||
dep_filename = dep_stmt.split('"')[1]
|
||||
if dep_filename not in self._all_dependencies[base_filename]:
|
||||
if dep_filename not in self._exclude_deps:
|
||||
self._all_dependencies[base_filename].append(dep_filename)
|
||||
data = data[strlen+startpos:]
|
||||
|
||||
def fetch_php_functions(self, src_filename):
|
||||
""" find php functions
|
||||
:param src_filename:
|
||||
:return:
|
||||
"""
|
||||
base_filename = os.path.basename(src_filename)
|
||||
if base_filename in self._all_functions:
|
||||
base_filename = '%s__%s' % (src_filename.split('/')[-2], base_filename)
|
||||
|
||||
function_list = []
|
||||
for line in open(src_filename,'r').read().split('\n'):
|
||||
if line.find('function ') > -1 and line.find('(') > -1:
|
||||
if line.find('*') > -1 and line.find('function') > line.find('*'):
|
||||
continue
|
||||
function_nm = line.split('(')[0].strip().split(' ')[-1].strip()
|
||||
function_list.append(function_nm)
|
||||
|
||||
self._all_functions[base_filename] = function_list
|
||||
|
||||
def find_files(self, analyse_dirs=('etc','www', 'captiveportal', 'sbin')):
|
||||
"""
|
||||
:param analyse_dirs: directories to analyse
|
||||
:return:
|
||||
"""
|
||||
for analyse_dir in analyse_dirs:
|
||||
analyse_dir = ('%s/%s' % (self.root, analyse_dir)).replace('//', '/')
|
||||
for wroot, wdirs, wfiles in os.walk(analyse_dir):
|
||||
for src_filename in wfiles:
|
||||
src_filename = '%s/%s' % (wroot, src_filename)
|
||||
if src_filename.split('.')[-1] in ('php', 'inc','class') \
|
||||
or open(src_filename).read(1024).find('/bin/php') > -1:
|
||||
yield src_filename
|
||||
|
||||
def crawl(self):
|
||||
""" Crawl through legacy code
|
||||
:param analyse_dirs: only analyse these directories
|
||||
:return: None
|
||||
"""
|
||||
for src_filename in self.find_files():
|
||||
self.fetch_php_modules(src_filename)
|
||||
self.fetch_php_functions(src_filename)
|
||||
|
||||
def where_used(self, src):
|
||||
"""
|
||||
:param src: source object name (base name)
|
||||
:return: dictionary containing files and functions
|
||||
"""
|
||||
where_used_lst={}
|
||||
for src_filename in self.find_files():
|
||||
data = open(src_filename,'r').read().replace('\n',' ').replace('\t',' ').replace('@',' ')
|
||||
use_list = []
|
||||
for function in self._all_functions[src]:
|
||||
if data.find(' %s(' % (function)) > -1 or \
|
||||
data.find('!%s ' % (function)) > -1 or \
|
||||
data.find('!%s(' % (function)) > -1 or \
|
||||
data.find('(%s(' % (function)) > -1 or \
|
||||
data.find('(%s ' % (function)) > -1 or \
|
||||
data.find(' %s ' % (function)) > -1:
|
||||
use_list.append(function)
|
||||
|
||||
if len(use_list) > 0:
|
||||
where_used_lst[src_filename] = sorted(use_list)
|
||||
|
||||
return where_used_lst
|
||||
|
||||
def get_total_files(self):
|
||||
""" get total number of analysed files
|
||||
:return: int
|
||||
"""
|
||||
return len(self._all_dependencies)
|
||||
|
||||
def get_total_dependencies(self):
|
||||
""" get total number of dependencies
|
||||
:return: int
|
||||
"""
|
||||
count = 0
|
||||
for src_filename in self._all_dependencies:
|
||||
count += len(self._all_dependencies[src_filename])
|
||||
return count
|
||||
|
||||
def get_files(self):
|
||||
""" retrieve all analysed files as iterator (ordered by name)
|
||||
:return: iterator
|
||||
"""
|
||||
for src_filename in sorted(self._all_dependencies):
|
||||
yield src_filename
|
||||
|
||||
def trace(self, src_filename, parent_filename=None, result=None, level=0):
|
||||
""" trace dependencies (recursive)
|
||||
:param src_filename:
|
||||
:param parent_filename:
|
||||
:param result:
|
||||
:param level:
|
||||
:return:
|
||||
"""
|
||||
if result is None:
|
||||
result = {}
|
||||
if src_filename not in result:
|
||||
result[src_filename] = {'level': level, 'dup': list(), 'parent': parent_filename}
|
||||
else:
|
||||
result[src_filename]['dup'].append(parent_filename)
|
||||
return
|
||||
|
||||
if src_filename in self._all_dependencies:
|
||||
for dependency in self._all_dependencies[src_filename]:
|
||||
self.trace(dependency, src_filename, result, level=level+1)
|
||||
|
||||
return result
|
||||
|
||||
def file_info(self, src_filename):
|
||||
""" retrieve file info, like maximum recursive depth and number of duplicate dependencies
|
||||
:param src_filename:
|
||||
:return:
|
||||
"""
|
||||
result = {'levels': 0,'dup_count':0}
|
||||
if src_filename in self._all_dependencies:
|
||||
data = self.trace(src_filename)
|
||||
for dep_filename in data:
|
||||
if data[dep_filename]['level'] > result['levels']:
|
||||
result['levels'] = data[dep_filename]['level']
|
||||
result['dup_count'] += len(data[dep_filename]['dup'])
|
||||
|
||||
return result
|
||||
|
||||
def generate_dot(self, filename_to_inspect):
|
||||
""" convert trace data to do graph
|
||||
:param filename_to_inspect: source filename to generate graph for
|
||||
:return: string (dot) data
|
||||
"""
|
||||
trace_data = self.trace(filename_to_inspect)
|
||||
result = list()
|
||||
result.append('digraph dependencies {')
|
||||
result.append('\toverlap=scale;')
|
||||
nodes = {}
|
||||
for level in range(100):
|
||||
for src_filename in trace_data:
|
||||
if trace_data[src_filename]['level'] == level:
|
||||
if trace_data[src_filename]['parent'] is not None:
|
||||
result.append('\tedge [color=black style=filled];')
|
||||
result.append('\t"%s" -> "%s" [weight=%d];' % (trace_data[src_filename]['parent'],
|
||||
src_filename, trace_data[src_filename]['level']))
|
||||
if len(trace_data[src_filename]['dup']) > 0:
|
||||
for target in trace_data[src_filename]['dup']:
|
||||
result.append('\tedge [color=red style=dotted];')
|
||||
result.append('\t"%s" -> "%s";' % (target, src_filename))
|
||||
|
||||
if trace_data[src_filename]['parent'] is None:
|
||||
nodes[src_filename] = '[shape=Mdiamond]'
|
||||
elif len(trace_data[src_filename]['dup']) > 0:
|
||||
nodes[src_filename] = '[shape=box,style=filled,color=".7 .3 1.0"]'
|
||||
else:
|
||||
nodes[src_filename] = '[shape=box]'
|
||||
|
||||
for node in nodes:
|
||||
result.append('\t"%s" %s;' % (node, nodes[node]))
|
||||
|
||||
result.append('}')
|
||||
return '\n'.join(result)
|
||||
|
||||
@staticmethod
|
||||
def generate_index_html(filelist):
|
||||
html_body = "<html><head><title></title></head><body><table><tr><th>Name</th></tr>\n%s</body>"
|
||||
html_row = '<tr><td><a href="%s">%s</a></td></tr>\n'
|
||||
html = html_body % ('\n'.join(map(lambda x: html_row % (x, x), sorted(filelist))))
|
||||
|
||||
return html
|
Loading…
Reference in New Issue