#!/usr/bin/python3.9 # # CDDL HEADER START # # The contents of this file are subject to the terms of the # Common Development and Distribution License (the "License"). # You may not use this file except in compliance with the License. # # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE # or http://www.opensolaris.org/os/licensing. # See the License for the specific language governing permissions # and limitations under the License. # # When distributing Covered Code, include this CDDL HEADER in each # file and include the License file at usr/src/OPENSOLARIS.LICENSE. # If applicable, add the following below this CDDL HEADER, with the # fields enclosed by brackets "[]" replaced with your own identifying # information: Portions Copyright [yyyy] [name of copyright owner] # # CDDL HEADER END # # Copyright (c) 2010, Oracle and/or it's affiliates. All rights reserved. # # # bass-o-matic.py # A simple program to enumerate components in the userland gate and report # on dependency related information. # import os import sys import re import glob import subprocess import argparse import logging import json import multiprocessing try: from scandir import walk except ImportError: from os import walk from bass.component import Component logger = logging.getLogger('bass-o-matic') # Locate SCM directories containing Userland components by searching from # the supplied top of tree for .p5m files. Once a .p5m file is located, # that directory is added to the list and no children are searched. def FindComponentPaths(path, debug=False, subdir='components', incremental=False, begin_commit=None, end_commit=None): expression = re.compile(r'.+\.p5m$', re.IGNORECASE) paths = [] if debug: logger.debug('searching %s for component directories', path) workspace_path = os.path.join(path, subdir) if incremental: cmd = ['git', '--no-pager', 'log', '--diff-filter=AMR', '--name-only', '--pretty=format:', '..'.join([begin_commit, end_commit])] proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=workspace_path, universal_newlines=True ) stdout, stderr = proc.communicate() if debug: if proc.returncode != 0: logger.debug('exit: %d, %s', proc.returncode, stderr) for line in stdout.splitlines(): line = line.rstrip() # git output might contain empty lines, so we skip them. if not line: continue # Every time component is added, modified or moved, Makefile has to be # present. However, this does not yet guarantee that the line is a # real component. filename = os.path.basename(line) dirname = os.path.dirname(line).rsplit(subdir + '/')[-1] if filename == 'Makefile': if glob.glob(os.path.join(workspace_path, dirname, '*.p5m')) and \ not os.path.isfile(os.path.join(workspace_path, dirname, 'pkg5.ignore')): paths.append(dirname) # Some components are using SCM checkout as a source code download method and # COMPONENT_REVISION is not bumped. With this, we will never rebuild them. # In order to rebuild them, we will look for such components and build them # every run. These components are located in openindiana category and we care # only about that category. One exception to this rule is meta-packages/history # component, which holds obsoleted components. We add it to paths manually for # that reason. cmd = ['git', 'grep', '-l', 'GIT_REPO *='] proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=workspace_path, universal_newlines=True ) stdout, stderr = proc.communicate() if debug: if proc.returncode != 0: logger.debug('exit: %d, %s', proc.returncode, stderr) for line in stdout.splitlines(): line = line.rstrip() # Only 'openindiana' category. category = line.split('/')[0] if category == 'openindiana': continue filename = os.path.basename(line) dirname = os.path.dirname(line) if filename == 'Makefile': if glob.glob(os.path.join(workspace_path, dirname, '*.p5m')) and \ not os.path.isfile(os.path.join(workspace_path, dirname, 'pkg5.ignore')): paths.append(os.path.dirname(line)) # Add meta-packages/history paths.append('meta-packages/history') paths = list(set(paths)) else: for dirpath, dirnames, filenames in walk(workspace_path): for name in filenames: if expression.match(name): if not os.path.isfile(os.path.join( dirpath, 'pkg5.ignore')): if debug: logger.debug('found %s', dirpath) paths.append(dirpath) del dirnames[:] break return sorted(paths) def main(): sys.stdout.flush() components = {} COMPONENTS_ALLOWED_PATHS = ['paths', 'dirs'] COMPONENTS_ALLOWED_FMRIS = ['fmris'] COMPONENTS_ALLOWED_DEPENDENCIES = ['dependencies'] COMPONENTS_ALLOWED_KEYWORDS = COMPONENTS_ALLOWED_PATHS + COMPONENTS_ALLOWED_FMRIS + COMPONENTS_ALLOWED_DEPENDENCIES parser = argparse.ArgumentParser() parser.add_argument('-w', '--workspace', default=os.getenv('WS_TOP'), help='Path to workspace') parser.add_argument('-l', '--components', default=None, choices=COMPONENTS_ALLOWED_KEYWORDS) parser.add_argument('--make', help='Makefile target to invoke') parser.add_argument('--pkg5', action='store_true',help='Invoke generation of metadata') parser.add_argument('--subdir', default='components', help='Directory holding components') parser.add_argument('-d', '--debug', action='store_true', default=False) parser.add_argument('--begin-commit', default=os.getenv('GIT_PREVIOUS_SUCCESSFUL_COMMIT', 'HEAD~1')) parser.add_argument('--end-commit', default='HEAD') args = parser.parse_args() workspace = args.workspace components_arg = args.components make_arg = args.make pkg5_arg = args.pkg5 subdir = args.subdir begin_commit = args.begin_commit end_commit = args.end_commit debug = args.debug log_level = logging.WARNING if debug: log_level = logging.DEBUG logging.basicConfig(level=log_level, format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',) if make_arg: MAKE=os.getenv("MAKE","gmake") # https://www.gnu.org/software/make/manual/html_node/POSIX-Jobserver.html JOBFLAGS=re.match('.* (--jobserver-auth=([0-9]+),([0-9]+)) ?.*',os.getenv("MAKEFLAGS","")) if JOBFLAGS: JOBFDS=( JOBFLAGS.group(2), JOBFLAGS.group(3) ) JOBFLAGS=[JOBFLAGS.group(1)] else: JOBFDS=() JOBFLAGS=[] proc = subprocess.Popen([MAKE, '-s'] + [make_arg] + JOBFLAGS,pass_fds=JOBFDS) rc = proc.wait() sys.exit(rc) if pkg5_arg: component_path = os.getcwd().split(os.path.join(workspace, subdir))[-1].replace('/', '', 1) # the component may not be built directly but as a dependency of another component if os.path.isfile(os.path.join( os.getcwd(), 'pkg5.ignore')): sys.exit(0) component_pkg5 = os.path.join( os.getcwd(), 'pkg5') if os.path.isfile(component_pkg5): os.remove(component_pkg5) Component(FindComponentPaths(path=workspace, debug=debug, subdir=os.path.join(subdir, component_path))[0]) sys.exit(0) incremental = False if os.getenv('BASS_O_MATIC_MODE') == 'incremental': incremental = True if incremental: component_paths = FindComponentPaths(path=workspace, debug=debug, subdir=subdir, incremental=incremental, begin_commit=begin_commit, end_commit=end_commit) else: component_paths = FindComponentPaths(path=workspace, debug=debug, subdir=subdir) if components_arg: if components_arg in COMPONENTS_ALLOWED_PATHS: for path in component_paths: print('{0}'.format(path)) elif components_arg in COMPONENTS_ALLOWED_FMRIS: pool = multiprocessing.Pool(processes=multiprocessing.cpu_count()) components = pool.map(Component, component_paths) for component in components: for fmri in component.supplied_packages: print('{0}'.format(fmri)) elif components_arg in COMPONENTS_ALLOWED_DEPENDENCIES: dependencies = {} pool = multiprocessing.Pool(processes=multiprocessing.cpu_count()) components = pool.map(Component, component_paths) with open(os.path.join(workspace, subdir, 'mapping.json'), "r") as f: data = json.loads(f.read()) component_path = {} for entry in data: component_path[entry['fmri']] = entry['path'] for component in components: selfpath = component.path.split(os.path.join(workspace, subdir))[-1].replace('/', '', 1) # Some packages from OpenSolaris only exist in binary form in the pkg repository paths = set([component_path.get(i, "https://pkg.openindiana.org/hipster") for i in component.required_packages]) # Remove self from the set of paths it depends on paths.discard(selfpath) dependencies[selfpath] = sorted(list(paths)) dependencies_file = os.path.join(workspace, subdir, 'dependencies.json') with open(dependencies_file, 'w') as f: f.write(json.dumps(dependencies, sort_keys=True, indent=4)) sys.exit(0) sys.exit(1) if __name__ == '__main__': main()