~olbohlen/oi-userland.git

Marcel Telka

2024-04-06 d5f77aba7114f0848f8116baa85c485375bc1385

commit \| author \| age
de89cf	1	#!/usr/bin/python3.9
a325d4	2	#
NJ	3	# CDDL HEADER START
	4	#
	5	# The contents of this file are subject to the terms of the
	6	# Common Development and Distribution License (the "License").
	7	# You may not use this file except in compliance with the License.
	8	#
	9	# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
	10	# or http://www.opensolaris.org/os/licensing.
	11	# See the License for the specific language governing permissions
	12	# and limitations under the License.
	13	#
	14	# When distributing Covered Code, include this CDDL HEADER in each
	15	# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
	16	# If applicable, add the following below this CDDL HEADER, with the
	17	# fields enclosed by brackets "[]" replaced with your own identifying
	18	# information: Portions Copyright [yyyy] [name of copyright owner]
	19	#
	20	# CDDL HEADER END
	21	#
	22	# Copyright (c) 2010, Oracle and/or it's affiliates. All rights reserved.
	23	#
	24	#
	25	# bass-o-matic.py
	26	# A simple program to enumerate components in the userland gate and report
	27	# on dependency related information.
	28	#
1018e3	29
a325d4	30	import os
NJ	31	import sys
	32	import re
1018e3	33	import glob
79b849	34	import subprocess
1018e3	35	import argparse
AŠ	36	import logging
476943	37	import json
AL	38	import multiprocessing
1018e3	39
AŠ	40	try:
	41	from scandir import walk
	42	except ImportError:
	43	from os import walk
	44
894a4d	45	from bass.component import Component
1e2fed	46
1018e3	47	logger = logging.getLogger('bass-o-matic')
a325d4	48
NJ	49	# Locate SCM directories containing Userland components by searching from
	50	# from a supplied top of tree for .p5m files. Once a .p5m file is located,
	51	# that directory is added to the list and no children are searched.
1018e3	52	def FindComponentPaths(path, debug=False, subdir='components',
AŠ	53	incremental=False, begin_commit=None, end_commit=None):
	54	expression = re.compile(r'.+\.p5m$', re.IGNORECASE)
a325d4	55
NJ	56	paths = []
	57
	58	if debug:
1018e3	59	logger.debug('searching %s for component directories', path)
a325d4	60
1018e3	61	workspace_path = os.path.join(path, subdir)
a325d4	62
1018e3	63	if incremental:
3b917d	64	cmd = ['git', '--no-pager', 'log', '--diff-filter=AMR', '--name-only', '--pretty=format:',
1018e3	65	'..'.join([begin_commit, end_commit])]
AŠ	66
	67	proc = subprocess.Popen(cmd,
	68	stdout=subprocess.PIPE,
	69	stderr=subprocess.PIPE,
3b917d	70	cwd=workspace_path,
fb10ba	71	universal_newlines=True
1018e3	72	)
3b917d	73	stdout, stderr = proc.communicate()
1018e3	74	if debug:
AŠ	75	if proc.returncode != 0:
3b917d	76	logger.debug('exit: %d, %s', proc.returncode, stderr)
1018e3	77
3b917d	78	for line in stdout.splitlines():
1018e3	79	line = line.rstrip()
AŠ	80	# git output might contain empty lines, so we skip them.
	81	if not line:
	82	continue
	83
	84	# Every time component is added, modified or moved, Makefile has to be
	85	# present. However, this does not yet guarantee that the line is a
	86	# real component.
	87	filename = os.path.basename(line)
	88	dirname = os.path.dirname(line).rsplit(subdir + '/')[-1]
	89
	90	if filename == 'Makefile':
afda44	91	if glob.glob(os.path.join(workspace_path, dirname, '*.p5m')) and \
AL	92	not os.path.isfile(os.path.join(workspace_path, dirname, 'pkg5.ignore')):
1018e3	93	paths.append(dirname)
476943	94
1018e3	95	# Some components are using SCM checkout as a source code download method and
AŠ	96	# COMPONENT_REVISION is not bumped. With this, we will never rebuild them.
	97	# In order to rebuild them, we will look for such components and build them
	98	# every run. These components are located in openindiana category and we care
abd63f	99	# only about that category. One exception to this rule is meta-packages/history
476943	100	# component, which holds obsoleted components. We add it to paths manually for
abd63f	101	# that reason.
1018e3	102	cmd = ['git', 'grep', '-l', 'GIT_REPO *=']
AŠ	103
	104	proc = subprocess.Popen(cmd,
	105	stdout=subprocess.PIPE,
	106	stderr=subprocess.PIPE,
	107	cwd=workspace_path,
fb10ba	108	universal_newlines=True
1018e3	109	)
AŠ	110
3b917d	111	stdout, stderr = proc.communicate()
1018e3	112	if debug:
AŠ	113	if proc.returncode != 0:
3b917d	114	logger.debug('exit: %d, %s', proc.returncode, stderr)
1018e3	115
3b917d	116	for line in stdout.splitlines():
1018e3	117	line = line.rstrip()
AŠ	118
	119	# Only 'openindiana' category.
	120	category = line.split('/')[0]
07188c	121	if category == 'openindiana':
1018e3	122	continue
AŠ	123
	124	filename = os.path.basename(line)
	125	dirname = os.path.dirname(line)
	126
	127	if filename == 'Makefile':
afda44	128	if glob.glob(os.path.join(workspace_path, dirname, '*.p5m')) and \
AL	129	not os.path.isfile(os.path.join(workspace_path, dirname, 'pkg5.ignore')):
1018e3	130	paths.append(os.path.dirname(line))
AŠ	131
9ff429	132	# Add meta-packages/history only if we build the main repository, where
AŠ	133	# subdir is equal to 'components'.
07188c	134	if subdir == 'components':
9ff429	135	paths.append('meta-packages/history')
aaf2df	136	# Add encumbered/meta-packages/history only if we build the encumbered repository
07188c	137	if subdir == 'components/encumbered':
aaf2df	138	paths.append('encumbered/meta-packages/history')
476943	139
1018e3	140	paths = list(set(paths))
AŠ	141
	142	else:
	143	for dirpath, dirnames, filenames in walk(workspace_path):
	144	for name in filenames:
	145	if expression.match(name):
afda44	146	if not os.path.isfile(os.path.join( dirpath, 'pkg5.ignore')):
AL	147	if debug:
	148	logger.debug('found %s', dirpath)
	149	paths.append(dirpath)
1018e3	150	del dirnames[:]
AŠ	151	break
a325d4	152
NJ	153	return sorted(paths)
	154
1018e3	155
a325d4	156	def main():
fb10ba	157	sys.stdout.flush()
a325d4	158
NJ	159	components = {}
	160
476943	161	COMPONENTS_ALLOWED_PATHS = ['paths', 'dirs']
AL	162	COMPONENTS_ALLOWED_FMRIS = ['fmris']
	163	COMPONENTS_ALLOWED_DEPENDENCIES = ['dependencies']
	164	COMPONENTS_ALLOWED_KEYWORDS = COMPONENTS_ALLOWED_PATHS + COMPONENTS_ALLOWED_FMRIS + COMPONENTS_ALLOWED_DEPENDENCIES
a325d4	165
1018e3	166	parser = argparse.ArgumentParser()
AŠ	167	parser.add_argument('-w', '--workspace', default=os.getenv('WS_TOP'), help='Path to workspace')
	168	parser.add_argument('-l', '--components', default=None, choices=COMPONENTS_ALLOWED_KEYWORDS)
	169	parser.add_argument('--make', help='Makefile target to invoke')
1e2fed	170	parser.add_argument('--pkg5', action='store_true',help='Invoke generation of metadata')
1018e3	171	parser.add_argument('--subdir', default='components', help='Directory holding components')
AŠ	172	parser.add_argument('-d', '--debug', action='store_true', default=False)
	173	parser.add_argument('--begin-commit', default=os.getenv('GIT_PREVIOUS_SUCCESSFUL_COMMIT', 'HEAD~1'))
	174	parser.add_argument('--end-commit', default='HEAD')
a325d4	175
1018e3	176	args = parser.parse_args()
AŠ	177
	178	workspace = args.workspace
	179	components_arg = args.components
	180	make_arg = args.make
1e2fed	181	pkg5_arg = args.pkg5
AL	182	subdir = args.subdir
1018e3	183	begin_commit = args.begin_commit
AŠ	184	end_commit = args.end_commit
	185	debug = args.debug
	186	log_level = logging.WARNING
	187
	188	if debug:
	189	log_level = logging.DEBUG
	190
	191	logging.basicConfig(level=log_level,
	192	format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',)
	193
aad0b2	194	if make_arg:
87ddce	195	MAKE=os.getenv("MAKE","gmake")
D	196
	197	# https://www.gnu.org/software/make/manual/html_node/POSIX-Jobserver.html
	198	JOBFLAGS=re.match('.* (--jobserver-auth=([0-9]+),([0-9]+)) ?.*',os.getenv("MAKEFLAGS",""))
	199	if JOBFLAGS:
	200	JOBFDS=( JOBFLAGS.group(2), JOBFLAGS.group(3) )
	201	JOBFLAGS=[JOBFLAGS.group(1)]
	202	else:
	203	JOBFDS=()
	204	JOBFLAGS=[]
	205	proc = subprocess.Popen([MAKE, '-s'] + [make_arg] + JOBFLAGS,pass_fds=JOBFDS)
aad0b2	206	rc = proc.wait()
AL	207	sys.exit(rc)
1e2fed	208
AL	209	if pkg5_arg:
	210	component_path = os.getcwd().split(os.path.join(workspace, subdir))[-1].replace('/', '', 1)
afda44	211	# the component may not be built directly but as a dependency of another component
AL	212	if os.path.isfile(os.path.join( os.getcwd(), 'pkg5.ignore')):
	213	sys.exit(0)
1e2fed	214	component_pkg5 = os.path.join( os.getcwd(), 'pkg5')
AL	215	if os.path.isfile(component_pkg5):
	216	os.remove(component_pkg5)
894a4d	217	Component(FindComponentPaths(path=workspace, debug=debug, subdir=os.path.join(subdir, component_path))[0])
1e2fed	218	sys.exit(0)
aad0b2	219
1018e3	220	incremental = False
AŠ	221	if os.getenv('BASS_O_MATIC_MODE') == 'incremental':
	222	incremental = True
	223
	224	if incremental:
	225	component_paths = FindComponentPaths(path=workspace, debug=debug, subdir=subdir,
	226	incremental=incremental, begin_commit=begin_commit, end_commit=end_commit)
	227	else:
	228	component_paths = FindComponentPaths(path=workspace, debug=debug, subdir=subdir)
79b849	229
a325d4	230	if components_arg:
1018e3	231	if components_arg in COMPONENTS_ALLOWED_PATHS:
a325d4	232	for path in component_paths:
1018e3	233	print('{0}'.format(path))
AŠ	234
476943	235	elif components_arg in COMPONENTS_ALLOWED_FMRIS:
AL	236	pool = multiprocessing.Pool(processes=multiprocessing.cpu_count())
894a4d	237	components = pool.map(Component, component_paths)
476943	238
AL	239	for component in components:
	240	for fmri in component.supplied_packages:
	241	print('{0}'.format(fmri))
	242
1018e3	243	elif components_arg in COMPONENTS_ALLOWED_DEPENDENCIES:
476943	244	dependencies = {}
a325d4	245
476943	246	pool = multiprocessing.Pool(processes=multiprocessing.cpu_count())
894a4d	247	components = pool.map(Component, component_paths)
a325d4	248
1e2fed	249	with open(os.path.join(workspace, subdir, 'mapping.json'), "r") as f:
AL	250	data = json.loads(f.read())
	251	component_path = {}
	252	for entry in data:
	253	component_path[entry['fmri']] = entry['path']
	254
476943	255	for component in components:
1fa515	256	selfpath = component.path.split(os.path.join(workspace, subdir))[-1].replace('/', '', 1)
AL	257	# Some packages from OpenSolaris only exist in binary form in the pkg repository
	258	paths = set([component_path.get(i, "https://pkg.openindiana.org/hipster") for i in component.required_packages])
	259	# Remove self from the set of paths it depends on
	260	paths.discard(selfpath)
	261	dependencies[selfpath] = sorted(list(paths))
a325d4	262
476943	263	dependencies_file = os.path.join(workspace, subdir, 'dependencies.json')
AL	264	with open(dependencies_file, 'w') as f:
	265	f.write(json.dumps(dependencies, sort_keys=True, indent=4))
a325d4	266	sys.exit(0)
NJ	267
	268	sys.exit(1)
	269
1018e3	270
AŠ	271	if __name__ == '__main__':
a325d4	272	main()