commit | author | age
|
de89cf
|
1 |
#!/usr/bin/python3.9 |
a325d4
|
2 |
# |
NJ |
3 |
# CDDL HEADER START |
|
4 |
# |
|
5 |
# The contents of this file are subject to the terms of the |
|
6 |
# Common Development and Distribution License (the "License"). |
|
7 |
# You may not use this file except in compliance with the License. |
|
8 |
# |
|
9 |
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE |
|
10 |
# or http://www.opensolaris.org/os/licensing. |
|
11 |
# See the License for the specific language governing permissions |
|
12 |
# and limitations under the License. |
|
13 |
# |
|
14 |
# When distributing Covered Code, include this CDDL HEADER in each |
|
15 |
# file and include the License file at usr/src/OPENSOLARIS.LICENSE. |
|
16 |
# If applicable, add the following below this CDDL HEADER, with the |
|
17 |
# fields enclosed by brackets "[]" replaced with your own identifying |
|
18 |
# information: Portions Copyright [yyyy] [name of copyright owner] |
|
19 |
# |
|
20 |
# CDDL HEADER END |
|
21 |
# |
|
22 |
# Copyright (c) 2010, Oracle and/or it's affiliates. All rights reserved. |
|
23 |
# |
|
24 |
# |
|
25 |
# bass-o-matic.py |
|
26 |
# A simple program to enumerate components in the userland gate and report |
|
27 |
# on dependency related information. |
|
28 |
# |
1018e3
|
29 |
|
a325d4
|
30 |
import os |
NJ |
31 |
import sys |
|
32 |
import re |
1018e3
|
33 |
import glob |
79b849
|
34 |
import subprocess |
1018e3
|
35 |
import argparse |
AŠ |
36 |
import logging |
476943
|
37 |
import json |
AL |
38 |
import multiprocessing |
1018e3
|
39 |
|
AŠ |
40 |
try: |
|
41 |
from scandir import walk |
|
42 |
except ImportError: |
|
43 |
from os import walk |
|
44 |
|
894a4d
|
45 |
from bass.component import Component |
1e2fed
|
46 |
|
1018e3
|
47 |
logger = logging.getLogger('bass-o-matic') |
a325d4
|
48 |
|
NJ |
49 |
# Locate SCM directories containing Userland components by searching from |
|
50 |
# from a supplied top of tree for .p5m files. Once a .p5m file is located, |
|
51 |
# that directory is added to the list and no children are searched. |
1018e3
|
52 |
def FindComponentPaths(path, debug=False, subdir='components', |
AŠ |
53 |
incremental=False, begin_commit=None, end_commit=None): |
|
54 |
expression = re.compile(r'.+\.p5m$', re.IGNORECASE) |
a325d4
|
55 |
|
NJ |
56 |
paths = [] |
|
57 |
|
|
58 |
if debug: |
1018e3
|
59 |
logger.debug('searching %s for component directories', path) |
a325d4
|
60 |
|
1018e3
|
61 |
workspace_path = os.path.join(path, subdir) |
a325d4
|
62 |
|
1018e3
|
63 |
if incremental: |
3b917d
|
64 |
cmd = ['git', '--no-pager', 'log', '--diff-filter=AMR', '--name-only', '--pretty=format:', |
1018e3
|
65 |
'..'.join([begin_commit, end_commit])] |
AŠ |
66 |
|
|
67 |
proc = subprocess.Popen(cmd, |
|
68 |
stdout=subprocess.PIPE, |
|
69 |
stderr=subprocess.PIPE, |
3b917d
|
70 |
cwd=workspace_path, |
fb10ba
|
71 |
universal_newlines=True |
1018e3
|
72 |
) |
3b917d
|
73 |
stdout, stderr = proc.communicate() |
1018e3
|
74 |
if debug: |
AŠ |
75 |
if proc.returncode != 0: |
3b917d
|
76 |
logger.debug('exit: %d, %s', proc.returncode, stderr) |
1018e3
|
77 |
|
3b917d
|
78 |
for line in stdout.splitlines(): |
1018e3
|
79 |
line = line.rstrip() |
AŠ |
80 |
# git output might contain empty lines, so we skip them. |
|
81 |
if not line: |
|
82 |
continue |
|
83 |
|
|
84 |
# Every time component is added, modified or moved, Makefile has to be |
|
85 |
# present. However, this does not yet guarantee that the line is a |
|
86 |
# real component. |
|
87 |
filename = os.path.basename(line) |
|
88 |
dirname = os.path.dirname(line).rsplit(subdir + '/')[-1] |
|
89 |
|
|
90 |
if filename == 'Makefile': |
afda44
|
91 |
if glob.glob(os.path.join(workspace_path, dirname, '*.p5m')) and \ |
AL |
92 |
not os.path.isfile(os.path.join(workspace_path, dirname, 'pkg5.ignore')): |
1018e3
|
93 |
paths.append(dirname) |
476943
|
94 |
|
1018e3
|
95 |
# Some components are using SCM checkout as a source code download method and |
AŠ |
96 |
# COMPONENT_REVISION is not bumped. With this, we will never rebuild them. |
|
97 |
# In order to rebuild them, we will look for such components and build them |
|
98 |
# every run. These components are located in openindiana category and we care |
abd63f
|
99 |
# only about that category. One exception to this rule is meta-packages/history |
476943
|
100 |
# component, which holds obsoleted components. We add it to paths manually for |
abd63f
|
101 |
# that reason. |
1018e3
|
102 |
cmd = ['git', 'grep', '-l', 'GIT_REPO *='] |
AŠ |
103 |
|
|
104 |
proc = subprocess.Popen(cmd, |
|
105 |
stdout=subprocess.PIPE, |
|
106 |
stderr=subprocess.PIPE, |
|
107 |
cwd=workspace_path, |
fb10ba
|
108 |
universal_newlines=True |
1018e3
|
109 |
) |
AŠ |
110 |
|
3b917d
|
111 |
stdout, stderr = proc.communicate() |
1018e3
|
112 |
if debug: |
AŠ |
113 |
if proc.returncode != 0: |
3b917d
|
114 |
logger.debug('exit: %d, %s', proc.returncode, stderr) |
1018e3
|
115 |
|
3b917d
|
116 |
for line in stdout.splitlines(): |
1018e3
|
117 |
line = line.rstrip() |
AŠ |
118 |
|
|
119 |
# Only 'openindiana' category. |
|
120 |
category = line.split('/')[0] |
07188c
|
121 |
if category == 'openindiana': |
1018e3
|
122 |
continue |
AŠ |
123 |
|
|
124 |
filename = os.path.basename(line) |
|
125 |
dirname = os.path.dirname(line) |
|
126 |
|
|
127 |
if filename == 'Makefile': |
afda44
|
128 |
if glob.glob(os.path.join(workspace_path, dirname, '*.p5m')) and \ |
AL |
129 |
not os.path.isfile(os.path.join(workspace_path, dirname, 'pkg5.ignore')): |
1018e3
|
130 |
paths.append(os.path.dirname(line)) |
AŠ |
131 |
|
9ff429
|
132 |
# Add meta-packages/history only if we build the main repository, where |
AŠ |
133 |
# subdir is equal to 'components'. |
07188c
|
134 |
if subdir == 'components': |
9ff429
|
135 |
paths.append('meta-packages/history') |
aaf2df
|
136 |
# Add encumbered/meta-packages/history only if we build the encumbered repository |
07188c
|
137 |
if subdir == 'components/encumbered': |
aaf2df
|
138 |
paths.append('encumbered/meta-packages/history') |
476943
|
139 |
|
1018e3
|
140 |
paths = list(set(paths)) |
AŠ |
141 |
|
|
142 |
else: |
|
143 |
for dirpath, dirnames, filenames in walk(workspace_path): |
|
144 |
for name in filenames: |
|
145 |
if expression.match(name): |
afda44
|
146 |
if not os.path.isfile(os.path.join( dirpath, 'pkg5.ignore')): |
AL |
147 |
if debug: |
|
148 |
logger.debug('found %s', dirpath) |
|
149 |
paths.append(dirpath) |
1018e3
|
150 |
del dirnames[:] |
AŠ |
151 |
break |
a325d4
|
152 |
|
NJ |
153 |
return sorted(paths) |
|
154 |
|
1018e3
|
155 |
|
a325d4
|
156 |
def main(): |
fb10ba
|
157 |
sys.stdout.flush() |
a325d4
|
158 |
|
NJ |
159 |
components = {} |
|
160 |
|
476943
|
161 |
COMPONENTS_ALLOWED_PATHS = ['paths', 'dirs'] |
AL |
162 |
COMPONENTS_ALLOWED_FMRIS = ['fmris'] |
|
163 |
COMPONENTS_ALLOWED_DEPENDENCIES = ['dependencies'] |
|
164 |
COMPONENTS_ALLOWED_KEYWORDS = COMPONENTS_ALLOWED_PATHS + COMPONENTS_ALLOWED_FMRIS + COMPONENTS_ALLOWED_DEPENDENCIES |
a325d4
|
165 |
|
1018e3
|
166 |
parser = argparse.ArgumentParser() |
AŠ |
167 |
parser.add_argument('-w', '--workspace', default=os.getenv('WS_TOP'), help='Path to workspace') |
|
168 |
parser.add_argument('-l', '--components', default=None, choices=COMPONENTS_ALLOWED_KEYWORDS) |
|
169 |
parser.add_argument('--make', help='Makefile target to invoke') |
1e2fed
|
170 |
parser.add_argument('--pkg5', action='store_true',help='Invoke generation of metadata') |
1018e3
|
171 |
parser.add_argument('--subdir', default='components', help='Directory holding components') |
AŠ |
172 |
parser.add_argument('-d', '--debug', action='store_true', default=False) |
|
173 |
parser.add_argument('--begin-commit', default=os.getenv('GIT_PREVIOUS_SUCCESSFUL_COMMIT', 'HEAD~1')) |
|
174 |
parser.add_argument('--end-commit', default='HEAD') |
a325d4
|
175 |
|
1018e3
|
176 |
args = parser.parse_args() |
AŠ |
177 |
|
|
178 |
workspace = args.workspace |
|
179 |
components_arg = args.components |
|
180 |
make_arg = args.make |
1e2fed
|
181 |
pkg5_arg = args.pkg5 |
AL |
182 |
subdir = args.subdir |
1018e3
|
183 |
begin_commit = args.begin_commit |
AŠ |
184 |
end_commit = args.end_commit |
|
185 |
debug = args.debug |
|
186 |
log_level = logging.WARNING |
|
187 |
|
|
188 |
if debug: |
|
189 |
log_level = logging.DEBUG |
|
190 |
|
|
191 |
logging.basicConfig(level=log_level, |
|
192 |
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',) |
|
193 |
|
aad0b2
|
194 |
if make_arg: |
87ddce
|
195 |
MAKE=os.getenv("MAKE","gmake") |
D |
196 |
|
|
197 |
# https://www.gnu.org/software/make/manual/html_node/POSIX-Jobserver.html |
|
198 |
JOBFLAGS=re.match('.* (--jobserver-auth=([0-9]+),([0-9]+)) ?.*',os.getenv("MAKEFLAGS","")) |
|
199 |
if JOBFLAGS: |
|
200 |
JOBFDS=( JOBFLAGS.group(2), JOBFLAGS.group(3) ) |
|
201 |
JOBFLAGS=[JOBFLAGS.group(1)] |
|
202 |
else: |
|
203 |
JOBFDS=() |
|
204 |
JOBFLAGS=[] |
|
205 |
proc = subprocess.Popen([MAKE, '-s'] + [make_arg] + JOBFLAGS,pass_fds=JOBFDS) |
aad0b2
|
206 |
rc = proc.wait() |
AL |
207 |
sys.exit(rc) |
1e2fed
|
208 |
|
AL |
209 |
if pkg5_arg: |
|
210 |
component_path = os.getcwd().split(os.path.join(workspace, subdir))[-1].replace('/', '', 1) |
afda44
|
211 |
# the component may not be built directly but as a dependency of another component |
AL |
212 |
if os.path.isfile(os.path.join( os.getcwd(), 'pkg5.ignore')): |
|
213 |
sys.exit(0) |
1e2fed
|
214 |
component_pkg5 = os.path.join( os.getcwd(), 'pkg5') |
AL |
215 |
if os.path.isfile(component_pkg5): |
|
216 |
os.remove(component_pkg5) |
894a4d
|
217 |
Component(FindComponentPaths(path=workspace, debug=debug, subdir=os.path.join(subdir, component_path))[0]) |
1e2fed
|
218 |
sys.exit(0) |
aad0b2
|
219 |
|
1018e3
|
220 |
incremental = False |
AŠ |
221 |
if os.getenv('BASS_O_MATIC_MODE') == 'incremental': |
|
222 |
incremental = True |
|
223 |
|
|
224 |
if incremental: |
|
225 |
component_paths = FindComponentPaths(path=workspace, debug=debug, subdir=subdir, |
|
226 |
incremental=incremental, begin_commit=begin_commit, end_commit=end_commit) |
|
227 |
else: |
|
228 |
component_paths = FindComponentPaths(path=workspace, debug=debug, subdir=subdir) |
79b849
|
229 |
|
a325d4
|
230 |
if components_arg: |
1018e3
|
231 |
if components_arg in COMPONENTS_ALLOWED_PATHS: |
a325d4
|
232 |
for path in component_paths: |
1018e3
|
233 |
print('{0}'.format(path)) |
AŠ |
234 |
|
476943
|
235 |
elif components_arg in COMPONENTS_ALLOWED_FMRIS: |
AL |
236 |
pool = multiprocessing.Pool(processes=multiprocessing.cpu_count()) |
894a4d
|
237 |
components = pool.map(Component, component_paths) |
476943
|
238 |
|
AL |
239 |
for component in components: |
|
240 |
for fmri in component.supplied_packages: |
|
241 |
print('{0}'.format(fmri)) |
|
242 |
|
1018e3
|
243 |
elif components_arg in COMPONENTS_ALLOWED_DEPENDENCIES: |
476943
|
244 |
dependencies = {} |
a325d4
|
245 |
|
476943
|
246 |
pool = multiprocessing.Pool(processes=multiprocessing.cpu_count()) |
894a4d
|
247 |
components = pool.map(Component, component_paths) |
a325d4
|
248 |
|
1e2fed
|
249 |
with open(os.path.join(workspace, subdir, 'mapping.json'), "r") as f: |
AL |
250 |
data = json.loads(f.read()) |
|
251 |
component_path = {} |
|
252 |
for entry in data: |
|
253 |
component_path[entry['fmri']] = entry['path'] |
|
254 |
|
476943
|
255 |
for component in components: |
1fa515
|
256 |
selfpath = component.path.split(os.path.join(workspace, subdir))[-1].replace('/', '', 1) |
AL |
257 |
# Some packages from OpenSolaris only exist in binary form in the pkg repository |
|
258 |
paths = set([component_path.get(i, "https://pkg.openindiana.org/hipster") for i in component.required_packages]) |
|
259 |
# Remove self from the set of paths it depends on |
|
260 |
paths.discard(selfpath) |
|
261 |
dependencies[selfpath] = sorted(list(paths)) |
a325d4
|
262 |
|
476943
|
263 |
dependencies_file = os.path.join(workspace, subdir, 'dependencies.json') |
AL |
264 |
with open(dependencies_file, 'w') as f: |
|
265 |
f.write(json.dumps(dependencies, sort_keys=True, indent=4)) |
a325d4
|
266 |
sys.exit(0) |
NJ |
267 |
|
|
268 |
sys.exit(1) |
|
269 |
|
1018e3
|
270 |
|
AŠ |
271 |
if __name__ == '__main__': |
a325d4
|
272 |
main() |