commit | author | age
|
de89cf
|
1 |
#!/usr/bin/python3.9 |
4158c0
|
2 |
# |
NJ |
3 |
# CDDL HEADER START |
|
4 |
# |
|
5 |
# The contents of this file are subject to the terms of the |
|
6 |
# Common Development and Distribution License (the "License"). |
|
7 |
# You may not use this file except in compliance with the License. |
|
8 |
# |
|
9 |
# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE |
f1351e
|
10 |
# or http://www.illumos.org/license/CDDL. |
4158c0
|
11 |
# See the License for the specific language governing permissions |
NJ |
12 |
# and limitations under the License. |
|
13 |
# |
|
14 |
# When distributing Covered Code, include this CDDL HEADER in each |
|
15 |
# file and include the License file at usr/src/OPENSOLARIS.LICENSE. |
|
16 |
# If applicable, add the following below this CDDL HEADER, with the |
|
17 |
# fields enclosed by brackets "[]" replaced with your own identifying |
|
18 |
# information: Portions Copyright [yyyy] [name of copyright owner] |
|
19 |
# |
|
20 |
# CDDL HEADER END |
|
21 |
# |
90e68e
|
22 |
# Copyright (c) 2011, 2012, Oracle and/or its affiliates. All rights reserved. |
4158c0
|
23 |
# |
NJ |
24 |
# |
|
25 |
# userland-mangler - a file mangling utility |
|
26 |
# |
|
27 |
# A simple program to mangle files to conform to Solaris WOS or Consoldation |
|
28 |
# requirements. |
|
29 |
# |
|
30 |
|
|
31 |
import os |
|
32 |
import sys |
|
33 |
import re |
df9898
|
34 |
import subprocess |
NJ |
35 |
import shutil |
237c88
|
36 |
import stat |
df9898
|
37 |
|
4158c0
|
38 |
|
NJ |
39 |
import pkg.fmri |
|
40 |
import pkg.manifest |
|
41 |
import pkg.actions |
|
42 |
import pkg.elf as elf |
90e68e
|
43 |
|
MS |
44 |
attribute_oracle_table_header = """ |
|
45 |
.\\\" Oracle has added the ARC stability level to this manual page""" |
4158c0
|
46 |
|
NJ |
47 |
attribute_table_header = """ |
|
48 |
.SH ATTRIBUTES |
|
49 |
See |
|
50 |
.BR attributes (5) |
|
51 |
for descriptions of the following attributes: |
|
52 |
.sp |
|
53 |
.TS |
|
54 |
box; |
|
55 |
cbp-1 | cbp-1 |
|
56 |
l | l . |
|
57 |
ATTRIBUTE TYPE ATTRIBUTE VALUE """ |
|
58 |
|
|
59 |
attribute_table_availability = """ |
|
60 |
= |
|
61 |
Availability %s""" |
|
62 |
|
|
63 |
attribute_table_stability = """ |
|
64 |
= |
|
65 |
Stability %s""" |
|
66 |
|
|
67 |
attribute_table_footer = """ |
404117
|
68 |
.TE |
4158c0
|
69 |
.PP |
NJ |
70 |
""" |
90e68e
|
71 |
def attributes_section_text(availability, stability, modified_date): |
fb10ba
|
72 |
result = '' |
2720d6
|
73 |
|
fb10ba
|
74 |
# is there anything to do? |
AP |
75 |
if availability is not None or stability is not None: |
|
76 |
result = attribute_oracle_table_header |
|
77 |
if modified_date is not None: |
|
78 |
result += ("\n.\\\" on %s" % modified_date) |
|
79 |
result += attribute_table_header |
4158c0
|
80 |
|
fb10ba
|
81 |
if availability is not None: |
AP |
82 |
result += (attribute_table_availability % availability) |
|
83 |
if stability is not None: |
|
84 |
result += (attribute_table_stability % stability.capitalize()) |
|
85 |
result += attribute_table_footer |
4158c0
|
86 |
|
fb10ba
|
87 |
return result |
90e68e
|
88 |
|
MS |
89 |
notes_oracle_comment = """ |
|
90 |
.\\\" Oracle has added source availability information to this manual page""" |
4158c0
|
91 |
|
NJ |
92 |
notes_header = """ |
|
93 |
.SH NOTES |
|
94 |
""" |
|
95 |
|
|
96 |
notes_community = """ |
|
97 |
Further information about this software can be found on the open source community website at %s. |
|
98 |
""" |
|
99 |
notes_source = """ |
f1351e
|
100 |
This software was built from source available at https://openindiana.org/. The original community source was downloaded from %s |
4158c0
|
101 |
""" |
NJ |
102 |
|
90e68e
|
103 |
def notes_section_text(header_seen, community, source, modified_date): |
fb10ba
|
104 |
result = '' |
2720d6
|
105 |
|
fb10ba
|
106 |
# is there anything to do? |
AP |
107 |
if community is not None or source is not None: |
|
108 |
if header_seen == False: |
|
109 |
result += notes_header |
|
110 |
result += notes_oracle_comment |
|
111 |
if modified_date is not None: |
|
112 |
result += ("\n.\\\" on %s" % modified_date) |
|
113 |
if source is not None: |
|
114 |
result += (notes_source % source) |
|
115 |
if community is not None: |
|
116 |
result += (notes_community % community) |
4158c0
|
117 |
|
fb10ba
|
118 |
return result |
4158c0
|
119 |
|
2720d6
|
120 |
so_re = re.compile('^\.so.+$', re.MULTILINE) |
4158c0
|
121 |
section_re = re.compile('\.SH "?([^"]+).*$', re.IGNORECASE) |
fbf173
|
122 |
TH_re = re.compile('\.TH\s+(?:"[^"]+"|\S+)\s+(\S+)', re.IGNORECASE) |
4158c0
|
123 |
# |
NJ |
124 |
# mangler.man.stability = (mangler.man.stability) |
90e68e
|
125 |
# mangler.man.modified_date = (mangler.man.modified-date) |
4158c0
|
126 |
# mangler.man.availability = (pkg.fmri) |
1de4f0
|
127 |
# mangler.man.source-url = (pkg.source-url) |
MS |
128 |
# mangler.man.upstream-url = (pkg.upstream-url) |
4158c0
|
129 |
# |
2720d6
|
130 |
def mangle_manpage(manifest, action, text): |
fb10ba
|
131 |
# manpages must have a taxonomy defined |
AP |
132 |
stability = action.attrs.pop('mangler.man.stability', None) |
|
133 |
if stability is None: |
|
134 |
sys.stderr.write("ERROR: manpage action missing mangler.man.stability: %s" % action) |
|
135 |
sys.exit(1) |
90e68e
|
136 |
|
fb10ba
|
137 |
# manpages may have a 'modified date' |
AP |
138 |
modified_date = action.attrs.pop('mangler.man.modified-date', None) |
fbf173
|
139 |
|
fb10ba
|
140 |
# Rewrite the section in the .TH line to match the section in which |
AP |
141 |
# we're delivering it. |
|
142 |
rewrite_sect = action.attrs.pop('mangler.man.rewrite-section', 'true') |
4158c0
|
143 |
|
fb10ba
|
144 |
attributes_written = False |
AP |
145 |
notes_seen = False |
4158c0
|
146 |
|
fb10ba
|
147 |
if 'pkg.fmri' in manifest.attributes: |
AP |
148 |
fmri = pkg.fmri.PkgFmri(manifest.attributes['pkg.fmri']) |
|
149 |
availability = fmri.pkg_name |
4158c0
|
150 |
|
fb10ba
|
151 |
community = None |
AP |
152 |
if 'info.upstream-url' in manifest.attributes: |
|
153 |
community = manifest.attributes['info.upstream-url'] |
4158c0
|
154 |
|
fb10ba
|
155 |
source = None |
AP |
156 |
if 'info.source-url' in manifest.attributes: |
|
157 |
source = manifest.attributes['info.source-url'] |
|
158 |
elif 'info.repository-url' in manifest.attributes: |
|
159 |
source = manifest.attributes['info.repository-url'] |
4158c0
|
160 |
|
fb10ba
|
161 |
# skip reference only pages |
AP |
162 |
if so_re.match(text) is not None: |
|
163 |
return text |
4158c0
|
164 |
|
fb10ba
|
165 |
# tell man that we want tables (and eqn) |
AP |
166 |
result = "'\\\" te\n" |
4158c0
|
167 |
|
fb10ba
|
168 |
# write the orginal data |
AP |
169 |
for line in text.split('\n'): |
|
170 |
match = section_re.match(line) |
|
171 |
if match is not None: |
|
172 |
section = match.group(1) |
|
173 |
if section in ['SEE ALSO', 'NOTES']: |
|
174 |
if attributes_written == False: |
|
175 |
result += attributes_section_text( |
|
176 |
availability, |
|
177 |
stability, |
|
178 |
modified_date) |
|
179 |
attributes_written = True |
|
180 |
if section == 'NOTES': |
|
181 |
notes_seen = True |
|
182 |
match = TH_re.match(line) |
|
183 |
if match and rewrite_sect.lower() == "true": |
|
184 |
# Use the section defined by the filename, rather than |
|
185 |
# the directory in which it sits. |
|
186 |
sect = os.path.splitext(action.attrs["path"])[1][1:] |
|
187 |
line = line[:match.span(1)[0]] + sect + \ |
|
188 |
line[match.span(1)[1]:] |
fbf173
|
189 |
|
fb10ba
|
190 |
result += ("%s\n" % line) |
4158c0
|
191 |
|
fb10ba
|
192 |
if attributes_written == False: |
AP |
193 |
result += attributes_section_text(availability, stability, |
|
194 |
modified_date) |
4158c0
|
195 |
|
fb10ba
|
196 |
result += notes_section_text(notes_seen, community, source, |
AP |
197 |
modified_date) |
4158c0
|
198 |
|
fb10ba
|
199 |
return result |
4158c0
|
200 |
|
NJ |
201 |
# |
df9898
|
202 |
# mangler.elf.strip_runpath = (true|false) |
4158c0
|
203 |
# |
NJ |
204 |
def mangle_elf(manifest, action, src, dest): |
fb10ba
|
205 |
strip_elf_runpath = action.attrs.pop('mangler.elf.strip_runpath', 'true') |
07188c
|
206 |
if strip_elf_runpath == 'false': |
fb10ba
|
207 |
return |
df9898
|
208 |
|
fb10ba
|
209 |
# |
AP |
210 |
# Strip any runtime linker default search path elements from the file |
|
211 |
# and replace relative paths with absolute paths |
|
212 |
# |
|
213 |
ELFEDIT = '/usr/bin/elfedit' |
df9898
|
214 |
|
fb10ba
|
215 |
# runtime linker default search path elements + /64 link |
AP |
216 |
rtld_default_dirs = [ '/lib', '/usr/lib', |
|
217 |
'/lib/64', '/usr/lib/64', |
|
218 |
'/lib/amd64', '/usr/lib/amd64', |
|
219 |
'/lib/sparcv9', '/usr/lib/sparcv9' ] |
df9898
|
220 |
|
fb10ba
|
221 |
runpath_re = re.compile('.+\s(RPATH|RUNPATH)\s+\S+\s+(\S+)') |
df9898
|
222 |
|
fb10ba
|
223 |
# Retreive the search path from the object file. Use elfedit(1) because pkg.elf only |
AP |
224 |
# retrieves the RUNPATH. Note that dyn:rpath and dyn:runpath return both values. |
|
225 |
# Both RPATH and RUNPATH are expected to be the same, but in an overabundand of caution, |
|
226 |
# process each element found separately. |
|
227 |
result = subprocess.Popen([ELFEDIT, '-re', 'dyn:runpath', src ], |
|
228 |
stdout=subprocess.PIPE, stderr=subprocess.PIPE, |
|
229 |
universal_newlines=True) |
df9898
|
230 |
result.wait() |
fb10ba
|
231 |
if result.returncode != 0: # no RUNPATH or RPATH to potentially strip |
AP |
232 |
return |
df9898
|
233 |
|
fb10ba
|
234 |
for line in result.stdout: |
AP |
235 |
result = runpath_re.match(line) |
|
236 |
if result != None: |
|
237 |
element = result.group(1) |
|
238 |
original_dirs = result.group(2).split(":") |
|
239 |
keep_dirs = [] |
|
240 |
matched_dirs = [] |
df9898
|
241 |
|
fb10ba
|
242 |
for dir in original_dirs: |
AP |
243 |
if dir not in rtld_default_dirs: |
|
244 |
if dir.startswith('$ORIGIN'): |
|
245 |
path = action.attrs['path'] |
|
246 |
dirname = os.path.dirname(path) |
|
247 |
if dirname[0] != '/': |
|
248 |
dirname = '/' + dirname |
|
249 |
corrected_dir = dir.replace('$ORIGIN', dirname) |
|
250 |
corrected_dir = os.path.realpath(corrected_dir) |
|
251 |
matched_dirs.append(dir) |
|
252 |
keep_dirs.append(corrected_dir) |
|
253 |
else: |
|
254 |
keep_dirs.append(dir) |
|
255 |
else: |
|
256 |
matched_dirs.append(dir) |
df9898
|
257 |
|
fb10ba
|
258 |
if len(matched_dirs) != 0: |
AP |
259 |
# Emit an "Error" message in case someone wants to look at the build log |
|
260 |
# and fix the component build so that this is a NOP. |
|
261 |
print("Stripping %s from %s in %s" % (":".join(matched_dirs), element, src), file=sys.stderr) |
df9898
|
262 |
|
fb10ba
|
263 |
# Make sure that there is a destdir to copy the file into for mangling. |
AP |
264 |
destdir = os.path.dirname(dest) |
|
265 |
if not os.path.exists(destdir): |
|
266 |
os.makedirs(destdir) |
a3ab10
|
267 |
# Create a copy to mangle |
AL |
268 |
# Earlier the code would check that the destination file does not exist |
|
269 |
# yet, however internal library versioning can be different while the |
|
270 |
# filename remains the same. |
|
271 |
# When publishing from a non-clean prototype directory older libraries |
|
272 |
# which may be ABI incompatible would then be republished in the new |
|
273 |
# package instead of the new version. |
|
274 |
shutil.copy2(src, dest) |
df9898
|
275 |
|
237c88
|
276 |
# Make sure we do have write permission before we try to modify the file |
MT |
277 |
os.chmod(dest, os.stat(dest).st_mode | stat.S_IWUSR) |
|
278 |
|
fb10ba
|
279 |
# Mangle the copy by deleting the tag if there is nothing left to keep |
AP |
280 |
# or replacing the value if there is something left. |
|
281 |
elfcmd = "dyn:delete %s" % element.lower() |
|
282 |
if len(keep_dirs) > 0: |
|
283 |
elfcmd = "dyn:%s '%s'" % (element.lower(), ":".join(keep_dirs)) |
|
284 |
subprocess.call([ELFEDIT, '-e', elfcmd, dest]) |
4158c0
|
285 |
|
NJ |
286 |
# |
|
287 |
# mangler.script.file-magic = |
|
288 |
# |
2720d6
|
289 |
def mangle_script(manifest, action, text): |
fb10ba
|
290 |
return text |
2720d6
|
291 |
|
NJ |
292 |
# |
|
293 |
# mangler.strip_cddl = false |
|
294 |
# |
|
295 |
def mangle_cddl(manifest, action, text): |
fb10ba
|
296 |
strip_cddl = action.attrs.pop('mangler.strip_cddl', 'false') |
07188c
|
297 |
if strip_cddl == 'false': |
fb10ba
|
298 |
return text |
AP |
299 |
cddl_re = re.compile('^[^\n]*CDDL HEADER START.+CDDL HEADER END[^\n]*$', |
|
300 |
re.MULTILINE|re.DOTALL) |
|
301 |
return cddl_re.sub('', text) |
4158c0
|
302 |
|
5f4d07
|
303 |
def do_ctfconvert(converter, file): |
BS |
304 |
args = [converter, '-i', '-m', '-k', file] |
|
305 |
print(*args, file=sys.stderr) |
|
306 |
subprocess.call(args) |
|
307 |
|
|
308 |
def mangle_path(manifest, action, src, dest, ctfconvert): |
fb10ba
|
309 |
if elf.is_elf_object(src): |
5f4d07
|
310 |
if ctfconvert is not None: |
BS |
311 |
do_ctfconvert(ctfconvert, src) |
fb10ba
|
312 |
mangle_elf(manifest, action, src, dest) |
AP |
313 |
else: |
|
314 |
# a 'text' document (script, man page, config file, ... |
|
315 |
# We treat all documents as latin-1 text to avoid |
|
316 |
# reencoding them and loosing data |
|
317 |
ifp = open(src, 'r', encoding='latin-1') |
|
318 |
text = ifp.read() |
|
319 |
ifp.close() |
2720d6
|
320 |
|
fb10ba
|
321 |
# remove the CDDL from files |
AP |
322 |
result = mangle_cddl(manifest, action, text) |
2720d6
|
323 |
|
fb10ba
|
324 |
if 'facet.doc.man' in action.attrs: |
AP |
325 |
result = mangle_manpage(manifest, action, result) |
|
326 |
elif 'mode' in action.attrs and int(action.attrs['mode'], 8) & 0o111 != 0: |
|
327 |
result = mangle_script(manifest, action, result) |
2720d6
|
328 |
|
fb10ba
|
329 |
if text != result: |
AP |
330 |
destdir = os.path.dirname(dest) |
|
331 |
if not os.path.exists(destdir): |
|
332 |
os.makedirs(destdir) |
|
333 |
with open(dest, 'w', encoding='latin-1') as ofp: |
|
334 |
ofp.write(result) |
4158c0
|
335 |
|
NJ |
336 |
# |
|
337 |
# mangler.bypass = (true|false) |
|
338 |
# |
5f4d07
|
339 |
def mangle_paths(manifest, search_paths, destination, ctfconvert): |
fb10ba
|
340 |
for action in manifest.gen_actions_by_type("file"): |
AP |
341 |
bypass = action.attrs.pop('mangler.bypass', 'false').lower() |
|
342 |
if bypass == 'true': |
|
343 |
continue |
4158c0
|
344 |
|
fb10ba
|
345 |
path = None |
AP |
346 |
if 'path' in action.attrs: |
|
347 |
path = action.attrs['path'] |
|
348 |
if action.hash and action.hash != 'NOHASH': |
|
349 |
path = action.hash |
|
350 |
if not path: |
|
351 |
continue |
4158c0
|
352 |
|
fb10ba
|
353 |
if not os.path.exists(destination): |
AP |
354 |
os.makedirs(destination) |
2720d6
|
355 |
|
fb10ba
|
356 |
dest = os.path.join(destination, path) |
AP |
357 |
for directory in search_paths: |
|
358 |
if directory != destination: |
|
359 |
src = os.path.join(directory, path) |
|
360 |
if os.path.isfile(src): |
5f4d07
|
361 |
mangle_path(manifest, action, |
BS |
362 |
src, dest, ctfconvert) |
fb10ba
|
363 |
break |
4158c0
|
364 |
|
NJ |
365 |
def load_manifest(manifest_file): |
fb10ba
|
366 |
manifest = pkg.manifest.Manifest() |
AP |
367 |
manifest.set_content(pathname=manifest_file) |
4158c0
|
368 |
|
fb10ba
|
369 |
return manifest |
4158c0
|
370 |
|
NJ |
371 |
def usage(): |
fb10ba
|
372 |
print("Usage: %s [-m|--manifest (file)] [-d|--search-directory (dir)] [-D|--destination (dir)] " % (sys.argv[0].split('/')[-1])) |
AP |
373 |
sys.exit(1) |
4158c0
|
374 |
|
NJ |
375 |
def main(): |
fb10ba
|
376 |
import getopt |
4158c0
|
377 |
|
fb10ba
|
378 |
sys.stdout.flush() |
4158c0
|
379 |
|
fb10ba
|
380 |
search_paths = [] |
AP |
381 |
destination = None |
|
382 |
manifests = [] |
5f4d07
|
383 |
ctfconvert = None |
4158c0
|
384 |
|
fb10ba
|
385 |
try: |
5f4d07
|
386 |
opts, args = getopt.getopt(sys.argv[1:], "c:D:d:m:", |
BS |
387 |
["ctf=", "destination=", "search-directory=", "manifest="]) |
fb10ba
|
388 |
except getopt.GetoptError as err: |
AP |
389 |
print(str(err)) |
|
390 |
usage() |
4158c0
|
391 |
|
fb10ba
|
392 |
for opt, arg in opts: |
AP |
393 |
if opt in [ "-D", "--destination" ]: |
|
394 |
destination = arg |
|
395 |
elif opt in [ "-d", "--search-directory" ]: |
|
396 |
search_paths.append(arg) |
|
397 |
elif opt in [ "-m", "--manifest" ]: |
|
398 |
try: |
|
399 |
manifest = load_manifest(arg) |
|
400 |
except IOError as err: |
|
401 |
print("oops, %s: %s" % (arg, str(err))) |
|
402 |
usage() |
|
403 |
else: |
|
404 |
manifests.append(manifest) |
5f4d07
|
405 |
elif opt in [ "-c", "--ctf" ]: |
BS |
406 |
ctfconvert = arg |
fb10ba
|
407 |
else: |
AP |
408 |
usage() |
4158c0
|
409 |
|
fb10ba
|
410 |
if destination == None: |
AP |
411 |
usage() |
4158c0
|
412 |
|
fb10ba
|
413 |
for manifest in manifests: |
5f4d07
|
414 |
mangle_paths(manifest, search_paths, destination, ctfconvert) |
fb10ba
|
415 |
print(manifest) |
4158c0
|
416 |
|
fb10ba
|
417 |
sys.exit(0) |
4158c0
|
418 |
|
NJ |
419 |
if __name__ == "__main__": |
fb10ba
|
420 |
main() |