587 lines
20 KiB
Python
587 lines
20 KiB
Python
#
|
|
# SPDX-License-Identifier: GPL-2.0-only
|
|
#
|
|
|
|
import subprocess
|
|
import multiprocessing
|
|
import traceback
|
|
|
|
def read_file(filename):
|
|
try:
|
|
f = open( filename, "r" )
|
|
except IOError as reason:
|
|
return "" # WARNING: can't raise an error now because of the new RDEPENDS handling. This is a bit ugly. :M:
|
|
else:
|
|
data = f.read().strip()
|
|
f.close()
|
|
return data
|
|
return None
|
|
|
|
def ifelse(condition, iftrue = True, iffalse = False):
|
|
if condition:
|
|
return iftrue
|
|
else:
|
|
return iffalse
|
|
|
|
def conditional(variable, checkvalue, truevalue, falsevalue, d):
|
|
if d.getVar(variable) == checkvalue:
|
|
return truevalue
|
|
else:
|
|
return falsevalue
|
|
|
|
def vartrue(var, iftrue, iffalse, d):
|
|
import oe.types
|
|
if oe.types.boolean(d.getVar(var)):
|
|
return iftrue
|
|
else:
|
|
return iffalse
|
|
|
|
def less_or_equal(variable, checkvalue, truevalue, falsevalue, d):
|
|
if float(d.getVar(variable)) <= float(checkvalue):
|
|
return truevalue
|
|
else:
|
|
return falsevalue
|
|
|
|
def version_less_or_equal(variable, checkvalue, truevalue, falsevalue, d):
|
|
result = bb.utils.vercmp_string(d.getVar(variable), checkvalue)
|
|
if result <= 0:
|
|
return truevalue
|
|
else:
|
|
return falsevalue
|
|
|
|
def both_contain(variable1, variable2, checkvalue, d):
|
|
val1 = d.getVar(variable1)
|
|
val2 = d.getVar(variable2)
|
|
val1 = set(val1.split())
|
|
val2 = set(val2.split())
|
|
if isinstance(checkvalue, str):
|
|
checkvalue = set(checkvalue.split())
|
|
else:
|
|
checkvalue = set(checkvalue)
|
|
if checkvalue.issubset(val1) and checkvalue.issubset(val2):
|
|
return " ".join(checkvalue)
|
|
else:
|
|
return ""
|
|
|
|
def set_intersect(variable1, variable2, d):
|
|
"""
|
|
Expand both variables, interpret them as lists of strings, and return the
|
|
intersection as a flattened string.
|
|
|
|
For example:
|
|
s1 = "a b c"
|
|
s2 = "b c d"
|
|
s3 = set_intersect(s1, s2)
|
|
=> s3 = "b c"
|
|
"""
|
|
val1 = set(d.getVar(variable1).split())
|
|
val2 = set(d.getVar(variable2).split())
|
|
return " ".join(val1 & val2)
|
|
|
|
def prune_suffix(var, suffixes, d):
|
|
# See if var ends with any of the suffixes listed and
|
|
# remove it if found
|
|
for suffix in suffixes:
|
|
if suffix and var.endswith(suffix):
|
|
var = var[:-len(suffix)]
|
|
|
|
prefix = d.getVar("MLPREFIX")
|
|
if prefix and var.startswith(prefix):
|
|
var = var[len(prefix):]
|
|
|
|
return var
|
|
|
|
def str_filter(f, str, d):
|
|
from re import match
|
|
return " ".join([x for x in str.split() if match(f, x, 0)])
|
|
|
|
def str_filter_out(f, str, d):
|
|
from re import match
|
|
return " ".join([x for x in str.split() if not match(f, x, 0)])
|
|
|
|
def build_depends_string(depends, task):
|
|
"""Append a taskname to a string of dependencies as used by the [depends] flag"""
|
|
return " ".join(dep + ":" + task for dep in depends.split())
|
|
|
|
def inherits(d, *classes):
|
|
"""Return True if the metadata inherits any of the specified classes"""
|
|
return any(bb.data.inherits_class(cls, d) for cls in classes)
|
|
|
|
def features_backfill(var,d):
|
|
# This construct allows the addition of new features to variable specified
|
|
# as var
|
|
# Example for var = "DISTRO_FEATURES"
|
|
# This construct allows the addition of new features to DISTRO_FEATURES
|
|
# that if not present would disable existing functionality, without
|
|
# disturbing distributions that have already set DISTRO_FEATURES.
|
|
# Distributions wanting to elide a value in DISTRO_FEATURES_BACKFILL should
|
|
# add the feature to DISTRO_FEATURES_BACKFILL_CONSIDERED
|
|
features = (d.getVar(var) or "").split()
|
|
backfill = (d.getVar(var+"_BACKFILL") or "").split()
|
|
considered = (d.getVar(var+"_BACKFILL_CONSIDERED") or "").split()
|
|
|
|
addfeatures = []
|
|
for feature in backfill:
|
|
if feature not in features and feature not in considered:
|
|
addfeatures.append(feature)
|
|
|
|
if addfeatures:
|
|
d.appendVar(var, " " + " ".join(addfeatures))
|
|
|
|
def all_distro_features(d, features, truevalue="1", falsevalue=""):
|
|
"""
|
|
Returns truevalue if *all* given features are set in DISTRO_FEATURES,
|
|
else falsevalue. The features can be given as single string or anything
|
|
that can be turned into a set.
|
|
|
|
This is a shorter, more flexible version of
|
|
bb.utils.contains("DISTRO_FEATURES", features, truevalue, falsevalue, d).
|
|
|
|
Without explicit true/false values it can be used directly where
|
|
Python expects a boolean:
|
|
if oe.utils.all_distro_features(d, "foo bar"):
|
|
bb.fatal("foo and bar are mutually exclusive DISTRO_FEATURES")
|
|
|
|
With just a truevalue, it can be used to include files that are meant to be
|
|
used only when requested via DISTRO_FEATURES:
|
|
require ${@ oe.utils.all_distro_features(d, "foo bar", "foo-and-bar.inc")
|
|
"""
|
|
return bb.utils.contains("DISTRO_FEATURES", features, truevalue, falsevalue, d)
|
|
|
|
def any_distro_features(d, features, truevalue="1", falsevalue=""):
|
|
"""
|
|
Returns truevalue if at least *one* of the given features is set in DISTRO_FEATURES,
|
|
else falsevalue. The features can be given as single string or anything
|
|
that can be turned into a set.
|
|
|
|
This is a shorter, more flexible version of
|
|
bb.utils.contains_any("DISTRO_FEATURES", features, truevalue, falsevalue, d).
|
|
|
|
Without explicit true/false values it can be used directly where
|
|
Python expects a boolean:
|
|
if not oe.utils.any_distro_features(d, "foo bar"):
|
|
bb.fatal("foo, bar or both must be set in DISTRO_FEATURES")
|
|
|
|
With just a truevalue, it can be used to include files that are meant to be
|
|
used only when requested via DISTRO_FEATURES:
|
|
require ${@ oe.utils.any_distro_features(d, "foo bar", "foo-or-bar.inc")
|
|
|
|
"""
|
|
return bb.utils.contains_any("DISTRO_FEATURES", features, truevalue, falsevalue, d)
|
|
|
|
def parallel_make(d, makeinst=False):
|
|
"""
|
|
Return the integer value for the number of parallel threads to use when
|
|
building, scraped out of PARALLEL_MAKE. If no parallelization option is
|
|
found, returns None
|
|
|
|
e.g. if PARALLEL_MAKE = "-j 10", this will return 10 as an integer.
|
|
"""
|
|
if makeinst:
|
|
pm = (d.getVar('PARALLEL_MAKEINST') or '').split()
|
|
else:
|
|
pm = (d.getVar('PARALLEL_MAKE') or '').split()
|
|
# look for '-j' and throw other options (e.g. '-l') away
|
|
while pm:
|
|
opt = pm.pop(0)
|
|
if opt == '-j':
|
|
v = pm.pop(0)
|
|
elif opt.startswith('-j'):
|
|
v = opt[2:].strip()
|
|
else:
|
|
continue
|
|
|
|
return int(v)
|
|
|
|
return ''
|
|
|
|
def parallel_make_argument(d, fmt, limit=None, makeinst=False):
|
|
"""
|
|
Helper utility to construct a parallel make argument from the number of
|
|
parallel threads specified in PARALLEL_MAKE.
|
|
|
|
Returns the input format string `fmt` where a single '%d' will be expanded
|
|
with the number of parallel threads to use. If `limit` is specified, the
|
|
number of parallel threads will be no larger than it. If no parallelization
|
|
option is found in PARALLEL_MAKE, returns an empty string
|
|
|
|
e.g. if PARALLEL_MAKE = "-j 10", parallel_make_argument(d, "-n %d") will return
|
|
"-n 10"
|
|
"""
|
|
v = parallel_make(d, makeinst)
|
|
if v:
|
|
if limit:
|
|
v = min(limit, v)
|
|
return fmt % v
|
|
return ''
|
|
|
|
def packages_filter_out_system(d):
|
|
"""
|
|
Return a list of packages from PACKAGES with the "system" packages such as
|
|
PN-dbg PN-doc PN-locale-eb-gb removed.
|
|
"""
|
|
pn = d.getVar('PN')
|
|
pkgfilter = [pn + suffix for suffix in ('', '-dbg', '-dev', '-doc', '-locale', '-staticdev', '-src')]
|
|
localepkg = pn + "-locale-"
|
|
pkgs = []
|
|
|
|
for pkg in d.getVar('PACKAGES').split():
|
|
if pkg not in pkgfilter and localepkg not in pkg:
|
|
pkgs.append(pkg)
|
|
return pkgs
|
|
|
|
def getstatusoutput(cmd):
|
|
return subprocess.getstatusoutput(cmd)
|
|
|
|
|
|
def trim_version(version, num_parts=2):
|
|
"""
|
|
Return just the first <num_parts> of <version>, split by periods. For
|
|
example, trim_version("1.2.3", 2) will return "1.2".
|
|
"""
|
|
if type(version) is not str:
|
|
raise TypeError("Version should be a string")
|
|
if num_parts < 1:
|
|
raise ValueError("Cannot split to parts < 1")
|
|
|
|
parts = version.split(".")
|
|
trimmed = ".".join(parts[:num_parts])
|
|
return trimmed
|
|
|
|
def cpu_count(at_least=1, at_most=64):
|
|
cpus = len(os.sched_getaffinity(0))
|
|
return max(min(cpus, at_most), at_least)
|
|
|
|
def execute_pre_post_process(d, cmds):
|
|
if cmds is None:
|
|
return
|
|
|
|
for cmd in cmds.strip().split(';'):
|
|
cmd = cmd.strip()
|
|
if cmd != '':
|
|
bb.note("Executing %s ..." % cmd)
|
|
bb.build.exec_func(cmd, d)
|
|
|
|
# For each item in items, call the function 'target' with item as the first
|
|
# argument, extraargs as the other arguments and handle any exceptions in the
|
|
# parent thread
|
|
def multiprocess_launch(target, items, d, extraargs=None):
|
|
|
|
class ProcessLaunch(multiprocessing.Process):
|
|
def __init__(self, *args, **kwargs):
|
|
multiprocessing.Process.__init__(self, *args, **kwargs)
|
|
self._pconn, self._cconn = multiprocessing.Pipe()
|
|
self._exception = None
|
|
self._result = None
|
|
|
|
def run(self):
|
|
try:
|
|
ret = self._target(*self._args, **self._kwargs)
|
|
self._cconn.send((None, ret))
|
|
except Exception as e:
|
|
tb = traceback.format_exc()
|
|
self._cconn.send((e, tb))
|
|
|
|
def update(self):
|
|
if self._pconn.poll():
|
|
(e, tb) = self._pconn.recv()
|
|
if e is not None:
|
|
self._exception = (e, tb)
|
|
else:
|
|
self._result = tb
|
|
|
|
@property
|
|
def exception(self):
|
|
self.update()
|
|
return self._exception
|
|
|
|
@property
|
|
def result(self):
|
|
self.update()
|
|
return self._result
|
|
|
|
max_process = int(d.getVar("BB_NUMBER_THREADS") or os.cpu_count() or 1)
|
|
launched = []
|
|
errors = []
|
|
results = []
|
|
items = list(items)
|
|
while (items and not errors) or launched:
|
|
if not errors and items and len(launched) < max_process:
|
|
args = (items.pop(),)
|
|
if extraargs is not None:
|
|
args = args + extraargs
|
|
p = ProcessLaunch(target=target, args=args)
|
|
p.start()
|
|
launched.append(p)
|
|
for q in launched:
|
|
# Have to manually call update() to avoid deadlocks. The pipe can be full and
|
|
# transfer stalled until we try and read the results object but the subprocess won't exit
|
|
# as it still has data to write (https://bugs.python.org/issue8426)
|
|
q.update()
|
|
# The finished processes are joined when calling is_alive()
|
|
if not q.is_alive():
|
|
if q.exception:
|
|
errors.append(q.exception)
|
|
if q.result:
|
|
results.append(q.result)
|
|
launched.remove(q)
|
|
# Paranoia doesn't hurt
|
|
for p in launched:
|
|
p.join()
|
|
if errors:
|
|
msg = ""
|
|
for (e, tb) in errors:
|
|
if isinstance(e, subprocess.CalledProcessError) and e.output:
|
|
msg = msg + str(e) + "\n"
|
|
msg = msg + "Subprocess output:"
|
|
msg = msg + e.output.decode("utf-8", errors="ignore")
|
|
else:
|
|
msg = msg + str(e) + ": " + str(tb) + "\n"
|
|
bb.fatal("Fatal errors occurred in subprocesses:\n%s" % msg)
|
|
return results
|
|
|
|
def squashspaces(string):
|
|
import re
|
|
return re.sub(r"\s+", " ", string).strip()
|
|
|
|
def rprovides_map(pkgdata_dir, pkg_dict):
|
|
# Map file -> pkg provider
|
|
rprov_map = {}
|
|
|
|
for pkg in pkg_dict:
|
|
path_to_pkgfile = os.path.join(pkgdata_dir, 'runtime-reverse', pkg)
|
|
if not os.path.isfile(path_to_pkgfile):
|
|
continue
|
|
with open(path_to_pkgfile) as f:
|
|
for line in f:
|
|
if line.startswith('RPROVIDES') or line.startswith('FILERPROVIDES'):
|
|
# List all components provided by pkg.
|
|
# Exclude version strings, i.e. those starting with (
|
|
provides = [x for x in line.split()[1:] if not x.startswith('(')]
|
|
for prov in provides:
|
|
if prov in rprov_map:
|
|
rprov_map[prov].append(pkg)
|
|
else:
|
|
rprov_map[prov] = [pkg]
|
|
|
|
return rprov_map
|
|
|
|
def format_pkg_list(pkg_dict, ret_format=None, pkgdata_dir=None):
|
|
output = []
|
|
|
|
if ret_format == "arch":
|
|
for pkg in sorted(pkg_dict):
|
|
output.append("%s %s" % (pkg, pkg_dict[pkg]["arch"]))
|
|
elif ret_format == "file":
|
|
for pkg in sorted(pkg_dict):
|
|
output.append("%s %s %s" % (pkg, pkg_dict[pkg]["filename"], pkg_dict[pkg]["arch"]))
|
|
elif ret_format == "ver":
|
|
for pkg in sorted(pkg_dict):
|
|
output.append("%s %s %s" % (pkg, pkg_dict[pkg]["arch"], pkg_dict[pkg]["ver"]))
|
|
elif ret_format == "deps":
|
|
rprov_map = rprovides_map(pkgdata_dir, pkg_dict)
|
|
for pkg in sorted(pkg_dict):
|
|
for dep in pkg_dict[pkg]["deps"]:
|
|
if dep in rprov_map:
|
|
# There could be multiple providers within the image
|
|
for pkg_provider in rprov_map[dep]:
|
|
output.append("%s|%s * %s [RPROVIDES]" % (pkg, pkg_provider, dep))
|
|
else:
|
|
output.append("%s|%s" % (pkg, dep))
|
|
else:
|
|
for pkg in sorted(pkg_dict):
|
|
output.append(pkg)
|
|
|
|
output_str = '\n'.join(output)
|
|
|
|
if output_str:
|
|
# make sure last line is newline terminated
|
|
output_str += '\n'
|
|
|
|
return output_str
|
|
|
|
|
|
# Helper function to get the host compiler version
|
|
# Do not assume the compiler is gcc
|
|
def get_host_compiler_version(d, taskcontextonly=False):
|
|
import re, subprocess
|
|
|
|
if taskcontextonly and d.getVar('BB_WORKERCONTEXT') != '1':
|
|
return
|
|
|
|
compiler = d.getVar("BUILD_CC")
|
|
# Get rid of ccache since it is not present when parsing.
|
|
if compiler.startswith('ccache '):
|
|
compiler = compiler[7:]
|
|
try:
|
|
env = os.environ.copy()
|
|
# datastore PATH does not contain session PATH as set by environment-setup-...
|
|
# this breaks the install-buildtools use-case
|
|
# env["PATH"] = d.getVar("PATH")
|
|
output = subprocess.check_output("%s --version" % compiler, \
|
|
shell=True, env=env, stderr=subprocess.STDOUT).decode("utf-8")
|
|
except subprocess.CalledProcessError as e:
|
|
bb.fatal("Error running %s --version: %s" % (compiler, e.output.decode("utf-8")))
|
|
|
|
match = re.match(r".* (\d+\.\d+)\.\d+.*", output.split('\n')[0])
|
|
if not match:
|
|
bb.fatal("Can't get compiler version from %s --version output" % compiler)
|
|
|
|
version = match.group(1)
|
|
return compiler, version
|
|
|
|
|
|
def host_gcc_version(d, taskcontextonly=False):
|
|
import re, subprocess
|
|
|
|
if taskcontextonly and d.getVar('BB_WORKERCONTEXT') != '1':
|
|
return
|
|
|
|
compiler = d.getVar("BUILD_CC")
|
|
# Get rid of ccache since it is not present when parsing.
|
|
if compiler.startswith('ccache '):
|
|
compiler = compiler[7:]
|
|
try:
|
|
env = os.environ.copy()
|
|
env["PATH"] = d.getVar("PATH")
|
|
output = subprocess.check_output("%s --version" % compiler, \
|
|
shell=True, env=env, stderr=subprocess.STDOUT).decode("utf-8")
|
|
except subprocess.CalledProcessError as e:
|
|
bb.fatal("Error running %s --version: %s" % (compiler, e.output.decode("utf-8")))
|
|
|
|
match = re.match(r".* (\d+\.\d+)\.\d+.*", output.split('\n')[0])
|
|
if not match:
|
|
bb.fatal("Can't get compiler version from %s --version output" % compiler)
|
|
|
|
version = match.group(1)
|
|
return "-%s" % version if version in ("4.8", "4.9") else ""
|
|
|
|
|
|
def get_multilib_datastore(variant, d):
|
|
localdata = bb.data.createCopy(d)
|
|
if variant:
|
|
overrides = localdata.getVar("OVERRIDES", False) + ":virtclass-multilib-" + variant
|
|
localdata.setVar("OVERRIDES", overrides)
|
|
localdata.setVar("MLPREFIX", variant + "-")
|
|
else:
|
|
origdefault = localdata.getVar("DEFAULTTUNE_MULTILIB_ORIGINAL")
|
|
if origdefault:
|
|
localdata.setVar("DEFAULTTUNE", origdefault)
|
|
overrides = localdata.getVar("OVERRIDES", False).split(":")
|
|
overrides = ":".join([x for x in overrides if not x.startswith("virtclass-multilib-")])
|
|
localdata.setVar("OVERRIDES", overrides)
|
|
localdata.setVar("MLPREFIX", "")
|
|
return localdata
|
|
|
|
#
|
|
# Python 2.7 doesn't have threaded pools (just multiprocessing)
|
|
# so implement a version here
|
|
#
|
|
|
|
from queue import Queue
|
|
from threading import Thread
|
|
|
|
class ThreadedWorker(Thread):
|
|
"""Thread executing tasks from a given tasks queue"""
|
|
def __init__(self, tasks, worker_init, worker_end, name=None):
|
|
Thread.__init__(self, name=name)
|
|
self.tasks = tasks
|
|
self.daemon = True
|
|
|
|
self.worker_init = worker_init
|
|
self.worker_end = worker_end
|
|
|
|
def run(self):
|
|
from queue import Empty
|
|
|
|
if self.worker_init is not None:
|
|
self.worker_init(self)
|
|
|
|
while True:
|
|
try:
|
|
func, args, kargs = self.tasks.get(block=False)
|
|
except Empty:
|
|
if self.worker_end is not None:
|
|
self.worker_end(self)
|
|
break
|
|
|
|
try:
|
|
func(self, *args, **kargs)
|
|
except Exception as e:
|
|
# Eat all exceptions
|
|
bb.mainlogger.debug("Worker task raised %s" % e, exc_info=e)
|
|
finally:
|
|
self.tasks.task_done()
|
|
|
|
class ThreadedPool:
|
|
"""Pool of threads consuming tasks from a queue"""
|
|
def __init__(self, num_workers, num_tasks, worker_init=None, worker_end=None, name="ThreadedPool-"):
|
|
self.tasks = Queue(num_tasks)
|
|
self.workers = []
|
|
|
|
for i in range(num_workers):
|
|
worker = ThreadedWorker(self.tasks, worker_init, worker_end, name=name + str(i))
|
|
self.workers.append(worker)
|
|
|
|
def start(self):
|
|
for worker in self.workers:
|
|
worker.start()
|
|
|
|
def add_task(self, func, *args, **kargs):
|
|
"""Add a task to the queue"""
|
|
self.tasks.put((func, args, kargs))
|
|
|
|
def wait_completion(self):
|
|
"""Wait for completion of all the tasks in the queue"""
|
|
self.tasks.join()
|
|
for worker in self.workers:
|
|
worker.join()
|
|
|
|
class ImageQAFailed(Exception):
|
|
def __init__(self, description, name=None, logfile=None):
|
|
self.description = description
|
|
self.name = name
|
|
self.logfile=logfile
|
|
|
|
def __str__(self):
|
|
msg = 'Function failed: %s' % self.name
|
|
if self.description:
|
|
msg = msg + ' (%s)' % self.description
|
|
|
|
return msg
|
|
|
|
def sh_quote(string):
|
|
import shlex
|
|
return shlex.quote(string)
|
|
|
|
def directory_size(root, blocksize=4096):
|
|
"""
|
|
Calculate the size of the directory, taking into account hard links,
|
|
rounding up every size to multiples of the blocksize.
|
|
"""
|
|
def roundup(size):
|
|
"""
|
|
Round the size up to the nearest multiple of the block size.
|
|
"""
|
|
import math
|
|
return math.ceil(size / blocksize) * blocksize
|
|
|
|
def getsize(filename):
|
|
"""
|
|
Get the size of the filename, not following symlinks, taking into
|
|
account hard links.
|
|
"""
|
|
stat = os.lstat(filename)
|
|
if stat.st_ino not in inodes:
|
|
inodes.add(stat.st_ino)
|
|
return stat.st_size
|
|
else:
|
|
return 0
|
|
|
|
inodes = set()
|
|
total = 0
|
|
for root, dirs, files in os.walk(root):
|
|
total += sum(roundup(getsize(os.path.join(root, name))) for name in files)
|
|
total += roundup(getsize(root))
|
|
return total
|