416 lines
16 KiB
Python
416 lines
16 KiB
Python
|
#
|
||
|
|
# SPDX-License-Identifier: GPL-2.0-only
|
||
|
|
#
|
||
|
|
from abc import ABCMeta, abstractmethod
|
||
|
|
from oe.utils import execute_pre_post_process
|
||
|
|
from oe.package_manager import *
|
||
|
|
from oe.manifest import *
|
||
|
|
import oe.path
|
||
|
|
import shutil
|
||
|
|
import os
|
||
|
|
import subprocess
|
||
|
|
import re
|
||
|
|
|
||
|
|
class Rootfs(object, metaclass=ABCMeta):
|
||
|
|
"""
|
||
|
|
This is an abstract class. Do not instantiate this directly.
|
||
|
|
"""
|
||
|
|
|
||
|
|
def __init__(self, d, progress_reporter=None, logcatcher=None):
|
||
|
|
self.d = d
|
||
|
|
self.pm = None
|
||
|
|
self.image_rootfs = self.d.getVar('IMAGE_ROOTFS')
|
||
|
|
self.deploydir = self.d.getVar('IMGDEPLOYDIR')
|
||
|
|
self.progress_reporter = progress_reporter
|
||
|
|
self.logcatcher = logcatcher
|
||
|
|
|
||
|
|
self.install_order = Manifest.INSTALL_ORDER
|
||
|
|
|
||
|
|
@abstractmethod
|
||
|
|
def _create(self):
|
||
|
|
pass
|
||
|
|
|
||
|
|
@abstractmethod
|
||
|
|
def _get_delayed_postinsts(self):
|
||
|
|
pass
|
||
|
|
|
||
|
|
@abstractmethod
|
||
|
|
def _save_postinsts(self):
|
||
|
|
pass
|
||
|
|
|
||
|
|
@abstractmethod
|
||
|
|
def _log_check(self):
|
||
|
|
pass
|
||
|
|
|
||
|
|
def _log_check_common(self, type, match):
|
||
|
|
# Ignore any lines containing log_check to avoid recursion, and ignore
|
||
|
|
# lines beginning with a + since sh -x may emit code which isn't
|
||
|
|
# actually executed, but may contain error messages
|
||
|
|
excludes = [ 'log_check', r'^\+' ]
|
||
|
|
if hasattr(self, 'log_check_expected_regexes'):
|
||
|
|
excludes.extend(self.log_check_expected_regexes)
|
||
|
|
# Insert custom log_check excludes
|
||
|
|
excludes += [x for x in (self.d.getVar("IMAGE_LOG_CHECK_EXCLUDES") or "").split(" ") if x]
|
||
|
|
excludes = [re.compile(x) for x in excludes]
|
||
|
|
r = re.compile(match)
|
||
|
|
log_path = self.d.expand("${T}/log.do_rootfs")
|
||
|
|
messages = []
|
||
|
|
with open(log_path, 'r') as log:
|
||
|
|
for line in log:
|
||
|
|
if self.logcatcher and self.logcatcher.contains(line.rstrip()):
|
||
|
|
continue
|
||
|
|
for ee in excludes:
|
||
|
|
m = ee.search(line)
|
||
|
|
if m:
|
||
|
|
break
|
||
|
|
if m:
|
||
|
|
continue
|
||
|
|
|
||
|
|
m = r.search(line)
|
||
|
|
if m:
|
||
|
|
messages.append('[log_check] %s' % line)
|
||
|
|
if messages:
|
||
|
|
if len(messages) == 1:
|
||
|
|
msg = '1 %s message' % type
|
||
|
|
else:
|
||
|
|
msg = '%d %s messages' % (len(messages), type)
|
||
|
|
msg = '[log_check] %s: found %s in the logfile:\n%s' % \
|
||
|
|
(self.d.getVar('PN'), msg, ''.join(messages))
|
||
|
|
if type == 'error':
|
||
|
|
bb.fatal(msg)
|
||
|
|
else:
|
||
|
|
bb.warn(msg)
|
||
|
|
|
||
|
|
def _log_check_warn(self):
|
||
|
|
self._log_check_common('warning', '^(warn|Warn|WARNING:)')
|
||
|
|
|
||
|
|
def _log_check_error(self):
|
||
|
|
self._log_check_common('error', self.log_check_regex)
|
||
|
|
|
||
|
|
def _insert_feed_uris(self):
|
||
|
|
if bb.utils.contains("IMAGE_FEATURES", "package-management",
|
||
|
|
True, False, self.d):
|
||
|
|
self.pm.insert_feeds_uris(self.d.getVar('PACKAGE_FEED_URIS') or "",
|
||
|
|
self.d.getVar('PACKAGE_FEED_BASE_PATHS') or "",
|
||
|
|
self.d.getVar('PACKAGE_FEED_ARCHS'))
|
||
|
|
|
||
|
|
|
||
|
|
"""
|
||
|
|
The _cleanup() method should be used to clean-up stuff that we don't really
|
||
|
|
want to end up on target. For example, in the case of RPM, the DB locks.
|
||
|
|
The method is called, once, at the end of create() method.
|
||
|
|
"""
|
||
|
|
@abstractmethod
|
||
|
|
def _cleanup(self):
|
||
|
|
pass
|
||
|
|
|
||
|
|
def _setup_dbg_rootfs(self, package_paths):
|
||
|
|
gen_debugfs = self.d.getVar('IMAGE_GEN_DEBUGFS') or '0'
|
||
|
|
if gen_debugfs != '1':
|
||
|
|
return
|
||
|
|
|
||
|
|
bb.note(" Renaming the original rootfs...")
|
||
|
|
try:
|
||
|
|
shutil.rmtree(self.image_rootfs + '-orig')
|
||
|
|
except:
|
||
|
|
pass
|
||
|
|
bb.utils.rename(self.image_rootfs, self.image_rootfs + '-orig')
|
||
|
|
|
||
|
|
bb.note(" Creating debug rootfs...")
|
||
|
|
bb.utils.mkdirhier(self.image_rootfs)
|
||
|
|
|
||
|
|
bb.note(" Copying back package database...")
|
||
|
|
for path in package_paths:
|
||
|
|
bb.utils.mkdirhier(self.image_rootfs + os.path.dirname(path))
|
||
|
|
if os.path.isdir(self.image_rootfs + '-orig' + path):
|
||
|
|
shutil.copytree(self.image_rootfs + '-orig' + path, self.image_rootfs + path, symlinks=True)
|
||
|
|
elif os.path.isfile(self.image_rootfs + '-orig' + path):
|
||
|
|
shutil.copyfile(self.image_rootfs + '-orig' + path, self.image_rootfs + path)
|
||
|
|
|
||
|
|
# Copy files located in /usr/lib/debug or /usr/src/debug
|
||
|
|
for dir in ["/usr/lib/debug", "/usr/src/debug"]:
|
||
|
|
src = self.image_rootfs + '-orig' + dir
|
||
|
|
if os.path.exists(src):
|
||
|
|
dst = self.image_rootfs + dir
|
||
|
|
bb.utils.mkdirhier(os.path.dirname(dst))
|
||
|
|
shutil.copytree(src, dst)
|
||
|
|
|
||
|
|
# Copy files with suffix '.debug' or located in '.debug' dir.
|
||
|
|
for root, dirs, files in os.walk(self.image_rootfs + '-orig'):
|
||
|
|
relative_dir = root[len(self.image_rootfs + '-orig'):]
|
||
|
|
for f in files:
|
||
|
|
if f.endswith('.debug') or '/.debug' in relative_dir:
|
||
|
|
bb.utils.mkdirhier(self.image_rootfs + relative_dir)
|
||
|
|
shutil.copy(os.path.join(root, f),
|
||
|
|
self.image_rootfs + relative_dir)
|
||
|
|
|
||
|
|
bb.note(" Install complementary '*-dbg' packages...")
|
||
|
|
self.pm.install_complementary('*-dbg')
|
||
|
|
|
||
|
|
if self.d.getVar('PACKAGE_DEBUG_SPLIT_STYLE') == 'debug-with-srcpkg':
|
||
|
|
bb.note(" Install complementary '*-src' packages...")
|
||
|
|
self.pm.install_complementary('*-src')
|
||
|
|
|
||
|
|
"""
|
||
|
|
Install additional debug packages. Possibility to install additional packages,
|
||
|
|
which are not automatically installed as complementary package of
|
||
|
|
standard one, e.g. debug package of static libraries.
|
||
|
|
"""
|
||
|
|
extra_debug_pkgs = self.d.getVar('IMAGE_INSTALL_DEBUGFS')
|
||
|
|
if extra_debug_pkgs:
|
||
|
|
bb.note(" Install extra debug packages...")
|
||
|
|
self.pm.install(extra_debug_pkgs.split(), True)
|
||
|
|
|
||
|
|
bb.note(" Removing package database...")
|
||
|
|
for path in package_paths:
|
||
|
|
if os.path.isdir(self.image_rootfs + path):
|
||
|
|
shutil.rmtree(self.image_rootfs + path)
|
||
|
|
elif os.path.isfile(self.image_rootfs + path):
|
||
|
|
os.remove(self.image_rootfs + path)
|
||
|
|
|
||
|
|
bb.note(" Rename debug rootfs...")
|
||
|
|
try:
|
||
|
|
shutil.rmtree(self.image_rootfs + '-dbg')
|
||
|
|
except:
|
||
|
|
pass
|
||
|
|
bb.utils.rename(self.image_rootfs, self.image_rootfs + '-dbg')
|
||
|
|
|
||
|
|
bb.note(" Restoring original rootfs...")
|
||
|
|
bb.utils.rename(self.image_rootfs + '-orig', self.image_rootfs)
|
||
|
|
|
||
|
|
def _exec_shell_cmd(self, cmd):
|
||
|
|
fakerootcmd = self.d.getVar('FAKEROOT')
|
||
|
|
if fakerootcmd is not None:
|
||
|
|
exec_cmd = [fakerootcmd, cmd]
|
||
|
|
else:
|
||
|
|
exec_cmd = cmd
|
||
|
|
|
||
|
|
try:
|
||
|
|
subprocess.check_output(exec_cmd, stderr=subprocess.STDOUT)
|
||
|
|
except subprocess.CalledProcessError as e:
|
||
|
|
return("Command '%s' returned %d:\n%s" % (e.cmd, e.returncode, e.output))
|
||
|
|
|
||
|
|
return None
|
||
|
|
|
||
|
|
def create(self):
|
||
|
|
bb.note("###### Generate rootfs #######")
|
||
|
|
pre_process_cmds = self.d.getVar("ROOTFS_PREPROCESS_COMMAND")
|
||
|
|
post_process_cmds = self.d.getVar("ROOTFS_POSTPROCESS_COMMAND")
|
||
|
|
rootfs_post_install_cmds = self.d.getVar('ROOTFS_POSTINSTALL_COMMAND')
|
||
|
|
|
||
|
|
execute_pre_post_process(self.d, pre_process_cmds)
|
||
|
|
|
||
|
|
if self.progress_reporter:
|
||
|
|
self.progress_reporter.next_stage()
|
||
|
|
|
||
|
|
# call the package manager dependent create method
|
||
|
|
self._create()
|
||
|
|
|
||
|
|
sysconfdir = self.image_rootfs + self.d.getVar('sysconfdir')
|
||
|
|
bb.utils.mkdirhier(sysconfdir)
|
||
|
|
with open(sysconfdir + "/version", "w+") as ver:
|
||
|
|
ver.write(self.d.getVar('BUILDNAME') + "\n")
|
||
|
|
|
||
|
|
execute_pre_post_process(self.d, rootfs_post_install_cmds)
|
||
|
|
|
||
|
|
self.pm.run_intercepts()
|
||
|
|
|
||
|
|
execute_pre_post_process(self.d, post_process_cmds)
|
||
|
|
|
||
|
|
if self.progress_reporter:
|
||
|
|
self.progress_reporter.next_stage()
|
||
|
|
|
||
|
|
if bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs",
|
||
|
|
True, False, self.d) and \
|
||
|
|
not bb.utils.contains("IMAGE_FEATURES",
|
||
|
|
"read-only-rootfs-delayed-postinsts",
|
||
|
|
True, False, self.d):
|
||
|
|
delayed_postinsts = self._get_delayed_postinsts()
|
||
|
|
if delayed_postinsts is not None:
|
||
|
|
bb.fatal("The following packages could not be configured "
|
||
|
|
"offline and rootfs is read-only: %s" %
|
||
|
|
delayed_postinsts)
|
||
|
|
|
||
|
|
if self.d.getVar('USE_DEVFS') != "1":
|
||
|
|
self._create_devfs()
|
||
|
|
|
||
|
|
self._uninstall_unneeded()
|
||
|
|
|
||
|
|
if self.progress_reporter:
|
||
|
|
self.progress_reporter.next_stage()
|
||
|
|
|
||
|
|
self._insert_feed_uris()
|
||
|
|
|
||
|
|
self._run_ldconfig()
|
||
|
|
|
||
|
|
if self.d.getVar('USE_DEPMOD') != "0":
|
||
|
|
self._generate_kernel_module_deps()
|
||
|
|
|
||
|
|
self._cleanup()
|
||
|
|
self._log_check()
|
||
|
|
|
||
|
|
if self.progress_reporter:
|
||
|
|
self.progress_reporter.next_stage()
|
||
|
|
|
||
|
|
|
||
|
|
def _uninstall_unneeded(self):
|
||
|
|
# Remove the run-postinsts package if no delayed postinsts are found
|
||
|
|
delayed_postinsts = self._get_delayed_postinsts()
|
||
|
|
if delayed_postinsts is None:
|
||
|
|
if os.path.exists(self.d.expand("${IMAGE_ROOTFS}${sysconfdir}/init.d/run-postinsts")) or os.path.exists(self.d.expand("${IMAGE_ROOTFS}${systemd_system_unitdir}/run-postinsts.service")):
|
||
|
|
self.pm.remove(["run-postinsts"])
|
||
|
|
|
||
|
|
image_rorfs = bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs",
|
||
|
|
True, False, self.d)
|
||
|
|
image_rorfs_force = self.d.getVar('FORCE_RO_REMOVE')
|
||
|
|
|
||
|
|
if image_rorfs or image_rorfs_force == "1":
|
||
|
|
# Remove components that we don't need if it's a read-only rootfs
|
||
|
|
unneeded_pkgs = self.d.getVar("ROOTFS_RO_UNNEEDED").split()
|
||
|
|
pkgs_installed = image_list_installed_packages(self.d)
|
||
|
|
# Make sure update-alternatives is removed last. This is
|
||
|
|
# because its database has to available while uninstalling
|
||
|
|
# other packages, allowing alternative symlinks of packages
|
||
|
|
# to be uninstalled or to be managed correctly otherwise.
|
||
|
|
provider = self.d.getVar("VIRTUAL-RUNTIME_update-alternatives")
|
||
|
|
pkgs_to_remove = sorted([pkg for pkg in pkgs_installed if pkg in unneeded_pkgs], key=lambda x: x == provider)
|
||
|
|
|
||
|
|
# update-alternatives provider is removed in its own remove()
|
||
|
|
# call because all package managers do not guarantee the packages
|
||
|
|
# are removed in the order they given in the list (which is
|
||
|
|
# passed to the command line). The sorting done earlier is
|
||
|
|
# utilized to implement the 2-stage removal.
|
||
|
|
if len(pkgs_to_remove) > 1:
|
||
|
|
self.pm.remove(pkgs_to_remove[:-1], False)
|
||
|
|
if len(pkgs_to_remove) > 0:
|
||
|
|
self.pm.remove([pkgs_to_remove[-1]], False)
|
||
|
|
|
||
|
|
if delayed_postinsts:
|
||
|
|
self._save_postinsts()
|
||
|
|
if image_rorfs:
|
||
|
|
bb.warn("There are post install scripts "
|
||
|
|
"in a read-only rootfs")
|
||
|
|
|
||
|
|
post_uninstall_cmds = self.d.getVar("ROOTFS_POSTUNINSTALL_COMMAND")
|
||
|
|
execute_pre_post_process(self.d, post_uninstall_cmds)
|
||
|
|
|
||
|
|
runtime_pkgmanage = bb.utils.contains("IMAGE_FEATURES", "package-management",
|
||
|
|
True, False, self.d)
|
||
|
|
if not runtime_pkgmanage:
|
||
|
|
# Remove the package manager data files
|
||
|
|
self.pm.remove_packaging_data()
|
||
|
|
|
||
|
|
def _run_ldconfig(self):
|
||
|
|
if self.d.getVar('LDCONFIGDEPEND'):
|
||
|
|
bb.note("Executing: ldconfig -r " + self.image_rootfs + " -c new -v -X")
|
||
|
|
self._exec_shell_cmd(['ldconfig', '-r', self.image_rootfs, '-c',
|
||
|
|
'new', '-v', '-X'])
|
||
|
|
|
||
|
|
image_rorfs = bb.utils.contains("IMAGE_FEATURES", "read-only-rootfs",
|
||
|
|
True, False, self.d)
|
||
|
|
ldconfig_in_features = bb.utils.contains("DISTRO_FEATURES", "ldconfig",
|
||
|
|
True, False, self.d)
|
||
|
|
if image_rorfs or not ldconfig_in_features:
|
||
|
|
ldconfig_cache_dir = os.path.join(self.image_rootfs, "var/cache/ldconfig")
|
||
|
|
if os.path.exists(ldconfig_cache_dir):
|
||
|
|
bb.note("Removing ldconfig auxiliary cache...")
|
||
|
|
shutil.rmtree(ldconfig_cache_dir)
|
||
|
|
|
||
|
|
def _check_for_kernel_modules(self, modules_dir):
|
||
|
|
for root, dirs, files in os.walk(modules_dir, topdown=True):
|
||
|
|
for name in files:
|
||
|
|
found_ko = name.endswith((".ko", ".ko.gz", ".ko.xz", ".ko.zst"))
|
||
|
|
if found_ko:
|
||
|
|
return found_ko
|
||
|
|
return False
|
||
|
|
|
||
|
|
def _generate_kernel_module_deps(self):
|
||
|
|
modules_dir = os.path.join(self.image_rootfs, 'lib', 'modules')
|
||
|
|
# if we don't have any modules don't bother to do the depmod
|
||
|
|
if not self._check_for_kernel_modules(modules_dir):
|
||
|
|
bb.note("No Kernel Modules found, not running depmod")
|
||
|
|
return
|
||
|
|
|
||
|
|
kernel_abi_ver_file = oe.path.join(self.d.getVar('PKGDATA_DIR'), "kernel-depmod",
|
||
|
|
'kernel-abiversion')
|
||
|
|
if not os.path.exists(kernel_abi_ver_file):
|
||
|
|
bb.fatal("No kernel-abiversion file found (%s), cannot run depmod, aborting" % kernel_abi_ver_file)
|
||
|
|
|
||
|
|
with open(kernel_abi_ver_file) as f:
|
||
|
|
kernel_ver = f.read().strip(' \n')
|
||
|
|
|
||
|
|
versioned_modules_dir = os.path.join(self.image_rootfs, modules_dir, kernel_ver)
|
||
|
|
|
||
|
|
bb.utils.mkdirhier(versioned_modules_dir)
|
||
|
|
|
||
|
|
self._exec_shell_cmd(['depmodwrapper', '-a', '-b', self.image_rootfs, kernel_ver])
|
||
|
|
|
||
|
|
"""
|
||
|
|
Create devfs:
|
||
|
|
* IMAGE_DEVICE_TABLE is the old name to an absolute path to a device table file
|
||
|
|
* IMAGE_DEVICE_TABLES is a new name for a file, or list of files, seached
|
||
|
|
for in the BBPATH
|
||
|
|
If neither are specified then the default name of files/device_table-minimal.txt
|
||
|
|
is searched for in the BBPATH (same as the old version.)
|
||
|
|
"""
|
||
|
|
def _create_devfs(self):
|
||
|
|
devtable_list = []
|
||
|
|
devtable = self.d.getVar('IMAGE_DEVICE_TABLE')
|
||
|
|
if devtable is not None:
|
||
|
|
devtable_list.append(devtable)
|
||
|
|
else:
|
||
|
|
devtables = self.d.getVar('IMAGE_DEVICE_TABLES')
|
||
|
|
if devtables is None:
|
||
|
|
devtables = 'files/device_table-minimal.txt'
|
||
|
|
for devtable in devtables.split():
|
||
|
|
devtable_list.append("%s" % bb.utils.which(self.d.getVar('BBPATH'), devtable))
|
||
|
|
|
||
|
|
for devtable in devtable_list:
|
||
|
|
self._exec_shell_cmd(["makedevs", "-r",
|
||
|
|
self.image_rootfs, "-D", devtable])
|
||
|
|
|
||
|
|
|
||
|
|
def get_class_for_type(imgtype):
|
||
|
|
import importlib
|
||
|
|
mod = importlib.import_module('oe.package_manager.' + imgtype + '.rootfs')
|
||
|
|
return mod.PkgRootfs
|
||
|
|
|
||
|
|
def variable_depends(d, manifest_dir=None):
|
||
|
|
img_type = d.getVar('IMAGE_PKGTYPE')
|
||
|
|
cls = get_class_for_type(img_type)
|
||
|
|
return cls._depends_list()
|
||
|
|
|
||
|
|
def create_rootfs(d, manifest_dir=None, progress_reporter=None, logcatcher=None):
|
||
|
|
env_bkp = os.environ.copy()
|
||
|
|
|
||
|
|
img_type = d.getVar('IMAGE_PKGTYPE')
|
||
|
|
|
||
|
|
cls = get_class_for_type(img_type)
|
||
|
|
cls(d, manifest_dir, progress_reporter, logcatcher).create()
|
||
|
|
os.environ.clear()
|
||
|
|
os.environ.update(env_bkp)
|
||
|
|
|
||
|
|
|
||
|
|
def image_list_installed_packages(d, rootfs_dir=None):
|
||
|
|
# Theres no rootfs for baremetal images
|
||
|
|
if bb.data.inherits_class('baremetal-image', d):
|
||
|
|
return ""
|
||
|
|
|
||
|
|
if not rootfs_dir:
|
||
|
|
rootfs_dir = d.getVar('IMAGE_ROOTFS')
|
||
|
|
|
||
|
|
img_type = d.getVar('IMAGE_PKGTYPE')
|
||
|
|
|
||
|
|
import importlib
|
||
|
|
cls = importlib.import_module('oe.package_manager.' + img_type)
|
||
|
|
return cls.PMPkgsList(d, rootfs_dir).list_pkgs()
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
"""
|
||
|
|
We should be able to run this as a standalone script, from outside bitbake
|
||
|
|
environment.
|
||
|
|
"""
|
||
|
|
"""
|
||
|
|
TBD
|
||
|
|
"""
|