300 lines
11 KiB
Python
300 lines
11 KiB
Python
|
# Copyright (C) 2016 Intel Corporation
|
||
|
|
# Copyright (C) 2020 Savoir-Faire Linux
|
||
|
|
#
|
||
|
|
# SPDX-License-Identifier: GPL-2.0-only
|
||
|
|
#
|
||
|
|
"""Recipe creation tool - npm module support plugin"""
|
||
|
|
|
||
|
|
import json
|
||
|
|
import logging
|
||
|
|
import os
|
||
|
|
import re
|
||
|
|
import sys
|
||
|
|
import tempfile
|
||
|
|
import bb
|
||
|
|
from bb.fetch2.npm import NpmEnvironment
|
||
|
|
from bb.fetch2.npmsw import foreach_dependencies
|
||
|
|
from recipetool.create import RecipeHandler
|
||
|
|
from recipetool.create import get_license_md5sums
|
||
|
|
from recipetool.create import guess_license
|
||
|
|
from recipetool.create import split_pkg_licenses
|
||
|
|
logger = logging.getLogger('recipetool')
|
||
|
|
|
||
|
|
TINFOIL = None
|
||
|
|
|
||
|
|
def tinfoil_init(instance):
|
||
|
|
"""Initialize tinfoil"""
|
||
|
|
global TINFOIL
|
||
|
|
TINFOIL = instance
|
||
|
|
|
||
|
|
class NpmRecipeHandler(RecipeHandler):
|
||
|
|
"""Class to handle the npm recipe creation"""
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def _npm_name(name):
|
||
|
|
"""Generate a Yocto friendly npm name"""
|
||
|
|
name = re.sub("/", "-", name)
|
||
|
|
name = name.lower()
|
||
|
|
name = re.sub(r"[^\-a-z0-9]", "", name)
|
||
|
|
name = name.strip("-")
|
||
|
|
return name
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def _get_registry(lines):
|
||
|
|
"""Get the registry value from the 'npm://registry' url"""
|
||
|
|
registry = None
|
||
|
|
|
||
|
|
def _handle_registry(varname, origvalue, op, newlines):
|
||
|
|
nonlocal registry
|
||
|
|
if origvalue.startswith("npm://"):
|
||
|
|
registry = re.sub(r"^npm://", "http://", origvalue.split(";")[0])
|
||
|
|
return origvalue, None, 0, True
|
||
|
|
|
||
|
|
bb.utils.edit_metadata(lines, ["SRC_URI"], _handle_registry)
|
||
|
|
|
||
|
|
return registry
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def _ensure_npm():
|
||
|
|
"""Check if the 'npm' command is available in the recipes"""
|
||
|
|
if not TINFOIL.recipes_parsed:
|
||
|
|
TINFOIL.parse_recipes()
|
||
|
|
|
||
|
|
try:
|
||
|
|
d = TINFOIL.parse_recipe("nodejs-native")
|
||
|
|
except bb.providers.NoProvider:
|
||
|
|
bb.error("Nothing provides 'nodejs-native' which is required for the build")
|
||
|
|
bb.note("You will likely need to add a layer that provides nodejs")
|
||
|
|
sys.exit(14)
|
||
|
|
|
||
|
|
bindir = d.getVar("STAGING_BINDIR_NATIVE")
|
||
|
|
npmpath = os.path.join(bindir, "npm")
|
||
|
|
|
||
|
|
if not os.path.exists(npmpath):
|
||
|
|
TINFOIL.build_targets("nodejs-native", "addto_recipe_sysroot")
|
||
|
|
|
||
|
|
if not os.path.exists(npmpath):
|
||
|
|
bb.error("Failed to add 'npm' to sysroot")
|
||
|
|
sys.exit(14)
|
||
|
|
|
||
|
|
return bindir
|
||
|
|
|
||
|
|
@staticmethod
|
||
|
|
def _npm_global_configs(dev):
|
||
|
|
"""Get the npm global configuration"""
|
||
|
|
configs = []
|
||
|
|
|
||
|
|
if dev:
|
||
|
|
configs.append(("also", "development"))
|
||
|
|
else:
|
||
|
|
configs.append(("only", "production"))
|
||
|
|
|
||
|
|
configs.append(("save", "false"))
|
||
|
|
configs.append(("package-lock", "false"))
|
||
|
|
configs.append(("shrinkwrap", "false"))
|
||
|
|
return configs
|
||
|
|
|
||
|
|
def _run_npm_install(self, d, srctree, registry, dev):
|
||
|
|
"""Run the 'npm install' command without building the addons"""
|
||
|
|
configs = self._npm_global_configs(dev)
|
||
|
|
configs.append(("ignore-scripts", "true"))
|
||
|
|
|
||
|
|
if registry:
|
||
|
|
configs.append(("registry", registry))
|
||
|
|
|
||
|
|
bb.utils.remove(os.path.join(srctree, "node_modules"), recurse=True)
|
||
|
|
|
||
|
|
env = NpmEnvironment(d, configs=configs)
|
||
|
|
env.run("npm install", workdir=srctree)
|
||
|
|
|
||
|
|
def _generate_shrinkwrap(self, d, srctree, dev):
|
||
|
|
"""Check and generate the 'npm-shrinkwrap.json' file if needed"""
|
||
|
|
configs = self._npm_global_configs(dev)
|
||
|
|
|
||
|
|
env = NpmEnvironment(d, configs=configs)
|
||
|
|
env.run("npm shrinkwrap", workdir=srctree)
|
||
|
|
|
||
|
|
return os.path.join(srctree, "npm-shrinkwrap.json")
|
||
|
|
|
||
|
|
def _handle_licenses(self, srctree, shrinkwrap_file, dev):
|
||
|
|
"""Return the extra license files and the list of packages"""
|
||
|
|
licfiles = []
|
||
|
|
packages = {}
|
||
|
|
|
||
|
|
# Handle the parent package
|
||
|
|
packages["${PN}"] = ""
|
||
|
|
|
||
|
|
def _licfiles_append_fallback_readme_files(destdir):
|
||
|
|
"""Append README files as fallback to license files if a license files is missing"""
|
||
|
|
|
||
|
|
fallback = True
|
||
|
|
readmes = []
|
||
|
|
basedir = os.path.join(srctree, destdir)
|
||
|
|
for fn in os.listdir(basedir):
|
||
|
|
upper = fn.upper()
|
||
|
|
if upper.startswith("README"):
|
||
|
|
fullpath = os.path.join(basedir, fn)
|
||
|
|
readmes.append(fullpath)
|
||
|
|
if upper.startswith("COPYING") or "LICENCE" in upper or "LICENSE" in upper:
|
||
|
|
fallback = False
|
||
|
|
if fallback:
|
||
|
|
for readme in readmes:
|
||
|
|
licfiles.append(os.path.relpath(readme, srctree))
|
||
|
|
|
||
|
|
# Handle the dependencies
|
||
|
|
def _handle_dependency(name, params, deptree):
|
||
|
|
suffix = "-".join([self._npm_name(dep) for dep in deptree])
|
||
|
|
destdirs = [os.path.join("node_modules", dep) for dep in deptree]
|
||
|
|
destdir = os.path.join(*destdirs)
|
||
|
|
packages["${PN}-" + suffix] = destdir
|
||
|
|
_licfiles_append_fallback_readme_files(destdir)
|
||
|
|
|
||
|
|
with open(shrinkwrap_file, "r") as f:
|
||
|
|
shrinkwrap = json.load(f)
|
||
|
|
|
||
|
|
foreach_dependencies(shrinkwrap, _handle_dependency, dev)
|
||
|
|
|
||
|
|
return licfiles, packages
|
||
|
|
|
||
|
|
def process(self, srctree, classes, lines_before, lines_after, handled, extravalues):
|
||
|
|
"""Handle the npm recipe creation"""
|
||
|
|
|
||
|
|
if "buildsystem" in handled:
|
||
|
|
return False
|
||
|
|
|
||
|
|
files = RecipeHandler.checkfiles(srctree, ["package.json"])
|
||
|
|
|
||
|
|
if not files:
|
||
|
|
return False
|
||
|
|
|
||
|
|
with open(files[0], "r") as f:
|
||
|
|
data = json.load(f)
|
||
|
|
|
||
|
|
if "name" not in data or "version" not in data:
|
||
|
|
return False
|
||
|
|
|
||
|
|
extravalues["PN"] = self._npm_name(data["name"])
|
||
|
|
extravalues["PV"] = data["version"]
|
||
|
|
|
||
|
|
if "description" in data:
|
||
|
|
extravalues["SUMMARY"] = data["description"]
|
||
|
|
|
||
|
|
if "homepage" in data:
|
||
|
|
extravalues["HOMEPAGE"] = data["homepage"]
|
||
|
|
|
||
|
|
dev = bb.utils.to_boolean(str(extravalues.get("NPM_INSTALL_DEV", "0")), False)
|
||
|
|
registry = self._get_registry(lines_before)
|
||
|
|
|
||
|
|
bb.note("Checking if npm is available ...")
|
||
|
|
# The native npm is used here (and not the host one) to ensure that the
|
||
|
|
# npm version is high enough to ensure an efficient dependency tree
|
||
|
|
# resolution and avoid issue with the shrinkwrap file format.
|
||
|
|
# Moreover the native npm is mandatory for the build.
|
||
|
|
bindir = self._ensure_npm()
|
||
|
|
|
||
|
|
d = bb.data.createCopy(TINFOIL.config_data)
|
||
|
|
d.prependVar("PATH", bindir + ":")
|
||
|
|
d.setVar("S", srctree)
|
||
|
|
|
||
|
|
bb.note("Generating shrinkwrap file ...")
|
||
|
|
# To generate the shrinkwrap file the dependencies have to be installed
|
||
|
|
# first. During the generation process some files may be updated /
|
||
|
|
# deleted. By default devtool tracks the diffs in the srctree and raises
|
||
|
|
# errors when finishing the recipe if some diffs are found.
|
||
|
|
git_exclude_file = os.path.join(srctree, ".git", "info", "exclude")
|
||
|
|
if os.path.exists(git_exclude_file):
|
||
|
|
with open(git_exclude_file, "r+") as f:
|
||
|
|
lines = f.readlines()
|
||
|
|
for line in ["/node_modules/", "/npm-shrinkwrap.json"]:
|
||
|
|
if line not in lines:
|
||
|
|
f.write(line + "\n")
|
||
|
|
|
||
|
|
lock_file = os.path.join(srctree, "package-lock.json")
|
||
|
|
lock_copy = lock_file + ".copy"
|
||
|
|
if os.path.exists(lock_file):
|
||
|
|
bb.utils.copyfile(lock_file, lock_copy)
|
||
|
|
|
||
|
|
self._run_npm_install(d, srctree, registry, dev)
|
||
|
|
shrinkwrap_file = self._generate_shrinkwrap(d, srctree, dev)
|
||
|
|
|
||
|
|
with open(shrinkwrap_file, "r") as f:
|
||
|
|
shrinkwrap = json.load(f)
|
||
|
|
|
||
|
|
if os.path.exists(lock_copy):
|
||
|
|
bb.utils.movefile(lock_copy, lock_file)
|
||
|
|
|
||
|
|
# Add the shrinkwrap file as 'extrafiles'
|
||
|
|
shrinkwrap_copy = shrinkwrap_file + ".copy"
|
||
|
|
bb.utils.copyfile(shrinkwrap_file, shrinkwrap_copy)
|
||
|
|
extravalues.setdefault("extrafiles", {})
|
||
|
|
extravalues["extrafiles"]["npm-shrinkwrap.json"] = shrinkwrap_copy
|
||
|
|
|
||
|
|
url_local = "npmsw://%s" % shrinkwrap_file
|
||
|
|
url_recipe= "npmsw://${THISDIR}/${BPN}/npm-shrinkwrap.json"
|
||
|
|
|
||
|
|
if dev:
|
||
|
|
url_local += ";dev=1"
|
||
|
|
url_recipe += ";dev=1"
|
||
|
|
|
||
|
|
# Add the npmsw url in the SRC_URI of the generated recipe
|
||
|
|
def _handle_srcuri(varname, origvalue, op, newlines):
|
||
|
|
"""Update the version value and add the 'npmsw://' url"""
|
||
|
|
value = origvalue.replace("version=" + data["version"], "version=${PV}")
|
||
|
|
value = value.replace("version=latest", "version=${PV}")
|
||
|
|
values = [line.strip() for line in value.strip('\n').splitlines()]
|
||
|
|
if "dependencies" in shrinkwrap:
|
||
|
|
values.append(url_recipe)
|
||
|
|
return values, None, 4, False
|
||
|
|
|
||
|
|
(_, newlines) = bb.utils.edit_metadata(lines_before, ["SRC_URI"], _handle_srcuri)
|
||
|
|
lines_before[:] = [line.rstrip('\n') for line in newlines]
|
||
|
|
|
||
|
|
# In order to generate correct licence checksums in the recipe the
|
||
|
|
# dependencies have to be fetched again using the npmsw url
|
||
|
|
bb.note("Fetching npm dependencies ...")
|
||
|
|
bb.utils.remove(os.path.join(srctree, "node_modules"), recurse=True)
|
||
|
|
fetcher = bb.fetch2.Fetch([url_local], d)
|
||
|
|
fetcher.download()
|
||
|
|
fetcher.unpack(srctree)
|
||
|
|
|
||
|
|
bb.note("Handling licences ...")
|
||
|
|
(licfiles, packages) = self._handle_licenses(srctree, shrinkwrap_file, dev)
|
||
|
|
|
||
|
|
def _guess_odd_license(licfiles):
|
||
|
|
import bb
|
||
|
|
|
||
|
|
md5sums = get_license_md5sums(d, linenumbers=True)
|
||
|
|
|
||
|
|
chksums = []
|
||
|
|
licenses = []
|
||
|
|
for licfile in licfiles:
|
||
|
|
f = os.path.join(srctree, licfile)
|
||
|
|
md5value = bb.utils.md5_file(f)
|
||
|
|
(license, beginline, endline, md5) = md5sums.get(md5value,
|
||
|
|
(None, "", "", ""))
|
||
|
|
if not license:
|
||
|
|
license = "Unknown"
|
||
|
|
logger.info("Please add the following line for '%s' to a "
|
||
|
|
"'lib/recipetool/licenses.csv' and replace `Unknown`, "
|
||
|
|
"`X`, `Y` and `MD5` with the license, begin line, "
|
||
|
|
"end line and partial MD5 checksum:\n" \
|
||
|
|
"%s,Unknown,X,Y,MD5" % (licfile, md5value))
|
||
|
|
chksums.append("file://%s%s%s;md5=%s" % (licfile,
|
||
|
|
";beginline=%s" % (beginline) if beginline else "",
|
||
|
|
";endline=%s" % (endline) if endline else "",
|
||
|
|
md5 if md5 else md5value))
|
||
|
|
licenses.append((license, licfile, md5value))
|
||
|
|
return (licenses, chksums)
|
||
|
|
|
||
|
|
(licenses, extravalues["LIC_FILES_CHKSUM"]) = _guess_odd_license(licfiles)
|
||
|
|
split_pkg_licenses([*licenses, *guess_license(srctree, d)], packages, lines_after)
|
||
|
|
|
||
|
|
classes.append("npm")
|
||
|
|
handled.append("buildsystem")
|
||
|
|
|
||
|
|
return True
|
||
|
|
|
||
|
|
def register_recipe_handlers(handlers):
|
||
|
|
"""Register the npm handler"""
|
||
|
|
handlers.append((NpmRecipeHandler(), 60))
|