Skip to content
Snippets Groups Projects

support for multiconfig

Merged Alberto Pianon requested to merge multiconfig into master
1 file
+ 82
47
Compare changes
  • Side-by-side
  • Inline
+ 82
47
@@ -96,7 +96,7 @@ def get_dir_sha256(dirpath, excludes=[]):
def parse_pkgdata(pkgdata_file):
pkgdata_text = pkgdata_file.read()
pkgdata = {}
p = re.compile('^([^:]+): (.*)$')
p = re.compile('^(.+): (.*)$')
lines = pkgdata_text.split('\n')
for line in lines:
m = p.match(line)
@@ -437,6 +437,13 @@ class BBRecipeItem(Model):
old = getattr(self.recipe.metadata, attr2check)
new = getattr(metadata, attr2check)
if new != old:
if attr2check == 'license':
if new in old:
self.recipe.metadata.license = old
continue
elif old in new:
metadata.license = new
continue
m = self.recipe.metadata
raise TinfoilHatException(
f"metadata mismatch in recipe"
@@ -516,24 +523,6 @@ class TinfoilHat:
self.bom = bom
self.project = project
self.release = release
self.distro = self.tf.config_data.getVar('DISTRO')
self.build_toolchain = self.tf.config_data.getVar('TOOLCHAIN')
self.buildtag = self.distro + (
f"-{self.build_toolchain}" if self.build_toolchain else ""
)
self.machine = self.tf.config_data.getVar('MACHINE')
self.package_classes = self.tf.config_data.getVar('PACKAGE_CLASSES')
cve_check_dir = self.tf.config_data.getVar("CVE_CHECK_DIR")
if not cve_check_dir or not os.path.isdir(cve_check_dir):
self.cve = {}
return
self.cve = {
# <recipe> : <cve_file_path>
f.replace("_cve.json",""): os.path.join(cve_check_dir, f)
for f in sorted(os.listdir(cve_check_dir))
if not f.endswith("-native_cve.json")
and f.endswith("_cve.json")
}
def __enter__(self):
return self
@@ -565,8 +554,8 @@ class TinfoilHat:
def get_source_files(self, recipe, tags, layer):
source_files = {}
search_paths = recipe.getVar('FILESPATH').split(':')
DL_DIR = self.tf.config_data.getVar('DL_DIR')
SOURCE_MIRROR_URL= self.tf.config_data.getVar('SOURCE_MIRROR_URL') or ''
DL_DIR = self.config_data.getVar('DL_DIR')
SOURCE_MIRROR_URL= self.config_data.getVar('SOURCE_MIRROR_URL') or ''
if SOURCE_MIRROR_URL and not SOURCE_MIRROR_URL.startswith('file://'):
raise TinfoilHatException(
"SOURCE_MIRROR_URLs with url scheme other than 'file://' "
@@ -583,9 +572,9 @@ class TinfoilHat:
shared_sources_recipe = None
if not src_uris and '/work-shared/' in recipe.getVar('S'):
shared_sources = recipe.getVar('S')
for r in self.tf.all_recipes():
if r.pv == version:
rr = self.tf.parse_recipe(r.pn)
for pn, pv in self.all_recipes.items():
if pv == version:
rr = self.tf.parse_recipe(self.mc_str + pn)
if (
rr.getVar('S') == shared_sources
and rr.getVar('SRC_URI')
@@ -786,13 +775,13 @@ class TinfoilHat:
))
return file_dir, files
def get_pkg_files(self, package_name, d):
def get_pkg_files(self, package_name, base_name, d):
PKG_NAME = {
"IPK": "%s_%s-%s_%s.ipk",
"RPM": "%s-%s-%s.%s.rpm",
}
PKG_EXTRACT_CMD = {
"IPK": "ar p %s data.tar.xz | tar xJf -",
"IPK": "ar p %s data.tar.xz | tar xJf - --exclude='dev/*'",
"RPM": "rpm2cpio %s | cpio -idmv"
}
pn = d.getVar('PN')
@@ -811,8 +800,8 @@ class TinfoilHat:
package_version = package_version.replace('-', '+')
if package_version.startswith('unpinnedgit'):
package_version = 'git'
package_revision = d.getVar("PKGR")
subd = read_subpkgdata(package_name, d)
subd = read_subpkgdata(base_name, d)
package_revision = subd.get("PKGR") or d.getVar("PKGR")
if subd.get(f'PKG:{package_name}'):
package_name = subd[f'PKG:{package_name}']
elif subd.get(f'PKG_{package_name}'):
@@ -830,10 +819,10 @@ class TinfoilHat:
def get_binary_package(self, package_name):
pkgdata_dir = self.tf.config_data.getVar('PKGDATA_DIR')
pkgdata_dir = self.config_data.getVar('PKGDATA_DIR')
pkgdata_file = os.path.join(pkgdata_dir, 'runtime-reverse',package_name)
if not os.path.isfile(pkgdata_file):
raise TinfoilHatException(f"Package '{package_name}' not found")
raise TinfoilHatException(f"Package '{package_name}' not found in {ps.path.dirname(pkgdata_file)}")
try:
with open(pkgdata_file, 'r') as f:
pkgdata = parse_pkgdata(f)
@@ -842,14 +831,14 @@ class TinfoilHat:
f"Can't parse pkgdata file for package {package_name}"
)
try:
recipe = self.tf.parse_recipe(pkgdata.get('PN'))
recipe = self.tf.parse_recipe(self.mc_str + pkgdata.get('PN'))
except:
raise TinfoilHatException(
f"Can't parse recipe for package '{package_name}'"
)
base_name = package_name
for k, v in pkgdata.items():
if k.startswith('PKG_') and v == package_name:
if (k.startswith('PKG_') or k.startswith('PKG:')) and v == package_name:
base_name = k[4:]
# it seems that there is no other way to get base package name
# (eg. dbus' main package is 'dbus-1', but its base name is 'dbus')
@@ -871,7 +860,7 @@ class TinfoilHat:
package_arch = recipe.getVar('PACKAGE_ARCH')
if self.package_classes in ['package_rpm', 'package_ipk']:
file_dir, files = self.get_pkg_files(package_name, recipe)
file_dir, files = self.get_pkg_files(package_name, base_name, recipe)
else:
file_dir, files = self.get_pkg_files_from_workdir(
base_name, recipe, pkgdata
@@ -950,11 +939,15 @@ class TinfoilHat:
return None
def get_recipe(self, recipe_name, tags):
recipe = self.tf.parse_recipe(recipe_name)
recipe_info = self.tf.get_recipe_info(recipe_name)
if recipe_name not in self.recipe_cache:
fn = self.tf.cooker.recipecaches[self.mc].pkg_pn[recipe_name][0]
recipe_fn = re.sub(r'^[^/]*', '', fn) # remove namespace before path
recipe = self.tf.parse_recipe(self.mc_str + recipe_name)
self.recipe_cache[recipe_name] = (recipe_fn, recipe)
else:
recipe_fn, recipe = self.recipe_cache[recipe_name]
layer_paths = recipe.getVar('BBLAYERS').split()
recipe_fn = re.sub(r'^[^/]*', '', recipe_info.fn)
# remove possible namespace before path
found = False
for layer_path in layer_paths:
if recipe_fn.startswith(layer_path):
@@ -1029,7 +1022,13 @@ class TinfoilHat:
else:
self.bom[recipe_id] = recipe_item
def get_all_recipes(self):
# workaround, Tinfoil.all_recipes(mc="<some_mc>") does not work
# FIXME upstream
self.all_recipes = {}
for pn, fn in self.tf.cooker.recipecaches[self.mc].pkg_pn.items():
pv = self.tf.cooker.recipecaches[self.mc].pkg_pepvpr[fn[0]][1]
self.all_recipes.update({pn: pv})
def process_packages(self, packages: Dict[str, List[str]]):
@@ -1058,10 +1057,11 @@ class TinfoilHat:
_stderr_rewrite_line("DONE!\n")
def process(self):
sys.stderr.write(f"Parsing build dir {self.build_dir}\n")
image_dir = self.tf.config_data.getVar('DEPLOY_DIR_IMAGE')
layer_paths = self.tf.config_data.getVar('BBLAYERS').split()
def process_images(self):
self.get_all_recipes()
self.recipe_cache = {}
image_dir = self.config_data.getVar('DEPLOY_DIR_IMAGE')
layer_paths = self.config_data.getVar('BBLAYERS').split()
self.layers = {}
for layer_path in layer_paths:
revision = None
@@ -1076,7 +1076,7 @@ class TinfoilHat:
self.layers[layer_path] = BBLayer(layer_path, remote, revision)
if not os.path.isdir(image_dir):
sys.stderr.write(
f'no images in build dir {self.build_dir}'
f'no images in build dir {self.build_dir}\n\n'
)
return
manifests = [
@@ -1113,21 +1113,55 @@ class TinfoilHat:
]
if not images:
sys.stderr.write(
f'no images in build dir {self.build_dir}'
f'no images in build dir {self.build_dir}\n\n'
)
return
elif len(images) > 1:
raise TinfoilHatException(f'Multiple elf images not supported yet')
# FIXME
sys.stderr.write(f'Multiple elf images not supported yet, skipping')
return
image = images[0]
sys.stderr.write('no manifests found or no packages found in manifests,'
' parsing pkgdata dir\n')
pkgdata_dir = self.tf.config_data.getVar('PKGDATA_DIR')
pkgdata_dir = self.config_data.getVar('PKGDATA_DIR')
pkgdata_runtime_reverse = os.path.join(pkgdata_dir, 'runtime-reverse')
package_names = os.listdir(pkgdata_runtime_reverse)
packages = {package_name: [image] for package_name in package_names}
self.process_packages(packages)
def process(self):
sys.stderr.write(f"Parsing build dir {self.build_dir}\n")
for mc in self.tf.cooker.multiconfigs:
if not self.tf.server_connection:
# memory optimization
self.tf.prepare()
if len(self.tf.cooker.multiconfigs) > 1 and not mc:
continue # skip default config, FIXME: is it right?
self.mc = mc
self.mc_str = f"mc:{mc}:" if mc else ""
pn = "cve-update-db-native" # FIXME find a better way
self.config_data = self.tf.parse_recipe(f"{self.mc_str}{pn}")
if len(self.tf.cooker.multiconfigs) > 1:
sys.stderr.write(f"Parsing multiconfig {self.mc}\n")
self.distro = self.config_data.getVar('DISTRO')
self.build_toolchain = self.config_data.getVar('TOOLCHAIN')
self.buildtag = self.distro + (
f"-{self.build_toolchain}" if self.build_toolchain else ""
)
self.machine = self.config_data.getVar('MACHINE')
self.package_classes = self.config_data.getVar('PACKAGE_CLASSES')
cve_check_dir = self.config_data.getVar("CVE_CHECK_DIR")
if not cve_check_dir or not os.path.isdir(cve_check_dir):
self.cve = {}
else:
self.cve = {
# <recipe> : <cve_file_path>
f.replace("_cve.json",""): os.path.join(cve_check_dir, f)
for f in sorted(os.listdir(cve_check_dir))
if not f.endswith("-native_cve.json")
and f.endswith("_cve.json")
}
self.process_images()
self.tf.shutdown() # memory optimization
if __name__ == "__main__":
@@ -1183,5 +1217,6 @@ if __name__ == "__main__":
f.write(recipe_bom.json)
_stderr_rewrite_line("DONE!\n")
except Exception as e:
raise e
sys.stderr.write(f"\n\nGot error {e.__class__.__name__}: {e}\n\n")
sys.exit(1)
Loading