Browse Source

build: refactor JSON info files to `profiles.json`

JSON info files contain machine readable information of built profiles
and resulting images. These files were added in commit 881ed09ee6
("build: create JSON files containing image info").

They are useful for firmware wizards and script checking for
reproducibility.

Currently all JSON files are stored next to the built images, resulting
in up to 168 individual files for the ath79/generic target.

This patch refactors the JSON creation to store individual per image
(not per profile) files in $(BUILD_DIR)/json_info_files and create an
single overview file called `profiles.json` in the target directory.

Storing per image files and not per profile solves the problem of
parallel file writes. If a profiles sysupgrade and factory image are
finished at the same time both processes would write to the same JSON
file, resulting in randomly broken outputs.

Some target like x86/64 do not use the image code yet, resulting in
missing JSON files. If no JSON info files were created, no
`profiles.json` files is created as it would be empty anyway.

As before, this creation is enabled by default only if `BUILDBOT` is set.

Tested via buildroot & ImageBuilder on ath79/generic, imx6 and x86/64.

Signed-off-by: Paul Spooren <mail@aparcar.org>
[json_info_files dir handling in Make, if case refactoring]
Signed-off-by: Petr Štetiar <ynezz@true.cz>
master
Paul Spooren Petr Štetiar 2 months ago
parent
commit
07449f692c
6 changed files with 136 additions and 66 deletions
  1. +9
    -0
      Makefile
  2. +5
    -4
      config/Config-build.in
  3. +32
    -29
      include/image.mk
  4. +41
    -33
      scripts/json_add_image_info.py
  5. +39
    -0
      scripts/json_overview_image_info.py
  6. +10
    -0
      target/imagebuilder/files/Makefile

+ 9
- 0
Makefile View File

@@ -88,6 +88,14 @@ prereq: $(target/stamp-prereq) tmp/.prereq_packages
exit 1; \
fi

$(BIN_DIR)/profiles.json: FORCE
$(if $(CONFIG_JSON_OVERVIEW_IMAGE_INFO), \
WORK_DIR=$(BUILD_DIR)/json_info_files \
$(SCRIPT_DIR)/json_overview_image_info.py $@ \
)

json_overview_image_info: $(BIN_DIR)/profiles.json

checksum: FORCE
$(call sha256sums,$(BIN_DIR),$(CONFIG_BUILDBOT))

@@ -109,6 +117,7 @@ prepare: .config $(tools/stamp-compile) $(toolchain/stamp-compile)

world: prepare $(target/stamp-compile) $(package/stamp-compile) $(package/stamp-install) $(target/stamp-install) FORCE
$(_SINGLE)$(SUBMAKE) -r package/index
$(_SINGLE)$(SUBMAKE) -r json_overview_image_info
$(_SINGLE)$(SUBMAKE) -r checksum

.PHONY: clean dirclean prereq prepare world package/symlinks package/symlinks-install package/symlinks-clean


+ 5
- 4
config/Config-build.in View File

@@ -7,12 +7,13 @@

menu "Global build settings"

config JSON_ADD_IMAGE_INFO
bool "Create JSON info files per build image"
config JSON_OVERVIEW_IMAGE_INFO
bool "Create JSON info file overview per target"
default BUILDBOT
help
The JSON info files contain information about the device and
build images, stored next to the firmware images.
Create a JSON info file called profiles.json in the target
directory containing machine readable list of built profiles
and resulting images.

config ALL_NONSHARED
bool "Select all target specific packages by default"


+ 32
- 29
include/image.mk View File

@@ -545,8 +545,11 @@ endef

define Device/Build/image
GZ_SUFFIX := $(if $(filter %dtb %gz,$(2)),,$(if $(and $(findstring ext4,$(1)),$(CONFIG_TARGET_IMAGES_GZIP)),.gz))
$$(_TARGET): $(BIN_DIR)/$(call IMAGE_NAME,$(1),$(2))$$(GZ_SUFFIX)
$$(_TARGET): $(if $(CONFIG_JSON_OVERVIEW_IMAGE_INFO), \
$(BUILD_DIR)/json_info_files/$(call IMAGE_NAME,$(1),$(2)).json, \
$(BIN_DIR)/$(call IMAGE_NAME,$(1),$(2))$$(GZ_SUFFIX))
$(eval $(call Device/Export,$(KDIR)/tmp/$(call IMAGE_NAME,$(1),$(2)),$(1)))

ROOTFS/$(1)/$(3) := \
$(KDIR)/root.$(1)$$(strip \
$$(if $$(FS_OPTIONS/$(1)),+fs=$$(call param_mangle,$$(FS_OPTIONS/$(1)))) \
@@ -568,32 +571,33 @@ define Device/Build/image

$(BIN_DIR)/$(call IMAGE_NAME,$(1),$(2)): $(KDIR)/tmp/$(call IMAGE_NAME,$(1),$(2))
cp $$^ $$@
$(if $(CONFIG_JSON_ADD_IMAGE_INFO), \
DEVICE_ID="$(DEVICE_NAME)" \
BIN_DIR="$(BIN_DIR)" \
IMAGE_NAME="$(IMAGE_NAME)" \
IMAGE_TYPE=$(word 1,$(subst ., ,$(2))) \
IMAGE_PREFIX="$(IMAGE_PREFIX)" \
DEVICE_VENDOR="$(DEVICE_VENDOR)" \
DEVICE_MODEL="$(DEVICE_MODEL)" \
DEVICE_VARIANT="$(DEVICE_VARIANT)" \
DEVICE_ALT0_VENDOR="$(DEVICE_ALT0_VENDOR)" \
DEVICE_ALT0_MODEL="$(DEVICE_ALT0_MODEL)" \
DEVICE_ALT0_VARIANT="$(DEVICE_ALT0_VARIANT)" \
DEVICE_ALT1_VENDOR="$(DEVICE_ALT1_VENDOR)" \
DEVICE_ALT1_MODEL="$(DEVICE_ALT1_MODEL)" \
DEVICE_ALT1_VARIANT="$(DEVICE_ALT1_VARIANT)" \
DEVICE_ALT2_VENDOR="$(DEVICE_ALT2_VENDOR)" \
DEVICE_ALT2_MODEL="$(DEVICE_ALT2_MODEL)" \
DEVICE_ALT2_VARIANT="$(DEVICE_ALT2_VARIANT)" \
DEVICE_TITLE="$(DEVICE_TITLE)" \
TARGET="$(BOARD)" \
SUBTARGET="$(if $(SUBTARGET),$(SUBTARGET),generic)" \
VERSION_NUMBER="$(VERSION_NUMBER)" \
VERSION_CODE="$(VERSION_CODE)" \
SUPPORTED_DEVICES="$(SUPPORTED_DEVICES)" \
$(TOPDIR)/scripts/json_add_image_info.py \
)

$(BUILD_DIR)/json_info_files/$(call IMAGE_NAME,$(1),$(2)).json: $(BIN_DIR)/$(call IMAGE_NAME,$(1),$(2))$$(GZ_SUFFIX)
@mkdir -p $$(shell dirname $$@)
DEVICE_ID="$(DEVICE_NAME)" \
BIN_DIR="$(BIN_DIR)" \
IMAGE_NAME="$(IMAGE_NAME)" \
IMAGE_TYPE=$(word 1,$(subst ., ,$(2))) \
IMAGE_PREFIX="$(IMAGE_PREFIX)" \
DEVICE_VENDOR="$(DEVICE_VENDOR)" \
DEVICE_MODEL="$(DEVICE_MODEL)" \
DEVICE_VARIANT="$(DEVICE_VARIANT)" \
DEVICE_ALT0_VENDOR="$(DEVICE_ALT0_VENDOR)" \
DEVICE_ALT0_MODEL="$(DEVICE_ALT0_MODEL)" \
DEVICE_ALT0_VARIANT="$(DEVICE_ALT0_VARIANT)" \
DEVICE_ALT1_VENDOR="$(DEVICE_ALT1_VENDOR)" \
DEVICE_ALT1_MODEL="$(DEVICE_ALT1_MODEL)" \
DEVICE_ALT1_VARIANT="$(DEVICE_ALT1_VARIANT)" \
DEVICE_ALT2_VENDOR="$(DEVICE_ALT2_VENDOR)" \
DEVICE_ALT2_MODEL="$(DEVICE_ALT2_MODEL)" \
DEVICE_ALT2_VARIANT="$(DEVICE_ALT2_VARIANT)" \
DEVICE_TITLE="$(DEVICE_TITLE)" \
TARGET="$(BOARD)" \
SUBTARGET="$(if $(SUBTARGET),$(SUBTARGET),generic)" \
VERSION_NUMBER="$(VERSION_NUMBER)" \
VERSION_CODE="$(VERSION_CODE)" \
SUPPORTED_DEVICES="$(SUPPORTED_DEVICES)" \
$(TOPDIR)/scripts/json_add_image_info.py $$@

endef

@@ -612,8 +616,6 @@ define Device/Build/artifact
endef

define Device/Build
$(shell rm -f $(BIN_DIR)/$(IMG_PREFIX)-$(1).json)

$(if $(CONFIG_TARGET_ROOTFS_INITRAMFS),$(call Device/Build/initramfs,$(1)))
$(call Device/Build/kernel,$(1))

@@ -699,6 +701,7 @@ define BuildImage

image_prepare: compile
mkdir -p $(BIN_DIR) $(KDIR)/tmp
rm -rf $(BUILD_DIR)/json_info_files
$(call Image/Prepare)

legacy-images-prepare-make: image_prepare


+ 41
- 33
scripts/json_add_image_info.py View File

@@ -1,18 +1,22 @@
#!/usr/bin/env python3

import json
import os
from os import getenv
from pathlib import Path
from sys import argv
import hashlib
import json

if len(argv) != 2:
print("ERROR: JSON info script requires output arg")
exit(1)

def e(variable, default=None):
return os.environ.get(variable, default)


json_path = "{}{}{}.json".format(e("BIN_DIR"), os.sep, e("IMAGE_PREFIX"))
json_path = Path(argv[1])
bin_dir = Path(getenv("BIN_DIR"))
image_file = bin_dir / getenv("IMAGE_NAME")

with open(os.path.join(e("BIN_DIR"), e("IMAGE_NAME")), "rb") as image_file:
image_hash = hashlib.sha256(image_file.read()).hexdigest()
if not image_file.is_file():
print("Skip JSON creation for non existing image ", image_file)
exit(0)


def get_titles():
@@ -20,36 +24,40 @@ def get_titles():
for prefix in ["", "ALT0_", "ALT1_", "ALT2_"]:
title = {}
for var in ["vendor", "model", "variant"]:
if e("DEVICE_{}{}".format(prefix, var.upper())):
title[var] = e("DEVICE_{}{}".format(prefix, var.upper()))
if getenv("DEVICE_{}{}".format(prefix, var.upper())):
title[var] = getenv("DEVICE_{}{}".format(prefix, var.upper()))

if title:
titles.append(title)

if not titles:
titles.append({"title": e("DEVICE_TITLE")})
titles.append({"title": getenv("DEVICE_TITLE")})

return titles


if not os.path.exists(json_path):
device_info = {
"id": e("DEVICE_ID"),
"image_prefix": e("IMAGE_PREFIX"),
"images": [],
"metadata_version": 1,
"supported_devices": e("SUPPORTED_DEVICES").split(),
"target": "{}/{}".format(e("TARGET"), e("SUBTARGET", "generic")),
"titles": get_titles(),
"version_commit": e("VERSION_CODE"),
"version_number": e("VERSION_NUMBER"),
}
else:
with open(json_path, "r") as json_file:
device_info = json.load(json_file)

image_info = {"type": e("IMAGE_TYPE"), "name": e("IMAGE_NAME"), "sha256": image_hash}
device_info["images"].append(image_info)

with open(json_path, "w") as json_file:
json.dump(device_info, json_file, sort_keys=True, indent=" ")
device_id = getenv("DEVICE_ID")
image_hash = hashlib.sha256(image_file.read_bytes()).hexdigest()

image_info = {
"metadata_version": 1,
"target": "{}/{}".format(getenv("TARGET"), getenv("SUBTARGET")),
"version_code": getenv("VERSION_CODE"),
"version_number": getenv("VERSION_NUMBER"),
"profiles": {
device_id: {
"image_prefix": getenv("IMAGE_PREFIX"),
"images": [
{
"type": getenv("IMAGE_TYPE"),
"name": getenv("IMAGE_NAME"),
"sha256": image_hash,
}
],
"supported_devices": getenv("SUPPORTED_DEVICES").split(),
"titles": get_titles(),
}
},
}

json_path.write_text(json.dumps(image_info, separators=(",", ":")))

+ 39
- 0
scripts/json_overview_image_info.py View File

@@ -0,0 +1,39 @@
#!/usr/bin/env python3

import json
from pathlib import Path
from os import getenv
from sys import argv

if len(argv) != 2:
print("JSON info files script requires ouput file as argument")
exit(1)

output_path = Path(argv[1])

assert getenv("WORK_DIR"), "$WORK_DIR required"

work_dir = Path(getenv("WORK_DIR"))

assert work_dir.is_dir(), "$WORK_DIR not a directory"

output = {}

for json_file in work_dir.glob("*.json"):
image_info = json.loads(json_file.read_text())
if not output:
output.update(image_info)
else:
# get first (and only) profile in json file
device_id = next(iter(image_info["profiles"].keys()))
if device_id not in output["profiles"]:
output["profiles"].update(image_info["profiles"])
else:
output["profiles"][device_id]["images"].append(
image_info["profiles"][device_id]["images"][0]
)

if output:
output_path.write_text(json.dumps(output, sort_keys=True, separators=(",", ":")))
else:
print("JSON info file script could not find any JSON files for target")

+ 10
- 0
target/imagebuilder/files/Makefile View File

@@ -118,6 +118,7 @@ _call_image: staging_dir/host/.prereq-build
$(MAKE) package_install
$(MAKE) -s prepare_rootfs
$(MAKE) -s build_image
$(MAKE) -s json_overview_image_info
$(MAKE) -s checksum

_call_manifest: FORCE
@@ -163,12 +164,21 @@ prepare_rootfs: FORCE
$(CP) $(TARGET_DIR) $(TARGET_DIR_ORIG)
$(call prepare_rootfs,$(TARGET_DIR),$(USER_FILES),$(DISABLED_SERVICES))


build_image: FORCE
@echo
@echo Building images...
$(NO_TRACE_MAKE) -C target/linux/$(BOARD)/image install TARGET_BUILD=1 IB=1 EXTRA_IMAGE_NAME="$(EXTRA_IMAGE_NAME)" \
$(if $(USER_PROFILE),PROFILE="$(USER_PROFILE)")

$(BIN_DIR)/profiles.json: FORCE
$(if $(CONFIG_JSON_OVERVIEW_IMAGE_INFO), \
WORK_DIR=$(BUILD_DIR)/json_info_files \
$(SCRIPT_DIR)/json_overview_image_info.py $@ \
)

json_overview_image_info: $(BIN_DIR)/profiles.json

checksum: FORCE
@echo
@echo Calculating checksums...


Loading…
Cancel
Save