domenico/openwrt_NSS_ACW
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

build: refactor JSON info files to profiles.json

Browse Source 
JSON info files contain machine readable information of built profiles
and resulting images. These files were added in commit 881ed09ee6
("build: create JSON files containing image info").

They are useful for firmware wizards and script checking for
reproducibility.

Currently all JSON files are stored next to the built images, resulting
in up to 168 individual files for the ath79/generic target.

This patch refactors the JSON creation to store individual per image
(not per profile) files in $(BUILD_DIR)/json_info_files and create an
single overview file called `profiles.json` in the target directory.

Storing per image files and not per profile solves the problem of
parallel file writes. If a profiles sysupgrade and factory image are
finished at the same time both processes would write to the same JSON
file, resulting in randomly broken outputs.

Some target like x86/64 do not use the image code yet, resulting in
missing JSON files. If no JSON info files were created, no
`profiles.json` files is created as it would be empty anyway.

As before, this creation is enabled by default only if `BUILDBOT` is set.

Tested via buildroot & ImageBuilder on ath79/generic, imx6 and x86/64.

Signed-off-by: Paul Spooren <mail@aparcar.org>
[json_info_files dir handling in Make, if case refactoring]
Signed-off-by: Petr Štetiar <ynezz@true.cz>
This commit is contained in:
Paul Spooren
2020-03-12 12:55:41 -10:00
committed by Petr Štetiar
parent fcd1401700
commit 07449f692c
6 changed files with 134 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
scripts/json_add_image_info.py
View File

@@ -1,18 +1,22 @@
#!/usr/bin/env python3
import json
import os
from os import getenv
from pathlib import Path
from sys import argv
import hashlib
import json
if len(argv) != 2:
print("ERROR: JSON info script requires output arg")
exit(1)
def e(variable, default=None):
return os.environ.get(variable, default)
json_path = Path(argv[1])
bin_dir = Path(getenv("BIN_DIR"))
image_file = bin_dir / getenv("IMAGE_NAME")
json_path = "{}{}{}.json".format(e("BIN_DIR"), os.sep, e("IMAGE_PREFIX"))
with open(os.path.join(e("BIN_DIR"), e("IMAGE_NAME")), "rb") as image_file:
image_hash = hashlib.sha256(image_file.read()).hexdigest()
if not image_file.is_file():
print("Skip JSON creation for non existing image ", image_file)
exit(0)
def get_titles():
@@ -20,36 +24,40 @@ def get_titles():
for prefix in ["", "ALT0_", "ALT1_", "ALT2_"]:
title = {}
for var in ["vendor", "model", "variant"]:
if e("DEVICE_{}{}".format(prefix, var.upper())):
title[var] = e("DEVICE_{}{}".format(prefix, var.upper()))
if getenv("DEVICE_{}{}".format(prefix, var.upper())):
title[var] = getenv("DEVICE_{}{}".format(prefix, var.upper()))
if title:
titles.append(title)
if not titles:
titles.append({"title": e("DEVICE_TITLE")})
titles.append({"title": getenv("DEVICE_TITLE")})
return titles
if not os.path.exists(json_path):
device_info = {
"id": e("DEVICE_ID"),
"image_prefix": e("IMAGE_PREFIX"),
"images": [],
"metadata_version": 1,
"supported_devices": e("SUPPORTED_DEVICES").split(),
"target": "{}/{}".format(e("TARGET"), e("SUBTARGET", "generic")),
"titles": get_titles(),
"version_commit": e("VERSION_CODE"),
"version_number": e("VERSION_NUMBER"),
}
else:
with open(json_path, "r") as json_file:
device_info = json.load(json_file)
device_id = getenv("DEVICE_ID")
image_hash = hashlib.sha256(image_file.read_bytes()).hexdigest()
image_info = {"type": e("IMAGE_TYPE"), "name": e("IMAGE_NAME"), "sha256": image_hash}
device_info["images"].append(image_info)
image_info = {
"metadata_version": 1,
"target": "{}/{}".format(getenv("TARGET"), getenv("SUBTARGET")),
"version_code": getenv("VERSION_CODE"),
"version_number": getenv("VERSION_NUMBER"),
"profiles": {
device_id: {
"image_prefix": getenv("IMAGE_PREFIX"),
"images": [
{
"type": getenv("IMAGE_TYPE"),
"name": getenv("IMAGE_NAME"),
"sha256": image_hash,
}
],
"supported_devices": getenv("SUPPORTED_DEVICES").split(),
"titles": get_titles(),
}
},
}
with open(json_path, "w") as json_file:
json.dump(device_info, json_file, sort_keys=True, indent=" ")
json_path.write_text(json.dumps(image_info, separators=(",", ":")))