Convert each time-stamped buildstats directory to a compressed tarball
using the hostname as a prefix (to the file name only) so that one can
identify the source machine. Move these tarballs to the directory:
testresults/<build_name>/buildstats/

The archiving is performed during the "collect results" step.

Signed-off-by: Aryaman Gupta <aryaman.gupta@...>
Signed-off-by: Randy MacLeod <Randy.MacLeod@...>
---
scripts/archive_buildstats.py | 32 ++++++++++++++++++++++++++++++++
scripts/run-config | 1 +
2 files changed, 33 insertions(+)
create mode 100755 scripts/archive_buildstats.py

diff --git a/scripts/archive_buildstats.py b/scripts/archive_buildstats.py
new file mode 100755
index 0000000..30ea289
--- /dev/null
+++ b/scripts/archive_buildstats.py
@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+import os, subprocess, socket, sys
+
+def usagedir():
+ print("Usage: " + sys.argv[0] + " <src> <dest> <target>")
+
+def main():
+ if len(sys.argv) != 4:
+ usage()
+ sys.exit()
+
+ builddir = sys.argv[1]
+ dest = sys.argv[2]
+ target = sys.argv[3]
+ dest_bsdir = os.path.join(dest, target, "buildstats")
+ subprocess.run(["mkdir", "-p", dest_bsdir])
+
+ build_bsdir = os.path.join(builddir, "tmp/buildstats")
+ hostname = socket.gethostname()
+ os.chdir(build_bsdir)
+ for timestamp in os.listdir(build_bsdir):
+ if hostname:
+ output = hostname + "-" + timestamp + ".tar.zst"
+ else:
+ output = "nohostname-"+ timestamp + ".tar.zst"
+ subprocess.check_call("tar -I zstd -cf "+output+" "+timestamp+"/*", shell=True)
+ subprocess.run(["mv", output, dest_bsdir])
+
+main()
diff --git a/scripts/run-config b/scripts/run-config
index 838847a..953977e 100755
--- a/scripts/run-config
+++ b/scripts/run-config
@@ -334,6 +334,7 @@ elif args.phase == "finish" and args.stepname == "collect-results":
hp.printheader("Running results collection")
runcmd([scriptsdir + "/collect-results", args.builddir, args.results_dir, args.target])
runcmd([scriptsdir + "/summarize_top_output.py", args.results_dir, args.target])
+ runcmd([scriptsdir + "/archive_buildstats.py", args.builddir, args.results_dir, args.target])
sys.exit(0)

if jcfg:
--
2.35.1

Starting with [1] kernel modules symbols is being slipped in OE-core
and this breaks the kernel modules sign, so disable it.

[1] https://git.openembedded.org/openembedded-core/commit/?id=e09a8fa931fe617afc05bd5e00dca5dd3fe386e8

Signed-off-by: Jose Quaresma <jose.quaresma@...>
---
meta-integrity/classes/kernel-modsign.bbclass | 2 ++
1 file changed, 2 insertions(+)

diff --git a/meta-integrity/classes/kernel-modsign.bbclass b/meta-integrity/classes/kernel-modsign.bbclass
index 093c358..d3aa7fb 100644
--- a/meta-integrity/classes/kernel-modsign.bbclass
+++ b/meta-integrity/classes/kernel-modsign.bbclass
@@ -13,7 +13,9 @@ MODSIGN_PRIVKEY ?= "${MODSIGN_KEY_DIR}/privkey_modsign.pem"
MODSIGN_X509 ?= "${MODSIGN_KEY_DIR}/x509_modsign.crt"

# If this class is enabled, disable stripping signatures from modules
+# as well disable the debug symbols split
INHIBIT_PACKAGE_STRIP = "1"
+INHIBIT_PACKAGE_DEBUG_SPLIT = "1"

kernel_do_configure:prepend() {
if [ -f "${MODSIGN_PRIVKEY}" -a -f "${MODSIGN_X509}" ]; then
--
2.36.1

Starting with [1] kernel modulus symbols is being slipped in OE-core
and this breaks the kernel module sign, so disable it.

[1] https://git.openembedded.org/openembedded-core/commit/?id=e09a8fa931fe617afc05bd5e00dca5dd3fe386e8

Signed-off-by: Jose Quaresma <jose.quaresma@...>
---
meta-integrity/classes/kernel-modsign.bbclass | 2 ++
1 file changed, 2 insertions(+)

diff --git a/meta-integrity/classes/kernel-modsign.bbclass b/meta-integrity/classes/kernel-modsign.bbclass
index 093c358..d3aa7fb 100644
--- a/meta-integrity/classes/kernel-modsign.bbclass
+++ b/meta-integrity/classes/kernel-modsign.bbclass
@@ -13,7 +13,9 @@ MODSIGN_PRIVKEY ?= "${MODSIGN_KEY_DIR}/privkey_modsign.pem"
MODSIGN_X509 ?= "${MODSIGN_KEY_DIR}/x509_modsign.crt"

# If this class is enabled, disable stripping signatures from modules
+# as well disable the debug symbols split
INHIBIT_PACKAGE_STRIP = "1"
+INHIBIT_PACKAGE_DEBUG_SPLIT = "1"

kernel_do_configure:prepend() {
if [ -f "${MODSIGN_PRIVKEY}" -a -f "${MODSIGN_X509}" ]; then
--
2.36.1

Hi,

Hi

I'm new to yocto, I wonder the recommended way to setup an environment for yocto using vagrant or docker to isolate from the host, since I've encounter many different error about python version or something like that. Some layers providede by vendor uses a tool called repo for version control, I always cannot install on my machine, following the steps form that page doesn't help either. 

I recommend taking a look at the kas project.[1] It allows you to manage Yocto layers in your project in a very easy
and pleasant way. You can also take a look at the Yocto Summit presentation [2] which talks about various ways of
managing Yocto layers. Good luck with learning the Yocto Project.
[1] https://github.com/siemens/kas; https://kas.readthedocs.io/en/latest/userguide.html?highlight=kas-container#usage
[2] https://www.youtube.com/watch?v=PLl8jwZOpS0&list=PLD4M5FoHz-TwZkOR-GoWjq5XlLpZIFt5-&index=12

Thanks,

CD

Regards
Tomasz Żyjewski
Embedded Systems Engineer
GPG: 5C495EA3EBEECA59
https://3mdeb.com | @3mdeb_com

We could do with wider exposure of testing of this so enable on the
autobuilder by default.

Signed-off-by: Richard Purdie <richard.purdie@...>
---
config.json | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/config.json b/config.json
index d01319f..de32fb1 100644
--- a/config.json
+++ b/config.json
@@ -66,7 +66,8 @@
"BB_LOG_HOST_STAT_ON_INTERVAL = '1'",
"BB_LOG_HOST_STAT_CMDS_INTERVAL = 'oe-time-dd-test.sh -c 100 -t 15'",
"BB_LOG_HOST_STAT_ON_FAILURE = '1'",
- "BB_LOG_HOST_STAT_CMDS_FAILURE = 'oe-time-dd-test.sh -l'"
+ "BB_LOG_HOST_STAT_CMDS_FAILURE = 'oe-time-dd-test.sh -l'",
+ "BB_SERVER_TIMEOUT = '60'"
]
},
"templates" : {
--
2.34.1

On 23 Jun 2022, at 23:40, Aurum Nitrogen via lists.yoctoproject.org <aurumnitrogen=gmail.com@...> wrote:

Hi,
I was wondering if there has been talk about support for source-less python on an image. Installing py and pyc files doubles the size of python on the rootfs. I can imagine this being implemented as an image feature.
I know that in buildroot it is supported.
Was this discussed and decided against? Is this an open issue?

There’s an open issue: https://bugzilla.yoctoproject.org/show_bug.cgi?id=6434

The easiest implementation would be a rootfs-time postprocessing step where you compile every .py file, and then delete the .py. Is this what buildroot does?

This does break feeds, but the alternative (of changing how packaging is done) would be a lot more invasive.

Ross
IMPORTANT NOTICE: The contents of this email and any attachments are confidential and may also be privileged. If you are not the intended recipient, please notify the sender immediately and do not disclose the contents to any other person, use it for any purpose, or store or copy the information in any medium. Thank you.

I don't remember any such discussion. If you want this to happen, you
are welcome to develop the feature.

Alex

Hi there,

I am looking for some kind of package or software, that creates an (USB thumb drive) installer for initial installation and deployment of a yocto image. So you can flash the image onto an USB thumb drive and boot from it (arch is x86-64 so this should be easy), and after asking if that is really what you want, will erase the local storage and partition the drive (root A/B, data, swap) and copies over the image from the yocto build. I know that there is some thing into this direction in the meta-intel layer, but that is not quite well documented nor do I think it will suit our needs (regarding partitioning for example).

So, before I write terrible bash scripts into an initramfs init (because this is my best idea so far), if someone could give me a hint were to look would be great.

Thanks and regards,

    Matthias

On Wed, 2022-06-22 at 17:12 -0400, Randy MacLeod wrote:

As a first pass, collect all the tmp/buildstats logs
using 'cp -a', to enable review and graphing of all builds.
The goal is to understand the trends in the /proc/pressure/{cpu|io}
data that is now gathered as part of the buildstats system.

Signed-off-by: Aryaman Gupta <Aryaman.Gupta@...>
Signed-off-by: Randy MacLeod <Randy.MacLeod@...>
---
scripts/archive_buildstats.py | 27 +++++++++++++++++++++++++++
scripts/run-config | 1 +
2 files changed, 28 insertions(+)
create mode 100755 scripts/archive_buildstats.py

The NAS couldn't cope with the number of small files these data trees
contained leading to timeouts. If you want to do something like this
you'll probably have to create tarball archives of the results.

Cheers,

Richard

refresh patch

Signed-off-by: Armin Kuster <akuster808@...>
---
.../lkrg/files/makefile_cleanup.patch | 72 +++++++------------
...g-module_0.9.2.bb => lkrg-module_0.9.3.bb} | 7 +-
2 files changed, 30 insertions(+), 49 deletions(-)
rename recipes-kernel/lkrg/{lkrg-module_0.9.2.bb => lkrg-module_0.9.3.bb} (90%)

diff --git a/recipes-kernel/lkrg/files/makefile_cleanup.patch b/recipes-kernel/lkrg/files/makefile_cleanup.patch
index 799b1a6..f29afbe 100644
--- a/recipes-kernel/lkrg/files/makefile_cleanup.patch
+++ b/recipes-kernel/lkrg/files/makefile_cleanup.patch
@@ -1,73 +1,53 @@
-Upstream-Status: Pending
-
-This needs more work. Its my starting point.
-
+Upstream-Status: Inappropriate [embedded specific]
+
Signed-off-by: Armin Kuster <akuster808@...>

+
Index: git/Makefile
===================================================================
--- git.orig/Makefile
+++ git/Makefile
-@@ -4,28 +4,10 @@
- # Author:
- # - Adam 'pi3' Zabrocki (http://pi3.com.pl)
- ##
--
--P_OUTPUT = output
+@@ -7,15 +7,8 @@
+
+ P_OUTPUT = output
P_PWD ?= $(shell pwd)
-P_KVER ?= $(shell uname -r)
--P_BOOTUP_SCRIPT ?= scripts/bootup/lkrg-bootup.sh
--TARGET := p_lkrg
+ P_BOOTUP_SCRIPT ?= scripts/bootup/lkrg-bootup.sh
+ TARGET := p_lkrg
-ifneq ($(KERNELRELEASE),)
- KERNEL := /lib/modules/$(KERNELRELEASE)/build
-else
- ## KERNELRELEASE not set.
- KERNEL := /lib/modules/$(P_KVER)/build
-endif
--
--#
--# Uncomment for debug compilation
--#
--# ccflags-m := -ggdb -DP_LKRG_DEBUG_BUILD -finstrument-functions
--# ccflags-y := ${ccflags-m}
--# p_lkrg-objs += src/modules/print_log/p_lkrg_debug_log.o

--obj-m += $(TARGET).o
--$(TARGET)-objs += src/modules/ksyms/p_resolve_ksym.o \
-+obj-m := p_lkrg.o
-+p_lkrg-y := src/modules/ksyms/p_resolve_ksym.o \
- src/modules/hashing/p_lkrg_fast_hash.o \
- src/modules/comm_channel/p_comm_channel.o \
- src/modules/integrity_timer/p_integrity_timer.o \
-@@ -92,23 +74,14 @@ $(TARGET)-objs += src/modules/ksyms/p_re
+ #
+ # Use DEBUG=on for debug build.
+@@ -94,14 +87,13 @@ $(TARGET)-objs += src/modules/ksyms/p_re
src/p_lkrg_main.o


-all:
-# $(MAKE) -C $(KERNEL) M=$(P_PWD) modules CONFIG_DEBUG_SECTION_MISMATCH=y
- $(MAKE) -C $(KERNEL) M=$(P_PWD) modules
-- mkdir -p $(P_OUTPUT)
-- cp $(P_PWD)/$(TARGET).ko $(P_OUTPUT)
--
--install:
-- $(MAKE) -C $(KERNEL) M=$(P_PWD) modules_install
-- depmod -a
-- $(P_PWD)/$(P_BOOTUP_SCRIPT) install
-
--uninstall:
-- $(P_PWD)/$(P_BOOTUP_SCRIPT) uninstall
+modules:
+ $(MAKE) -C $(KERNEL_SRC) M=$(P_PWD) modules
-+
-+modules_install:
+ mkdir -p $(P_OUTPUT)
+ cp $(P_PWD)/$(TARGET).ko $(P_OUTPUT)
+
+-install:
+- $(MAKE) -C $(KERNEL) M=$(P_PWD) modules_install
++moduled_install:
+ $(MAKE) -C $(KERNEL_SRC) M=$(P_PWD) modules_install
+ depmod -a
+ $(P_PWD)/$(P_BOOTUP_SCRIPT) install
+
+@@ -109,7 +101,7 @@ uninstall:
+ $(P_PWD)/$(P_BOOTUP_SCRIPT) uninstall

clean:
- $(MAKE) -C $(KERNEL) M=$(P_PWD) clean
-- $(RM) Module.markers modules.order
-- $(RM) $(P_PWD)/src/modules/kmod/client/kmod/Module.markers
-- $(RM) $(P_PWD)/src/modules/kmod/client/kmod/modules.order
-- $(RM) -rf $(P_OUTPUT)
-+ rm -f *.o *~ core .depend .*.cmd *.ko *.mod.c
-+ rm -f Module.markers Module.symvers modules.order
-+ rm -rf .tmp_versions Modules.symvers
++ $(MAKE) -C $(KERNEL_SRC) M=$(P_PWD) clean
+ $(RM) Module.markers modules.order
+ $(RM) $(P_PWD)/src/modules/kmod/client/kmod/Module.markers
+ $(RM) $(P_PWD)/src/modules/kmod/client/kmod/modules.order
diff --git a/recipes-kernel/lkrg/lkrg-module_0.9.2.bb b/recipes-kernel/lkrg/lkrg-module_0.9.3.bb
similarity index 90%
rename from recipes-kernel/lkrg/lkrg-module_0.9.2.bb
rename to recipes-kernel/lkrg/lkrg-module_0.9.3.bb
index 85f7d44..2553974 100644
--- a/recipes-kernel/lkrg/lkrg-module_0.9.2.bb
+++ b/recipes-kernel/lkrg/lkrg-module_0.9.3.bb
@@ -9,10 +9,11 @@ LIC_FILES_CHKSUM = "file://LICENSE;md5=5105ead24b08a32954f34cbaa7112432"

DEPENDS = "virtual/kernel elfutils"

-SRCREV = "43db5f19fca259feb1962f6db33382348cbc8320"
-
SRC_URI = "git://github.com/lkrg-org/lkrg.git;protocol=https;branch=main \
- file://makefile_cleanup.patch "
+ file://makefile_cleanup.patch \
+"
+
+SRCREV = "c578e9f786299b67ffd62057b4534b0bf4fb7ece"

S = "${WORKDIR}/git"

--
2.25.1

As a first pass, collect all the tmp/buildstats logs
using 'cp -a', to enable review and graphing of all builds.
The goal is to understand the trends in the /proc/pressure/{cpu|io}
data that is now gathered as part of the buildstats system.

Signed-off-by: Aryaman Gupta <Aryaman.Gupta@...>
Signed-off-by: Randy MacLeod <Randy.MacLeod@...>
---
scripts/archive_buildstats.py | 27 +++++++++++++++++++++++++++
scripts/run-config | 1 +
2 files changed, 28 insertions(+)
create mode 100755 scripts/archive_buildstats.py

diff --git a/scripts/archive_buildstats.py b/scripts/archive_buildstats.py
new file mode 100755
index 0000000..bdbe9f4
--- /dev/null
+++ b/scripts/archive_buildstats.py
@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+
+import os, sys
+
+def usage():
+ print("Usage: " + sys.argv[0] + " <src> <dest> <target>")
+
+def main():
+ if len(sys.argv) != 4:
+ usage()
+ sys.exit()
+
+ builddir = sys.argv[1]
+ dest = sys.argv[2]
+ target = sys.argv[3]
+
+ bsdir = os.path.join(builddir, "tmp/buildstats")
+ directory = os.path.join(dest, target)
+
+ os.system('mkdir -p ' + directory)
+ # cp -a <path-to-worker-buildstats> <directory>
+ os.system('cp -a ' + builddir + ' ' + directory)
+
+main()
diff --git a/scripts/run-config b/scripts/run-config
index 838847a..953977e 100755
--- a/scripts/run-config
+++ b/scripts/run-config
@@ -334,6 +334,7 @@ elif args.phase == "finish" and args.stepname == "collect-results":
hp.printheader("Running results collection")
runcmd([scriptsdir + "/collect-results", args.builddir, args.results_dir, args.target])
runcmd([scriptsdir + "/summarize_top_output.py", args.results_dir, args.target])
+ runcmd([scriptsdir + "/archive_buildstats.py", args.builddir, args.results_dir, args.target])
sys.exit(0)

if jcfg:
--
2.32.0