[PATCH yocto-autobuilder-helper 2/2] scripts/archive_buildstats.py: archive buildstats to tar.zst


Aryaman Gupta <aryaman.gupta@...>
 

Archive the buildstats of every failed build and 1% of random builds. Convert
the time-stamped buildstats directory to a compressed tarball using the
hostname as a prefix (to the file name only) so that one can identify the
source machine. Move these tarballs to the directory:
testresults/<build_name>/buildstats/

The archiving is performed during the "collect results" step.

Signed-off-by: Aryaman Gupta <aryaman.gupta@...>
Signed-off-by: Randy MacLeod <Randy.MacLeod@...>
---
scripts/archive_buildstats.py | 37 +++++++++++++++++++++++++++++++++++
scripts/run-config | 1 +
2 files changed, 38 insertions(+)
create mode 100755 scripts/archive_buildstats.py

diff --git a/scripts/archive_buildstats.py b/scripts/archive_buildstats.py
new file mode 100755
index 0000000..de866e5
--- /dev/null
+++ b/scripts/archive_buildstats.py
@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+#
+# SPDX-License-Identifier: GPL-2.0-only
+#
+import glob, os, random, subprocess, socket, sys
+
+def usage():
+ print("Usage: " + sys.argv[0] + " <src> <dest> <target>")
+
+def main():
+ if len(sys.argv) != 4:
+ usage()
+ sys.exit()
+
+ builddir = sys.argv[1]
+ dest = sys.argv[2]
+ target = sys.argv[3]
+ dest_bsdir = os.path.join(dest, target, "buildstats")
+ subprocess.run(["mkdir", "-p", dest_bsdir])
+
+ build_bsdir = os.path.join(builddir, "tmp/buildstats")
+ hostname = socket.gethostname()
+ os.chdir(build_bsdir)
+ fail_path = os.path.join(dest, target, "intermittent_failure_host_data")
+ fail_output = glob.glob(fail_path + '/*top_summary.txt')
+
+ #archive the buildstats of failures and 1% of random builds
+ if fail_output or random.randint(1,100)%100 == 0:
+ for timestamp in os.listdir(build_bsdir):
+ if hostname:
+ output = hostname + "-" + timestamp + ".tar.zst"
+ else:
+ output = "nohostname-"+ timestamp + ".tar.zst"
+ subprocess.check_call("tar -I zstd -cf "+output+" "+timestamp+"/*", shell=True)
+ subprocess.run(["mv", output, dest_bsdir])
+
+main()
\ No newline at end of file
diff --git a/scripts/run-config b/scripts/run-config
index 838847a..953977e 100755
--- a/scripts/run-config
+++ b/scripts/run-config
@@ -334,6 +334,7 @@ elif args.phase == "finish" and args.stepname == "collect-results":
hp.printheader("Running results collection")
runcmd([scriptsdir + "/collect-results", args.builddir, args.results_dir, args.target])
runcmd([scriptsdir + "/summarize_top_output.py", args.results_dir, args.target])
+ runcmd([scriptsdir + "/archive_buildstats.py", args.builddir, args.results_dir, args.target])
sys.exit(0)

if jcfg:
--
2.35.3