Hi Richard,
thanks for the feedback

On 1/26/23 23:47, Richard Purdie wrote:

On Tue, 2023-01-24 at 18:30 +0100, Alexis Lothoré via lists.yoctoproject.org wrote:

report = subprocess.check_output([resulttool, "report", args.results_dir])
with open(args.results_dir + "/testresult-report.txt", "wb") as f:
f.write(report)

There looks to be a potential issue here since you drop the use of
comparebranch entirely. This comes from config.json:

"BUILD_HISTORY_FORKPUSH" : {"poky-contrib:ross/mut" : "poky:master", "poky:master-next" : "poky:master"},

and basically what it says is that master-next is a force push branch
which should be compared against master (and ross/mut should be
compared against master too).

We need this code so that when we build test branches, we run the
regression comparison against the last known master test results. We
should probably put Alexandre's test branch in there too.

I may have missed that. I need to dig a bit to understand exactly what is
expected here in order to fix it. Noted, I will take a look at it.

Regards

--
Alexis Lothoré, Bootlin
Embedded Linux and Kernel engineering
https://bootlin.com

On Tue, 2023-01-24 at 18:30 +0100, Alexis Lothoré via lists.yoctoproject.org wrote:

Instead of only generating regressions reports against HEAD of relevant branch, compute
most relevant tag (ie : release) against which we can check for regressions. General rules
introduced are the following :
- milestone release is checked against previous milestone if possible, otherwise
against major release
- point release is checked against previous point release if possible,
otherwise against major release
- major release is checked against previous major release
- a non release build is checked against base branch
Examples :
- 4.1.2.rc1 is checked against yocto-4.1.1
- 4.1.2 is checked against yocto-4.1.1
- 4.1.1.rc1 is checked against yocto-4.1
- 4.1.1 is checked against yocto-4.1
- 4.1 is checked against yocto-4.0
- 4.1.rc4 is checked against yocto-4.0
- 4.1_M2.rc1 is checked against 4.1_M1
- 4.1_M2 is checked against 4.1_M1
- 4.1_M1.rc1 is checked against yocto-4.0
- 4.1_M1 is checked against yocto-4.0

Signed-off-by: Alexis Lothoré <alexis.lothore@...>
---
scripts/send_qa_email.py | 86 +++++++++++++++++++++++++++++++++-------
scripts/utils.py | 47 ++++++++++++++++++++++
2 files changed, 118 insertions(+), 15 deletions(-)

diff --git a/scripts/send_qa_email.py b/scripts/send_qa_email.py
index 4023918..199fe4e 100755
--- a/scripts/send_qa_email.py
+++ b/scripts/send_qa_email.py
@@ -9,11 +9,79 @@ import json
import os
import sys
import subprocess
-import errno
import tempfile
+import re

import utils

+def is_non_release_version(version):
+ p = re.compile('\d{8}-\d+')
+ return p.match(version) is not None
+
+def get_previous_tag(targetrepodir, version):
+ previousversion = None
+ previousmilestone = None
+ if version:
+ if is_non_release_version(version):
+ return subprocess.check_output(["git", "describe", "--abbrev=0"], cwd=targetrepodir).decode('utf-8').strip()
+ compareversion, comparemilestone, _ = utils.get_version_from_string(version)
+ compareversionminor = compareversion[-1]
+ # After ignoring rc part, if we get a minor to 0 on point release (e.g 4.0.0),
+ # reject last digit since such versions do not exist
+ if len(compareversion) == 3 and compareversionminor == 0:
+ compareversion = compareversion[:-1]
+
+ # Process milestone if not first in current release
+ if comparemilestone and comparemilestone > 1:
+ previousversion = compareversion
+ previousmilestone = comparemilestone-1
+ # Process first milestone or release if not first in major release
+ elif compareversionminor > 0:
+ previousversion = compareversion[:-1] + [compareversion[-1] - 1]
+ # Otherwise : format it as tag (which must exist) and search previous tag
+ else:
+ comparetagstring = utils.get_tag_from_version(compareversion, comparemilestone)
+ return subprocess.check_output(["git", "describe", "--abbrev=0", comparetagstring + "^"], cwd=targetrepodir).decode('utf-8').strip()
+
+ return utils.get_tag_from_version(previousversion, previousmilestone)
+
+ # All other cases : merely check against latest tag reachable
+ defaultbaseversion, _, _ = utils.get_version_from_string(subprocess.check_output(["git", "describe", "--abbrev=0"], cwd=targetrepodir).decode('utf-8').strip())
+ return utils.get_tag_from_version(defaultbaseversion, None)
+
+def get_sha1(targetrepodir, revision):
+ return subprocess.check_output(["git", "rev-list", "-n", "1", revision], cwd=targetrepodir).decode('utf-8').strip()
+
+def fetch_testresults(resultdir, revision):
+ rawtags = subprocess.check_output(["git", "ls-remote", "--refs", "--tags", "origin", f"*{revision}*"], cwd=resultdir).decode('utf-8').strip()
+ if not rawtags:
+ raise Exception(f"No reference found for commit {revision} in {resultdir}")
+ for ref in [rawtag.split()[1] for rawtag in rawtags.splitlines()]:
+ print(f"Fetching matching revisions: {ref}")
+ subprocess.check_call(["git", "fetch", "--depth", "1", "origin", f"{ref}:{ref}"], cwd=resultdir)
+
+
+def generate_regression_report(resulttool, targetrepodir, basebranch, resultdir, outputdir, yoctoversion):
+ baseversion = get_previous_tag(targetrepodir, yoctoversion)
+ baserevision = get_sha1(targetrepodir, baseversion)
+ comparerevision = get_sha1(targetrepodir, basebranch)
+ print(f"Compare version : {basebranch} ({comparerevision})")
+ print(f"Base tag : {baseversion} ({baserevision})")
+
+ try:
+ """
+ Results directory is likely a shallow clone :
+ we need to fetch results corresponding to base revision before
+ running resulttool
+ """
+ fetch_testresults(resultdir, baserevision)
+ regreport = subprocess.check_output([resulttool, "regression-git", "-B", basebranch, "--commit", baserevision, "--commit2", comparerevision, resultdir])
+ with open(outputdir + "/testresult-regressions-report.txt", "wb") as f:
+ f.write(regreport)
+ except subprocess.CalledProcessError as e:
+ error = str(e)
+ print(f"Error while generating report between {baserevision} and {comparerevision} : {error}")
+

def send_qa_email():
parser = utils.ArgParser(description='Process test results and optionally send an email about the build to prompt QA to begin testing.')
@@ -57,16 +125,7 @@ def send_qa_email():
branch = repos['poky']['branch']
repo = repos['poky']['url']

- extraopts = None
basebranch, comparebranch = utils.getcomparisonbranch(ourconfig, repo, branch)
- if basebranch:
- extraopts = " --branch %s --commit %s" % (branch, revision)
- if comparebranch:
- extraopts = extraopts + " --branch2 %s" % (comparebranch)
- elif basebranch:
- print("No comparision branch found, comparing to %s" % basebranch)
- extraopts = extraopts + " --branch2 %s" % basebranch
-
report = subprocess.check_output([resulttool, "report", args.results_dir])
with open(args.results_dir + "/testresult-report.txt", "wb") as f:
f.write(report)

There looks to be a potential issue here since you drop the use of
comparebranch entirely. This comes from config.json:

"BUILD_HISTORY_FORKPUSH" : {"poky-contrib:ross/mut" : "poky:master", "poky:master-next" : "poky:master"},

and basically what it says is that master-next is a force push branch
which should be compared against master (and ross/mut should be
compared against master too).

We need this code so that when we build test branches, we run the
regression comparison against the last known master test results. We
should probably put Alexandre's test branch in there too.

Cheers,

Richard

Instead of only generating regressions reports against HEAD of relevant branch, compute
most relevant tag (ie : release) against which we can check for regressions. General rules
introduced are the following :
- milestone release is checked against previous milestone if possible, otherwise
against major release
- point release is checked against previous point release if possible,
otherwise against major release
- major release is checked against previous major release
- a non release build is checked against base branch
Examples :
- 4.1.2.rc1 is checked against yocto-4.1.1
- 4.1.2 is checked against yocto-4.1.1
- 4.1.1.rc1 is checked against yocto-4.1
- 4.1.1 is checked against yocto-4.1
- 4.1 is checked against yocto-4.0
- 4.1.rc4 is checked against yocto-4.0
- 4.1_M2.rc1 is checked against 4.1_M1
- 4.1_M2 is checked against 4.1_M1
- 4.1_M1.rc1 is checked against yocto-4.0
- 4.1_M1 is checked against yocto-4.0

Signed-off-by: Alexis Lothoré <alexis.lothore@...>
---
scripts/send_qa_email.py | 86 +++++++++++++++++++++++++++++++++-------
scripts/utils.py | 47 ++++++++++++++++++++++
2 files changed, 118 insertions(+), 15 deletions(-)

diff --git a/scripts/send_qa_email.py b/scripts/send_qa_email.py
index 4023918..199fe4e 100755
--- a/scripts/send_qa_email.py
+++ b/scripts/send_qa_email.py
@@ -9,11 +9,79 @@ import json
import os
import sys
import subprocess
-import errno
import tempfile
+import re

import utils

+def is_non_release_version(version):
+ p = re.compile('\d{8}-\d+')
+ return p.match(version) is not None
+
+def get_previous_tag(targetrepodir, version):
+ previousversion = None
+ previousmilestone = None
+ if version:
+ if is_non_release_version(version):
+ return subprocess.check_output(["git", "describe", "--abbrev=0"], cwd=targetrepodir).decode('utf-8').strip()
+ compareversion, comparemilestone, _ = utils.get_version_from_string(version)
+ compareversionminor = compareversion[-1]
+ # After ignoring rc part, if we get a minor to 0 on point release (e.g 4.0.0),
+ # reject last digit since such versions do not exist
+ if len(compareversion) == 3 and compareversionminor == 0:
+ compareversion = compareversion[:-1]
+
+ # Process milestone if not first in current release
+ if comparemilestone and comparemilestone > 1:
+ previousversion = compareversion
+ previousmilestone = comparemilestone-1
+ # Process first milestone or release if not first in major release
+ elif compareversionminor > 0:
+ previousversion = compareversion[:-1] + [compareversion[-1] - 1]
+ # Otherwise : format it as tag (which must exist) and search previous tag
+ else:
+ comparetagstring = utils.get_tag_from_version(compareversion, comparemilestone)
+ return subprocess.check_output(["git", "describe", "--abbrev=0", comparetagstring + "^"], cwd=targetrepodir).decode('utf-8').strip()
+
+ return utils.get_tag_from_version(previousversion, previousmilestone)
+
+ # All other cases : merely check against latest tag reachable
+ defaultbaseversion, _, _ = utils.get_version_from_string(subprocess.check_output(["git", "describe", "--abbrev=0"], cwd=targetrepodir).decode('utf-8').strip())
+ return utils.get_tag_from_version(defaultbaseversion, None)
+
+def get_sha1(targetrepodir, revision):
+ return subprocess.check_output(["git", "rev-list", "-n", "1", revision], cwd=targetrepodir).decode('utf-8').strip()
+
+def fetch_testresults(resultdir, revision):
+ rawtags = subprocess.check_output(["git", "ls-remote", "--refs", "--tags", "origin", f"*{revision}*"], cwd=resultdir).decode('utf-8').strip()
+ if not rawtags:
+ raise Exception(f"No reference found for commit {revision} in {resultdir}")
+ for ref in [rawtag.split()[1] for rawtag in rawtags.splitlines()]:
+ print(f"Fetching matching revisions: {ref}")
+ subprocess.check_call(["git", "fetch", "--depth", "1", "origin", f"{ref}:{ref}"], cwd=resultdir)
+
+
+def generate_regression_report(resulttool, targetrepodir, basebranch, resultdir, outputdir, yoctoversion):
+ baseversion = get_previous_tag(targetrepodir, yoctoversion)
+ baserevision = get_sha1(targetrepodir, baseversion)
+ comparerevision = get_sha1(targetrepodir, basebranch)
+ print(f"Compare version : {basebranch} ({comparerevision})")
+ print(f"Base tag : {baseversion} ({baserevision})")
+
+ try:
+ """
+ Results directory is likely a shallow clone :
+ we need to fetch results corresponding to base revision before
+ running resulttool
+ """
+ fetch_testresults(resultdir, baserevision)
+ regreport = subprocess.check_output([resulttool, "regression-git", "-B", basebranch, "--commit", baserevision, "--commit2", comparerevision, resultdir])
+ with open(outputdir + "/testresult-regressions-report.txt", "wb") as f:
+ f.write(regreport)
+ except subprocess.CalledProcessError as e:
+ error = str(e)
+ print(f"Error while generating report between {baserevision} and {comparerevision} : {error}")
+

def send_qa_email():
parser = utils.ArgParser(description='Process test results and optionally send an email about the build to prompt QA to begin testing.')
@@ -57,16 +125,7 @@ def send_qa_email():
branch = repos['poky']['branch']
repo = repos['poky']['url']

- extraopts = None
basebranch, comparebranch = utils.getcomparisonbranch(ourconfig, repo, branch)
- if basebranch:
- extraopts = " --branch %s --commit %s" % (branch, revision)
- if comparebranch:
- extraopts = extraopts + " --branch2 %s" % (comparebranch)
- elif basebranch:
- print("No comparision branch found, comparing to %s" % basebranch)
- extraopts = extraopts + " --branch2 %s" % basebranch
-
report = subprocess.check_output([resulttool, "report", args.results_dir])
with open(args.results_dir + "/testresult-report.txt", "wb") as f:
f.write(report)
@@ -95,7 +154,6 @@ def send_qa_email():
subprocess.check_call(["git", "checkout", "master"], cwd=tempdir)
subprocess.check_call(["git", "branch", basebranch], cwd=tempdir)
subprocess.check_call(["git", "checkout", basebranch], cwd=tempdir)
- extraopts = None

subprocess.check_call([resulttool, "store", args.results_dir, tempdir])
if comparebranch:
@@ -105,10 +163,8 @@ def send_qa_email():
subprocess.check_call(["git", "push", "--all"], cwd=tempdir)
subprocess.check_call(["git", "push", "--tags"], cwd=tempdir)

- if extraopts:
- regreport = subprocess.check_output([resulttool, "regression-git", tempdir] + extraopts.split())
- with open(args.results_dir + "/testresult-regressions-report.txt", "wb") as f:
- f.write(regreport)
+ if basebranch:
+ generate_regression_report(resulttool, targetrepodir, basebranch, tempdir, args.results_dir, args.release)

finally:
subprocess.check_call(["rm", "-rf", tempdir])
diff --git a/scripts/utils.py b/scripts/utils.py
index c0ad14e..444b3ab 100644
--- a/scripts/utils.py
+++ b/scripts/utils.py
@@ -478,3 +478,50 @@ def setup_buildtools_tarball(ourconfig, workername, btdir, checkonly=False):
pass
subprocess.check_call(["bash", btdlpath, "-d", btdir, "-y"])
enable_buildtools_tarball(btdir)
+
+def get_string_from_version(version, milestone=None, rc=None):
+ """ Point releases finishing by 0 (e.g 4.0.0, 4.1.0) do no exists,
+ those are major releases
+ """
+ if len(version) == 3 and version[-1] == 0:
+ version = version[:-1]
+
+ result = ".".join(list(map(str, version)))
+ if milestone:
+ result += "_M" + str(milestone)
+ if rc:
+ result += ".rc" + str(rc)
+ return result
+
+def get_tag_from_version(version, milestone):
+ if not milestone:
+ return "yocto-" + get_string_from_version(version, milestone)
+ return get_string_from_version(version, milestone)
+
+
+def get_version_from_string(raw_version):
+ """ Get version as list of int from raw_version.
+
+ Raw version _can_ be prefixed by "yocto-",
+ Raw version _can_ be suffixed by "_MX"
+ Raw version _can_ be suffixed by ".rcY"
+ """
+ version = None
+ milestone = None
+ rc = None
+ if raw_version[:6] == "yocto-":
+ raw_version = raw_version[6:]
+ raw_version = raw_version.split(".")
+ if raw_version[-1][:2] == "rc":
+ rc = int(raw_version[-1][-1])
+ raw_version = raw_version[:-1]
+ if raw_version[-1][-3:-1] == "_M":
+ milestone = int(raw_version[-1][-1])
+ raw_version = raw_version[:-1] + [raw_version[-1][:-3]]
+ version = list(map(int, raw_version))
+ """ Point releases finishing by 0 (e.g 4.0.0, 4.1.0) do no exists,
+ those are major releases
+ """
+ if len(version) == 3 and version[-1] == 0:
+ version = version[:-1]
+ return version, milestone, rc
\ No newline at end of file
--
2.39.0