diff --git a/.github/actions/test-job-summary/action.yml b/.github/actions/test-job-summary/action.yml index ae191d516..3343acc9e 100644 --- a/.github/actions/test-job-summary/action.yml +++ b/.github/actions/test-job-summary/action.yml @@ -15,6 +15,13 @@ inputs: summary_title: description: "Title for the summary part of the report" required: false + known_failures_file: + description: > + Path to the YAML file listing the known failures of the tested build + variant. Relative to the workspace. Missing file means no known failure. + See .github/known-failures/README.md for the file format. + required: false + default: "" outputs: artifact_id: description: "ID of the uploaded artifact" @@ -43,6 +50,7 @@ runs: INPUTS_PREFIX: ${{ inputs.prefix }} INPUTS_SUMMARY_FILE_NAME: ${{ inputs.summary_file_name }} INPUTS_SUMMARY_TITLE: ${{ inputs.summary_title }} + INPUTS_KNOWN_FAILURES_FILE: ${{ inputs.known_failures_file }} run: | set -x # When a failed LAVA job is re-run, GitHub keeps the original failed @@ -70,6 +78,9 @@ runs: do JOB_ID=$(cat "$TESTJOB" | jq ".id") JOB_URL="https://lava.infra.foundries.io/results/$JOB_ID" + # the job log renders every line as , so a test + # result can be linked straight to the line it was logged on + JOB_LOG_URL="https://lava.infra.foundries.io/scheduler/job/$JOB_ID" JOB_DETAILS=$(curl -s "https://lava.infra.foundries.io/api/v0.2/jobs/$JOB_ID/") JOB_HEALTH=$(echo "$JOB_DETAILS" | jq -r ".health") JOB_STATE=$(echo "$JOB_DETAILS" | jq -r ".state") @@ -89,7 +100,12 @@ runs: SUITE_ID=$(echo "$SUITE" | jq -r ".id") SUITE_TESTS=$(curl -s "https://lava.infra.foundries.io/api/v0.2/jobs/$JOB_ID/suites/$SUITE_ID/tests/") if [ "$SUITE_NAME" != "lava" ]; then - echo "$SUITE_TESTS" | jq ".results[] | {(.name): {result: .result, url: .resource_uri}} | to_entries | .[]" + # link to the log line the result was reported on + # rather than to the API resource of the test case + echo "$SUITE_TESTS" | jq --arg log "$JOB_LOG_URL" '.results[] + | {(.name): {result: .result, + url: (if .start_log_line then "\($log)#L\(.start_log_line)" else $log end)}} + | to_entries | .[]' fi fi else @@ -99,7 +115,9 @@ runs: TEST_RESULT_BOOT="fail" fi TEST_NAME="boot" - TEST_URL="${JOB_URL}" + # a boot job has no test case to point at, so link to the + # log itself, which is what a boot failure is read from + TEST_URL="${JOB_LOG_URL}" TEST_RESULT_OBJ=$(jq --arg url "$TEST_URL" --arg result "$TEST_RESULT_BOOT" -n -c '$ARGS.named') jq -n -c --arg key "$TEST_NAME" --argjson value "$TEST_RESULT_OBJ" '$ARGS.named' fi @@ -108,6 +126,57 @@ runs: # combine entries from boot jobs and pre-merge jobs done | jq -s -c 'reduce .[] as $i ({}; .[$i.key] = ((.[$i.key] // {}) + $i.value))') echo "$INPUT" + + # Load the list of known failures of this build variant. Every build + # variant has its own list, see .github/known-failures/README.md. + # A missing or empty file simply means "no known failure". + # An entry is either a bare test name or a mapping with a "test" and an + # optional "comment" holding the issue it is tracked in. Both forms are + # normalised to {device: {test name: comment}}. + KNOWN_FAILURES="{}" + if [ -n "${INPUTS_KNOWN_FAILURES_FILE}" ]; then + if [ -f "${GITHUB_WORKSPACE}/${INPUTS_KNOWN_FAILURES_FILE}" ]; then + KNOWN_FAILURES_RAW=$(yq -o=json -I=0 "${GITHUB_WORKSPACE}/${INPUTS_KNOWN_FAILURES_FILE}" | jq -c '. // {}') + KNOWN_FAILURES=$(echo "${KNOWN_FAILURES_RAW}" | jq -c ' + with_entries(.value |= ((. // []) | map( + if type == "string" then {key: ., value: ""} + elif type == "object" and (.test | type) == "string" then {key: .test, value: (.comment // "" | tostring)} + else empty + end) | from_entries))') + # entries that match neither form are dropped by the mapping above + RAW_COUNT=$(echo "${KNOWN_FAILURES_RAW}" | jq '[.[] | (. // []) | length] | add // 0') + KNOWN_COUNT=$(echo "${KNOWN_FAILURES}" | jq '[.[] | length] | add // 0') + if [ "${RAW_COUNT}" != "${KNOWN_COUNT}" ]; then + echo "::warning::${INPUTS_KNOWN_FAILURES_FILE}: ignored $((RAW_COUNT - KNOWN_COUNT)) malformed entry/entries" + fi + else + echo "::warning::known failures file ${INPUTS_KNOWN_FAILURES_FILE} not found" + fi + fi + echo "${KNOWN_FAILURES}" + + # Turn the LAVA result of every test into the status that is reported. + # A failure that is on the known failures list is reported as "known + # failure" and counted as a pass. A test that passes while it is on + # the list is reported as "unexpected pass" and counted as a failure, + # so that stale entries get noticed and removed from the list. The + # comment of the entry is carried over so it can be reported too. + INPUT=$(echo "$INPUT" | jq -c --argjson known "${KNOWN_FAILURES}" ' + with_entries( + # a device specific entry overrides the one listed for all devices + (($known["*"] // {}) + ($known[.key] // {})) as $known_tests + | .value |= with_entries( + .key as $test + | ($known_tests | has($test)) as $is_known + | .value.status = ( + if .value.result == "fail" and $is_known then "known failure" + elif .value.result == "pass" and $is_known then "unexpected pass" + else .value.result + end) + | if $is_known then .value.comment = $known_tests[$test] else . end) + )') + echo "$INPUT" + DEVICES=$(echo "$INPUT" | jq -r 'keys[]' | sort) RESULTS=$(echo "$INPUT" | jq -r '.[] | keys[]?' | sort -u) @@ -115,14 +184,17 @@ runs: # Print collapsible box echo "
" >> "${INPUTS_SUMMARY_FILE_NAME}" # Print section summary - TOTAL_PASS=$(echo "${INPUT}" | jq '[.. | objects | select(.result? == "pass")] | length') - TOTAL_FAIL=$(echo "${INPUT}" | jq '[.. | objects | select(.result? == "fail")] | length') + TOTAL_PASS=$(echo "${INPUT}" | jq '[.. | objects | select(.status? == "pass")] | length') + TOTAL_KNOWN_FAIL=$(echo "${INPUT}" | jq '[.. | objects | select(.status? == "known failure")] | length') + # an unexpected pass counts as a failure: the known failures list is + # out of date and has to be updated + TOTAL_FAIL=$(echo "${INPUT}" | jq '[.. | objects | select(.status? == "fail" or .status? == "unexpected pass")] | length') TOTAL=$(echo "${INPUT}" | jq '[.. | objects | select(.result)] | length') SUMMARY_TITLE="" if [ "${INPUTS_SUMMARY_TITLE}" != "" ]; then SUMMARY_TITLE="${INPUTS_SUMMARY_TITLE}
" fi - echo "${SUMMARY_TITLE}Pass: $TOTAL_PASS | Fail: $TOTAL_FAIL | Total: $TOTAL " >> "${INPUTS_SUMMARY_FILE_NAME}" + echo "${SUMMARY_TITLE}Pass: $TOTAL_PASS | Known failures: $TOTAL_KNOWN_FAIL | Fail: $TOTAL_FAIL | Total: $TOTAL " >> "${INPUTS_SUMMARY_FILE_NAME}" echo "" >> "${INPUTS_SUMMARY_FILE_NAME}" echo "" >> "${INPUTS_SUMMARY_FILE_NAME}" @@ -151,10 +223,12 @@ runs: printf "| %s |" "$R" >> "${INPUTS_SUMMARY_FILE_NAME}" for D in $DEVICES; do - VALUE=$(echo "$INPUT" | jq -r --arg d "$D" --arg r "$R" '.[$d][$r].result // ""') + VALUE=$(echo "$INPUT" | jq -r --arg d "$D" --arg r "$R" '.[$d][$r].status // ""') URL=$(echo "$INPUT" | jq -r --arg d "$D" --arg r "$R" '.[$d][$r].url // ""') CHECKMARK=":white_check_mark:" if [ "${VALUE}" = "fail" ]; then CHECKMARK=":x:"; fi + if [ "${VALUE}" = "unexpected pass" ]; then CHECKMARK=":x:"; fi + if [ "${VALUE}" = "known failure" ]; then CHECKMARK=":ballot_box_with_check:"; fi if [ "${VALUE}" = "skip" ]; then CHECKMARK=":warning:"; fi if [ -z "${VALUE}" ]; then CHECKMARK=":no_entry_sign:"; fi printf " %s [%s](%s) |" "$CHECKMARK" "$VALUE" "$URL" @@ -164,6 +238,24 @@ runs: done echo "" >> "${INPUTS_SUMMARY_FILE_NAME}" echo "
" >> "${INPUTS_SUMMARY_FILE_NAME}" + + # Print the entries of the known failures list that were applied, + # together with the issue each one is annotated with + KNOWN_ROWS=$(echo "$INPUT" | jq -r ' + to_entries[] | .key as $device | .value | to_entries[] + | select(.value.status == "known failure" or .value.status == "unexpected pass") + | "| \($device) | \(.key) | \(.value.status) | \(.value.comment // "") |"' | sort) + if [ -n "${KNOWN_ROWS}" ]; then + echo "
" >> "${INPUTS_SUMMARY_FILE_NAME}" + echo "Known failures ($TOTAL_KNOWN_FAIL)" >> "${INPUTS_SUMMARY_FILE_NAME}" + echo "" >> "${INPUTS_SUMMARY_FILE_NAME}" + echo "| Device | Test | Status | Comment |" >> "${INPUTS_SUMMARY_FILE_NAME}" + echo "| ---- | ---- | ---- | ---- |" >> "${INPUTS_SUMMARY_FILE_NAME}" + echo "${KNOWN_ROWS}" >> "${INPUTS_SUMMARY_FILE_NAME}" + echo "" >> "${INPUTS_SUMMARY_FILE_NAME}" + echo "
" >> "${INPUTS_SUMMARY_FILE_NAME}" + fi + echo "
" >> "${INPUTS_SUMMARY_FILE_NAME}" echo "All jobs summary" >> "${INPUTS_SUMMARY_FILE_NAME}" echo "" >> "${INPUTS_SUMMARY_FILE_NAME}" diff --git a/.github/known-failures/README.md b/.github/known-failures/README.md new file mode 100644 index 000000000..26cf02373 --- /dev/null +++ b/.github/known-failures/README.md @@ -0,0 +1,139 @@ +# Known test failures + +Each build variant tested by `.github/workflows/test.yml` has its own list of +known failures in this directory. The file name is +`.yaml`, i.e. the same value that is used to name +the test job summary: + +| Build variant | File | +| --------------------------- | ---------------------------------- | +| nodistro | `nodistro.yaml` | +| qcom-distro | `qcom-distro.yaml` | +| qcom-distro_linux-qcom-6.18 | `qcom-distro_linux-qcom-6.18.yaml` | + +## Format + +The file is a YAML mapping of LAVA device type to the list of tests that are +expected to fail on that device. Every entry carries the test name and a +`comment` pointing at the issue the failure is tracked in: + +```yaml +qcm6490-idp: + - test: some_failing_test + comment: https://github.com/qualcomm-linux/meta-qcom/issues/1234 + - test: another_failing_test + comment: "waiting for the firmware uprev, issue #1235" + +# "*" applies to every device tested in this variant +"*": + - test: test_failing_everywhere + comment: https://github.com/qualcomm-linux/meta-qcom/issues/1236 +``` + +`comment` is free text. A link to the reported issue is what makes the list +reviewable, so add one for every entry. Quote a comment that contains a `#`, +otherwise YAML treats the rest of the line as a comment of its own. A bare test +name is also accepted for an entry that has nothing to say: + +```yaml +qcm6490-idp: + - some_failing_test +``` + +The test name is the name reported by LAVA, i.e. the value shown in the first +column of the test job summary table. The device type is the LAVA +`requested_device_type`, i.e. the column header of that table. When a test is +listed both for a device and under `"*"`, the device entry and its comment win. + +An empty file (or a file containing only comments) means that no failure is +known for that variant. + +## How the list is used + +Two consumers apply the list. + +### The test job summary + +`.github/actions/test-job-summary` applies the list when it renders the summary +table of a build variant: + +| Result in LAVA | Listed as known failure | Reported as | +| -------------- | ----------------------- | --------------------------------------- | +| `fail` | no | :x: `fail` | +| `fail` | yes | :ballot_box_with_check: `known failure` | +| `pass` | no | :white_check_mark: `pass` | +| `pass` | yes | :x: `unexpected pass` | + +A known failure is counted like a pass, so it does not add to the failure +count. A test that passes while it is listed as a known failure is counted as +a failure - that is the signal to remove the entry from this list. + +Every entry that was applied is listed with its comment in a "Known failures" +section below the table. + +The summary also reports the result of the boot test under the name `boot`, so +`boot` can be listed here like any other test name. + +### The "Test Results" check + +`.github/workflows/publish-results.yml` publishes the JUnit XML files that LAVA +produced for every test job. Before they are published, +`.github/scripts/apply-known-failures.py` rewrites them in place: a known +failure becomes a skipped test, so it no longer fails the check, and a test +that passes while it is listed becomes a failure. The comment of the entry is +appended to the message of the rewritten test, so the issue is one click away +in the check report. The script maps a result file to a variant and a device +through the file name, which lava-test-plans builds as +`---.yaml`. + +The XML files only contain the tests that ran inside a LAVA job, so a `boot` +entry has no effect on this check. + +## Which lists are used + +Both consumers apply the lists of the pull request under test, not the ones +already merged, so a pull request that fixes a listed failure removes the entry +in the same change, and a pull request that hits a new one can list it right +away. + +The test chain of a pull request runs on the `workflow_run` event, so its own +checkout is the base branch. It therefore checks this directory out a second +time from the branch or fork the pull request is built from, into +`known-failures-pr/`, with a sparse checkout that fetches nothing else. Those +lists are only ever read as data: the scripts applying them, and everything +else the chain runs, still come from the base branch. + +Before they are applied to the "Test Results" check the lists are checked with +`--validate --syntax-only`, i.e. for their syntax alone - which variants exist +is a property of the base branch, and it is `known-failures.yml`, running on +the pull request itself, that checks the lists against them. A list that does +not parse, or a fork that is no longer reachable, falls back to the lists of +the base branch. The run log says which lists were applied. + +A push, a nightly build and a manual run have no pull request to take lists +from and simply use the ones of their own checkout. + +## Validation + +`.github/workflows/known-failures.yml` validates the lists on every change to +this directory, to the script, or to the test workflow. It rejects: + +* a file that is not valid YAML, or that does not follow the format above, +* a test listed twice for the same device, +* a file whose name is not one of the build variants tested by + `.github/workflows/test.yml`, and a variant that has no file at all, +* a device that the variant does not test. + +The last two matter because such an entry is not an error at test time, it is +simply never applied - the list would look like it suppresses a failure while +the check stays red. An entry without a comment is reported as a warning. + +Run the same check locally with: + +```shell +python3 .github/scripts/apply-known-failures.py --validate +``` + +Add `--syntax-only` to check the format of the lists without checking them +against the build variants of this branch. That is what the test chain uses on +the lists it takes from a pull request. diff --git a/.github/known-failures/nodistro.yaml b/.github/known-failures/nodistro.yaml new file mode 100644 index 000000000..f7c4890f3 --- /dev/null +++ b/.github/known-failures/nodistro.yaml @@ -0,0 +1,16 @@ +--- +# Known test failures for the "nodistro" build variant. +# +# Mapping of LAVA device type to the list of test names that are expected to +# fail. Use "*" as the device type for a failure seen on every device. Annotate +# every entry with the issue it is tracked in, so that the list stays reviewable. +# See README.md in this directory for details. +# +# Example: +# +# qcm6490-idp: +# - test: some_failing_test +# comment: https://github.com/qualcomm-linux/meta-qcom/issues/1234 +# "*": +# - test: test_failing_everywhere +# comment: "flaky since the 6.18 kernel uprev, issue #1235" diff --git a/.github/known-failures/qcom-distro.yaml b/.github/known-failures/qcom-distro.yaml new file mode 100644 index 000000000..4f65500e4 --- /dev/null +++ b/.github/known-failures/qcom-distro.yaml @@ -0,0 +1,30 @@ +--- +# Known test failures for the "qcom-distro" build variant. +# +# Mapping of LAVA device type to the list of test names that are expected to +# fail. Use "*" as the device type for a failure seen on every device. Annotate +# every entry with the issue it is tracked in, so that the list stays reviewable. +# See README.md in this directory for details. +# +# Example: +# +# qcm6490-idp: +# - test: some_failing_test +# comment: https://github.com/qualcomm-linux/meta-qcom/issues/1234 +# "*": +# - test: test_failing_everywhere +# comment: "flaky since the 6.18 kernel uprev, issue #1235" + +iq-x7181-evk: + - test: AudioRecord + comment: "" + - test: Ethernet + comment: "" + - test: GStreamer_Video_Decode_h264_480p + comment: "" + - test: GStreamer_Video_Decode_h265_480p + comment: "" + - test: GStreamer_Video_Encode_h264_480p + comment: "" + - test: GStreamer_Video_Encode_h265_480p + comment: "" diff --git a/.github/known-failures/qcom-distro_linux-qcom-6.18.yaml b/.github/known-failures/qcom-distro_linux-qcom-6.18.yaml new file mode 100644 index 000000000..40af42f30 --- /dev/null +++ b/.github/known-failures/qcom-distro_linux-qcom-6.18.yaml @@ -0,0 +1,22 @@ +--- +# Known test failures for the "qcom-distro_linux-qcom-6.18" build variant. +# +# Mapping of LAVA device type to the list of test names that are expected to +# fail. Use "*" as the device type for a failure seen on every device. Annotate +# every entry with the issue it is tracked in, so that the list stays reviewable. +# See README.md in this directory for details. +# +# Example: +# +# qcm6490-idp: +# - test: some_failing_test +# comment: https://github.com/qualcomm-linux/meta-qcom/issues/1234 +# "*": +# - test: test_failing_everywhere +# comment: "flaky since the 6.18 kernel uprev, issue #1235" + +rb3gen2-core-kit: + - test: Camera_RDI_FrameCapture + comment: "" + - test: Libcamera_cam + comment: "" diff --git a/.github/scripts/apply-known-failures.py b/.github/scripts/apply-known-failures.py new file mode 100755 index 000000000..bb570fc58 --- /dev/null +++ b/.github/scripts/apply-known-failures.py @@ -0,0 +1,347 @@ +#!/usr/bin/env python3 +"""Apply the known failures lists to the LAVA JUnit XML result files. + +The LAVA JUnit files are published as a check by publish-unit-test-result-action, +which fails the workflow on any . Without this script a failure that is +already known and accepted for a build variant would keep that check red forever. + +Every build variant has its own list of known failures in +.github/known-failures/.yaml, see the README in that directory. This +script rewrites the result files in place so that: + + * a failing test that is on the list becomes ("known failure"), so + it no longer fails the check but stays visible in the report, + * a passing test that is on the list becomes ("unexpected pass"), + which is the signal that the entry has to be removed from the list. + +The variant and the device a result file belongs to are taken from its name. +lava-test-plans names the result files "---.yaml" +(see .github/actions/lava-test-plans/action.yml), and lava-action saves them +with an .xml suffix under an artifact directory of the same name. + +With --validate the script does not touch any result file. It only checks the +lists themselves: their syntax, and that every file and every device in them +matches a build variant that the test workflow really tests. An entry that names +a variant or a device that is not tested would silently never be applied. Add +--syntax-only to check the syntax alone, which is all that can be asked of lists +that do not come from this branch. +""" + +import argparse +import os +import pathlib +import sys +import xml.etree.ElementTree as ET + +import yaml + +# suite holding the LAVA infrastructure steps rather than the actual tests +LAVA_SUITE = "lava" + +# workflow calling test-distro.yml once per tested build variant +TEST_WORKFLOW = ".github/workflows/test.yml" + +# device type standing for every device tested in a variant +ANY_DEVICE = "*" + + +def annotation(path, message): + """Format a message as a GitHub annotation when running in a workflow.""" + if os.environ.get("GITHUB_ACTIONS") == "true": + return f"::error file={path}::{message}" + return f"{path}: {message}" + + +def fail(path, message): + sys.exit(annotation(path, message)) + + +def load_entry(path, device, entry): + """Return (test name, comment) for one entry of a known failures list. + + An entry is either a bare test name or a mapping with a "test" and an + optional "comment" naming the issue the failure is tracked in. + """ + if isinstance(entry, str): + return entry, "" + if not isinstance(entry, dict): + fail(path, f"{device}: expected a test name or a mapping, got {entry!r}") + unknown = set(entry) - {"test", "comment"} + if unknown: + fail(path, f"{device}: unknown key(s) {', '.join(sorted(unknown))}") + test = entry.get("test") + if not isinstance(test, str) or not test: + fail(path, f"{device}: entry {entry!r} is missing a test name") + return test, str(entry.get("comment", "")) + + +def load_known_failures(known_failures_dir): + """Return {variant: {device: {test name: comment}}} for every list found.""" + known_failures = {} + for path in sorted(pathlib.Path(known_failures_dir).glob("*.yaml")): + try: + content = yaml.safe_load(path.read_text()) or {} + except yaml.YAMLError as error: + fail(path, f"not valid YAML: {error}") + if not isinstance(content, dict): + fail(path, "expected a mapping of device to list of tests") + variant = {} + for device, entries in content.items(): + if not isinstance(entries, list): + fail(path, f"{device}: expected a list of test names") + tests = {} + for entry in entries: + test, comment = load_entry(path, device, entry) + if test in tests: + fail(path, f"{device}: {test} is listed more than once") + tests[test] = comment + variant[str(device)] = tests + known_failures[path.stem] = variant + return known_failures + + +def match_result_file(name, known_failures): + """Return the {test name: comment} of the known failures of a result file. + + The longest matching variant wins so that "qcom-distro_linux-qcom-6.18" is + not shadowed by "qcom-distro". Returns None when the file does not belong to + any variant that has a known failures list. + """ + for variant in sorted(known_failures, key=len, reverse=True): + marker = f"-{variant}-" + if marker not in name: + continue + devices = known_failures[variant] + # everything after the variant is "-" + remainder = name.split(marker, 1)[1] + tests = dict(devices.get("*", {})) + for device, device_tests in devices.items(): + # a device specific entry overrides the one listed for all devices + if device != "*" and remainder.startswith(f"{device}-"): + tests.update(device_tests) + return tests + return None + + +def annotate(message, comment): + """Append the comment of a known failures entry to a JUnit message.""" + return f"{message} ({comment})" if comment else message + + +def apply_to_testcase(testcase, suite_name, comment): + """Rewrite a single testcase. Returns a description of the change or None.""" + name = testcase.get("name") + failure = testcase.find("failure") + if failure is None: + failure = testcase.find("error") + if failure is not None: + # known failure: report it as skipped so it does not fail the check + original = failure.get("message", "failed") + testcase.remove(failure) + skipped = ET.SubElement(testcase, "skipped") + skipped.set("type", "known failure") + skipped.set( + "message", + annotate( + "known failure: listed in .github/known-failures, " + f"original result: {original}", + comment, + ), + ) + return f"{suite_name}/{name}: fail -> known failure (skipped)" + if testcase.find("skipped") is not None: + # the test did not run, nothing to say about the known failure + return None + # the test passed although it is expected to fail + failure = ET.SubElement(testcase, "failure") + failure.set("type", "unexpected pass") + failure.set( + "message", + annotate( + "unexpected pass: the test is listed as a known failure in " + ".github/known-failures, remove it from the list", + comment, + ), + ) + return f"{suite_name}/{name}: pass -> unexpected pass (failure)" + + +def count(testsuite, tag): + return len(testsuite.findall(f"./testcase/{tag}")) + + +def refresh_counters(testsuites): + """Recompute the counters of every testsuite and of the root element.""" + totals = {"tests": 0, "failures": 0, "errors": 0, "skipped": 0} + for testsuite in testsuites.findall("testsuite"): + counters = { + "tests": len(testsuite.findall("testcase")), + "failures": count(testsuite, "failure"), + "errors": count(testsuite, "error"), + "skipped": count(testsuite, "skipped"), + } + for key, value in counters.items(): + testsuite.set(key, str(value)) + totals[key] += value + for key, value in totals.items(): + # the root element of the LAVA JUnit output carries no "skipped" counter + if key != "skipped" or "skipped" in testsuites.attrib: + testsuites.set(key, str(value)) + + +def process(path, known_tests): + """Apply known_tests to one result file. Returns the list of changes.""" + tree = ET.parse(path) + testsuites = tree.getroot() + changes = [] + for testsuite in testsuites.findall("testsuite"): + suite_name = testsuite.get("name", "") + if suite_name == LAVA_SUITE: + continue + for testcase in testsuite.findall("testcase"): + name = testcase.get("name") + if name in known_tests: + change = apply_to_testcase(testcase, suite_name, known_tests[name]) + if change: + changes.append(change) + if changes: + refresh_counters(testsuites) + tree.write(path, encoding="utf-8", xml_declaration=True) + return changes + + +def load_tested_variants(workflow_path): + """Return {variant: set(devices)} for every variant tested by the workflow. + + The build variants are the jobs of the test workflow that call + test-distro.yml, named after the distro and its suffix, exactly like the + known failures lists. + """ + workflow = yaml.safe_load(pathlib.Path(workflow_path).read_text()) + variants = {} + for job in workflow.get("jobs", {}).values(): + if not str(job.get("uses", "")).endswith("test-distro.yml"): + continue + inputs = job.get("with", {}) + variant = f"{inputs.get('distro_name', '')}{inputs.get('distro_suffix', '')}" + devices = set() + for key in ("devices", "devices_premerge"): + devices |= { + device.strip() + for device in str(inputs.get(key, "")).split(",") + if device.strip() + } + variants[variant] = devices + return variants + + +def validate(known_failures_dir, known_failures, workflow_path): + """Check the lists against the build variants the test workflow tests.""" + variants = load_tested_variants(workflow_path) + if not variants: + sys.exit(f"{workflow_path}: no build variant found, is it still the test workflow?") + print(f"{workflow_path} tests {len(variants)} build variant(s): {', '.join(sorted(variants))}") + + errors = [] + for variant in sorted(set(variants) - set(known_failures)): + errors.append( + annotation( + pathlib.Path(known_failures_dir) / f"{variant}.yaml", + f"missing known failures list for the {variant} build variant, " + "add the file even when it holds no entry", + ) + ) + for variant in sorted(set(known_failures) - set(variants)): + errors.append( + annotation( + pathlib.Path(known_failures_dir) / f"{variant}.yaml", + f"{variant} is not a build variant tested by {workflow_path}, " + "the entries of this file would never be applied. Tested: " + f"{', '.join(sorted(variants))}", + ) + ) + for variant, devices in sorted(known_failures.items()): + for device in sorted(set(devices) - {ANY_DEVICE} - variants.get(variant, set())): + errors.append( + annotation( + pathlib.Path(known_failures_dir) / f"{variant}.yaml", + f"{device} is not tested in the {variant} build variant, the " + "entries listed for it would never be applied", + ) + ) + for device, tests in sorted(devices.items()): + for test, comment in sorted(tests.items()): + if not comment: + print( + f"::warning file={known_failures_dir}/{variant}.yaml::" + f"{device}: {test} has no comment, add the issue it is " + "tracked in" + ) + + for error in errors: + print(error) + if errors: + sys.exit(f"{len(errors)} error(s) in {known_failures_dir}") + print(f"{known_failures_dir}: all lists are valid") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--known-failures-dir", + default=".github/known-failures", + help="directory holding the per variant known failures lists", + ) + parser.add_argument( + "--results-dir", + help="directory holding the downloaded LAVA JUnit result files", + ) + parser.add_argument( + "--validate", + action="store_true", + help="only check the known failures lists, do not touch any result file", + ) + parser.add_argument( + "--test-workflow", + default=TEST_WORKFLOW, + help="workflow the tested build variants are read from with --validate", + ) + parser.add_argument( + "--syntax-only", + action="store_true", + help="with --validate, only check that the lists are well formed, not " + "that they match the build variants of the test workflow", + ) + args = parser.parse_args() + if not args.validate and not args.results_dir: + parser.error("either --results-dir or --validate is required") + + known_failures = load_known_failures(args.known_failures_dir) + if not known_failures: + sys.exit(f"no known failures list found in {args.known_failures_dir}") + for variant, devices in sorted(known_failures.items()): + listed = sum(len(tests) for tests in devices.values()) + print(f"{variant}: {listed} known failure(s) listed") + + if args.validate: + # The syntax is a property of the lists themselves, the build variants + # are a property of this branch. Lists coming from somewhere else - the + # branch or fork a pull request is built from - are only checked for + # syntax, so that they can add or drop a variant without being rejected. + if not args.syntax_only: + validate(args.known_failures_dir, known_failures, args.test_workflow) + return + + total = 0 + for path in sorted(pathlib.Path(args.results_dir).rglob("*.xml")): + known_tests = match_result_file(path.name, known_failures) + if not known_tests: + continue + for change in process(path, known_tests): + print(f"{path.name}: {change}") + total += 1 + print(f"applied the known failures lists to {total} test result(s)") + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/known-failures.yml b/.github/workflows/known-failures.yml new file mode 100644 index 000000000..4765bd0cc --- /dev/null +++ b/.github/workflows/known-failures.yml @@ -0,0 +1,43 @@ +name: Validate known failures + +# The known failures lists are only read when a test run publishes its results, +# so a typo in one of them stays invisible until it silently fails to suppress a +# known failure. Validate them on every change instead, both to the lists and to +# the test workflow that defines the build variants and devices they refer to. +on: + push: + branches: [ master ] + paths: + - '.github/known-failures/**' + - '.github/scripts/apply-known-failures.py' + - '.github/workflows/known-failures.yml' + - '.github/workflows/test.yml' + pull_request: + paths: + - '.github/known-failures/**' + - '.github/scripts/apply-known-failures.py' + - '.github/workflows/known-failures.yml' + - '.github/workflows/test.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + validate: + name: "Validate known failures lists" + runs-on: ubuntu-latest + steps: + - name: Checkout Repo + uses: actions/checkout@v6 + with: + persist-credentials: false + + - uses: actions/setup-python@v6 + with: + python-version: 3.11 + + - name: Validate + run: | + pip install --quiet pyyaml + python3 .github/scripts/apply-known-failures.py --validate diff --git a/.github/workflows/publish-results.yml b/.github/workflows/publish-results.yml index 9685e8421..2e8ab5acc 100644 --- a/.github/workflows/publish-results.yml +++ b/.github/workflows/publish-results.yml @@ -15,6 +15,20 @@ on: commit: required: true type: string + known_failures_repository: + required: false + type: string + default: "" + description: > + Repository the known failures lists are taken from, "/". + Set it to the head repository of a pull request so its own lists are + applied. Empty (the default) applies the lists of the checkout. + known_failures_ref: + required: false + type: string + default: "" + description: > + Ref or commit of known_failures_repository to take the lists from. secrets: TEST_REPORTING_APP_TOKEN: required: true @@ -31,6 +45,33 @@ jobs: name: "Publish Tests Results" runs-on: ubuntu-latest steps: + # Checked out before the artifacts are downloaded: actions/checkout wipes + # untracked files from the workspace, which would take the artifacts with + # it. For the PR test chain this is a workflow_run run, so the checkout is + # the base branch: it provides the script below, which always comes from + # the base branch, and the known failures lists to fall back on. + - uses: actions/checkout@v6 + with: + persist-credentials: false + + # A pull request may fix a failure that is still listed, or list one it + # found, so it has to be judged by its own lists rather than by the ones + # already merged. Only the lists are checked out, and only ever read as + # data - no code from the pull request runs in this privileged chain. + # A fork that has become unreachable is not fatal: the lists of the base + # branch are applied instead. + - name: Check out the known failures lists of the pull request + if: ${{ inputs.known_failures_repository != '' }} + continue-on-error: true + uses: actions/checkout@v6 + with: + repository: ${{ inputs.known_failures_repository }} + ref: ${{ inputs.known_failures_ref }} + path: known-failures-pr + sparse-checkout: .github/known-failures + sparse-checkout-cone-mode: false + persist-credentials: false + - name: Download result files uses: actions/download-artifact@v7 with: @@ -55,6 +96,36 @@ jobs: exit 1 fi + - uses: actions/setup-python@v6 + with: + python-version: 3.11 + + # A failure that is already known and accepted for a build variant must + # not keep the "Test Results" check red. The result files are rewritten + # before they are published: a known failure becomes a skipped test, a + # test that passes while it is listed as a known failure becomes a + # failure. See .github/known-failures/README.md. + - name: Apply known failures + env: + PR_KNOWN_FAILURES: known-failures-pr/.github/known-failures + run: | + pip install --quiet pyyaml + KNOWN_FAILURES_DIR=.github/known-failures + # Only the syntax of the lists of the pull request is required here: + # which build variants exist is a property of this branch, and it is + # known-failures.yml, running on the pull request itself, that checks + # the lists against it. A list that does not even parse would abort + # the run that applies it, so fall back to the reviewed lists. + if [ -d "${PR_KNOWN_FAILURES}" ] && python3 \ + .github/scripts/apply-known-failures.py --validate --syntax-only \ + --known-failures-dir "${PR_KNOWN_FAILURES}"; then + KNOWN_FAILURES_DIR="${PR_KNOWN_FAILURES}" + fi + echo "::notice title=Known failures::Applying ${KNOWN_FAILURES_DIR}" + python3 .github/scripts/apply-known-failures.py \ + --known-failures-dir "${KNOWN_FAILURES_DIR}" \ + --results-dir "${GITHUB_WORKSPACE}/artifacts" + - id: app_token uses: actions/create-github-app-token@v3 if: always() diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 7cbbd5a58..c0f820278 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -4,6 +4,8 @@ on: push: branches: - master + - wrynose + - next permissions: contents: read diff --git a/.github/workflows/test-distro.yml b/.github/workflows/test-distro.yml index 11194c463..9854fa9b3 100644 --- a/.github/workflows/test-distro.yml +++ b/.github/workflows/test-distro.yml @@ -24,6 +24,29 @@ on: required: false type: string description: "Suffix for the distro name" + known_failures_file: + required: false + type: string + default: "" + description: > + Path to the YAML file listing the known failures of this build + variant. Defaults to .yaml, taken from + known_failures_repository when it has one and from + .github/known-failures otherwise. + known_failures_repository: + required: false + type: string + default: "" + description: > + Repository the known failures lists are taken from, "/". + Set it to the head repository of a pull request so its own lists are + applied. Empty (the default) applies the lists of the checkout. + known_failures_ref: + required: false + type: string + default: "" + description: > + Ref or commit of known_failures_repository to take the lists from. lava_test_plans_ref: type: string description: "Ref in lava-test-plans repository: commit, tag or branch" @@ -267,6 +290,38 @@ jobs: with: fetch-depth: 0 + # A pull request may fix a failure that is still listed, or list one it + # found, so the summary has to report it against its own lists. The test + # chain runs on workflow_run, so the checkout above is the base branch; + # this adds the lists of the branch or fork the pull request comes from, + # read as data only. An unreachable fork falls back to the base branch. + - name: Check out the known failures lists of the pull request + if: ${{ inputs.known_failures_repository != '' }} + continue-on-error: true + uses: actions/checkout@v6 + with: + repository: ${{ inputs.known_failures_repository }} + ref: ${{ inputs.known_failures_ref }} + path: known-failures-pr + sparse-checkout: .github/known-failures + sparse-checkout-cone-mode: false + persist-credentials: false + + - name: Select known failures list + id: known_failures + env: + INPUTS_KNOWN_FAILURES_FILE: "${{ inputs.known_failures_file }}" + VARIANT: "${{ inputs.distro_name }}${{ inputs.distro_suffix }}" + run: | + FILE="known-failures-pr/.github/known-failures/${VARIANT}.yaml" + if [ -n "${INPUTS_KNOWN_FAILURES_FILE}" ]; then + FILE="${INPUTS_KNOWN_FAILURES_FILE}" + elif [ ! -f "${FILE}" ]; then + FILE=".github/known-failures/${VARIANT}.yaml" + fi + echo "::notice title=Known failures::Applying ${FILE}" + echo "file=${FILE}" >> "${GITHUB_OUTPUT}" + - name: Generate Summary id: generate-summary uses: ./.github/actions/test-job-summary @@ -276,6 +331,7 @@ jobs: prefix: "${{ inputs.distro_name }}${{ inputs.distro_suffix }}-test-job*" summary_file_name: "${{ inputs.distro_name }}${{ inputs.distro_suffix }}-test_job_summary" summary_title: "${{ inputs.distro_name }}${{ inputs.distro_suffix }}" + known_failures_file: "${{ steps.known_failures.outputs.file }}" - name: Download Summary uses: actions/download-artifact@v7 diff --git a/.github/workflows/test-pr.yml b/.github/workflows/test-pr.yml index cb8257d52..6f422ca55 100644 --- a/.github/workflows/test-pr.yml +++ b/.github/workflows/test-pr.yml @@ -74,6 +74,11 @@ jobs: build_id: ${{ github.event.workflow_run.id }} pr_number: ${{ needs.determine-target-branch.outputs.pr_number }} pr_url: ${{ needs.determine-target-branch.outputs.pr_url }} + # The known failures lists come from the branch or fork the pull request + # is built from, so that it can drop an entry it fixes, or add one it + # found, and be reported against them right away instead of after merge. + known_failures_repository: ${{ github.event.workflow_run.head_repository.full_name }} + known_failures_ref: ${{ github.event.workflow_run.head_sha }} comment-on-pr: name: "Comment on PR" @@ -146,3 +151,5 @@ jobs: event_file: artifacts/Event File/event.json event_name: ${{ github.event.workflow_run.event }} workflow_id: ${{ github.event.workflow_run.id }} + known_failures_repository: ${{ github.event.workflow_run.head_repository.full_name }} + known_failures_ref: ${{ github.event.workflow_run.head_sha }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 810bb3753..95726075b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,6 +14,20 @@ on: required: false type: string description: "Pull request URL. Added to the LAVA job metadata for PR builds" + known_failures_repository: + required: false + type: string + default: "" + description: > + Repository the known failures lists are taken from, "/". + Set it to the head repository of a pull request so its own lists are + applied. Empty (the default) applies the lists of the checkout. + known_failures_ref: + required: false + type: string + default: "" + description: > + Ref or commit of known_failures_repository to take the lists from. outputs: summary_ids: description: "IDs of the test summary artifact" @@ -26,7 +40,7 @@ env: # git sha, tag or branch in lava-test-plans repository LAVA_TEST_PLANS_REF: "6070e41a1472e3fd595ea7ac849f7c30f86a2282" # CalVer tag in qcom-linux-testkit repository (format: testkit-YYYY.MM.DD) - TESTKIT_REF: "testkit-2026.08.14" + TESTKIT_REF: "testkit-2026.08.23" jobs: prepare-env: @@ -55,6 +69,8 @@ jobs: testkit_ref: "${{ needs.prepare-env.outputs.testkit-ref }}" pr_number: "${{ inputs.pr_number }}" pr_url: "${{ inputs.pr_url }}" + known_failures_repository: "${{ inputs.known_failures_repository }}" + known_failures_ref: "${{ inputs.known_failures_ref }}" test-qcom-distro: needs: [prepare-env] name: "Test qcom-distro" @@ -71,6 +87,8 @@ jobs: testkit_ref: "${{ needs.prepare-env.outputs.testkit-ref }}" pr_number: "${{ inputs.pr_number }}" pr_url: "${{ inputs.pr_url }}" + known_failures_repository: "${{ inputs.known_failures_repository }}" + known_failures_ref: "${{ inputs.known_failures_ref }}" test-qcom-distro_linux-qcom-6-18: needs: [prepare-env] name: "Test qcom-distro_linux-qcom-6.18" @@ -88,6 +106,8 @@ jobs: testkit_ref: "${{ needs.prepare-env.outputs.testkit-ref }}" pr_number: "${{ inputs.pr_number }}" pr_url: "${{ inputs.pr_url }}" + known_failures_repository: "${{ inputs.known_failures_repository }}" + known_failures_ref: "${{ inputs.known_failures_ref }}" collect-ids: name: "Collect Summary artifact IDs" diff --git a/README.md b/README.md index da6efd502..3b782a332 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,7 @@ for the implementation details. Qualcomm Linux 2.x. - **all stable branches up until styhead:** Legacy branches maintained by Linaro, prior to the migration to [Qualcomm-linux](https://github.com/qualcomm-linux). +- **next:** For testing workflow changes before being merged to master. ## Machine Support @@ -103,6 +104,13 @@ For a manual build without KAS, refer to the [Yocto Project Quick Build](https:/ For instructions on building the QDL tool, preparing the board, and flashing images over USB (EDL mode), see [Flashing images](docs/flashing.md). +## Testing + +Images are booted and tested on real hardware in a LAVA lab on every pull +request, on every push to `master` and nightly. See [Testing](TESTING.md) for +the tested build variants and devices, where the results are reported, and how +to use the known failures lists. + ## Security recommendations for production Please refer to the security recommendations for production builds documented here: diff --git a/TESTING.md b/TESTING.md new file mode 100644 index 000000000..5e349a649 --- /dev/null +++ b/TESTING.md @@ -0,0 +1,184 @@ +# Testing + +Images built by CI are booted and tested on real hardware in a +[LAVA](https://lava.infra.foundries.io) lab. This document describes which +build variants are tested, where the results are reported, and how to use the +known failures lists to keep an accepted failure from turning the results red. + +## When tests run + +| Trigger | Workflow | Notes | +| ------------------------ | ------------------------------------ | -------------------------------------------------------------------------- | +| Pull request to `master` | `pr.yml` builds, `test-pr.yml` tests | The test chain runs on `workflow_run` so that the build stays unprivileged | +| Push to `master` | `push.yml` | Builds, tests and publishes in one run | +| Nightly | `nightly-build.yml` | Scheduled at 00:22 UTC, Sunday to Friday | +| Weekly | `weekly-build.yml` | Saturday's slot, building a superset of the nightly matrix | + +All four end up in the same place: `test.yml` runs one `test-distro.yml` job +per tested build variant, and `publish-results.yml` publishes the results. + +## Tested build variants + +CI builds many more distro and kernel combinations than it tests. Only the +three variants below are booted and tested on hardware. A variant is named +after the distro and the kernel directory, ``, and +that name is used consistently for its build artifacts, its test job summary +and its known failures list. + +| Build variant | Distro | Kernel | Image | Known failures list | +| ----------------------------- | ----------------------------- | ------------------------------------------------ | ----------------------- | --------------------------------------------------------- | +| `nodistro` | plain OpenEmbedded, no distro | default | `core-image-base` | `.github/known-failures/nodistro.yaml` | +| `qcom-distro` | `ci/qcom-distro.yml` | default | `qcom-multimedia-image` | `.github/known-failures/qcom-distro.yaml` | +| `qcom-distro_linux-qcom-6.18` | `ci/qcom-distro.yml` | `linux-qcom` 6.18, from `ci/linux-qcom-6.18.yml` | `qcom-multimedia-image` | `.github/known-failures/qcom-distro_linux-qcom-6.18.yaml` | + +To change the set of variants, or the devices a variant runs on, edit the +`test-*` jobs in `.github/workflows/test.yml`. A new variant needs a known +failures list of its own, even an empty one - the validation workflow described +below fails until it exists. + +### Devices per variant + +| Device | `nodistro` | `qcom-distro` | `qcom-distro_linux-qcom-6.18` | +| ------------------ | ---------- | ---------------- | ----------------------------- | +| `dragonboard-410c` | boot | - | - | +| `dragonboard-820c` | boot | - | - | +| `glymur-crd` | - | boot + pre-merge | - | +| `iq-8275-evk` | boot | boot + pre-merge | boot + pre-merge | +| `iq-9075-evk` | boot | boot + pre-merge | boot + pre-merge | +| `iq-x7181-evk` | boot | boot + pre-merge | boot + pre-merge | +| `kaanapali-mtp` | - | boot + pre-merge | - | +| `qcm6490-idp` | boot | boot + pre-merge | boot + pre-merge | +| `qcs615-ride` | boot | boot + pre-merge | boot + pre-merge | +| `qcs8300-ride-sx` | boot | boot + pre-merge | boot + pre-merge | +| `qcs9100-ride-sx` | boot | boot + pre-merge | boot + pre-merge | +| `rb1-core-kit` | boot | boot + pre-merge | boot + pre-merge | +| `rb3gen2-core-kit` | boot | boot + pre-merge | boot + pre-merge | +| `shikra-evk` | - | boot | - | +| `sm8750-mtp` | - | boot + pre-merge | - | + +The device name is the LAVA device type. It is what a known failures list keys +on, and what the columns of the test job summary are named after. + +## Test stages + +Each variant runs in two stages: + +1. **boot** - one LAVA job per device that flashes the image and boots it. The + summary reports the outcome as a test named `boot`. +2. **pre-merge** - the actual test suites, run only when every boot job of the + variant passed. A variant with no `devices_premerge` never reaches this + stage. + +The test jobs themselves come from two pinned external repositories, both +referenced at the top of `.github/workflows/test.yml`: + +- [`qualcomm-linux/lava-test-plans`](https://github.com/qualcomm-linux/lava-test-plans) + (`LAVA_TEST_PLANS_REF`) renders the LAVA job definitions from the + `meta-qcom//boot` and `meta-qcom//pre-merge` test plans. +- [`qualcomm-linux/qcom-linux-testkit`](https://github.com/qualcomm-linux/qcom-linux-testkit) + (`TESTKIT_REF`) holds the test scripts that run on the device. + +Adding or changing a test case is done in those repositories; this repository +only pins the revision to use. + +## Where results are reported + +- **Test job summary** - one collapsible section per variant on the workflow + run summary page, with a test-by-device table, a "Known failures" section and + the list of all LAVA jobs with links. +- **"Test Results" check** - published from the JUnit XML that LAVA produced, + by `publish-results.yml`. This is the check that turns a pull request red. +- **PR comment** - a single comment per pull request, updated in place. + +## Known failures + +A test that is known to fail, and whose failure is accepted for now, can be +listed as a *known failure*. It then no longer fails the build, but it stays +visible in the reports, and it starts failing again as soon as it passes - so +the list cannot silently rot. + +Every build variant has its own list in `.github/known-failures/`, named after +the variant. Full details are in +[`.github/known-failures/README.md`](.github/known-failures/README.md); what +follows is what you need to use them. + +### Adding a known failure + +1. Find the failing test in the test job summary: the row gives the test name, + the column gives the device type. +2. Open the list of the variant, e.g. `.github/known-failures/qcom-distro.yaml` + for the `qcom-distro` section of the summary. +3. Add an entry under the device, with a link to the issue it is tracked in: + + ```yaml + qcm6490-idp: + - test: AudioRecord_Config01 + comment: https://github.com/qualcomm-linux/meta-qcom/issues/1234 + ``` + + Use `"*"` instead of a device name when the test fails on every device of + the variant. A device entry overrides the `"*"` entry for the same test. + Quote a comment containing a `#`, otherwise YAML swallows the rest of the + line. +4. Run `python3 .github/scripts/apply-known-failures.py --validate` before + pushing. + +The entry takes effect on the pull request that adds it: the test chain checks +the lists out from the branch or fork the pull request is built from, so a +pull request that fixes a listed failure deletes the entry in the same change +and stays green. The lists of the base branch are used only when the pull +request has none, or when the ones it has do not parse - the run log says which +of the two was applied. + +### What it changes + +| Result in LAVA | Listed | Test job summary | "Test Results" check | +| -------------- | ------ | --------------------------------------- | ------------------------------------------ | +| `fail` | no | :x: `fail` | failed | +| `fail` | yes | :ballot_box_with_check: `known failure` | skipped, with the comment in the message | +| `pass` | no | :white_check_mark: `pass` | passed | +| `pass` | yes | :x: `unexpected pass` | failed, asking for the entry to be removed | + +A known failure counts as a pass in the summary totals, and is reported as a +skipped test in the check, so neither turns red because of it. + +### Removing a known failure + +When the underlying issue is fixed, the test starts passing while it is still +listed. Both reports then flag it as an *unexpected pass* and the check goes +red. Delete the entry (and close the issue it points at) to make it green +again. This is deliberate: it is what stops the lists from growing forever. + +### Validation + +`.github/workflows/known-failures.yml` validates the lists on every change to +them, to the script, or to `test.yml`. Besides the syntax it checks that every +list matches a tested variant and every device is one the variant actually +tests, because such an entry is never applied and would otherwise look like it +suppressed a failure. Run the same check locally with: + +```shell +python3 .github/scripts/apply-known-failures.py --validate +``` + +### Limitations + +- A `boot` entry only affects the test job summary. The JUnit XML holds the + tests that ran inside a LAVA job, so it has nothing to suppress for a boot + failure, and the boot job itself still fails the workflow. +- The lists apply to the variants of this branch. The `wrynose` branch runs its + own test workflow with its own variants and keeps its own lists. +- Only the syntax of the lists taken from a pull request is checked before they + are applied. That they name a tested variant and a tested device is checked by + `known-failures.yml` on the pull request itself, against the variants of the + branch it targets. + +## Reproducing a failure + +Every entry of the summary table links into the log of its LAVA job, at the +line the result was reported on. The surrounding lines hold the output of the +test itself, and the rest of the page the serial console log and the exact job +definition, which is the fastest way to tell a real regression from lab +flakiness. A boot entry links to the top of the log, there being no test case +to point at. A job can be resubmitted from that page to check whether a failure +is reproducible.