From 6b99655712e6aef081564e09c9c37baf59c9667a Mon Sep 17 00:00:00 2001 From: Katelyn Gadd Date: Thu, 7 May 2026 08:53:48 -0700 Subject: [PATCH 1/7] Add update_github script for generating issues and comments --- .../scripts/update_github.py | 212 ++++++++++++++++++ 1 file changed, 212 insertions(+) create mode 100644 .github/skills/ci-pipeline-monitor/scripts/update_github.py diff --git a/.github/skills/ci-pipeline-monitor/scripts/update_github.py b/.github/skills/ci-pipeline-monitor/scripts/update_github.py new file mode 100644 index 00000000000000..e53635324a569b --- /dev/null +++ b/.github/skills/ci-pipeline-monitor/scripts/update_github.py @@ -0,0 +1,212 @@ +"""CI Pipeline Monitor — Issue Generator + +Generates github issues and comments from the contents of the ci pipeline monitor's database. + +Usage: + python update_github.py --db monitor.db +""" + +import argparse +import os +import sqlite3 +import sys +import tempfile +import subprocess +from datetime import datetime + +# --- Configuration --- +ADO_ORG = "dnceng-public" +ADO_PROJECT = "public" +DEFAULT_DB = "monitor.db" + +class IssueGenerator: + def __init__(self, db_path): + if not os.path.isfile(db_path): + print(f"Error: database {db_path} not found.", file=sys.stderr) + sys.exit(1) + self.conn = sqlite3.connect(db_path) + self.conn.row_factory = sqlite3.Row + + def generate(self, go): + self.generate_issues(go) + self.conn.close() + + def generate_issues(self, go): + cur = self.conn.cursor() + + for fail in cur.execute("SELECT * FROM failures ORDER BY id"): + self._one_failure(fail, go) + + def _one_failure(self, fail, go): + cur = self.conn.cursor() + out = [] + + scope = fail["scope"] + if scope: + title_line = f"FAILURE {fail['id']}: {fail['title']} ({scope})" + else: + title_line = f"FAILURE {fail['id']}: {fail['title']}" + print(f"--- {title_line} ---") + + gh_issue_command = ["gh", "issue"] + creating_new_issue = False + + # GitHub issue line + if fail["github_issue_number"]: + issue_url = fail['github_issue_url'] or f"https://github.com/dotnet/runtime/issues/{fail['github_issue_number']}" + print( + f"GitHub Issue: #{fail['github_issue_number']} " + f"({issue_url}) — {fail['github_issue_state']}" + ) + gh_issue_command.append('comment') + gh_issue_command.append(f'"{issue_url}"') + else: + print("GitHub Issue: NEW — creating new issue") + gh_issue_command.append("create") + creating_new_issue = True + + if fail["labels"]: + if creating_new_issue: + for label in fail["labels"].split(','): + gh_issue_command.append('--label') + gh_issue_command.append(f'"{label.strip()}"') + + # Title / Labels / Milestone for issue filing + test_name = fail["test_name"] + if creating_new_issue: + gh_issue_command.append('--title') + gh_issue_command.append(f'"Test Failure: {test_name}"') + gh_issue_command.append('--milestone') + gh_issue_command.append(f'"{fail['milestone']}"') + + # --- Body block --- + # Summary + if creating_new_issue: + out.append("**Summary:**") + out.append(f" {fail['summary']}") + out.append("") + + # Failed in — JOIN with pipelines to guarantee build_id/build_number + affected = list(cur.execute( + "SELECT fp.pipeline_name, " + "COALESCE(fp.build_id, p.build_id) AS build_id, " + "COALESCE(fp.build_number, p.build_number) AS build_number " + "FROM failure_pipelines fp " + "LEFT JOIN pipelines p ON fp.pipeline_name = p.name " + "WHERE fp.failure_id = ? ORDER BY fp.pipeline_name", + (fail["id"],) + )) + out.append(f"**Failed in ({len(affected)}):**") + for ap in affected: + bn = ap["build_number"] or "" + out.append( + f"- [{ap['pipeline_name']} {bn}]" + f"(https://dev.azure.com/{ADO_ORG}/{ADO_PROJECT}/_build/results?buildId={ap['build_id']})" + ) + out.append("") + + # Console Log and Source + if fail["console_log_url"]: + out.append(f"**Console Log:** [Console Log]({fail['console_log_url']})") + # Show which test result the error_message/stack_trace came from + if fail["source_test_result_id"]: + src = cur.execute( + """SELECT tr.pipeline_name, tr.build_id, tr.run_name, tr.test_name + FROM test_results tr WHERE tr.id = ?""", + (fail["source_test_result_id"],) + ).fetchone() + if src: + src_url = f"https://dev.azure.com/{ADO_ORG}/{ADO_PROJECT}/_build/results?buildId={src['build_id']}&view=ms.vss-test-web.build-test-results-tab" + out.append(f"**Source:** [{src['pipeline_name']} / {src['run_name']} / {src['test_name']}]({src_url})") + out.append("") + + if creating_new_issue: + # Failed tests — group by pipeline if 2+ pipelines + out.append("**Failed tests:**") + out.append("```") + + if len(affected) >= 2: + for ap in affected: + out.append(ap["pipeline_name"]) + for ft in cur.execute( + "SELECT DISTINCT run_name FROM failure_tests " + "WHERE failure_id = ? AND pipeline_name = ? ORDER BY run_name", + (fail["id"], ap["pipeline_name"]) + ): + out.append(f"- {ft['run_name']}") + else: + for ft in cur.execute( + "SELECT DISTINCT run_name FROM failure_tests WHERE failure_id = ? ORDER BY run_name", + (fail["id"],) + ): + out.append(f"- {ft['run_name']}") + + # Unique test names (cap at 5) + test_names = [row["test_name"] for row in cur.execute( + "SELECT DISTINCT test_name FROM failure_tests WHERE failure_id = ? ORDER BY test_name", + (fail["id"],) + )] + for tn in test_names[:5]: + out.append(f" - {tn}") + if len(test_names) > 5: + out.append(f" - ... and {len(test_names) - 5} more") + out.append("```") + out.append("") + + # Error message + out.append("**Error Message:**") + out.append("```") + out.append(fail["error_message"] or "N/A") + out.append("```") + out.append("") + + # Stack trace + out.append("**Stack Trace:**") + out.append("```") + out.append(fail["stack_trace"] or "N/A") + out.append("```") + out.append("") + + # Analysis + out.append("**Analysis:**") + out.append(fail["analysis"] or "") + + out.append("") + out.append("**Generated by ci-pipeline-monitor/scripts/generate_report.py**") + + temp_file = tempfile.NamedTemporaryFile('w', encoding="utf8", delete_on_close=False, newline="\r\n") + body_path = temp_file.name + for line in out: + temp_file.write(line) + temp_file.write("\r\n") + temp_file.flush() + temp_file.close() + gh_issue_command.append("--body-file") + gh_issue_command.append(f'"{body_path}"') + + print(gh_issue_command) + if (go): + subprocess.run(gh_issue_command, check=True) + else: + print(f"--- {body_path} ---") + for line in out: + print(line) + print(f"--- END {body_path} ---") + + print("") + print("") + +def main(): + parser = argparse.ArgumentParser( + description="CI Pipeline Monitor — github updater" + ) + parser.add_argument("--db", default=DEFAULT_DB, help=f"Database path (default: {DEFAULT_DB})") + parser.add_argument("--go", default=False, help=f"Actually file issues and comments instead of performing a dry run (default: False)") + args = parser.parse_args() + + gen = IssueGenerator(args.db) + gen.generate(args.go) + + +if __name__ == "__main__": + main() From eee07a65ea9f7e7e47d489debf1ef6e28f611ca3 Mon Sep 17 00:00:00 2001 From: Katelyn Gadd Date: Thu, 7 May 2026 09:05:49 -0700 Subject: [PATCH 2/7] Fix argument formatting --- .../ci-pipeline-monitor/scripts/update_github.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/skills/ci-pipeline-monitor/scripts/update_github.py b/.github/skills/ci-pipeline-monitor/scripts/update_github.py index e53635324a569b..ac2f21e586fc04 100644 --- a/.github/skills/ci-pipeline-monitor/scripts/update_github.py +++ b/.github/skills/ci-pipeline-monitor/scripts/update_github.py @@ -59,7 +59,7 @@ def _one_failure(self, fail, go): f"({issue_url}) — {fail['github_issue_state']}" ) gh_issue_command.append('comment') - gh_issue_command.append(f'"{issue_url}"') + gh_issue_command.append(f'{issue_url}') else: print("GitHub Issue: NEW — creating new issue") gh_issue_command.append("create") @@ -69,15 +69,15 @@ def _one_failure(self, fail, go): if creating_new_issue: for label in fail["labels"].split(','): gh_issue_command.append('--label') - gh_issue_command.append(f'"{label.strip()}"') + gh_issue_command.append(f'{label.strip()}') # Title / Labels / Milestone for issue filing test_name = fail["test_name"] if creating_new_issue: gh_issue_command.append('--title') - gh_issue_command.append(f'"Test Failure: {test_name}"') + gh_issue_command.append(f'Test Failure: {test_name}') gh_issue_command.append('--milestone') - gh_issue_command.append(f'"{fail['milestone']}"') + gh_issue_command.append(f'{fail['milestone']}') # --- Body block --- # Summary @@ -182,7 +182,7 @@ def _one_failure(self, fail, go): temp_file.flush() temp_file.close() gh_issue_command.append("--body-file") - gh_issue_command.append(f'"{body_path}"') + gh_issue_command.append(f'{body_path}') print(gh_issue_command) if (go): From d4ae65efcc594a9a484a7c37724f7726b0633d9f Mon Sep 17 00:00:00 2001 From: Katelyn Gadd Date: Thu, 7 May 2026 11:28:20 -0700 Subject: [PATCH 3/7] Make --go a flag --- .../skills/ci-pipeline-monitor/scripts/update_github.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/skills/ci-pipeline-monitor/scripts/update_github.py b/.github/skills/ci-pipeline-monitor/scripts/update_github.py index ac2f21e586fc04..12a29e704e90cb 100644 --- a/.github/skills/ci-pipeline-monitor/scripts/update_github.py +++ b/.github/skills/ci-pipeline-monitor/scripts/update_github.py @@ -2,8 +2,10 @@ Generates github issues and comments from the contents of the ci pipeline monitor's database. -Usage: +Usage (dry run): python update_github.py --db monitor.db +Usage (generate issues and comments): + python update_github.py --db monitor.db --go """ import argparse @@ -174,7 +176,7 @@ def _one_failure(self, fail, go): out.append("") out.append("**Generated by ci-pipeline-monitor/scripts/generate_report.py**") - temp_file = tempfile.NamedTemporaryFile('w', encoding="utf8", delete_on_close=False, newline="\r\n") + temp_file = tempfile.NamedTemporaryFile('w', encoding="utf8", delete_on_close=False) body_path = temp_file.name for line in out: temp_file.write(line) @@ -201,7 +203,7 @@ def main(): description="CI Pipeline Monitor — github updater" ) parser.add_argument("--db", default=DEFAULT_DB, help=f"Database path (default: {DEFAULT_DB})") - parser.add_argument("--go", default=False, help=f"Actually file issues and comments instead of performing a dry run (default: False)") + parser.add_argument("--go", action="store_true", help=f"Actually file issues and comments instead of performing a dry run (default: False)") args = parser.parse_args() gen = IssueGenerator(args.db) From af9e144932ee7a744729d422ae01b4b2507d990e Mon Sep 17 00:00:00 2001 From: Katelyn Gadd Date: Thu, 7 May 2026 11:39:50 -0700 Subject: [PATCH 4/7] Probe whether `gh issue` will work --- .../skills/ci-pipeline-monitor/scripts/update_github.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/skills/ci-pipeline-monitor/scripts/update_github.py b/.github/skills/ci-pipeline-monitor/scripts/update_github.py index 12a29e704e90cb..0942c17d0d5034 100644 --- a/.github/skills/ci-pipeline-monitor/scripts/update_github.py +++ b/.github/skills/ci-pipeline-monitor/scripts/update_github.py @@ -1,6 +1,7 @@ """CI Pipeline Monitor — Issue Generator Generates github issues and comments from the contents of the ci pipeline monitor's database. +You must have done `gh repo set-default` at least once on the current checkout for this to work. Usage (dry run): python update_github.py --db monitor.db @@ -30,9 +31,15 @@ def __init__(self, db_path): self.conn.row_factory = sqlite3.Row def generate(self, go): + self.probe_configuration() self.generate_issues(go) self.conn.close() + def probe_configuration(self): + probe_result = subprocess.run(["gh", "repo", "set-default", "-v"], check=True, capture_output=True) + if (len(probe_result.stderr)): + raise Exception("You need to perform gh repo set-default") + def generate_issues(self, go): cur = self.conn.cursor() From 40b1b9b0ae15974591f41095e7445869fe664205 Mon Sep 17 00:00:00 2001 From: Katelyn Gadd Date: Thu, 7 May 2026 11:42:06 -0700 Subject: [PATCH 5/7] Cleanup --- .../skills/ci-pipeline-monitor/scripts/update_github.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/skills/ci-pipeline-monitor/scripts/update_github.py b/.github/skills/ci-pipeline-monitor/scripts/update_github.py index 0942c17d0d5034..c7243e4757b679 100644 --- a/.github/skills/ci-pipeline-monitor/scripts/update_github.py +++ b/.github/skills/ci-pipeline-monitor/scripts/update_github.py @@ -68,7 +68,7 @@ def _one_failure(self, fail, go): f"({issue_url}) — {fail['github_issue_state']}" ) gh_issue_command.append('comment') - gh_issue_command.append(f'{issue_url}') + gh_issue_command.append(issue_url) else: print("GitHub Issue: NEW — creating new issue") gh_issue_command.append("create") @@ -78,7 +78,7 @@ def _one_failure(self, fail, go): if creating_new_issue: for label in fail["labels"].split(','): gh_issue_command.append('--label') - gh_issue_command.append(f'{label.strip()}') + gh_issue_command.append(label.strip()) # Title / Labels / Milestone for issue filing test_name = fail["test_name"] @@ -86,7 +86,7 @@ def _one_failure(self, fail, go): gh_issue_command.append('--title') gh_issue_command.append(f'Test Failure: {test_name}') gh_issue_command.append('--milestone') - gh_issue_command.append(f'{fail['milestone']}') + gh_issue_command.append(fail['milestone']) # --- Body block --- # Summary @@ -191,7 +191,7 @@ def _one_failure(self, fail, go): temp_file.flush() temp_file.close() gh_issue_command.append("--body-file") - gh_issue_command.append(f'{body_path}') + gh_issue_command.append(body_path) print(gh_issue_command) if (go): From c62f07d874446036a255de6659084a5fe19c590d Mon Sep 17 00:00:00 2001 From: Katelyn Gadd Date: Fri, 8 May 2026 09:44:31 -0700 Subject: [PATCH 6/7] Address copilot feedback --- .../ci-pipeline-monitor/scripts/update_github.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/skills/ci-pipeline-monitor/scripts/update_github.py b/.github/skills/ci-pipeline-monitor/scripts/update_github.py index c7243e4757b679..02296b547e8635 100644 --- a/.github/skills/ci-pipeline-monitor/scripts/update_github.py +++ b/.github/skills/ci-pipeline-monitor/scripts/update_github.py @@ -15,7 +15,6 @@ import sys import tempfile import subprocess -from datetime import datetime # --- Configuration --- ADO_ORG = "dnceng-public" @@ -181,9 +180,9 @@ def _one_failure(self, fail, go): out.append(fail["analysis"] or "") out.append("") - out.append("**Generated by ci-pipeline-monitor/scripts/generate_report.py**") + out.append("**Generated by ci-pipeline-monitor/scripts/update_github.py**") - temp_file = tempfile.NamedTemporaryFile('w', encoding="utf8", delete_on_close=False) + temp_file = tempfile.NamedTemporaryFile('w', encoding="utf8", delete=False) body_path = temp_file.name for line in out: temp_file.write(line) @@ -205,6 +204,13 @@ def _one_failure(self, fail, go): print("") print("") + # NOTE: This isn't in a try/finally because we want it to be possible to manually + # retry a failed command invocation after script failure for troubleshooting + try: + os.unlink(body_path) + except: + # ignore + def main(): parser = argparse.ArgumentParser( description="CI Pipeline Monitor — github updater" From 45516b63753efcf3c6155be170d46749dd182ab6 Mon Sep 17 00:00:00 2001 From: Katelyn Gadd Date: Fri, 8 May 2026 10:02:19 -0700 Subject: [PATCH 7/7] Address copilot feedback --- .../scripts/update_github.py | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/.github/skills/ci-pipeline-monitor/scripts/update_github.py b/.github/skills/ci-pipeline-monitor/scripts/update_github.py index 02296b547e8635..23efa9bbb960fe 100644 --- a/.github/skills/ci-pipeline-monitor/scripts/update_github.py +++ b/.github/skills/ci-pipeline-monitor/scripts/update_github.py @@ -1,6 +1,6 @@ """CI Pipeline Monitor — Issue Generator -Generates github issues and comments from the contents of the ci pipeline monitor's database. +Generates GitHub issues and comments from the contents of the ci pipeline monitor's database. You must have done `gh repo set-default` at least once on the current checkout for this to work. Usage (dry run): @@ -76,16 +76,19 @@ def _one_failure(self, fail, go): if fail["labels"]: if creating_new_issue: for label in fail["labels"].split(','): - gh_issue_command.append('--label') - gh_issue_command.append(label.strip()) + stripped_label = label.strip() + if stripped_label: + gh_issue_command.append('--label') + gh_issue_command.append(stripped_label) # Title / Labels / Milestone for issue filing test_name = fail["test_name"] if creating_new_issue: gh_issue_command.append('--title') gh_issue_command.append(f'Test Failure: {test_name}') - gh_issue_command.append('--milestone') - gh_issue_command.append(fail['milestone']) + if fail['milestone']: + gh_issue_command.append('--milestone') + gh_issue_command.append(fail['milestone']) # --- Body block --- # Summary @@ -182,11 +185,9 @@ def _one_failure(self, fail, go): out.append("") out.append("**Generated by ci-pipeline-monitor/scripts/update_github.py**") - temp_file = tempfile.NamedTemporaryFile('w', encoding="utf8", delete=False) + temp_file = tempfile.NamedTemporaryFile('w', encoding="utf8", delete=False, newline='\n') body_path = temp_file.name - for line in out: - temp_file.write(line) - temp_file.write("\r\n") + temp_file.write("\n".join(out)) temp_file.flush() temp_file.close() gh_issue_command.append("--body-file") @@ -208,12 +209,13 @@ def _one_failure(self, fail, go): # retry a failed command invocation after script failure for troubleshooting try: os.unlink(body_path) - except: - # ignore + except OSError: + # ignore, failures to unlink the temp file due to it being locked by virus scanner etc are unimportant + return def main(): parser = argparse.ArgumentParser( - description="CI Pipeline Monitor — github updater" + description="CI Pipeline Monitor — GitHub updater" ) parser.add_argument("--db", default=DEFAULT_DB, help=f"Database path (default: {DEFAULT_DB})") parser.add_argument("--go", action="store_true", help=f"Actually file issues and comments instead of performing a dry run (default: False)")