1b8b6801d4f896409a765d302e7e0d50ff089692
- Author
- Nick Brassel <nick@tzarc.org>
- Committer
- GitHub <noreply@github.com>
- Date
Message
[CI] Add index page generator. (#23737)
Diff
This diff is truncated to protect this page.
1diff --git a/.github/workflows/ci_build_major_branch.yml b/.github/workflows/ci_build_major_branch.yml
2index 77755ba71f4183be597683de8b757139b3b6a21f..0e772c7d53613ae38cfd7ac08d58b9e7a9cc3ed0 100644
3--- a/.github/workflows/ci_build_major_branch.yml
4+++ b/.github/workflows/ci_build_major_branch.yml
5@@ -77,44 +77,56 @@ jobs:
6 runs-on: ubuntu-latest
7
8 steps:
9+ - name: Disable safe.directory check
10+ run: |
11+ git config --global --add safe.directory '*'
12+
13+ - name: Checkout QMK Firmware
14+ uses: actions/checkout@v4
15+
16 - name: Download firmwares
17 uses: actions/download-artifact@v4
18 with:
19 pattern: firmware-*
20- path: firmwares
21+ path: .
22 merge-multiple: true
23
24+ - name: Generate index page
25+ run: |
26+ python3 -m pip install -r ./util/ci/requirements.txt
27+ ./util/ci/index_generator.py > index.html
28+
29 - name: Upload to https://ci.qmk.fm/${{ inputs.branch || github.ref_name }}/${{ github.sha }}
30 uses: jakejarvis/s3-sync-action@master
31 with:
32- args: --acl public-read --follow-symlinks --delete
33+ args: --acl public-read --follow-symlinks --delete --exclude '*' --include 'index.html' --include '*.hex' --include '*.bin' --include '*.uf2'
34 env:
35 AWS_S3_BUCKET: ${{ vars.CI_QMK_FM_SPACES_BUCKET }}
36 AWS_ACCESS_KEY_ID: ${{ secrets.CI_QMK_FM_SPACES_KEY }}
37 AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_QMK_FM_SPACES_SECRET }}
38 AWS_REGION: ${{ vars.CI_QMK_FM_SPACES_REGION }}
39 AWS_S3_ENDPOINT: ${{ vars.CI_QMK_FM_SPACES_ENDPOINT }}
40- SOURCE_DIR: firmwares
41+ SOURCE_DIR: .
42 DEST_DIR: ${{ inputs.branch || github.ref_name }}/${{ github.sha }}
43
44 - name: Upload to https://ci.qmk.fm/${{ inputs.branch || github.ref_name }}/latest
45 uses: jakejarvis/s3-sync-action@master
46 with:
47- args: --acl public-read --follow-symlinks --delete
48+ args: --acl public-read --follow-symlinks --delete --exclude '*' --include 'index.html' --include '*.hex' --include '*.bin' --include '*.uf2'
49 env:
50 AWS_S3_BUCKET: ${{ vars.CI_QMK_FM_SPACES_BUCKET }}
51 AWS_ACCESS_KEY_ID: ${{ secrets.CI_QMK_FM_SPACES_KEY }}
52 AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_QMK_FM_SPACES_SECRET }}
53 AWS_REGION: ${{ vars.CI_QMK_FM_SPACES_REGION }}
54 AWS_S3_ENDPOINT: ${{ vars.CI_QMK_FM_SPACES_ENDPOINT }}
55- SOURCE_DIR: firmwares
56+ SOURCE_DIR: .
57 DEST_DIR: ${{ inputs.branch || github.ref_name }}/latest
58
59 - name: Check if failure marker file exists
60 id: check_failure_marker
61 uses: andstor/file-existence-action@v3
62 with:
63- files: firmwares/.failed
64+ files: ./.failed
65
66 - name: Fail build if needed
67 if: steps.check_failure_marker.outputs.files_exists == 'true'
68diff --git a/.gitignore b/.gitignore
69index 35b128606d7d633c3b2c8e0dd091d03be5780393..5de38c161cdd3c14575741eef4e1ff052db641f0 100644
70--- a/.gitignore
71+++ b/.gitignore
72@@ -25,6 +25,7 @@
73 *.la
74 *.stackdump
75 *.sym
76+index.html
77
78 # QMK-specific
79 api_data/v1
80diff --git a/util/ci/index_generator.py b/util/ci/index_generator.py
81new file mode 100755
82index 0000000000000000000000000000000000000000..b6440bbffb71308130f9289e9d07a8a339b920bd
83--- /dev/null
84+++ b/util/ci/index_generator.py
85@@ -0,0 +1,109 @@
86+#!/usr/bin/env python3
87+
88+import os
89+import re
90+import shlex
91+import subprocess
92+from pathlib import Path
93+
94+from ansi2html import Ansi2HTMLConverter
95+from jinja2 import Environment, FileSystemLoader, select_autoescape
96+
97+orig_cwd = os.getcwd()
98+qmk_firmware_dir = Path(os.path.realpath(__file__)).parents[2]
99+build_dir = qmk_firmware_dir / ".build"
100+
101+KEYBOARD_PATTERN = re.compile("CI Metadata: KEYBOARD=(?P<keyboard>.*)\r?\n")
102+KEYMAP_PATTERN = re.compile("CI Metadata: KEYMAP=(?P<keymap>.*)\r?\n")
103+
104+env = Environment(
105+ loader=FileSystemLoader(Path(os.path.realpath(__file__)).parent / "templates"),
106+ autoescape=select_autoescape(),
107+)
108+
109+
110+def _run(command, capture_output=True, combined_output=False, text=True, **kwargs):
111+ if isinstance(command, str):
112+ command = shlex.split(command)
113+ if capture_output:
114+ kwargs["stdout"] = subprocess.PIPE
115+ kwargs["stderr"] = subprocess.PIPE
116+ if combined_output:
117+ kwargs["stderr"] = subprocess.STDOUT
118+ if "stdin" in kwargs and kwargs["stdin"] is None:
119+ del kwargs["stdin"]
120+ if text:
121+ kwargs["universal_newlines"] = True
122+ return subprocess.run(command, **kwargs)
123+
124+
125+def _ansi2html(value):
126+ return Ansi2HTMLConverter().convert(value, full=False)
127+
128+
129+def _ansi2html_styles():
130+ from ansi2html.style import get_styles
131+
132+ styles = get_styles(scheme="dracula")
133+ return "\n".join([str(s) for s in styles])
134+
135+
136+def _git_log(count = 4):
137+ os.chdir(qmk_firmware_dir)
138+ ret = _run(f"git log -n {count} --color=always --no-merges --topo-order --stat").stdout.strip()
139+ os.chdir(orig_cwd)
140+ return ret
141+
142+def _git_describe():
143+ os.chdir(qmk_firmware_dir)
144+ ret = _run("git describe --tags --always --dirty").stdout.strip()
145+ os.chdir(orig_cwd)
146+ return ret
147+
148+def _git_revision():
149+ os.chdir(qmk_firmware_dir)
150+ ret = _run("git rev-parse HEAD").stdout.strip()
151+ os.chdir(orig_cwd)
152+ return ret
153+
154+
155+env.filters["ansi2html"] = _ansi2html
156+
157+binaries = []
158+binaries.extend(qmk_firmware_dir.glob("*.bin"))
159+binaries.extend(qmk_firmware_dir.glob("*.hex"))
160+binaries.extend(qmk_firmware_dir.glob("*.uf2"))
161+binaries = list(sorted(binaries))
162+
163+failures = []
164+for mdfile in list(sorted(build_dir.glob("failed.log.*"))):
165+ txt = Path(mdfile).read_text()
166+
167+ m_kb = KEYBOARD_PATTERN.search(txt)
168+ if not m_kb:
169+ raise Exception("Couldn't determine the keyboard from the failure output")
170+ m_km = KEYMAP_PATTERN.search(txt)
171+ if not m_km:
172+ raise Exception("Couldn't determine the keymap from the failure output")
173+
174+ txt = KEYBOARD_PATTERN.sub("", KEYMAP_PATTERN.sub("", txt)).strip()
175+
176+ failures.append(
177+ {
178+ "stdout": txt,
179+ "keyboard": m_kb.group("keyboard"),
180+ "keymap": m_km.group("keymap"),
181+ }
182+ )
183+
184+template = env.get_template("index.html.j2")
185diff --git a/util/ci/requirements.txt b/util/ci/requirements.txt
186index 3196568e1a7c5ca2018db28206554d12bd878750..47acf85644f550f012c601702400864d5f3b84d4 100644
187--- a/util/ci/requirements.txt
188+++ b/util/ci/requirements.txt
189@@ -1 +1,3 @@
190 discord-webhook
191+Jinja2
192+ansi2html
193diff --git a/util/ci/templates/index.html.j2 b/util/ci/templates/index.html.j2
194new file mode 100644
195index 0000000000000000000000000000000000000000..18086987db0648d92bc789be23ccb32e43dce50a
196--- /dev/null
197+++ b/util/ci/templates/index.html.j2
198@@ -0,0 +1,139 @@
199+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
200+<html>
201+
202+<head>
203+ <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
204+ <style type="text/css">
205+ {{ ansi2html_styles }}
206+
207+ body {
208+ color: #FFF;
209+ background-color: #111;
210+ }
211+
212+ h3 {
213+ font-family: sans-serif;
214+ margin-top: 0;
215+ }
216+
217+ a {
218+ color: #00e1ff;
219+ text-decoration: none;
220+ }
221+
222+ a:hover {
223+ color: #f700ff;
224+ text-decoration: underline;
225+ }
226+
227+ a:visited {
228+ color: #00e1ff;
229+ }
230+
231+ .build-target {
232+ background-color: #333;
233+ font-family: monospace;
234+ padding-left: 0.3em;
235+ padding-right: 0.3em;
236+ }
237+
238+ .binary-link {
239+ margin: 0;
240+ margin-top: 0.25em;
241+ }
242+
243+ .container {
244+ background-color: #222;
245+ padding: 2.0em;
246+ margin: 2.0em;
247+ border-radius: 2.0em;
248+ }
249+
250+ .header-container {
251+ display: table-cell;
252+ padding-left: 1.5em;
253+ vertical-align: middle;
254+ font-family: sans-serif;
255+ }
256+
257+ .header-container div:not(:first-child) {
258+ margin-top: 0.25em;
259+ }
260+ </style>
261+</head>
262+
263+<body class="" style="font-size: normal;">
264+ <div style="float: left">
265+ <div class="container">
266+ <div style="display: table-cell; vertical-align: middle;">
267+ <a href="https://qmk.fm/"><img src="https://qmk.fm/assets/images/badge-community-dark.svg" style="width: 30em;" /></a>
268+ </div>
269+ <div class="header-container">
270+ <div>
271+ <span style="font-size: 175%; font-weight: bold;">CI Build {% if failures | length > 0 %}❌{% else %}✅{% endif %}</span>
272+ </div>
273+ <div>
274+ <span style="font-size: 100%; font-family: monospace;">{{ git_describe }}</span>
275+ </div>
276+ <div>
277+ <span style="font-size: 100%; font-family: monospace;">{{ git_revision }}</span>
278+ </div>
279+ <div>
280+ <span style="font-size: 80%; font-weight: bold; color: {% if failures | length > 0 %}#F00{% else %}#0F0{% endif %};">{{ failures | length }} failure{% if failures | length != 1 %}s{% endif %}</span>
281+ </div>
282+ <div>
283+ <span style="font-size: 80%">
284+ {% if binaries | length > 0 %}<a href="#firmwares">Firmwares</a> | {% endif %}
285+ <a href="#commit_info">Commit info</a>
286+ {% if failures | length > 0 %} | <a href="#failure_logs">Failure Logs</a>{% endif %}
287+ </span>
288+ </div>
289+ </div>
290+ </div>
291+ <a name="commit_info"></a>
292+ <div class="container">
293+ <h3>Git commit</h3>
294+ <div class="body_foreground body_background"
295+ style="display: table-cell; padding: 1.0em; border-radius: 1.0em;">
296+ <pre class="ansi2html-content">{{ git_log | ansi2html }}</pre>
297+ </div>