380e0c9cad72ac29f858bef85c8b8eb35b6931f0
- Author
- Nick Brassel <nick@tzarc.org>
- Committer
- GitHub <noreply@github.com>
- Date
Message
Userspace: add support for adding environment variables during build (#22887)
Diff
This diff is truncated to protect this page.
1diff --git a/builddefs/build_keyboard.mk b/builddefs/build_keyboard.mk
2index f0788e55c99cf1dc234ec6cbb8874382b3796b5c..e6bb1bcb5b9fbd3496742ce8a747d717881c1a77 100644
3--- a/builddefs/build_keyboard.mk
4+++ b/builddefs/build_keyboard.mk
5@@ -34,10 +34,16 @@ ifeq ($(strip $(DUMP_CI_METADATA)),yes)
6 endif
7
8 # Force expansion
9-TARGET := $(TARGET)
10+override TARGET := $(TARGET)
11+$(info TARGET=$(TARGET))
12
13 ifneq ($(FORCE_LAYOUT),)
14- TARGET := $(TARGET)_$(FORCE_LAYOUT)
15+ override TARGET := $(TARGET)_$(FORCE_LAYOUT)
16+ $(info TARGET=$(TARGET))
17+endif
18+ifneq ($(CONVERT_TO),)
19+ override TARGET := $(TARGET)_$(CONVERT_TO)
20+ $(info TARGET=$(TARGET))
21 endif
22
23 # Object files and generated keymap directory
24@@ -58,9 +64,6 @@ ifdef SKIP_GIT
25 VERSION_H_FLAGS += --skip-git
26 endif
27
28-# Generate the board's version.h file.
29-$(shell $(QMK_BIN) generate-version-h $(VERSION_H_FLAGS) -q -o $(INTERMEDIATE_OUTPUT)/src/version.h)
30-
31 # Determine which subfolders exist.
32 KEYBOARD_FOLDER_PATH_1 := $(KEYBOARD)
33 KEYBOARD_FOLDER_PATH_2 := $(patsubst %/,%,$(dir $(KEYBOARD_FOLDER_PATH_1)))
34@@ -218,6 +221,9 @@ endif
35
36 include $(BUILDDEFS_PATH)/converters.mk
37
38+# Generate the board's version.h file.
39+$(shell $(QMK_BIN) generate-version-h $(VERSION_H_FLAGS) -q -o $(INTERMEDIATE_OUTPUT)/src/version.h)
40+
41 MCU_ORIG := $(MCU)
42 include $(wildcard $(PLATFORM_PATH)/*/mcu_selection.mk)
43
44diff --git a/builddefs/converters.mk b/builddefs/converters.mk
45index 3e77a070f2fbdac23d57186f7f9e95a566f49806..b1e5a1bed27bfa355f83d11b9eb67c0d32017b77 100644
46--- a/builddefs/converters.mk
47+++ b/builddefs/converters.mk
48@@ -32,9 +32,6 @@ ifneq ($(CONVERT_TO),)
49
50 PLATFORM_KEY = $(shell echo $(CONVERTER) | cut -d "/" -f2)
51
52- # force setting as value can be from environment
53- override TARGET := $(TARGET)_$(CONVERT_TO)
54-
55 # Configure any defaults
56 OPT_DEFS += -DCONVERT_TO_$(shell echo $(CONVERT_TO) | tr '[:lower:]' '[:upper:]')
57 OPT_DEFS += -DCONVERTER_TARGET=\"$(CONVERT_TO)\"
58diff --git a/data/schemas/definitions.jsonschema b/data/schemas/definitions.jsonschema
59index ef44915244c97dfba33b836157befa708489824c..76ea8520ac982222b5686b721bf77ad4b9f47470 100644
60--- a/data/schemas/definitions.jsonschema
61+++ b/data/schemas/definitions.jsonschema
62@@ -16,12 +16,6 @@
63 "type": "object",
64 "additionalProperties": {"type": "boolean"}
65 },
66- "build_target": {
67- "oneOf": [
68- {"$ref": "#/keyboard_keymap_tuple"},
69- {"$ref": "#/json_file_path"}
70- ]
71- },
72 "filename": {
73 "type": "string",
74 "minLength": 1,
75@@ -53,6 +47,19 @@
76 {"$ref": "#/keyboard"},
77 {"$ref": "#/filename"}
78 ],
79+ "minItems": 2,
80+ "maxItems": 2,
81+ "unevaluatedItems": false
82+ },
83+ "keyboard_keymap_env": {
84+ "type": "array",
85+ "prefixItems": [
86+ {"$ref": "#/keyboard"},
87+ {"$ref": "#/filename"},
88+ {"$ref": "#/kvp_object"}
89+ ],
90+ "minItems": 3,
91+ "maxItems": 3,
92 "unevaluatedItems": false
93 },
94 "keycode": {
95@@ -87,6 +94,10 @@
96 "maxLength": 7,
97 "pattern": "^[A-Z][A-Zs_0-9]*$"
98 },
99+ "kvp_object": {
100+ "type": "object",
101+ "additionalProperties": {"type": "string"}
102+ },
103 "layout_macro": {
104 "oneOf": [
105 {
106diff --git a/data/schemas/user_repo_v1.jsonschema b/data/schemas/user_repo_v1.jsonschema
107index 6cdf758685c5a885918ba8e8b916eb1e054d5755..69a59bce00725fafa59f7408c097da3faedb653b 100644
108--- a/data/schemas/user_repo_v1.jsonschema
109+++ b/data/schemas/user_repo_v1.jsonschema
110@@ -3,6 +3,14 @@
111 "$id": "qmk.user_repo.v1",
112 "title": "User Repository Information",
113 "type": "object",
114+ "definitions": {
115+ "build_target": {
116+ "oneOf": [
117+ {"$ref": "qmk.definitions.v1#/keyboard_keymap_tuple"},
118+ {"$ref": "qmk.definitions.v1#/json_file_path"}
119+ ]
120+ },
121+ },
122 "required": [
123 "userspace_version",
124 "build_targets"
125@@ -15,7 +23,7 @@
126 "build_targets": {
127 "type": "array",
128 "items": {
129- "$ref": "qmk.definitions.v1#/build_target"
130+ "$ref": "#/definitions/build_target"
131 }
132 }
133 }
134diff --git a/data/schemas/user_repo_v1_1.jsonschema b/data/schemas/user_repo_v1_1.jsonschema
135new file mode 100644
136index 0000000000000000000000000000000000000000..5a7ccce0633c02b8bbd84d1d52da010b7afb6b1e
137--- /dev/null
138+++ b/data/schemas/user_repo_v1_1.jsonschema
139@@ -0,0 +1,31 @@
140+{
141+ "$schema": "https://json-schema.org/draft/2020-12/schema#",
142+ "$id": "qmk.user_repo.v1_1",
143+ "title": "User Repository Information",
144+ "type": "object",
145+ "definitions": {
146+ "build_target": {
147+ "oneOf": [
148+ {"$ref": "qmk.definitions.v1#/keyboard_keymap_tuple"},
149+ {"$ref": "qmk.definitions.v1#/keyboard_keymap_env"},
150+ {"$ref": "qmk.definitions.v1#/json_file_path"}
151+ ]
152+ },
153+ },
154+ "required": [
155+ "userspace_version",
156+ "build_targets"
157+ ],
158+ "properties": {
159+ "userspace_version": {
160+ "type": "string",
161+ "enum": ["1.1"]
162+ },
163+ "build_targets": {
164+ "type": "array",
165+ "items": {
166+ "$ref": "#/definitions/build_target"
167+ }
168+ }
169+ }
170+}
171diff --git a/lib/python/qmk/build_targets.py b/lib/python/qmk/build_targets.py
172index d974d04020c48c3feaf1e933635073829c29d058..e2df0294907a8c0d8bd85f4b5fae6d311754105d 100644
173--- a/lib/python/qmk/build_targets.py
174+++ b/lib/python/qmk/build_targets.py
175@@ -1,8 +1,8 @@
176-# Copyright 2023 Nick Brassel (@tzarc)
177+# Copyright 2023-2024 Nick Brassel (@tzarc)
178 # SPDX-License-Identifier: GPL-2.0-or-later
179 import json
180 import shutil
181-from typing import List, Union
182+from typing import Dict, List, Union
183 from pathlib import Path
184 from dotty_dict import dotty, Dotty
185 from milc import cli
186@@ -13,6 +13,9 @@ from qmk.info import keymap_json
187 from qmk.keymap import locate_keymap
188 from qmk.path import is_under_qmk_firmware, is_under_qmk_userspace
189
190+# These must be kept in the order in which they're applied to $(TARGET) in the makefiles in order to ensure consistency.
191+TARGET_FILENAME_MODIFIERS = ['FORCE_LAYOUT', 'CONVERT_TO']
192+
193
194 class BuildTarget:
195 def __init__(self, keyboard: str, keymap: str, json: Union[dict, Dotty] = None):
196@@ -22,25 +25,25 @@ class BuildTarget:
197 self._parallel = 1
198 self._clean = False
199 self._compiledb = False
200- self._target = f'{self._keyboard_safe}_{self.keymap}'
201- self._intermediate_output = Path(f'{INTERMEDIATE_OUTPUT_PREFIX}{self._target}')
202- self._generated_files_path = self._intermediate_output / 'src'
203+ self._extra_args = {}
204 self._json = json.to_dict() if isinstance(json, Dotty) else json
205
206 def __str__(self):
207 return f'{self.keyboard}:{self.keymap}'
208
209 def __repr__(self):
210+ if len(self._extra_args.items()) > 0:
211+ return f'BuildTarget(keyboard={self.keyboard}, keymap={self.keymap}, extra_args={json.dumps(self._extra_args, sort_keys=True)})'
212 return f'BuildTarget(keyboard={self.keyboard}, keymap={self.keymap})'
213
214+ def __lt__(self, __value: object) -> bool:
215+ return self.__repr__() < __value.__repr__()
216+
217 def __eq__(self, __value: object) -> bool:
218 if not isinstance(__value, BuildTarget):
219 return False
220 return self.__repr__() == __value.__repr__()
221
222- def __ne__(self, __value: object) -> bool:
223- return not self.__eq__(__value)
224-
225 def __hash__(self) -> int:
226 return self.__repr__().__hash__()
227
228@@ -72,7 +75,34 @@ class BuildTarget:
229 def dotty(self) -> Dotty:
230 return dotty(self.json)
231
232- def _common_make_args(self, dry_run: bool = False, build_target: str = None):
233+ @property
234+ def extra_args(self) -> Dict[str, str]:
235+ return {k: v for k, v in self._extra_args.items()}
236+
237+ @extra_args.setter
238+ def extra_args(self, ex_args: Dict[str, str]):
239+ if ex_args is not None and isinstance(ex_args, dict):
240+ self._extra_args = {k: v for k, v in ex_args.items()}
241+
242+ def target_name(self, **env_vars) -> str:
243+ # Work out the intended target name
244+ target = f'{self._keyboard_safe}_{self.keymap}'
245+ vars = self._all_vars(**env_vars)
246+ for modifier in TARGET_FILENAME_MODIFIERS:
247+ if modifier in vars:
248+ target += f"_{vars[modifier]}"
249+ return target
250+
251+ def _all_vars(self, **env_vars) -> Dict[str, str]:
252+ vars = {k: v for k, v in env_vars.items()}
253+ for k, v in self._extra_args.items():
254+ vars[k] = v
255+ return vars
256+
257+ def _intermediate_output(self, **env_vars) -> Path:
258+ return Path(f'{INTERMEDIATE_OUTPUT_PREFIX}{self.target_name(**env_vars)}')
259+
260+ def _common_make_args(self, dry_run: bool = False, build_target: str = None, **env_vars):
261 compile_args = [
262 find_make(),
263 *get_make_parallel_args(self._parallel),
264@@ -98,14 +128,17 @@ class BuildTarget:
265 f'KEYBOARD={self.keyboard}',
266 f'KEYMAP={self.keymap}',
267 f'KEYBOARD_FILESAFE={self._keyboard_safe}',
268- f'TARGET={self._target}',
269- f'INTERMEDIATE_OUTPUT={self._intermediate_output}',
270+ f'TARGET={self._keyboard_safe}_{self.keymap}', # don't use self.target_name() here, it's rebuilt on the makefile side
271 f'VERBOSE={verbose}',
272 f'COLOR={color}',
273 'SILENT=false',
274 'QMK_BIN="qmk"',
275diff --git a/lib/python/qmk/cli/format/json.py b/lib/python/qmk/cli/format/json.py
276index 87a3837d10e6cbbb33903b8f254039c79eb88da4..3670294434504c0f0e9db0b2369d102f563968e6 100755
277--- a/lib/python/qmk/cli/format/json.py
278+++ b/lib/python/qmk/cli/format/json.py
279@@ -18,11 +18,18 @@ def _detect_json_format(file, json_data):
280 """
281 json_encoder = None
282 try:
283- validate(json_data, 'qmk.user_repo.v1')
284+ validate(json_data, 'qmk.user_repo.v1_1')
285 json_encoder = UserspaceJSONEncoder
286 except ValidationError:
287 pass
288
289+ if json_encoder is None:
290+ try:
291+ validate(json_data, 'qmk.user_repo.v1')
292+ json_encoder = UserspaceJSONEncoder
293+ except ValidationError:
294+ pass
295+
296 if json_encoder is None:
297 try:
298 validate(json_data, 'qmk.keyboard.v1')
299diff --git a/lib/python/qmk/cli/mass_compile.py b/lib/python/qmk/cli/mass_compile.py
300index d13afc614329ad56f8ef00f64b842d40d4bebdc1..cf9be0fd1e2969f9e2a54362c9bc6030ee6e6f4b 100755
301--- a/lib/python/qmk/cli/mass_compile.py
302+++ b/lib/python/qmk/cli/mass_compile.py
303@@ -7,6 +7,7 @@ from typing import List
304 from pathlib import Path
305 from subprocess import DEVNULL
306 from milc import cli
307+import shlex
308
309 from qmk.constants import QMK_FIRMWARE
310 from qmk.commands import find_make, get_make_parallel_args, build_environment
311@@ -26,7 +27,8 @@ def mass_compile_targets(targets: List[BuildTarget], clean: bool, dry_run: bool,
312 if dry_run:
313 cli.log.info('Compilation targets:')
314 for target in sorted(targets, key=lambda t: (t.keyboard, t.keymap)):
315- cli.log.info(f"{{fg_cyan}}qmk compile -kb {target.keyboard} -km {target.keymap}{{fg_reset}}")
316+ extra_args = ' '.join([f"-e {shlex.quote(f'{k}={v}')}" for k, v in target.extra_args.items()])
317+ cli.log.info(f"{{fg_cyan}}qmk compile -kb {target.keyboard} -km {target.keymap} {extra_args}{{fg_reset}}")
318 else:
319 if clean:
320 cli.run([make_cmd, 'clean'], capture_output=False, stdin=DEVNULL)
321@@ -36,18 +38,26 @@ def mass_compile_targets(targets: List[BuildTarget], clean: bool, dry_run: bool,
322 for target in sorted(targets, key=lambda t: (t.keyboard, t.keymap)):
323 keyboard_name = target.keyboard
324 keymap_name = target.keymap
325+ keyboard_safe = keyboard_name.replace('/', '_')
326+ target_filename = target.target_name(**env)
327 target.configure(parallel=1) # We ignore parallelism on a per-build basis as we defer to the parent make invocation
328 target.prepare_build(**env) # If we've got json targets, allow them to write out any extra info to .build before we kick off `make`
329 command = target.compile_command(**env)
330 command[0] = '+@$(MAKE)' # Override the make so that we can use jobserver to handle parallelism
331- keyboard_safe = keyboard_name.replace('/', '_')
332+ extra_args = '_'.join([f"{k}_{v}" for k, v in target.extra_args.items()])
333 build_log = f"{QMK_FIRMWARE}/.build/build.log.{os.getpid()}.{keyboard_safe}.{keymap_name}"
334 failed_log = f"{QMK_FIRMWARE}/.build/failed.log.{os.getpid()}.{keyboard_safe}.{keymap_name}"
335+ target_suffix = ''
336+ if len(extra_args) > 0:
337+ build_log += f".{extra_args}"
338+ failed_log += f".{extra_args}"
339+ target_suffix = f"_{extra_args}"
340 # yapf: disable
341 f.write(
342 f"""\
343-all: {keyboard_safe}_{keymap_name}_binary
344-{keyboard_safe}_{keymap_name}_binary:
345+.PHONY: {target_filename}{target_suffix}_binary
346+all: {target_filename}{target_suffix}_binary
347+{target_filename}{target_suffix}_binary:
348 @rm -f "{build_log}" || true
349 @echo "Compiling QMK Firmware for target: '{keyboard_name}:{keymap_name}'..." >>"{build_log}"
350 {' '.join(command)} \\
351@@ -65,9 +75,9 @@ all: {keyboard_safe}_{keymap_name}_binary
352 # yapf: disable
353 f.write(
354 f"""\
355- @rm -rf "{QMK_FIRMWARE}/.build/{keyboard_safe}_{keymap_name}.elf" 2>/dev/null || true
356- @rm -rf "{QMK_FIRMWARE}/.build/{keyboard_safe}_{keymap_name}.map" 2>/dev/null || true
357- @rm -rf "{QMK_FIRMWARE}/.build/obj_{keyboard_safe}_{keymap_name}" || true
358+ @rm -rf "{QMK_FIRMWARE}/.build/{target_filename}.elf" 2>/dev/null || true
359+ @rm -rf "{QMK_FIRMWARE}/.build/{target_filename}.map" 2>/dev/null || true
360+ @rm -rf "{QMK_FIRMWARE}/.build/obj_{target_filename}" || true
361 """# noqa
362 )
363 # yapf: enable
364diff --git a/lib/python/qmk/cli/userspace/add.py b/lib/python/qmk/cli/userspace/add.py
365index 8993d54dba5222158669714bb68de0a7959cd80c..0d6f32cd11db3f96b5f92d10b211965355414735 100644
366--- a/lib/python/qmk/cli/userspace/add.py
367+++ b/lib/python/qmk/cli/userspace/add.py
368@@ -1,8 +1,9 @@
369-# Copyright 2023 Nick Brassel (@tzarc)
370+# Copyright 2023-2024 Nick Brassel (@tzarc)
371 # SPDX-License-Identifier: GPL-2.0-or-later
372 from pathlib import Path
373 from milc import cli
374
375+from qmk.commands import parse_env_vars
376 from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE
377 from qmk.keyboard import keyboard_completer, keyboard_folder_or_all
378 from qmk.keymap import keymap_completer, is_keymap_target
379@@ -12,12 +13,15 @@ from qmk.userspace import UserspaceDefs
380 @cli.argument('builds', nargs='*', arg_only=True, help="List of builds in form <keyboard>:<keymap>, or path to a keymap JSON file.")
381 @cli.argument('-kb', '--keyboard', type=keyboard_folder_or_all, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.')
382 @cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.')
383+@cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Extra variables to set during build. May be passed multiple times.")
384 @cli.subcommand('Adds a build target to userspace `qmk.json`.')
385 def userspace_add(cli):
386 if not HAS_QMK_USERSPACE:
387 cli.log.error('Could not determine QMK userspace location. Please run `qmk doctor` or `qmk userspace-doctor` to diagnose.')
388 return False
389
390+ build_env = None if len(cli.args.env) == 0 else parse_env_vars(cli.args.env)
391+
392 userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
393
394 if len(cli.args.builds) > 0:
395@@ -44,8 +48,8 @@ def userspace_add(cli):
396 cli.config.new_keymap.keyboard = cli.args.keyboard
397 cli.config.new_keymap.keymap = cli.args.keymap
398 if new_keymap(cli) is not False:
399- userspace.add_target(keyboard=cli.args.keyboard, keymap=cli.args.keymap)
400+ userspace.add_target(keyboard=cli.args.keyboard, keymap=cli.args.keymap, build_env=build_env)
401 else:
402- userspace.add_target(keyboard=cli.args.keyboard, keymap=cli.args.keymap)
403+ userspace.add_target(keyboard=cli.args.keyboard, keymap=cli.args.keymap, build_env=build_env)
404
405 return userspace.save()
406diff --git a/lib/python/qmk/cli/userspace/compile.py b/lib/python/qmk/cli/userspace/compile.py
407index e8cdf6cd9716c9dd28b78094b06e386936b998de..f164ca2ef12a7f21df7765b22030ece7c01e158d 100644
408--- a/lib/python/qmk/cli/userspace/compile.py
409+++ b/lib/python/qmk/cli/userspace/compile.py
410@@ -1,4 +1,4 @@
411-# Copyright 2023 Nick Brassel (@tzarc)
412+# Copyright 2023-2024 Nick Brassel (@tzarc)
413 # SPDX-License-Identifier: GPL-2.0-or-later
414 from pathlib import Path
415 from milc import cli
416@@ -12,6 +12,10 @@ from qmk.cli.mass_compile import mass_compile_targets
417 from qmk.util import maybe_exit_config
418
419
420+def _extra_arg_setter(target, extra_args):
421+ target.extra_args = extra_args
422+
423+
424 @cli.argument('-t', '--no-temp', arg_only=True, action='store_true', help="Remove temporary files during build.")
425 @cli.argument('-j', '--parallel', type=int, default=1, help="Set the number of parallel make jobs; 0 means unlimited.")
426 @cli.argument('-c', '--clean', arg_only=True, action='store_true', help="Remove object files before compiling.")
427@@ -33,8 +37,8 @@ def userspace_compile(cli):
428 if isinstance(e, Path):
429 build_targets.append(JsonKeymapBuildTarget(e))
430 elif isinstance(e, dict):
431- keyboard_keymap_targets.append((e['keyboard'], e['keymap']))
432-
433+ f = e['env'] if 'env' in e else None
434+ keyboard_keymap_targets.append((e['keyboard'], e['keymap'], f))
435 if len(keyboard_keymap_targets) > 0:
436 build_targets.extend(search_keymap_targets(keyboard_keymap_targets))
437
438diff --git a/lib/python/qmk/cli/userspace/list.py b/lib/python/qmk/cli/userspace/list.py
439index 8689c80a76968407881123e2b8e081a17e7f1e87..9f83a14a2a6d15e567111d435b8a899fe9ab6bd6 100644
440--- a/lib/python/qmk/cli/userspace/list.py
441+++ b/lib/python/qmk/cli/userspace/list.py
442@@ -1,4 +1,4 @@
443-# Copyright 2023 Nick Brassel (@tzarc)
444+# Copyright 2023-2024 Nick Brassel (@tzarc)
445 # SPDX-License-Identifier: GPL-2.0-or-later
446 from pathlib import Path
447 from dotty_dict import Dotty
448@@ -13,6 +13,10 @@ from qmk.search import search_keymap_targets
449 from qmk.util import maybe_exit_config
450
451
452+def _extra_arg_setter(target, extra_args):
453+ target.extra_args = extra_args
454+
455+
456 @cli.argument('-e', '--expand', arg_only=True, action='store_true', help="Expands any use of `all` for either keyboard or keymap.")
457 @cli.subcommand('Lists the build targets specified in userspace `qmk.json`.')
458 def userspace_list(cli):
459@@ -26,11 +30,15 @@ def userspace_list(cli):
460
461 if cli.args.expand:
462 build_targets = []
463+ keyboard_keymap_targets = []
464 for e in userspace.build_targets:
465 if isinstance(e, Path):
466 build_targets.append(e)
467 elif isinstance(e, dict) or isinstance(e, Dotty):
468- build_targets.extend(search_keymap_targets([(e['keyboard'], e['keymap'])]))
469+ f = e['env'] if 'env' in e else None
470+ keyboard_keymap_targets.append((e['keyboard'], e['keymap'], f))
471+ if len(keyboard_keymap_targets) > 0:
472+ build_targets.extend(search_keymap_targets(keyboard_keymap_targets))
473 else:
474 build_targets = userspace.build_targets
475
476@@ -43,12 +51,19 @@ def userspace_list(cli):
477 # keyboard/keymap dict from userspace
478 keyboard = e['keyboard']
479 keymap = e['keymap']
480+ extra_args = e.get('env')
481 elif isinstance(e, BuildTarget):
482 # BuildTarget from search_keymap_targets()
483 keyboard = e.keyboard
484 keymap = e.keymap
485+ extra_args = e.extra_args
486+
487+ extra_args_str = ''
488+ if extra_args is not None and len(extra_args) > 0:
489+ extra_args_str = ', '.join([f'{{fg_cyan}}{k}={v}{{fg_reset}}' for k, v in extra_args.items()])
490+ extra_args_str = f' ({{fg_cyan}}{extra_args_str}{{fg_reset}})'
491
492 if is_all_keyboards(keyboard) or is_keymap_target(keyboard_folder(keyboard), keymap):
493- cli.log.info(f'Keyboard: {{fg_cyan}}{keyboard}{{fg_reset}}, keymap: {{fg_cyan}}{keymap}{{fg_reset}}')
494+ cli.log.info(f'Keyboard: {{fg_cyan}}{keyboard}{{fg_reset}}, keymap: {{fg_cyan}}{keymap}{{fg_reset}}{extra_args_str}')
495 else:
496- cli.log.warn(f'Keyboard: {{fg_cyan}}{keyboard}{{fg_reset}}, keymap: {{fg_cyan}}{keymap}{{fg_reset}} -- not found!')
497+ cli.log.warn(f'Keyboard: {{fg_cyan}}{keyboard}{{fg_reset}}, keymap: {{fg_cyan}}{keymap}{{fg_reset}}{extra_args_str} -- not found!')
498diff --git a/lib/python/qmk/cli/userspace/remove.py b/lib/python/qmk/cli/userspace/remove.py
499index c7d180bfd1234735993723bb2ba713e929f304d4..b2da08a98ecd44e49c8a3305334eb8a79bdc2902 100644
500--- a/lib/python/qmk/cli/userspace/remove.py
501+++ b/lib/python/qmk/cli/userspace/remove.py
502@@ -1,8 +1,9 @@
503-# Copyright 2023 Nick Brassel (@tzarc)
504+# Copyright 2023-2024 Nick Brassel (@tzarc)
505 # SPDX-License-Identifier: GPL-2.0-or-later
506 from pathlib import Path
507 from milc import cli
508
509+from qmk.commands import parse_env_vars
510 from qmk.constants import QMK_USERSPACE, HAS_QMK_USERSPACE
511 from qmk.keyboard import keyboard_completer, keyboard_folder_or_all
512 from qmk.keymap import keymap_completer
513@@ -12,12 +13,15 @@ from qmk.userspace import UserspaceDefs
514 @cli.argument('builds', nargs='*', arg_only=True, help="List of builds in form <keyboard>:<keymap>, or path to a keymap JSON file.")
515 @cli.argument('-kb', '--keyboard', type=keyboard_folder_or_all, completer=keyboard_completer, help='The keyboard to build a firmware for. Ignored when a configurator export is supplied.')
516 @cli.argument('-km', '--keymap', completer=keymap_completer, help='The keymap to build a firmware for. Ignored when a configurator export is supplied.')
517+@cli.argument('-e', '--env', arg_only=True, action='append', default=[], help="Extra variables to set during build. May be passed multiple times.")
518 @cli.subcommand('Removes a build target from userspace `qmk.json`.')
519 def userspace_remove(cli):
520 if not HAS_QMK_USERSPACE:
521 cli.log.error('Could not determine QMK userspace location. Please run `qmk doctor` or `qmk userspace-doctor` to diagnose.')
522 return False
523
524+ build_env = None if len(cli.args.env) == 0 else parse_env_vars(cli.args.env)
525+
526 userspace = UserspaceDefs(QMK_USERSPACE / 'qmk.json')
527
528 if len(cli.args.builds) > 0:
529@@ -29,9 +33,9 @@ def userspace_remove(cli):
530
531 for e in make_like_targets:
532 s = e.split(':')
533- userspace.remove_target(keyboard=s[0], keymap=s[1])
534+ userspace.remove_target(keyboard=s[0], keymap=s[1], build_env=build_env)
535
536 else:
537- userspace.remove_target(keyboard=cli.args.keyboard, keymap=cli.args.keymap)
538+ userspace.remove_target(keyboard=cli.args.keyboard, keymap=cli.args.keymap, build_env=build_env)
539
540 return userspace.save()
541diff --git a/lib/python/qmk/commands.py b/lib/python/qmk/commands.py
542index df6ed6b88ab0d67d5e4a3547d1e1847c9c724ab5..459858ab2c97e71e5eae2c3f8f32ccf391163302 100644
543--- a/lib/python/qmk/commands.py
544+++ b/lib/python/qmk/commands.py
545@@ -68,7 +68,7 @@ def parse_configurator_json(configurator_file):
546 return user_keymap
547
548
549-def build_environment(args):
550+def parse_env_vars(args):
551 """Common processing for cli.args.env
552 """
553 envs = {}
554@@ -78,6 +78,11 @@ def build_environment(args):
555 envs[key] = value
556 else:
557 cli.log.warning('Invalid environment variable: %s', env)
558+ return envs
559+
560+
561+def build_environment(args):
562+ envs = parse_env_vars(args)
563
564 if HAS_QMK_USERSPACE:
565 envs['QMK_USERSPACE'] = Path(QMK_USERSPACE).resolve()
566diff --git a/lib/python/qmk/info.py b/lib/python/qmk/info.py
567index 833271c09cc7486de19166425cc7e7d10ce95de8..e295c5cfe0dbb82c9f845d9770ab2108b8fdee24 100644
568--- a/lib/python/qmk/info.py
569+++ b/lib/python/qmk/info.py
570@@ -212,7 +212,7 @@ def _validate(keyboard, info_data):
571 maybe_exit(1)
572
573
574-def info_json(keyboard):
575+def info_json(keyboard, force_layout=None):
576 """Generate the info.json data for a specific keyboard.
577 """
578 cur_dir = Path('keyboards')
579@@ -255,6 +255,11 @@ def info_json(keyboard):
580 # Merge in data from <keyboard.c>
581 info_data = _extract_led_config(info_data, str(keyboard))
582
583+ # Force a community layout if requested
584+ community_layouts = info_data.get("community_layouts", [])
585+ if force_layout in community_layouts:
586+ info_data["community_layouts"] = [force_layout]
587+
588 # Validate
589 _validate(keyboard, info_data)
590
591@@ -988,25 +993,25 @@ def find_info_json(keyboard):
592 return [info_json for info_json in info_jsons if info_json.exists()]
593
594
595-def keymap_json_config(keyboard, keymap):
596+def keymap_json_config(keyboard, keymap, force_layout=None):
597 """Extract keymap level config
598 """
599 # TODO: resolve keymap.py and info.py circular dependencies
600 from qmk.keymap import locate_keymap
601
602- keymap_folder = locate_keymap(keyboard, keymap).parent
603+ keymap_folder = locate_keymap(keyboard, keymap, force_layout=force_layout).parent
604
605 km_info_json = parse_configurator_json(keymap_folder / 'keymap.json')
606 return km_info_json.get('config', {})
607
608
609-def keymap_json(keyboard, keymap):
610+def keymap_json(keyboard, keymap, force_layout=None):
611 """Generate the info.json data for a specific keymap.
612 """
613 # TODO: resolve keymap.py and info.py circular dependencies
614 from qmk.keymap import locate_keymap
615
616- keymap_folder = locate_keymap(keyboard, keymap).parent
617+ keymap_folder = locate_keymap(keyboard, keymap, force_layout=force_layout).parent
618
619 # Files to scan
620 keymap_config = keymap_folder / 'config.h'
621@@ -1014,10 +1019,10 @@ def keymap_json(keyboard, keymap):
622 keymap_file = keymap_folder / 'keymap.json'
623
624 # Build the info.json file
625- kb_info_json = info_json(keyboard)
626+ kb_info_json = info_json(keyboard, force_layout=force_layout)
627
628 # Merge in the data from keymap.json
629- km_info_json = keymap_json_config(keyboard, keymap) if keymap_file.exists() else {}
630+ km_info_json = keymap_json_config(keyboard, keymap, force_layout=force_layout) if keymap_file.exists() else {}
631 deep_update(kb_info_json, km_info_json)
632
633 # Merge in the data from config.h, and rules.mk
634diff --git a/lib/python/qmk/keymap.py b/lib/python/qmk/keymap.py
635index b7bf897377c501439d00e483116ade144ba5ad28..91075f1bdc512ff8fd1e238ad71661d30a62b1de 100644
636--- a/lib/python/qmk/keymap.py
637+++ b/lib/python/qmk/keymap.py
638@@ -420,7 +420,7 @@ def write(keymap_json):
639 return write_file(keymap_file, keymap_content)
640
641
642-def locate_keymap(keyboard, keymap):
643+def locate_keymap(keyboard, keymap, force_layout=None):
644 """Returns the path to a keymap for a specific keyboard.
645 """
646 if not qmk.path.is_keyboard(keyboard):
647@@ -459,7 +459,7 @@ def locate_keymap(keyboard, keymap):
648 return keymap_path
649
650 # Check community layouts as a fallback
651- info = info_json(keyboard)
652+ info = info_json(keyboard, force_layout=force_layout)
653
654 community_parents = list(Path('layouts').glob('*/'))
655 if HAS_QMK_USERSPACE and (Path(QMK_USERSPACE) / "layouts").exists():
656diff --git a/lib/python/qmk/search.py b/lib/python/qmk/search.py
657index 2afb3033fcca00b57259b7be7dc87d0e6fbec8e5..baaf11eb340638c658c168eaefd06f69ad4d8928 100644
658--- a/lib/python/qmk/search.py
659+++ b/lib/python/qmk/search.py
660@@ -1,11 +1,13 @@
661 """Functions for searching through QMK keyboards and keymaps.
662 """
663+from dataclasses import dataclass
664 import contextlib
665 import functools
666 import fnmatch
667+import json
668 import logging
669 import re
670-from typing import Callable, List, Optional, Tuple
671+from typing import Callable, Dict, List, Optional, Tuple, Union
672 from dotty_dict import dotty, Dotty
673 from milc import cli
674
675@@ -15,7 +17,32 @@ from qmk.keyboard import list_keyboards, keyboard_folder
676 from qmk.keymap import list_keymaps, locate_keymap
677 from qmk.build_targets import KeyboardKeymapBuildTarget, BuildTarget
678
679-TargetInfo = Tuple[str, str, dict]
680+
681+@dataclass
682+class KeyboardKeymapDesc:
683+ keyboard: str
684+ keymap: str
685+ data: dict = None
686+ extra_args: dict = None
687+
688+ def __hash__(self) -> int:
689+ return self.keyboard.__hash__() ^ self.keymap.__hash__() ^ json.dumps(self.extra_args, sort_keys=True).__hash__()
690+
691+ def __lt__(self, other) -> bool:
692+ return (self.keyboard, self.keymap, json.dumps(self.extra_args, sort_keys=True)) < (other.keyboard, other.keymap, json.dumps(other.extra_args, sort_keys=True))
693+
694+ def load_data(self):
695+ data = keymap_json(self.keyboard, self.keymap)
696+ self.data = data.to_dict() if isinstance(data, Dotty) else data
697+
698+ @property
699+ def dotty(self) -> Dotty:
700+ return dotty(self.data) if self.data is not None else None
701+
702+ def to_build_target(self) -> KeyboardKeymapBuildTarget:
703+ target = KeyboardKeymapBuildTarget(keyboard=self.keyboard, keymap=self.keymap, json=self.data)
704+ target.extra_args = self.extra_args
705+ return target
706
707
708 # by using a class for filters, we dont need to worry about capturing values
709@@ -36,7 +63,7 @@ class FilterFunction:
710 value: Optional[str]
711
712 func_name: str
713- apply: Callable[[TargetInfo], bool]
714+ apply: Callable[[KeyboardKeymapDesc], bool]
715
716 def __init__(self, key, value):
717 self.key = key
718@@ -46,33 +73,29 @@ class FilterFunction:
719 class Exists(FilterFunction):
720 func_name = "exists"
721
722- def apply(self, target_info: TargetInfo) -> bool:
723- _kb, _km, info = target_info
724- return self.key in info
725+ def apply(self, target_info: KeyboardKeymapDesc) -> bool:
726+ return self.key in target_info.data
727
728
729 class Absent(FilterFunction):
730 func_name = "absent"
731
732- def apply(self, target_info: TargetInfo) -> bool:
733- _kb, _km, info = target_info
734- return self.key not in info
735+ def apply(self, target_info: KeyboardKeymapDesc) -> bool:
736+ return self.key not in target_info.data
737
738
739 class Length(FilterFunction):
740 func_name = "length"
741
742- def apply(self, target_info: TargetInfo) -> bool:
743- _kb, _km, info = target_info
744- return (self.key in info and len(info[self.key]) == int(self.value))
745+ def apply(self, target_info: KeyboardKeymapDesc) -> bool:
746+ return (self.key in target_info.data and len(target_info.data[self.key]) == int(self.value))
747
748
749 class Contains(FilterFunction):
750 func_name = "contains"
751
752- def apply(self, target_info: TargetInfo) -> bool:
753- _kb, _km, info = target_info
754- return (self.key in info and self.value in info[self.key])
755+ def apply(self, target_info: KeyboardKeymapDesc) -> bool:
756+ return (self.key in target_info.data and self.value in target_info.data[self.key])
757
758
759 def _get_filter_class(func_name: str, key: str, value: str) -> Optional[FilterFunction]:
760diff --git a/lib/python/qmk/userspace.py b/lib/python/qmk/userspace.py
761index 1e5823b229198670c0d59fc124a5c27c0caffffa..1c2a97f9c1b240334bac06c90e11aba05cf7c7f3 100644
762--- a/lib/python/qmk/userspace.py
763+++ b/lib/python/qmk/userspace.py
764@@ -1,4 +1,4 @@
765-# Copyright 2023 Nick Brassel (@tzarc)
766+# Copyright 2023-2024 Nick Brassel (@tzarc)
767 # SPDX-License-Identifier: GPL-2.0-or-later
768 from os import environ
769 from pathlib import Path
770@@ -77,31 +77,43 @@ class UserspaceDefs:
771 raise exception
772
773 # Iterate through each version of the schema, starting with the latest and decreasing to v1
774- try:
775- validate(json, 'qmk.user_repo.v1')
776- self.__load_v1(json)
777- success = True
778- except jsonschema.ValidationError as err:
779- exception.add('qmk.user_repo.v1', err)
780+ schema_versions = [
781+ ('qmk.user_repo.v1_1', self.__load_v1_1), #
782+ ('qmk.user_repo.v1', self.__load_v1) #
783+ ]
784+
785+ for v in schema_versions:
786+ schema = v[0]
787+ loader = v[1]
788+ try:
789+ validate(json, schema)
790+ loader(json)
791+ success = True
792+ break
793+ except jsonschema.ValidationError as err:
794+ exception.add(schema, err)
795
796 if not success:
797 raise exception
798
799 def save(self):
800 target_json = {
801- "userspace_version": "1.0", # Needs to match latest version
802+ "userspace_version": "1.1", # Needs to match latest version
803 "build_targets": []
804 }
805
806 for e in self.build_targets:
807 if isinstance(e, dict):
808- target_json['build_targets'].append([e['keyboard'], e['keymap']])
809+ entry = [e['keyboard'], e['keymap']]
810+ if 'env' in e:
811+ entry.append(e['env'])
812+ target_json['build_targets'].append(entry)
813 elif isinstance(e, Path):
814 target_json['build_targets'].append(str(e.relative_to(self.path.parent)))
815
816 try:
817 # Ensure what we're writing validates against the latest version of the schema
818- validate(target_json, 'qmk.user_repo.v1')
819+ validate(target_json, 'qmk.user_repo.v1_1')
820 except jsonschema.ValidationError as err:
821 cli.log.error(f'Could not save userspace file: {err}')
822 return False
823@@ -114,7 +126,7 @@ class UserspaceDefs:
824 cli.log.info(f'Saved userspace file to {self.path}.')
825 return True
826
827- def add_target(self, keyboard=None, keymap=None, json_path=None, do_print=True):
828+ def add_target(self, keyboard=None, keymap=None, build_env=None, json_path=None, do_print=True):
829 if json_path is not None:
830 # Assume we're adding a json filename/path
831 json_path = Path(json_path)
832@@ -128,6 +140,8 @@ class UserspaceDefs:
833 elif keyboard is not None and keymap is not None:
834 # Both keyboard/keymap specified
835 e = {"keyboard": keyboard, "keymap": keymap}
836+ if build_env is not None:
837+ e['env'] = build_env
838 if e not in self.build_targets:
839 self.build_targets.append(e)
840 if do_print:
841@@ -136,7 +150,7 @@ class UserspaceDefs:
842 if do_print:
843 cli.log.info(f'{keyboard}:{keymap} is already a userspace build target.')
844
845- def remove_target(self, keyboard=None, keymap=None, json_path=None, do_print=True):
846+ def remove_target(self, keyboard=None, keymap=None, build_env=None, json_path=None, do_print=True):
847 if json_path is not None:
848 # Assume we're removing a json filename/path
849 json_path = Path(json_path)
850@@ -150,6 +164,8 @@ class UserspaceDefs:
851 elif keyboard is not None and keymap is not None:
852 # Both keyboard/keymap specified
853 e = {"keyboard": keyboard, "keymap": keymap}
854+ if build_env is not None:
855+ e['env'] = build_env
856 if e in self.build_targets:
857 self.build_targets.remove(e)
858 if do_print:
859@@ -160,12 +176,26 @@ class UserspaceDefs:
860
861 def __load_v1(self, json):
862 for e in json['build_targets']:
863- if isinstance(e, list) and len(e) == 2: