forked from mirrors/qmk_userspace
		
	CLI parallel search updates (#22525)
This commit is contained in:
		
					parent
					
						
							
								7e27d72cbc
							
						
					
				
			
			
				commit
				
					
						46b996a55e
					
				
			
		
					 5 changed files with 26 additions and 13 deletions
				
			
		| 
						 | 
					@ -1,11 +1,8 @@
 | 
				
			||||||
"""Validates the list of keyboard aliases.
 | 
					"""Validates the list of keyboard aliases.
 | 
				
			||||||
"""
 | 
					"""
 | 
				
			||||||
from pathlib import Path
 | 
					 | 
				
			||||||
 | 
					 | 
				
			||||||
from milc import cli
 | 
					from milc import cli
 | 
				
			||||||
 | 
					
 | 
				
			||||||
from qmk.json_schema import json_load
 | 
					from qmk.keyboard import resolve_keyboard, keyboard_folder, keyboard_alias_definitions
 | 
				
			||||||
from qmk.keyboard import resolve_keyboard, keyboard_folder
 | 
					 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
def _safe_keyboard_folder(target):
 | 
					def _safe_keyboard_folder(target):
 | 
				
			||||||
| 
						 | 
					@ -34,7 +31,7 @@ def _target_keyboard_exists(target):
 | 
				
			||||||
 | 
					
 | 
				
			||||||
@cli.subcommand('Validates the list of keyboard aliases.', hidden=True)
 | 
					@cli.subcommand('Validates the list of keyboard aliases.', hidden=True)
 | 
				
			||||||
def ci_validate_aliases(cli):
 | 
					def ci_validate_aliases(cli):
 | 
				
			||||||
    aliases = json_load(Path('data/mappings/keyboard_aliases.hjson'))
 | 
					    aliases = keyboard_alias_definitions()
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    success = True
 | 
					    success = True
 | 
				
			||||||
    for alias in aliases.keys():
 | 
					    for alias in aliases.keys():
 | 
				
			||||||
| 
						 | 
					
 | 
				
			||||||
| 
						 | 
					@ -10,7 +10,7 @@ from qmk.datetime import current_datetime
 | 
				
			||||||
from qmk.info import info_json
 | 
					from qmk.info import info_json
 | 
				
			||||||
from qmk.json_schema import json_load
 | 
					from qmk.json_schema import json_load
 | 
				
			||||||
from qmk.keymap import list_keymaps
 | 
					from qmk.keymap import list_keymaps
 | 
				
			||||||
from qmk.keyboard import find_readme, list_keyboards
 | 
					from qmk.keyboard import find_readme, list_keyboards, keyboard_alias_definitions
 | 
				
			||||||
from qmk.keycodes import load_spec, list_versions, list_languages
 | 
					from qmk.keycodes import load_spec, list_versions, list_languages
 | 
				
			||||||
 | 
					
 | 
				
			||||||
DATA_PATH = Path('data')
 | 
					DATA_PATH = Path('data')
 | 
				
			||||||
| 
						 | 
					@ -166,7 +166,7 @@ def generate_api(cli):
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    # Generate data for the global files
 | 
					    # Generate data for the global files
 | 
				
			||||||
    keyboard_list = sorted(kb_all)
 | 
					    keyboard_list = sorted(kb_all)
 | 
				
			||||||
    keyboard_aliases = json_load(Path('data/mappings/keyboard_aliases.hjson'))
 | 
					    keyboard_aliases = keyboard_alias_definitions()
 | 
				
			||||||
    keyboard_metadata = {
 | 
					    keyboard_metadata = {
 | 
				
			||||||
        'last_updated': current_datetime(),
 | 
					        'last_updated': current_datetime(),
 | 
				
			||||||
        'keyboards': keyboard_list,
 | 
					        'keyboards': keyboard_list,
 | 
				
			||||||
| 
						 | 
					
 | 
				
			||||||
| 
						 | 
					@ -3,12 +3,12 @@
 | 
				
			||||||
import os
 | 
					import os
 | 
				
			||||||
import sys
 | 
					import sys
 | 
				
			||||||
import shutil
 | 
					import shutil
 | 
				
			||||||
from pathlib import Path
 | 
					 | 
				
			||||||
 | 
					
 | 
				
			||||||
from milc import cli
 | 
					from milc import cli
 | 
				
			||||||
import jsonschema
 | 
					import jsonschema
 | 
				
			||||||
 | 
					
 | 
				
			||||||
from qmk.json_schema import json_load, validate
 | 
					from qmk.json_schema import json_load, validate
 | 
				
			||||||
 | 
					from qmk.keyboard import keyboard_alias_definitions
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
def find_make():
 | 
					def find_make():
 | 
				
			||||||
| 
						 | 
					@ -53,7 +53,7 @@ def parse_configurator_json(configurator_file):
 | 
				
			||||||
        exit(1)
 | 
					        exit(1)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    keyboard = user_keymap['keyboard']
 | 
					    keyboard = user_keymap['keyboard']
 | 
				
			||||||
    aliases = json_load(Path('data/mappings/keyboard_aliases.hjson'))
 | 
					    aliases = keyboard_alias_definitions()
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    while keyboard in aliases:
 | 
					    while keyboard in aliases:
 | 
				
			||||||
        last_keyboard = keyboard
 | 
					        last_keyboard = keyboard
 | 
				
			||||||
| 
						 | 
					
 | 
				
			||||||
| 
						 | 
					@ -70,6 +70,11 @@ class AllKeyboards:
 | 
				
			||||||
base_path = os.path.join(os.getcwd(), "keyboards") + os.path.sep
 | 
					base_path = os.path.join(os.getcwd(), "keyboards") + os.path.sep
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					@lru_cache(maxsize=1)
 | 
				
			||||||
 | 
					def keyboard_alias_definitions():
 | 
				
			||||||
 | 
					    return json_load(Path('data/mappings/keyboard_aliases.hjson'))
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
def is_all_keyboards(keyboard):
 | 
					def is_all_keyboards(keyboard):
 | 
				
			||||||
    """Returns True if the keyboard is an AllKeyboards object.
 | 
					    """Returns True if the keyboard is an AllKeyboards object.
 | 
				
			||||||
    """
 | 
					    """
 | 
				
			||||||
| 
						 | 
					@ -112,7 +117,7 @@ def keyboard_folder(keyboard):
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    This checks aliases and DEFAULT_FOLDER to resolve the actual path for a keyboard.
 | 
					    This checks aliases and DEFAULT_FOLDER to resolve the actual path for a keyboard.
 | 
				
			||||||
    """
 | 
					    """
 | 
				
			||||||
    aliases = json_load(Path('data/mappings/keyboard_aliases.hjson'))
 | 
					    aliases = keyboard_alias_definitions()
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    while keyboard in aliases:
 | 
					    while keyboard in aliases:
 | 
				
			||||||
        last_keyboard = keyboard
 | 
					        last_keyboard = keyboard
 | 
				
			||||||
| 
						 | 
					
 | 
				
			||||||
| 
						 | 
					@ -6,7 +6,7 @@ import fnmatch
 | 
				
			||||||
import logging
 | 
					import logging
 | 
				
			||||||
import re
 | 
					import re
 | 
				
			||||||
from typing import List, Tuple
 | 
					from typing import List, Tuple
 | 
				
			||||||
from dotty_dict import dotty
 | 
					from dotty_dict import dotty, Dotty
 | 
				
			||||||
from milc import cli
 | 
					from milc import cli
 | 
				
			||||||
 | 
					
 | 
				
			||||||
from qmk.util import parallel_map
 | 
					from qmk.util import parallel_map
 | 
				
			||||||
| 
						 | 
					@ -107,13 +107,22 @@ def expand_keymap_targets(targets: List[Tuple[str, str]]) -> List[Tuple[str, str
 | 
				
			||||||
    return list(sorted(set(overall_targets)))
 | 
					    return list(sorted(set(overall_targets)))
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					def _construct_build_target_kb_km(e):
 | 
				
			||||||
 | 
					    return KeyboardKeymapBuildTarget(keyboard=e[0], keymap=e[1])
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					def _construct_build_target_kb_km_json(e):
 | 
				
			||||||
 | 
					    return KeyboardKeymapBuildTarget(keyboard=e[0], keymap=e[1], json=e[2])
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					
 | 
				
			||||||
def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str] = []) -> List[BuildTarget]:
 | 
					def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str] = []) -> List[BuildTarget]:
 | 
				
			||||||
    """Filter a list of (keyboard, keymap) tuples based on the supplied filters.
 | 
					    """Filter a list of (keyboard, keymap) tuples based on the supplied filters.
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    Optionally includes the values of the queried info.json keys.
 | 
					    Optionally includes the values of the queried info.json keys.
 | 
				
			||||||
    """
 | 
					    """
 | 
				
			||||||
    if len(filters) == 0:
 | 
					    if len(filters) == 0:
 | 
				
			||||||
        targets = [KeyboardKeymapBuildTarget(keyboard=kb, keymap=km) for kb, km in target_list]
 | 
					        cli.log.info('Preparing target list...')
 | 
				
			||||||
 | 
					        targets = list(parallel_map(_construct_build_target_kb_km, target_list))
 | 
				
			||||||
    else:
 | 
					    else:
 | 
				
			||||||
        cli.log.info('Parsing data for all matching keyboard/keymap combinations...')
 | 
					        cli.log.info('Parsing data for all matching keyboard/keymap combinations...')
 | 
				
			||||||
        valid_keymaps = [(e[0], e[1], dotty(e[2])) for e in parallel_map(_load_keymap_info, target_list)]
 | 
					        valid_keymaps = [(e[0], e[1], dotty(e[2])) for e in parallel_map(_load_keymap_info, target_list)]
 | 
				
			||||||
| 
						 | 
					@ -172,7 +181,9 @@ def _filter_keymap_targets(target_list: List[Tuple[str, str]], filters: List[str
 | 
				
			||||||
                cli.log.warning(f'Unrecognized filter expression: {filter_expr}')
 | 
					                cli.log.warning(f'Unrecognized filter expression: {filter_expr}')
 | 
				
			||||||
                continue
 | 
					                continue
 | 
				
			||||||
 | 
					
 | 
				
			||||||
        targets = [KeyboardKeymapBuildTarget(keyboard=e[0], keymap=e[1], json=e[2]) for e in valid_keymaps]
 | 
					        cli.log.info('Preparing target list...')
 | 
				
			||||||
 | 
					        valid_keymaps = [(e[0], e[1], e[2].to_dict() if isinstance(e[2], Dotty) else e[2]) for e in valid_keymaps]  # need to convert dotty_dict back to dict because it doesn't survive parallelisation
 | 
				
			||||||
 | 
					        targets = list(parallel_map(_construct_build_target_kb_km_json, list(valid_keymaps)))
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    return targets
 | 
					    return targets
 | 
				
			||||||
 | 
					
 | 
				
			||||||
| 
						 | 
					
 | 
				
			||||||
		Loading…
	
	Add table
		Add a link
		
	
		Reference in a new issue