diff --git a/prototype/args.py b/prototype/args.py new file mode 100644 index 0000000..9bb22a0 --- /dev/null +++ b/prototype/args.py @@ -0,0 +1,42 @@ +class Args: + def __init__(self, argv): + self.__argv = argv + + + def get_positional(self, index): + if len(self.__argv) > index: + return self.__argv[index] + return None + + + def get_named(self, name, index, nr_args): + found_name = False + values = [] + + for s in self.__argv: + if found_name: + if nr_args > 0: + nr_args -= 1 + values.append(s) + continue + else: + break + + if s == '--{}'.format(name): + if index > 0: + index -= 1 + continue + + found_name = True + continue + + if nr_args == 0: + return found_name + + if nr_args > 0 or not found_name: + return None + + if len(values) == 1: + return values[0] + + return values diff --git a/prototype/build.py b/prototype/build.py new file mode 100644 index 0000000..321355b --- /dev/null +++ b/prototype/build.py @@ -0,0 +1,146 @@ +import os +import shutil +from env import Environment +from path import Path +from pathlib import Path as SysPath + + +class UnknownBuildSystemError(Exception): + pass + + +class BuildConfigurationError(Exception): + pass + + +class BuildError(Exception): + pass + + +class BuildSystem: + def __init__(self, **kwargs): + self.__name = None + self.__source_dir = None + self.__bin_dir = None + self.__parent = None + self.__install = None + + if 'name' in kwargs: + self.__name = kwargs['name'] + if 'source_dir' in kwargs: + self.__source_dir = kwargs['source_dir'] + if 'bin_dir' in kwargs: + self.__bin_dir = kwargs['bin_dir'] + if 'parent' in kwargs: + self.__parent = kwargs['parent'] + if 'install' in kwargs: + self.__install = kwargs['install'] + + + def name(self): + return self.__name + + + def source_dir(self): + return self.__source_dir + + + def binary_dir(self): + return self.__bin_dir + + + def parent(self): + return self.__parent + + + def install_manifest(self): + return self.__install + + + def build_for(self): + return 'target' + + + def get_path_namespaces(self): + ns = { + 'src': os.path.abspath(self.source_dir()), + 'root-src': Environment.source_root(), + 'bin': os.path.abspath(self.binary_dir()), + } + + if self.build_for() == 'host': + ns['install'] = Environment.native_root() + else: + ns['install'] = Environment.system_root() + + return ns + + + def check_out_of_date(self): + self.__config_out_of_date = self._is_configuration_out_of_date() + self.__build_out_of_date = self._is_build_out_of_date() + + + def is_configuration_out_of_date(self): + return self.__config_out_of_date + + + def is_build_out_of_date(self): + return self.__build_out_of_date + + + def configure(self): + return + + + def build(self): + return + + + def __copy_dir(self, src, dst): + dst_dir = os.path.dirname(dst) + SysPath(dst_dir).mkdir(parents=True, exist_ok=True) + shutil.copytree(src, dst, dirs_exist_ok=True) + + + def __copy_file(self, src, dst): + dst_dir = os.path.dirname(dst) + SysPath(dst_dir).mkdir(parents=True, exist_ok=True) + shutil.copyfile(src, dst) + + + def install_from_seed(self): + manifest = self.install_manifest() + if manifest is None: + return + + ns = self.get_path_namespaces() + source_dir = self.source_dir() + + dest_dir = Environment.system_root() + if self.build_for() == 'host': + dest_dir = Environment.native_root() + + for dest, src in manifest.items(): + if dest[0] != '/': + continue + + dest = Path(dest).remove_root().get_path() + + for p in src: + path = Path(p) + files = path.expand_path(relative_to=ns) + + for p2 in files: + file_name = os.path.basename(p2) + rel_path = os.path.relpath(p2, start=source_dir) + src_path = p2 + dst_path = os.path.join(dest_dir, dest, file_name) + if os.path.isdir(src_path): + self.__copy_dir(src_path, dst_path) + else: + self.__copy_file(src_path, dst_path) + + + def install(self): + return diff --git a/prototype/builtin.py b/prototype/builtin.py new file mode 100644 index 0000000..4eb8672 --- /dev/null +++ b/prototype/builtin.py @@ -0,0 +1,237 @@ +import os +import shutil +from command import Command +from config import Config +from cache import Cache +from seed import Seed +from target import Target +from env import Environment +from platform import Platform +from colorama import Fore, Style + + +class SetCommand(Command): + def __init__(self): + super().__init__( + name='set', + description='Set a configuration key value.') + + + def run(self, args): + key = args.get_positional(0) + value = args.get_positional(1) + + if key is None or value is None: + print('usage: meadow set ') + return -1 + + try: + value_i = int(value) + Config.set(key, value_i) + except: + Config.set(key, value) + + return 0 + + +class SeedsCommand(Command): + def __init__(self): + super().__init__( + name='seeds', + description='List all available seeds.') + + + def write_dot(self, path): + f = open(path, mode='w') + f.write('digraph seeds {\n') + f.write(' node [shape=Mrecord];\n') + nodes = {} + + seeds = Seed.all_seeds() + targets = Target.all_targets() + for s in seeds.values(): + if s.name() in nodes: + continue + + sname = s.name().replace('-', '_') + nodes[s.name()] = s + f.write(' {} [label="{}"];\n'.format(sname, s.name())) + + for d in s.dependencies().values(): + continue + + + for t in targets.values(): + if t.name() in nodes: + continue + + t.resolve() + tname = t.name().replace('-', '_') + nodes[t.name()] = t + f.write(' {} [label="{}",shape=hexagon];\n'.format(tname, t.name())) + + for s in seeds.values(): + for k, d in s.dependencies().items(): + if d.name() not in nodes: + nodes[d.name()] = d + dname = d.name().replace('-', '_') + f.write(' {} [label="{}",shape=diamond];\n'.format(dname, d.name())) + + sname = s.name().replace('-', '_') + dname = d.name().replace('-', '_') + + if d.is_resolved(): + f.write(' {} -> {};\n'.format(dname, sname)) + else: + f.write(' {} -> {}[style=dotted];\n'.format(dname, sname)) + + for t in targets.values(): + for k, d in t.dependencies().items(): + tname = t.name().replace('-', '_') + dname = d.name().replace('-', '_') + + if d.is_resolved(): + f.write(' {} -> {};\n'.format(dname, tname)) + else: + f.write(' {} -> {}[style=dotted];\n'.format(dname, tname)) + + f.write('}\n') + f.close() + + + def run(self, args): + seeds = Seed.all_seeds() + + dot_path = args.get_named('to-dot-file', 0, 1) + if dot_path is not None: + self.write_dot(dot_path) + return 0 + + for s in seeds.values(): + print('{}'.format(s.name())) + + +class PlatformsCommand(Command): + def __init__(self): + super().__init__( + name='platforms', + description='List all available platforms') + + + def run(self, args): + print('Supported platforms:') + + platforms = Platform.all_platforms() + current_platform = Platform.current_platform() + + for p in platforms.values(): + if current_platform is not None and p.name() == current_platform.name(): + print(f'* {Fore.GREEN}{p.name()}{Style.RESET_ALL}') + else: + print(f' {p.name()}') + + +class HelpCommand(Command): + def __init__(self): + super().__init__( + name='help', + description='List all available commands.') + + + def run(self, args): + print('Available commands:') + + commands = Command.all_commands() + max_len = 0 + + for c in commands.values(): + if len(c.name()) > max_len: + max_len = len(c.name()) + + max_len += 5 + for c in commands.values(): + print(' ', end='') + print(c.name(), end='') + + for i in range(len(c.name()), max_len): + print(' ', end='') + + print(c.description()) + + +class BuildCommand(Command): + def __init__(self): + super().__init__( + name='build', + description='Build the currently selected target.') + + + def run(self, args): + verbose = args.get_named('verbose', 0, 0) + + current_target = Target.current_target() + if current_target is None: + print(f'{Fore.RED}Err:{Style.RESET_ALL} no build target has been set. Use `meadow set target.name <...>` to specify which target to build') + return -1 + + current_platform = Platform.current_platform() + if current_platform is None: + print(f'{Fore.RED}Err:{Style.RESET_ALL} no target platform has been set. Use `meadow set target.platform <...>` to specify which platform to target') + return -1 + + components = current_target.resolve() + + for c in components: + build_system = None + if isinstance(c, Seed): + build_system = c.build_system() + build_system.check_out_of_date() + + for c in components: + build_system = None + if isinstance(c, Seed): + build_system = c.build_system() + + if build_system is not None: + out_of_date = \ + build_system.is_configuration_out_of_date() \ + or build_system.is_build_out_of_date() + + if out_of_date: + print(f'{Fore.GREEN}>>>{Style.RESET_ALL} Building {c.name()}') + + if build_system.is_configuration_out_of_date(): + print(f'{Fore.CYAN} >>{Style.RESET_ALL} Configuring build...') + build_system.configure(verbose=verbose) + + if out_of_date: + print(f'{Fore.CYAN} >>{Style.RESET_ALL} Building...') + build_system.build(verbose=verbose) + print(f'{Fore.CYAN} >>{Style.RESET_ALL} Installing...') + build_system.install(verbose=verbose) + + print(f'{Style.BRIGHT}{Fore.GREEN}>>> Build Complete!{Style.RESET_ALL}') + + +class CleanCommand(Command): + def __init__(self): + super().__init__( + name='clean', + description='Delete all build-related files.') + + + def run(self, args): + build_dir = Environment.build_root() + + for i in os.listdir(build_dir): + if i.startswith('meadow') and ('cache' not in i): + continue + + ipath = os.path.join(build_dir, i) + if os.path.isdir(ipath): + shutil.rmtree(ipath) + else: + os.remove(ipath) + + Cache.reset() + return 0 diff --git a/prototype/cache.py b/prototype/cache.py new file mode 100644 index 0000000..3a90295 --- /dev/null +++ b/prototype/cache.py @@ -0,0 +1,69 @@ +import json +import os + + +cache_path = 'build/meadow-cache.json' +cache_data = None + + +class Cache: + def load(): + global cache_data + + if not os.path.exists(cache_path): + cache_data = {} + return + + with open(cache_path, mode='r') as f: + cache_data = json.load(f) + + + def reset(): + global cache_data + cache_data = {} + + + def save(): + global cache_data + + with open(cache_path, mode='w') as f: + json.dump(cache_data, f, indent=4) + + + def get(path): + parts = path.split('.') + container = cache_data + + for p in parts[:-1]: + if p not in container: + return None + + if not isinstance(container[p], dict): + print('Err: element "{}" in cache key "{}" is not a dictionary'.format(p, path)) + return + + container = container[p] + + + if parts[-1] in container: + return container[parts[-1]] + + return None + + + def set(path, value): + parts = path.split('.') + container = cache_data + + for p in parts[:-1]: + if p not in container: + container[p] = {} + + if not isinstance(container[p], dict): + print('Err: element "{}" in cache key "{}" is not a dictionary'.format(p, path)) + return + + container = container[p] + + + container[parts[-1]] = value diff --git a/prototype/cmake.py b/prototype/cmake.py new file mode 100644 index 0000000..eafc911 --- /dev/null +++ b/prototype/cmake.py @@ -0,0 +1,316 @@ +import os +import json +import subprocess +from build import * +from pathlib import Path as SysPath +from cache import Cache +from env import Environment +from platform import Platform +from target import Target +from path import Path + +class CMakeBuildSystem(BuildSystem): + def __init__(self, **kwargs): + name = kwargs.get('name', None) + source_dir = kwargs.get('source_dir', None) + bin_dir = kwargs.get('bin_dir', None) + parent = kwargs.get('parent', None) + self.__data = kwargs.get('data', None) + + super().__init__(**kwargs) + + + def build_for(self): + if self.__data is None or 'build_for' not in self.__data: + return 'target' + return self.__data['build_for'] + + + def __get_watched_files(self): + result = [] + ns = self.get_path_namespaces() + + if 'watch_files' in self.__data: + watch_list = self.__data['watch_files'] + for p in watch_list: + path = Path(p) + + real_paths = path.expand_path(relative_to=ns) + for p2 in real_paths: + result.append(p2) + + manifest_path = os.path.join(self.binary_dir(), 'compile_commands.json') + manifest_file = None + manifest = None + try: + manifest_file = open(manifest_path, mode='r') + except: + return result + + try: + manifest = json.load(manifest_file) + except: + return result + + for v in manifest: + if 'file' not in v: + continue + + path = v['file'] + result.append(path) + + return result + + + def __get_build_mtime(self): + files = self.__get_watched_files() + result = 0 + + for path in files: + mtime = os.path.getmtime(path) + + if mtime > result: + result = mtime + + return result + + + def __get_configure_mtime(self): + source_dir = os.path.abspath(self.source_dir()) + result = 0 + for root, dirs, files in os.walk(source_dir): + for f in files: + if f != 'CMakeLists.txt' and not f.endswith('.cmake'): + continue + + target_path = os.path.join(root, f) + mtime = os.path.getmtime(target_path) + if mtime > result: + result = mtime + + return result + + + def _is_configuration_out_of_date(self): + cache_file = os.path.join(self.binary_dir(), 'CMakeCache.txt') + if not os.path.isfile(cache_file): + return True + + current_mtime = self.__get_configure_mtime() + target_mtime = Cache.get(f'component.{self.name()}.configure-mtime') + if target_mtime is None: + target_mtime = 0 + + if current_mtime > target_mtime: + return True + + for d in self.parent().collect_dependencies(): + build_system = d.build_system() + if build_system is None: + continue + + if build_system.is_build_out_of_date() or build_system.is_configuration_out_of_date(): + return True + + return False + + + def _is_build_out_of_date(self): + current_mtime = self.__get_build_mtime() + target_mtime = Cache.get(f'component.{self.name()}.build-mtime') + if target_mtime is None: + target_mtime = 0 + + if current_mtime > target_mtime: + return True + + for d in self.parent().collect_dependencies(): + build_system = d.build_system() + if build_system is None: + continue + + if build_system.is_build_out_of_date() or build_system.is_configuration_out_of_date(): + return True + + return False + + + def __get_published_items(self): + published = {} + + for d in self.parent().collect_dependencies(): + build_sys = d.build_system() + ns = {} + if build_sys is not None: + ns = build_sys.get_path_namespaces() + + p = d.publish() + if p is None: + continue + + path = p.path() + if path is not None: + if 'path' not in published: + published['path'] = [] + + path = Path.expand_all(path, relative_to=ns) + published['path'].extend(path) + + cmake = p.cmake() + if cmake is not None: + if 'cmake' not in published: + published['cmake'] = {} + + if 'module_paths' in cmake: + if 'module_paths' not in published['cmake']: + published['cmake']['module_paths'] = [] + + module_paths = cmake['module_paths'] + module_paths = Path.expand_all(module_paths, relative_to=ns) + published['cmake']['module_paths'].extend(module_paths) + + return published + + + def configure(self, **kwargs): + verbose = kwargs.get('verbose', False) + ns = self.get_path_namespaces() + stdout = None + if not verbose: + stdout = subprocess.DEVNULL + + target = Target.current_target() + platform = Platform.current_platform() + cmake_platform = platform.get_component('cmake') + cmake_args = [ + 'cmake', + '-DCMAKE_EXPORT_COMPILE_COMMANDS=ON', + '--no-warn-unused-cli' + ] + + if 'configure_args' in self.__data: + cmake_args.extend(self.__data['configure_args']) + + published = self.__get_published_items() + env = os.environ.copy() + if 'path' in published: + path = '' + if 'PATH' in env: + path = env['PATH'] + + if len(path): + path += ':' + + path += ':'.join(published['path']) + env['PATH'] = path + + module_paths = [] + if 'cmake' in published: + cmake_publish = published['cmake'] + for p in cmake_publish.get('module_paths', []): + path = Path(p) + for p2 in path.expand_path(relative_to=ns): + module_paths.append(p2) + + install_details = self.parent().install() + install_prefix = '' + if self.build_for() == 'host': + install_prefix = Environment.native_root() + cmake_platform = None + seed_install_prefix = '/' + if install_details is not None: + seed_install_prefix = install_details.get('prefix', seed_install_prefix) + seed_install_prefix = Path(seed_install_prefix).remove_root().get_path() + install_prefix = os.path.join(install_prefix, seed_install_prefix) + else: + install_prefix = Environment.system_root() + env['ROSETTA_SYSROOT'] = Environment.system_root() + + seed_install_prefix = target.install_prefix() + if install_details is not None: + seed_install_prefix = install_details.get('prefix', seed_install_prefix) + seed_install_prefix = Path(seed_install_prefix).remove_root().get_path() + install_prefix = os.path.join(install_prefix, seed_install_prefix) + + cmake_args.append(f'-DCMAKE_INSTALL_PREFIX={install_prefix}') + + if cmake_platform is not None: + if 'module_paths' in cmake_platform: + for p in cmake_platform['module_paths']: + module_paths.append(os.path.join(platform.base_directory(), p)) + if 'toolchain_file' in cmake_platform: + toolchain_file = os.path.join(platform.base_directory(), cmake_platform['toolchain_file']) + cmake_args.append(f'-DCMAKE_TOOLCHAIN_FILE={toolchain_file}') + if 'system_name' in cmake_platform: + cmake_args.append(f'-DCMAKE_SYSTEM_NAME={cmake_platform['system_name']}') + + if 'module_paths' in self.__data: + for p in self.__data['module_paths']: + path = Path(p) + for p2 in path.expand_path(relative_to=ns): + module_paths.append(p2) + + if len(module_paths) > 0: + cmake_args.append(f'-DCMAKE_MODULE_PATH={';'.join(module_paths)}') + + source_dir = os.path.abspath(self.source_dir()) + bin_dir = os.path.abspath(self.binary_dir()) + cmake_args.append(source_dir) + + SysPath(bin_dir).mkdir(parents=True, exist_ok=True) + result = subprocess.run(cmake_args, cwd=bin_dir, env=env, stdout=stdout) + + if result.returncode == 0: + Cache.set(f'component.{self.name()}.configure-mtime', self.__get_configure_mtime()) + else: + raise BuildConfigurationError(f'CMake configuration for {self.name()} failed. See log for details') + + + def __find_build_command(self): + bin_dir = os.path.abspath(self.binary_dir()) + if os.path.isfile(os.path.join(bin_dir, 'build.ninja')): + return 'ninja' + + return None + + + def build(self, **kwargs): + verbose = kwargs.get('verbose', False) + stdout = None + if not verbose: + stdout = subprocess.DEVNULL + + bin_dir = os.path.abspath(self.binary_dir()) + build_cmd = self.__find_build_command() + if build_cmd is None: + raise BuildError(f'Cannot determine which command to use to build {self.name()}') + + env = os.environ.copy() + if self.build_for() != 'host': + env['ROSETTA_SYSROOT'] = Environment.system_root() + + args = [ build_cmd ] + result = subprocess.run(args, cwd=bin_dir, stdout=stdout, env=env) + + if result.returncode != 0: + raise BuildConfigurationError(f'{build_cmd} build for {self.name()} failed. See log for details') + + + def install(self, **kwargs): + self.install_from_seed() + + verbose = kwargs.get('verbose', False) + stdout = None + if not verbose: + stdout = subprocess.DEVNULL + + bin_dir = os.path.abspath(self.binary_dir()) + build_cmd = self.__find_build_command() + if build_cmd is None: + raise BuildError(f'Cannot determine which command to use to build {self.name()}') + + args = [ build_cmd, 'install' ] + result = subprocess.run(args, cwd=bin_dir, stdout=stdout) + + if result.returncode == 0: + Cache.set(f'component.{self.name()}.build-mtime', self.__get_build_mtime()) diff --git a/prototype/command.py b/prototype/command.py new file mode 100644 index 0000000..de60b78 --- /dev/null +++ b/prototype/command.py @@ -0,0 +1,205 @@ +import os +import sys +import tomli +import tempfile +import subprocess +from dependency import * +from colorama import Fore, Style +from env import Environment +from path import Path +from config import Config + + +commands = {} + + +class CommandConfigurationError(Exception): + pass + + +class CommandExecutionError(Exception): + pass + + +class Command(DependencyItem): + def all_commands(): + return commands + + + def __init__(self, **kwargs): + super().__init__() + + if 'name' in kwargs: + self.__name = kwargs['name'] + if 'description' in kwargs: + self.__description = kwargs['description'] + + + def name(self): + return self.__name + + + def description(self): + return self.__description + + + def dependencies(self): + return None + + + def register(cmd): + commands[cmd.name()] = cmd + + + def get(name): + if name in commands: + return commands[name] + return None + + + def run(self, args): + return + + +class UserCommand(Command): + def load_file(self, path): + with open(path, mode='rb') as f: + self.__data = tomli.load(f) + + + def __init_dependencies(self): + if 'dependency' not in self.__data: + return + + dependency = self.__data['dependency'] + self.__dependencies = {} + if 'seeds' in dependency: + seeds = dependency['seeds'] + for s in seeds: + self.__dependencies[s] = SeedDependency(parent=self.name(), name=s) + + if 'commands' in dependency: + commands = dependency['commands'] + for c in commands: + self.__dependencies[c] = ProgramDependency(parent=self.name(), name=c) + + + def __init__(self, **kwargs): + if 'path' not in kwargs: + return + + self.__tmpdir = tempfile.TemporaryDirectory() + self.__path = kwargs['path'] + self.load_file(self.__path) + cmd = self.__data['command'] + super().__init__(name=cmd['name'], description=cmd['description']) + + self.__init_dependencies() + + + def dependencies(self): + return self.__dependencies + + + def scan_directory(paths): + for p in paths: + for root, dirs, files in os.walk(p): + for f in files: + if not f.endswith('.command'): + continue + cmd_path = os.path.join(root, f) + cmd = None + try: + cmd = UserCommand(path=cmd_path) + except tomli.TOMLDecodeError as e: + print('Err: Failed to parse command file {}\n {}'.format(seed_path, e)) + return -1 + + Command.register(cmd) + + + def get_path_namespaces(self): + ns = { + 'src': os.path.dirname(self.__path), + 'root-src': Environment.source_root(), + 'install': Environment.system_root(), + } + + return ns + + + def get_environment(self): + return { + 'MEADOW_SYSROOT': Environment.system_root(), + 'MEADOW_NATIVE_ROOT': Environment.native_root(), + 'MEADOW_SOURCE_ROOT': Environment.source_root(), + 'MEADOW_TEMP': self.__tmpdir.name, + 'MEADOW_TARGET_PLATFORM': Config.get('target.platform'), + } + + + def run(self, args): + from seed import Seed + deps = self.resolve() + + for c in deps: + build_system = None + if isinstance(c, Seed): + build_system = c.build_system() + build_system.check_out_of_date() + + for c in deps: + build_system = None + if isinstance(c, Seed): + build_system = c.build_system() + + if build_system is not None: + out_of_date = \ + build_system.is_configuration_out_of_date() \ + or build_system.is_build_out_of_date() + + if out_of_date: + print(f'{Fore.GREEN}>>>{Style.RESET_ALL} Building {c.name()}') + + if build_system.is_configuration_out_of_date(): + print(f'{Fore.CYAN} >>{Style.RESET_ALL} Configuring build...') + build_system.configure(verbose=False) + + if out_of_date: + print(f'{Fore.CYAN} >>{Style.RESET_ALL} Building...') + build_system.build(verbose=False) + print(f'{Fore.CYAN} >>{Style.RESET_ALL} Installing...') + build_system.install(verbose=False) + + exec_info = self.__data.get('exec', None) + if exec_info is None: + raise CommandConfigurationError(f'Command {self.name()} has no executable configured') + + args = exec_info.get('args', None) + if args is None or len(args) == 0: + raise CommandConfigurationError(f'Command {self.name()} has no executable configured') + + ns = self.get_path_namespaces() + expanded_args = [] + for p in args: + parts = p.split(':') + if len(parts) < 2 or parts[0] not in ns: + expanded_args.append(p) + continue + + paths = Path(p).expand_path(relative_to=ns) + expanded_args.extend(paths) + + if expanded_args[0].endswith('.py'): + expanded_args.insert(0, sys.executable) + + env = os.environ | self.get_environment() + result = None + + try: + result = subprocess.run(expanded_args, env=env) + except KeyboardInterrupt: + pass + + if result is not None and result.returncode != 0: + raise CommandExecutionError(f'Execution of {self.name()} failed. See log for details') diff --git a/prototype/config.py b/prototype/config.py new file mode 100644 index 0000000..81771a0 --- /dev/null +++ b/prototype/config.py @@ -0,0 +1,64 @@ +import json +import os + + +config_path = 'build/meadow-config.json' +config_data = None + + +class Config: + def load(): + global config_data + + if not os.path.exists(config_path): + config_data = {} + return + + with open(config_path, mode='r') as f: + config_data = json.load(f) + + + def save(): + global config_data + + with open(config_path, mode='w') as f: + json.dump(config_data, f, indent=4) + + + def get(path): + parts = path.split('.') + container = config_data + + for p in parts[:-1]: + if p not in container: + return None + + if not isinstance(container[p], dict): + print('Err: element "{}" in config key "{}" is not a dictionary'.format(p, path)) + return + + container = container[p] + + + if parts[-1] in container: + return container[parts[-1]] + + return None + + + def set(path, value): + parts = path.split('.') + container = config_data + + for p in parts[:-1]: + if p not in container: + container[p] = {} + + if not isinstance(container[p], dict): + print('Err: element "{}" in config key "{}" is not a dictionary'.format(p, path)) + return + + container = container[p] + + + container[parts[-1]] = value diff --git a/prototype/dependency.py b/prototype/dependency.py new file mode 100644 index 0000000..472b7f6 --- /dev/null +++ b/prototype/dependency.py @@ -0,0 +1,156 @@ +import os + + +class DependencyResolutionError(Exception): + pass + + +class DependencyItem: + def __init__(self): + self.__target = None + self.__dep_list = None + + + def name(self): + return None + + + def target(self): + return self.__target + + + def dependencies(self): + return None + + + def dependency_list(self): + return self.__dep_list + + + def publish(self): + return None + + + def reset_resolution(self): + self.__dep_list = None + + + def resolve(self, **kwargs): + if self.__dep_list is not None: + return self.__dep_list + + ctx = None + dep_list = None + if 'ctx' in kwargs: + ctx = kwargs['ctx'] + else: + ctx = {} + + if 'dep_list' in kwargs: + dep_list = kwargs['dep_list'] + else: + dep_list = [] + + if self.name() in ctx: + return + + ctx[self.name()] = self + + deps = self.dependencies() + if deps is None: + return + + for d in deps.values(): + d_value = d.resolve_self() + if d_value is not None: + d_value.resolve(ctx=ctx, dep_list=dep_list) + + dep_list.append(self) + return dep_list + + +class Dependency: + def __init__(self, parent): + self.__parent = parent + + + def name(self, parent): + return None + + + def parent_name(self): + return self.__parent + + + def resolve_self(self): + return None + + + def is_resolved(self): + return False + + +class SeedDependency(Dependency): + def __init__(self, **kwargs): + self.__target_seed = None + if 'name' in kwargs: + self.__name = kwargs['name'] + if 'parent' in kwargs: + super().__init__(kwargs['parent']) + + + def name(self): + return self.__name + + + def target(self): + return self.__target_seed + + + def resolve_self(self): + from seed import Seed + if self.__target_seed is not None: + return self.__target_seed + + target = Seed.get(self.__name) + if target is None: + raise DependencyResolutionError( + '{} depends on seed "{}", but the required seed cannot be found'.format(self.parent_name(), self.__name)) + + self.__target_seed = target + return self.__target_seed + + + def is_resolved(self): + return self.__target_seed is not None + + +class ProgramDependency(Dependency): + def __init__(self, **kwargs): + if 'name' in kwargs: + self.__name = kwargs['name'] + if 'parent' in kwargs: + super().__init__(kwargs['parent']) + + + def __is_exe(self, fpath): + return os.path.isfile(fpath) and os.access(fpath, os.X_OK) + + + def name(self): + return self.__name + + + def resolve_self(self): + if self.__is_exe(self.__name): + return None + + path_string = os.environ['PATH'] + path_list = path_string.split(':') + for d in path_list: + exe_path = os.path.join(d, self.__name) + if self.__is_exe(exe_path): + return None + + raise DependencyResolutionError(f'{self.parent_name()} depends on command `{self.__name}`, but the command cannot be found. Make sure that the named command exists, the containing directory is in $PATH if necessary, and the executable file has the correct permissions.') + diff --git a/prototype/env.py b/prototype/env.py new file mode 100644 index 0000000..714da18 --- /dev/null +++ b/prototype/env.py @@ -0,0 +1,28 @@ +import os + + +source_root = None + + +class Environment: + def set_source_root(path): + global source_root + source_root = path + + + def source_root(): + global source_root + return source_root + + + def build_root(): + global source_root + return os.path.join(source_root, 'build') + + + def system_root(): + return os.path.join(Environment.build_root(), 'target-root') + + + def native_root(): + return os.path.join(Environment.build_root(), 'host-root') diff --git a/prototype/generic.py b/prototype/generic.py new file mode 100644 index 0000000..1832202 --- /dev/null +++ b/prototype/generic.py @@ -0,0 +1,140 @@ +import os +import json +import shutil +import subprocess +from build import * +from pathlib import Path as SysPath +from cache import Cache +from env import Environment +from platform import Platform +from target import Target +from path import Path + +class GenericBuildSystem(BuildSystem): + def __init__(self, **kwargs): + self.__data = kwargs.get('data', None) + super().__init__(**kwargs) + + + def __get_watched_files(self): + result = [] + ns = self.get_path_namespaces() + + if 'watch_files' in self.__data: + watch_list = self.__data['watch_files'] + for p in watch_list: + path = Path(p) + + real_paths = path.expand_path(relative_to=ns) + for p2 in real_paths: + result.append(p2) + + manifest_path = os.path.join(self.binary_dir(), 'compile_commands.json') + manifest_file = None + manifest = None + try: + manifest_file = open(manifest_path, mode='r') + except: + return result + + try: + manifest = json.load(manifest_file) + except: + return result + + for v in manifest: + if 'file' not in v: + continue + + path = v['file'] + result.append(path) + + return result + + + def __get_mtime(self): + files = self.__get_watched_files() + result = 0 + + for path in files: + mtime = os.path.getmtime(path) + + if mtime > result: + result = mtime + + return result + + + def _is_configuration_out_of_date(self): + return False + + + def _is_build_out_of_date(self): + current_mtime = self.__get_mtime() + target_mtime = Cache.get(f'component.{self.name()}.build-mtime') + if target_mtime is None: + return True + + if current_mtime > target_mtime: + return True + + for d in self.parent().collect_dependencies(): + build_system = d.build_system() + if build_system is None: + continue + + if build_system.is_build_out_of_date() or build_system.is_configuration_out_of_date(): + return True + + return False + + + def __run_command(self, name, **kwargs): + verbose = kwargs.get('verbose', False) + ns = self.get_path_namespaces() + stdout = None + if not verbose: + stdout = subprocess.DEVNULL + + cmd = self.__data.get(name, None) + if cmd is None: + return + + args = cmd.get('args', None) + if args is None: + raise BuildConfigurationError(f'{self.name()} is incorrectly configured. build_command has no args defined') + + + expanded_args = [] + for p in args: + parts = p.split(':') + if len(parts) < 2 or parts[0] not in ns: + expanded_args.append(p) + continue + + paths = Path(p).expand_path(relative_to=ns) + expanded_args.extend(paths) + + result = subprocess.run(expanded_args, stdout=stdout) + if result.returncode != 0: + raise BuildConfigurationError(f'Execution of {name} for {self.name()} failed. See log for details') + + + def configure(self, **kwargs): + self.__run_command('configure_command', **kwargs) + + + def build(self, **kwargs): + self.__run_command('build_command', **kwargs) + + + def install(self, **kwargs): + self.install_from_seed() + + previous_mtime = Cache.get(f'component.{self.name()}.build-mtime') + if previous_mtime is None: + previous_mtime = 0 + + new_mtime = self.__get_mtime() + if new_mtime > previous_mtime: + Cache.set(f'component.{self.name()}.build-mtime', new_mtime) diff --git a/prototype/main.py b/prototype/main.py new file mode 100644 index 0000000..c7c1b8b --- /dev/null +++ b/prototype/main.py @@ -0,0 +1,95 @@ +import sys +import os +import tomli +from seed import Seed +from target import Target +from platform import Platform +from config import Config +from cache import Cache +from command import Command, UserCommand +from args import Args +from env import Environment +from dependency import SeedDependency, DependencyResolutionError +from builtin import * +from colorama import init as colorama_init +from colorama import Fore, Style + + +def main(): + colorama_init() + Command.register(SetCommand()) + Command.register(SeedsCommand()) + Command.register(PlatformsCommand()) + Command.register(HelpCommand()) + Command.register(BuildCommand()) + Command.register(CleanCommand()) + Config.load() + Cache.load() + Environment.set_source_root(os.getcwd()) + + Seed.scan_directory([ + 'base', + 'external', + 'interface', + 'toolchain', + 'lib', + 'programs', + 'services', + 'sys' + ]) + + UserCommand.scan_directory([ + 'toolchain' + ]) + + Target.scan_directory('./target') + platforms = Platform.scan_directory('./arch') + + current_platform = Platform.current_platform() + if current_platform: + current_platform.load_extensions() + + if len(sys.argv) < 2: + print(f'{Fore.CYAN}Usage:{Style.RESET_ALL} {sys.argv[0]} [args...]') + return -1 + + command_name = sys.argv[1] + command = Command.get(command_name) + if command is None: + print(f'{Fore.RED}Err:{Style.RESET_ALL} unknown command "{command_name}"') + return -1 + + current_target = Target.current_target() + seeds = Seed.all_seeds() + + if current_target is not None: + system_dep_names = current_target.base_components() + + for s in seeds.values(): + if s.name() in system_dep_names or not s.should_include_standard_dependencies(): + continue + + for n in system_dep_names: + s.add_dependency(n, SeedDependency(parent=s.name(), name=n)) + + try: + current_target.resolve() + except DependencyResolutionError as e: + print(f'{Fore.RED}Err:{Style.RESET_ALL} {e}') + return -1 + + args = Args(sys.argv[2:]) + result = None + try: + result = command.run(args) + except Exception as e: + print(f'{Fore.RED}Err:{Style.RESET_ALL} {e}') + finally: + Cache.save() + Config.save() + + return result + + +if __name__ == '__main__': + exit(main()) diff --git a/prototype/path.py b/prototype/path.py new file mode 100644 index 0000000..92f81c2 --- /dev/null +++ b/prototype/path.py @@ -0,0 +1,88 @@ +import glob +import os + + +class Path: + def __init__(self, value): + self.__path = value + self.__namespace = None + + parts = value.split(':') + + if len(parts) != 2: + return + + if len(parts[0]) < 3: + return + + self.__namespace = parts[0] + self.__path = parts[1] + + + def get_path(self): + return self.__path + + + def remove_root(self): + while len(self.__path) > 0 and self.__path[0] == '/': + self.__path = self.__path [1:] + + if len(self.__path) == 0: + self.__path= '.' + + return self + + + def expand_path(self, **kwargs): + relative_to = None + + if 'relative_to' in kwargs: + relative_to = kwargs['relative_to'] + + if isinstance(relative_to, dict): + if self.__namespace is not None: + relative_to = relative_to[self.__namespace] + else: + relative_to = None + + paths = glob.glob(self.__path, root_dir=relative_to, recursive=True) + if len(paths) == 0 and '*' not in self.__path: + if relative_to is not None: + return [os.path.join(relative_to, self.__path)] + return [self.__path] + + if relative_to is not None: + abs_paths = [] + + for p in paths: + abs_paths.append(os.path.join(relative_to, p)) + + return abs_paths + + return paths + + + def expand_all(paths, **kwargs): + relative_to = None + + if 'relative_to' in kwargs: + relative_to = kwargs['relative_to'] + + result = [] + for p in paths: + if not isinstance(p, Path): + p = Path(p) + r = p.expand_path(relative_to=relative_to) + result.extend(r) + + return result + + + def get_namespace(self, **kwargs): + if self.__namespace is not None: + return self.__namespace + + if 'default' in kwargs: + return kwargs['default'] + + return None diff --git a/prototype/platform.py b/prototype/platform.py new file mode 100644 index 0000000..d14f64f --- /dev/null +++ b/prototype/platform.py @@ -0,0 +1,119 @@ +import os +import sys +import tomli +from config import Config +from command import Command, UserCommand + + +all_platforms = {} + + +class Platform: + def all_platforms(): + return all_platforms + + + def current_platform(): + name = Config.get('target.platform') + if not name or name not in all_platforms: + return None + + return all_platforms[name] + + + def load_file(self, path): + self.__path = path + with open(path, mode='rb') as f: + self.__data = tomli.load(f) + + + def __init__(self, **kwargs): + if 'path' in kwargs: + self.__platform_file = os.path.abspath(kwargs['path']) + self.load_file(kwargs['path']) + + + def base_directory(self): + return os.path.dirname(self.__platform_file) + + + def name(self): + if 'platform' not in self.__data: + return None + + platform = self.__data['platform'] + if 'name' in platform: + return platform['name'] + + return None + + + def arch(self): + if 'platform' not in self.__data: + return None + + platform = self.__data['platform'] + if 'arch' in platform: + return platform['arch'] + + return None + + + def load_extensions(self): + from seed import Seed + if 'extension' not in self.__data: + return + + + all_seeds = Seed.all_seeds() + + extension_dir = os.path.dirname(self.__path) + extension = self.__data['extension'] + + if 'seeds' in extension: + seeds = extension['seeds'] + for seed_filename in seeds: + seed_path = os.path.join(extension_dir, seed_filename) + seed = None + try: + seed = Seed(path=seed_path) + except tomli.TOMLDecodeError as e: + print('Err: Failed to parse seed file {}\n {}'.format(seed_path, e)) + return -1 + + all_seeds[seed.name()] = seed + + if 'commands' in extension: + commands = extension['commands'] + for cmd_filename in commands: + cmd_path = os.path.join(extension_dir, cmd_filename) + cmd = None + try: + cmd = UserCommand(path=cmd_path) + except tomli.TOMLDecodeError as e: + print('Err: Failed to parse command file {}\n {}'.format(seed_path, e)) + return -1 + + Command.register(cmd) + + + def get_component(self, name): + if name in self.__data: + return self.__data[name] + return None + + + def scan_directory(root_path): + for root, dirs, files in os.walk(root_path): + for f in files: + if not f.endswith('.platform'): + continue + platform_path = os.path.join(root, f) + platform = None + try: + platform = Platform(path=platform_path) + except tomli.TOMLDecodeError as e: + print('Err: Failed to parse platform file {}\n {}'.format(platform_path, e)) + continue + + all_platforms[platform.name()] = platform diff --git a/prototype/publish.py b/prototype/publish.py new file mode 100644 index 0000000..92bd48c --- /dev/null +++ b/prototype/publish.py @@ -0,0 +1,21 @@ +import os + + +class Publish: + def __init__(self, **kwargs): + self.__data = None + self.__install_root = None + + if 'data' in kwargs: + self.__data = kwargs['data'] + + if 'install_path' in kwargs: + self.__install_root = kwargs['install_path'] + + + def path(self): + return self.__data.get('path', None) + + + def cmake(self): + return self.__data.get('cmake', None) diff --git a/prototype/requirements.txt b/prototype/requirements.txt new file mode 100644 index 0000000..487942e --- /dev/null +++ b/prototype/requirements.txt @@ -0,0 +1,2 @@ +tomli +colorama diff --git a/prototype/seed.py b/prototype/seed.py new file mode 100644 index 0000000..5fc7428 --- /dev/null +++ b/prototype/seed.py @@ -0,0 +1,219 @@ +import os +import sys +import tomli +from dependency import * +from cmake import CMakeBuildSystem +from publish import Publish +from build import UnknownBuildSystemError +from env import Environment +from source_only import SourceOnlyBuildSystem +from generic import GenericBuildSystem + + +all_seeds = {} + + +class SeedParseError(Exception): + pass + + +class Seed(DependencyItem): + def all_seeds(): + return all_seeds + + + def get(name): + if name in all_seeds: + return all_seeds[name] + + return None + + + def load_file(self, path): + with open(path, mode='rb') as f: + self.__data = tomli.load(f) + + + def __init_dependencies(self): + self.__dependencies = {} + + if 'dependency' not in self.__data: + return + + dependency = self.__data['dependency'] + if 'seeds' in dependency: + seeds = dependency['seeds'] + self.__dependencies = {} + for s in seeds: + self.__dependencies[s] = SeedDependency(parent=self.name(), name=s) + + + def __init_build_system(self): + self.__build_system = None + if 'build' not in self.__data: + return + + build = self.__data['build'] + if 'build_system' not in build: + return + + source_dir = os.path.dirname(self.__seed_path) + bin_dir = os.path.join(Environment.build_root(), self.name()) + install = self.__data.get('install', None) + + if 'source_dir' in build: + source_dir = build['source_dir'] + + match build['build_system']: + case 'none': + self.__build_system = None + case 'cmake': + self.__build_system = CMakeBuildSystem( + parent=self, + name=self.name(), + source_dir=source_dir, + bin_dir=bin_dir, + install=install, + data=build) + case 'source-only': + self.__build_system = SourceOnlyBuildSystem( + parent=self, + name=self.name(), + source_dir=source_dir, + bin_dir=bin_dir, + install=install, + data=build) + case 'generic': + self.__build_system = GenericBuildSystem( + parent=self, + name=self.name(), + source_dir=source_dir, + bin_dir=bin_dir, + install=install, + data=build) + case _: + raise UnknownBuildSystemError( + 'Seed {} has unrecognised build system "{}"'.format(self.name(), build['build_system'])) + + + def __init_publish(self): + install_path = Environment.system_root() + if 'build' in self.__data and 'build_for' in self.__data['build']: + build_for = self.__data['build']['build_for'] + if build_for == 'host': + install_path = Environment.native_root() + + self.__publish = None + if 'publish' not in self.__data: + return + + self.__publish = Publish(install_path=install_path, data=self.__data['publish']) + + + def add_dependency(self, name, dep): + self.__dependencies[name] = dep + + + def collect_dependencies(self, **kwargs): + ctx = None + if 'ctx' in kwargs: + ctx = kwargs['ctx'] + else: + ctx = {} + + for k, d in self.__dependencies.items(): + if k in ctx: + continue + + ctx[d.name()] = d.target() + if isinstance(d.target(), Seed): + d.target().collect_dependencies(ctx=ctx) + + return list(ctx.values()) + + + def __init__(self, **kwargs): + super().__init__() + + if not 'path' in kwargs: + return + + self.__seed_path = kwargs['path'] + self.load_file(kwargs['path']) + self.__init_dependencies() + self.__init_build_system() + self.__init_publish() + + + def seed_path(self): + return self.__seed_path + + + def name(self): + if not 'seed' in self.__data: + return None + + s = self.__data['seed'] + if 'name' in s: + return s['name'] + return None + + + def type(self): + if not 'seed' in self.__data: + return None + + s = self.__data['seed'] + if 'type' in s: + return s['type'] + return 'component' + + + def dependencies(self): + return self.__dependencies + + + def publish(self): + return self.__publish + + + def install(self): + return self.__data.get('install', None) + + + def should_include_standard_dependencies(self): + if 'dependency' not in self.__data: + return True + + s = self.__data['dependency'] + if 'no_standard_dependencies' in s: + return not s['no_standard_dependencies'] + + return True + + + def build_system(self): + return self.__build_system + + + def scan_directory(paths): + for p in paths: + for root, dirs, files in os.walk(p): + for f in files: + if not f.endswith('.seed'): + continue + seed_path = os.path.join(root, f) + seed = None + try: + seed = Seed(path=seed_path) + except tomli.TOMLDecodeError as e: + raise SeedParseError('Err: Failed to parse seed file {}: {}'.format(seed_path, e)) + + all_seeds[seed.name()] = seed + + + def resolve_all(): + for s in all_seeds.values(): + deps = s.dependencies() + for k, v in deps.items(): + v.resolve() diff --git a/prototype/source_only.py b/prototype/source_only.py new file mode 100644 index 0000000..7d92db2 --- /dev/null +++ b/prototype/source_only.py @@ -0,0 +1,119 @@ +import os +import json +import shutil +import subprocess +from build import * +from pathlib import Path as SysPath +from cache import Cache +from env import Environment +from platform import Platform +from target import Target +from path import Path + +class SourceOnlyBuildSystem(BuildSystem): + def __init__(self, **kwargs): + self.__data = kwargs.get('data', None) + super().__init__(**kwargs) + + + def __get_watched_files(self): + result = [] + ns = self.get_path_namespaces() + + if 'watch_files' in self.__data: + watch_list = self.__data['watch_files'] + for p in watch_list: + path = Path(p) + + real_paths = path.expand_path(relative_to=ns) + for p2 in real_paths: + result.append(p2) + + manifest_path = os.path.join(self.binary_dir(), 'compile_commands.json') + manifest_file = None + manifest = None + try: + manifest_file = open(manifest_path, mode='r') + except: + return result + + try: + manifest = json.load(manifest_file) + except: + return result + + for v in manifest: + if 'file' not in v: + continue + + path = v['file'] + result.append(path) + + return result + + + def __get_mtime(self): + files = self.__get_watched_files() + result = 0 + + for path in files: + mtime = os.path.getmtime(path) + + if mtime > result: + result = mtime + + return result + + + def _is_configuration_out_of_date(self): + parent = self.parent() + seed_path = parent.seed_path() + seed_mtime = os.path.getmtime(seed_path) + target_mtime = Cache.get(f'component.{self.name()}.configure-mtime') + if target_mtime is None: + return True + + return seed_mtime > target_mtime + + + def _is_build_out_of_date(self): + current_mtime = self.__get_mtime() + target_mtime = Cache.get(f'component.{self.name()}.build-mtime') + if target_mtime is None: + return True + + if current_mtime > target_mtime: + return True + + for d in self.parent().collect_dependencies(): + build_system = d.build_system() + if build_system is None: + continue + + if build_system.is_build_out_of_date() or build_system.is_configuration_out_of_date(): + return True + + return False + + + def configure(self, **kwargs): + parent = self.parent() + seed_path = parent.seed_path() + seed_mtime = os.path.getmtime(seed_path) + Cache.set(f'component.{self.name()}.configure-mtime', seed_mtime) + + + def build(self, **kwargs): + pass + + + def install(self, **kwargs): + self.install_from_seed() + + previous_mtime = Cache.get(f'component.{self.name()}.build-mtime') + if previous_mtime is None: + previous_mtime = 0 + + new_mtime = self.__get_mtime() + if new_mtime > previous_mtime: + Cache.set(f'component.{self.name()}.build-mtime', new_mtime) diff --git a/prototype/target.py b/prototype/target.py new file mode 100644 index 0000000..8924b6d --- /dev/null +++ b/prototype/target.py @@ -0,0 +1,105 @@ +import os +import sys +import tomli +from dependency import * +from config import Config +from path import Path + + +all_targets = {} + + +class Target(DependencyItem): + def all_targets(): + global all_targets + return all_targets + + + def current_target(): + name = Config.get('target.name') + if not name or name not in all_targets: + return None + + return all_targets[name] + + + def load_file(self, path): + with open(path, mode='rb') as f: + self.__data = tomli.load(f) + + + def __init_dependencies(self): + self.__dependencies = {} + + if 'dependency' not in self.__data: + return + + dependency = self.__data['dependency'] + if 'seeds' in dependency: + seeds = dependency['seeds'] + self.__dependencies = {} + for s in seeds: + self.__dependencies[s] = SeedDependency(parent=self.name(), name=s) + + + def __init__(self, **kwargs): + super().__init__() + + if 'path' not in kwargs: + return + + self.load_file(kwargs['path']) + self.__init_dependencies() + + + def name(self): + if not 'target' in self.__data: + return None + + s = self.__data['target'] + if 'name' in s: + return s['name'] + return None + + + def dependencies(self): + return self.__dependencies + + + def base_components(self): + if not 'target' in self.__data: + return None + + s = self.__data['target'] + if 'base_components' in s: + return s['base_components'] + + return [] + + + def install_prefix(self, **kwargs): + make_relative = kwargs.get('make_relative', False) + path = self.__data.get('target', {}).get('default_prefix', '/') + + if make_relative: + path = Path(path).remove_root().get_path + + return path + + + def scan_directory(root_path): + global all_targets + + for root, dirs, files in os.walk(root_path): + for f in files: + if not f.endswith('.target'): + continue + target_path = os.path.join(root, f) + target = None + try: + target = Target(path=target_path) + except tomli.TOMLDecodeError as e: + print('Err: Failed to parse target file {}\n {}'.format(target_path, e)) + continue + + all_targets[target.name()] = target