From 2f91c5023b758de32858998d8c120302aa1c722f Mon Sep 17 00:00:00 2001 From: Edwin Z <20777515+Lucien950@users.noreply.github.com> Date: Thu, 22 May 2025 15:10:31 -0700 Subject: [PATCH 1/4] ui callback location --- bootloader.py | 43 +++++++++++++++++++++++-------------------- update.py | 34 ++++++++++++++++++---------------- 2 files changed, 41 insertions(+), 36 deletions(-) diff --git a/bootloader.py b/bootloader.py index 5a8235b..83d4421 100644 --- a/bootloader.py +++ b/bootloader.py @@ -40,13 +40,13 @@ class Bootloader: ih: intelhex.IntelHex board: boards.Board timeout: int - ui_callback: Callable + # ui_callback: Callable def __init__( self, bus: can.Bus, board: boards.Board, - ui_callback: Callable, + # ui_callback: Callable, ih: intelhex.IntelHex = None, timeout: int = 5, ) -> None: @@ -54,7 +54,7 @@ def __init__( self.ih: intelhex.IntelHex = ih self.board: boards.Board = board self.timeout: int = timeout - self.ui_callback: Callable = ui_callback + # self.ui_callback: Callable = ui_callback def goto_bootloader(self) -> bool: """ @@ -130,7 +130,7 @@ def _validator(msg: can.Message) -> bool: self._await_can_msg(validator=_validator, timeout=self.timeout) is not None ) - def erase_sectors(self, sectors) -> bool: + def erase_sectors(self, sectors, ui_callback: Optional[Callable] = None) -> bool: """ Erase specific sectors of FLASH, sets all bytes to 0xFF. FLASH memory must first be erased before it can be programmed. @@ -154,8 +154,8 @@ def _validator(msg: can.Message): erase_progress = 0 for sector in sectors: - if self.ui_callback: - self.ui_callback("Erasing FLASH sectors", erase_size, erase_progress) + if ui_callback is not None: + ui_callback("Erasing FLASH sectors", erase_size, erase_progress) if sector.write_protect: raise RuntimeError("Attempted to write to a readonly memory sector!") @@ -173,12 +173,12 @@ def _validator(msg: can.Message): erase_progress += sector.size - if self.ui_callback: - self.ui_callback("Erasing FLASH sectors", erase_size, erase_size) + if ui_callback is not None: + ui_callback("Erasing FLASH sectors", erase_size, erase_size) return True - def program(self) -> None: + def program(self, ui_callback: Optional[Callable] = None) -> None: """ Program the binary into flash. There is no CAN handshake here to reduce latency during programming. Also, the bootloader will verify the app's code is valid @@ -188,8 +188,8 @@ def program(self) -> None: for i, address in enumerate( range(self.ih.minaddr(), self.ih.minaddr() + self.size_bytes(), 8) ): - if self.ui_callback and i % 128 == 0: - self.ui_callback("Programming data", self.size_bytes(), i * 8) + if ui_callback is not None and i % 128 == 0: + ui_callback("Programming data", self.size_bytes(), i * 8) data = [self.ih[address + i] for i in range(0, 8)] @@ -207,8 +207,8 @@ def program(self) -> None: success = True except can.interfaces.vector.exceptions.VectorOperationError: pass - if self.ui_callback: - self.ui_callback("Programming data", self.size_bytes(), self.size_bytes()) + if ui_callback is not None: + ui_callback("Programming data", self.size_bytes(), self.size_bytes()) def status(self) -> Optional[int]: """ @@ -247,7 +247,7 @@ def _validator(msg: can.Message): return rx_msg.data[0] - def update(self) -> None: + def update(self, ui_callback: Optional[Callable] = None) -> None: """ Run the update procedure for this bootloader. @@ -284,7 +284,8 @@ def _intersect(a_min, a_max, b_min, b_max): self.program() time.sleep(0.5) - self.ui_callback("Verifying programming", self.size_bytes(), 0) + if ui_callback is not None: + ui_callback("Verifying programming", self.size_bytes(), 0) boot_status = self.status() if boot_status is not None: if boot_status != BOOT_STATUS_APP_VALID: @@ -296,10 +297,11 @@ def _intersect(a_min, a_max, b_min, b_max): f"Bootloader for {self.board.name} did not respond to command to verify application integrity." ) - self.ui_callback("Verifying programming", self.size_bytes(), self.size_bytes()) + if ui_callback is not None: + ui_callback("Verifying programming", self.size_bytes(), self.size_bytes()) time.sleep(0.5) - def erase(self) -> None: + def erase(self, ui_callback: Optional[Callable] = None) -> None: """ Erase this bootloader's application. @@ -322,7 +324,8 @@ def erase(self) -> None: ) time.sleep(0.5) - self.ui_callback("Verifying erase", erase_size, 0) + if ui_callback is not None: + ui_callback("Verifying erase", erase_size, 0) boot_status = self.status() if boot_status is not None: if boot_status != BOOT_STATUS_NO_APP: @@ -333,8 +336,8 @@ def erase(self) -> None: raise RuntimeError( f"Bootloader for {self.board.name} did not respond to command to erase flash." ) - - self.ui_callback("Verifying erase", erase_size, erase_size) + if ui_callback is not None: + ui_callback("Verifying erase", erase_size, erase_size) time.sleep(0.5) def _await_can_msg( diff --git a/update.py b/update.py index 2d96bdb..a4b5d14 100644 --- a/update.py +++ b/update.py @@ -69,17 +69,10 @@ def all_goto_app(live: Live, bootloaders: List[bootloader.Bootloader]): def update(configs: List[boards.Board], build_dir: str) -> None: """Update and handle UI.""" num_boards = len(configs) - steps_task = progress.add_task("Steps") bootloaders: List[bootloader.Bootloader] = [ bootloader.Bootloader( bus=bus, board=board, - ui_callback=lambda description, total, completed: progress.update( - task_id=steps_task, - total=total, - description=description, - completed=completed, - ), ih=intelhex.IntelHex(os.path.join(build_dir, board.path)), ) for board in configs @@ -92,6 +85,7 @@ def update(configs: List[boards.Board], build_dir: str) -> None: live.console.log( f"Updating firmware for boards: [blue bold]{', '.join(board.name for board in configs)}" ) + steps_task = progress.add_task("Steps") for b_idx, bootload_board in enumerate(bootloaders): # TODO do this in parallel progress.update( @@ -103,7 +97,14 @@ def update(configs: List[boards.Board], build_dir: str) -> None: status.update( f"Updating board [yellow]{b_idx + 1}/{num_boards}[/]: [blue bold]{bootload_board.board.name}" ) - bootload_board.update() + bootload_board.update( + ui_callback=lambda description, total, completed: progress.update( + task_id=steps_task, + total=total, + description=description, + completed=completed, + ) + ) live.console.log(f"[green]{bootload_board.board.name} updated successfully") progress.remove_task(steps_task) live.console.log( @@ -117,17 +118,10 @@ def erase(configs: List[boards.Board]) -> None: """Erase and handle UI.""" # push all boards into bootloader num_boards = len(configs) - steps_task = progress.add_task("Steps") bootloaders = [ bootloader.Bootloader( bus=bus, board=board, - ui_callback=lambda description, total, completed: progress.update( - task_id=steps_task, - total=total, - description=description, - completed=completed, - ), ) for board in configs ] @@ -138,13 +132,21 @@ def erase(configs: List[boards.Board]) -> None: live.console.log( f"Erasing with config: [blue bold]{', '.join(board.name for board in configs)}" ) + steps_task = progress.add_task("Steps") for b_idx, bootloader_board in enumerate(bootloaders): # TODO do this in parallel status.update(f"Sending board {bootloader_board.board.name} to bootloader") status.update( f"Erasing board [yellow]{b_idx + 1}/{num_boards}[/]: [blue bold]{bootloader_board.board.name}" ) - bootloader_board.erase() + bootloader_board.erase( + ui_callback=lambda description, total, completed: progress.update( + task_id=steps_task, + total=total, + description=description, + completed=completed, + ) + ) live.console.log( f"[green]{bootloader_board.board.name} erased successfully" ) From 4bdd8e356a9ca04fbf96ab9b51cca1e1656ac340 Mon Sep 17 00:00:00 2001 From: Edwin Z <20777515+Lucien950@users.noreply.github.com> Date: Thu, 22 May 2025 15:29:27 -0700 Subject: [PATCH 2/4] parallelize --- update.py | 96 +++++++++++++++++++++++++------------------------------ 1 file changed, 44 insertions(+), 52 deletions(-) diff --git a/update.py b/update.py index a4b5d14..87b553b 100644 --- a/update.py +++ b/update.py @@ -7,6 +7,7 @@ import argparse import os +from concurrent.futures import ThreadPoolExecutor from typing import List import can @@ -27,7 +28,7 @@ ) -def all_goto_bootloader(live: Live, bootloaders: List[bootloader.Bootloader]): +def all_goto_bootloader(bootloaders: List[bootloader.Bootloader], live: Live): live.console.log("Putting all boards into bootloader mode") # first put everybody into bootloader mode bootload_task = progress.add_task("Jump to Bootloader") @@ -46,7 +47,7 @@ def all_goto_bootloader(live: Live, bootloaders: List[bootloader.Bootloader]): live.console.log(f"[bold green]All boards pushed into bootloader mode successfully") -def all_goto_app(live: Live, bootloaders: List[bootloader.Bootloader]): +def all_goto_app(bootloaders: List[bootloader.Bootloader], live: Live): live.console.log("Pushing all boards out of bootloader mode") app_task = progress.add_task("Jump to App") for b_idx, bootload_board in enumerate(bootloaders): @@ -66,9 +67,24 @@ def all_goto_app(live: Live, bootloaders: List[bootloader.Bootloader]): ) +def update_board(bootload_board: bootloader.Bootloader, live: Live): + steps_task = progress.add_task( + f"Updating board [blue bold]{bootload_board.board.name}" + ) + bootload_board.update( + ui_callback=lambda description, total, completed: progress.update( + task_id=steps_task, + total=total, + description=description, + completed=completed, + ) + ) + live.console.log(f"[green]{bootload_board.board.name} updated successfully") + progress.remove_task(steps_task) + + def update(configs: List[boards.Board], build_dir: str) -> None: """Update and handle UI.""" - num_boards = len(configs) bootloaders: List[bootloader.Bootloader] = [ bootloader.Bootloader( bus=bus, @@ -81,43 +97,38 @@ def update(configs: List[boards.Board], build_dir: str) -> None: # push all boards into bootloader with Live(Group(status, progress), transient=True) as live: # push all boards into bootloader - all_goto_bootloader(live, bootloaders) + all_goto_bootloader(bootloaders, live) live.console.log( f"Updating firmware for boards: [blue bold]{', '.join(board.name for board in configs)}" ) - steps_task = progress.add_task("Steps") - for b_idx, bootload_board in enumerate(bootloaders): - # TODO do this in parallel - progress.update( - task_id=steps_task, - total=0, - completed=0, - description=f"Starting update for {bootload_board.board.name}", - ) - status.update( - f"Updating board [yellow]{b_idx + 1}/{num_boards}[/]: [blue bold]{bootload_board.board.name}" - ) - bootload_board.update( - ui_callback=lambda description, total, completed: progress.update( - task_id=steps_task, - total=total, - description=description, - completed=completed, - ) - ) - live.console.log(f"[green]{bootload_board.board.name} updated successfully") - progress.remove_task(steps_task) + with ThreadPoolExecutor(max_workers=(len(configs))) as executor: + executor.map(update_board, [(b, live) for b in bootloaders]) live.console.log( - f"[bold green]Firmware update successfully ({num_boards} board{'s' if num_boards > 1 else ''} updated)" + f"[bold green]Firmware update successfully ({len(configs)} board{'s' if len(configs) > 1 else ''} updated)" ) # push all boards out of bootloader - all_goto_app(live, bootloaders) + all_goto_app(bootloaders, live) + + +def erase_board(bootloader_board: bootloader.Bootloader, live: Live): + steps_task = progress.add_task( + f"Erasing board [blue bold]{bootloader_board.board.name}" + ) + bootloader_board.erase( + ui_callback=lambda description, total, completed: progress.update( + task_id=steps_task, + total=total, + description=description, + completed=completed, + ) + ) + live.console.log(f"[green]{bootloader_board.board.name} erased successfully") + progress.remove_task(steps_task) def erase(configs: List[boards.Board]) -> None: """Erase and handle UI.""" # push all boards into bootloader - num_boards = len(configs) bootloaders = [ bootloader.Bootloader( bus=bus, @@ -125,34 +136,15 @@ def erase(configs: List[boards.Board]) -> None: ) for board in configs ] - with Live(Group(status, progress), transient=True) as live: - all_goto_bootloader(live, bootloaders) - + all_goto_bootloader(bootloaders, live) live.console.log( f"Erasing with config: [blue bold]{', '.join(board.name for board in configs)}" ) - steps_task = progress.add_task("Steps") - for b_idx, bootloader_board in enumerate(bootloaders): - # TODO do this in parallel - status.update(f"Sending board {bootloader_board.board.name} to bootloader") - status.update( - f"Erasing board [yellow]{b_idx + 1}/{num_boards}[/]: [blue bold]{bootloader_board.board.name}" - ) - bootloader_board.erase( - ui_callback=lambda description, total, completed: progress.update( - task_id=steps_task, - total=total, - description=description, - completed=completed, - ) - ) - live.console.log( - f"[green]{bootloader_board.board.name} erased successfully" - ) - progress.remove_task(steps_task) + with ThreadPoolExecutor(max_workers=(len(configs))) as executor: + executor.map(update_board, [(b, live) for b in bootloaders]) live.console.log( - f"[bold green]Erase successful ({num_boards} board{'s' if num_boards > 1 else ''} erased)" + f"[bold green]Erase successful ({len(configs)} board{'s' if len(configs) > 1 else ''} erased)" ) From ef91ae0a79472a46b59b23b4c668a88305f25182 Mon Sep 17 00:00:00 2001 From: Edwin Z <20777515+Lucien950@users.noreply.github.com> Date: Thu, 22 May 2025 15:31:02 -0700 Subject: [PATCH 3/4] remove comments --- bootloader.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/bootloader.py b/bootloader.py index 83d4421..0140586 100644 --- a/bootloader.py +++ b/bootloader.py @@ -40,13 +40,11 @@ class Bootloader: ih: intelhex.IntelHex board: boards.Board timeout: int - # ui_callback: Callable def __init__( self, bus: can.Bus, board: boards.Board, - # ui_callback: Callable, ih: intelhex.IntelHex = None, timeout: int = 5, ) -> None: @@ -54,7 +52,6 @@ def __init__( self.ih: intelhex.IntelHex = ih self.board: boards.Board = board self.timeout: int = timeout - # self.ui_callback: Callable = ui_callback def goto_bootloader(self) -> bool: """ From 1d62b6ba46664e752480f38b5d4626d14f1ead24 Mon Sep 17 00:00:00 2001 From: Edwin Z <20777515+Lucien950@users.noreply.github.com> Date: Thu, 22 May 2025 15:36:06 -0700 Subject: [PATCH 4/4] use thread safe bus --- update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/update.py b/update.py index 87b553b..3cb7fb1 100644 --- a/update.py +++ b/update.py @@ -181,7 +181,7 @@ def erase(configs: List[boards.Board]) -> None: for board in boards.CONFIGS[config_name.strip()] } ) - with can.interface.Bus( + with can.ThreadSafeBus( interface=args.bus, channel=args.channel, bitrate=args.bit_rate ) as bus: if args.erase: