Add composable config system with includes support

scripts/cluster_utils.py

@@ -1,5 +1,6 @@
 """Utility functions for interacting with clusters."""
 
+import copy
 import os
 import subprocess
 from dataclasses import dataclass
@@ -74,13 +75,56 @@ def config_to_cmd_flags(cfg: RunConfig) -> str:
     return args_and_flags_str
 
 
+def _deep_merge(base: Dict[str, Any], override: Dict[str, Any]) -> Dict[str, Any]:
+    """Recursively merge override into base. Lists are concatenated, dicts are
+    merged, scalars are overwritten by override."""
+    result = copy.deepcopy(base)
+    for key, value in override.items():
+        if key in result:
+            if isinstance(result[key], dict) and isinstance(value, dict):
+                result[key] = _deep_merge(result[key], value)
+            elif isinstance(result[key], list) and isinstance(value, list):
+                result[key] = result[key] + value
+            else:
+                result[key] = value
+        else:
+            result[key] = copy.deepcopy(value)
+    return result
+
+
+def _resolve_config(config_filepath: str) -> Dict[str, Any]:
+    """Load a single YAML file and recursively resolve its 'includes'."""
+    with open(config_filepath, "r", encoding="utf-8") as f:
+        config = yaml.safe_load(f) or {}
+    includes = config.pop("includes", [])
+    # Resolve includes relative to the directory of the including file.
+    base_dir = os.path.dirname(config_filepath)
+    merged: Dict[str, Any] = {}
+    for inc_path in includes:
+        inc_filepath = os.path.join(base_dir, inc_path)
+        inc_config = _resolve_config(inc_filepath)
+        merged = _deep_merge(merged, inc_config)
+    # The including file's own keys override the included ones.
+    merged = _deep_merge(merged, config)
+    return merged
+
+
 def parse_configs(config_filename: str) -> Iterator[Dict[str, Any]]:
-    """Parse the YAML config file."""
+    """Parse the YAML config file, resolving any 'includes' directives."""
     scripts_dir = os.path.dirname(os.path.realpath(__file__))
     configs_dir = os.path.join(scripts_dir, "configs")
     config_filepath = os.path.join(configs_dir, config_filename)
+    # If the file uses includes, it must be a single document (no multi-doc).
+    # Try resolving includes first; fall back to multi-doc for legacy files.
     with open(config_filepath, "r", encoding="utf-8") as f:
-        for config in yaml.safe_load_all(f):
+        raw_docs = list(yaml.safe_load_all(f))
+    for raw_config in raw_docs:
+        if raw_config and "includes" in raw_config:
+            yield _resolve_config(config_filepath)
+            return  # includes-based files are single-document
+    # Legacy path: no includes, yield each document as-is.
+    for config in raw_docs:
+        if config is not None:
             yield config
 
 

scripts/configs/predicatorv3/agents.yaml

@@ -0,0 +1,7 @@
+# Composed config: agent approaches
+# Usage: python scripts/local/launch_simp.py -c predicatorv3/agents.yaml
+---
+includes:
+  - common.yaml
+  - approaches/agents.yaml
+  - envs/all.yaml

scripts/configs/predicatorv3/approaches/agents.yaml

@@ -0,0 +1,37 @@
+APPROACHES:
+  # agent_planner:
+  #   NAME: "agent_planner"
+  #   FLAGS:
+  #     explorer: "agent"
+  #     demonstrator: "oracle_process_planning"
+  #     terminate_on_goal_reached_and_option_terminated: True
+  #     agent_sdk_use_local_sandbox: True
+  #     option_model_terminate_on_repeat: False
+  #     agent_sdk_max_agent_turns_per_iteration: 50
+  #     agent_planner_use_scratchpad: False
+  #     agent_planner_use_visualize_state: True
+  #     agent_planner_use_annotate_scene: True
+  #     option_model_use_gui: True
+  agent_bilevel:
+    NAME: "agent_bilevel"
+    FLAGS:
+      explorer: "agent"
+      demonstrator: "oracle_process_planning"
+      terminate_on_goal_reached_and_option_terminated: True
+      agent_sdk_use_local_sandbox: True
+      option_model_terminate_on_repeat: False
+      agent_sdk_max_agent_turns_per_iteration: 50
+      agent_planner_use_scratchpad: False
+      agent_planner_use_visualize_state: True
+      agent_planner_use_annotate_scene: True
+      option_model_use_gui: True
+      agent_bilevel_log_state: False
+  # agent_option_learning:
+  #   NAME: "agent_option_learning"
+  #   FLAGS:
+  #     explorer: "agent"
+  #     option_learner: "agent"
+  #     demonstrator: "oracle_process_planning"
+  #     terminate_on_goal_reached_and_option_terminated: True
+  #     agent_sdk_use_local_sandbox: True
+  #     agent_sdk_max_agent_turns_per_iteration: 50

scripts/configs/predicatorv3/approaches/oracle.yaml

@@ -0,0 +1,15 @@
+APPROACHES:
+  oracle:
+    NAME: "oracle_process_planning"
+    FLAGS:
+      demonstrator: "oracle_process_planning"
+      terminate_on_goal_reached_and_option_terminated: True
+      bilevel_plan_without_sim: True
+  # human_interaction:
+  #   NAME: "human_interaction"
+  #   FLAGS:
+  #     human_interaction_approach_use_scripted_option: True
+  #     human_interaction_approach_use_all_options: True
+  #     scripted_option_dir: "scripted_option_policies"
+  #     skill_phase_use_motion_planning: True
+  #     terminate_on_goal_reached_and_option_terminated: True

scripts/configs/predicatorv3/common.yaml

@@ -0,0 +1,32 @@
+ARGS:
+  - "debug"
+  # - "use_gui"
+  - "make_failure_videos"
+  - "make_test_videos"
+  # - "make_demo_videos"
+  # - "make_demo_images"  # support images
+  # - "make_failure_images"  # query images
+  # - "make_test_images"  # query images
+  # - "save_atoms"
+FLAGS:
+  max_initial_demos: 0
+  num_online_learning_cycles: 0
+  online_nsrt_learning_requests_per_cycle: 1
+  skill_phase_use_motion_planning: True
+  max_num_steps_interaction_request: 300
+  pretrained_model_service_provider: "openrouter"
+  llm_model_name: "google/gemini-2.5-pro"
+  llm_openai_max_response_tokens: 1e6
+  terminate_on_goal_reached: False
+  pybullet_ik_validate: False
+  num_train_tasks: 1
+  num_test_tasks: 1
+  video_fps: 20
+  pybullet_camera_height: 900
+  pybullet_camera_width: 900
+  planning_filter_unreachable_nsrt: False
+  timeout: 600
+  log: 'logs/'
+  no_repeated_arguments_in_grounding: True
+START_SEED: 0
+NUM_SEEDS: 1

scripts/configs/predicatorv3/envs/all.yaml

@@ -0,0 +1,64 @@
+ENVS:
+  # grow:
+  #   NAME: "pybullet_grow"
+  #   FLAGS:
+  #     excluded_predicates: "HandTilted"
+  #     coffee_use_pixelated_jug: True
+  #     grow_weak_pour_terminate_condition: True
+  #     grow_place_option_no_sampler: True
+  #     horizon: 400
+  # domino:
+  #   NAME: "pybullet_domino"
+  #   FLAGS:
+  #     excluded_objects_in_state_str: "loc,rot,angle,direction"
+  #     horizon: 200
+  #     domino_initialize_at_finished_state: False
+  #     domino_use_domino_blocks_as_target: True
+  #     domino_use_grid: True
+  #     domino_include_connected_predicate: False
+  #     domino_use_continuous_place: True
+  #     domino_restricted_push: True
+  #     domino_prune_actions: False
+  #     process_planning_heuristic_weight: 2.0
+  #     process_planning_use_abstract_policy: False
+  #     domino_has_glued_dominos: False
+  #     keep_failed_demos: True
+  #     env_has_impossible_goals: True
+  #     process_param_learning_use_empirical: True
+  #     process_learning_use_empirical: True
+  #     predicate_invent_invent_derived_predicates: True
+  #     script_option_file_name: "domino2.txt"
+  # coffee:
+  #   NAME: "pybullet_coffee"
+  #   FLAGS:
+  #     excluded_predicates: "Twisting,PressingButton,NotSameCup,RobotAboveCup"
+  #     coffee_rotated_jug_ratio: 0
+  #     coffee_num_cups_train: [1]
+  #     coffee_machine_have_light_bar: False
+  #     coffee_move_back_after_place_and_push: True
+  #     coffee_machine_has_plug: False
+  #     coffee_combined_move_and_twist_policy: True
+  #     coffee_use_pixelated_jug: True
+  #     coffee_fill_jug_gradually: True
+  #     max_num_steps_option_rollout: 100
+  #     horizon: 300
+  #     script_option_file_name: "coffee.txt"
+  boil:
+    NAME: "pybullet_boil"
+    FLAGS:
+      excluded_objects_in_state_str: "switch"
+      max_num_steps_option_rollout: 100
+      horizon: 500
+      boil_goal: "simple"
+      boil_require_jug_full_to_heatup: True
+      script_option_file_name: "boil.txt"
+      boil_water_fill_speed: 0.0015
+      pybullet_birrt_path_subsample_ratio: 2
+  # fan:
+  #   NAME: "pybullet_fan"
+  #   FLAGS:
+  #     excluded_objects_in_state_str: "switch"
+  #     terminate_on_goal_reached: True
+  #     process_planning_heuristic_weight: 10.0
+  #     horizon: 500
+  #     pybullet_birrt_path_subsample_ratio: 2

scripts/configs/predicatorv3/oracle.yaml

@@ -0,0 +1,7 @@
+# Composed config: oracle approach
+# Usage: python scripts/local/launch_simp.py -c predicatorv3/oracle.yaml
+---
+includes:
+  - common.yaml
+  - approaches/oracle.yaml
+  - envs/all.yaml

scripts/configs/predicatorv3/predicator_v3.yaml

@@ -1,6 +1,5 @@
-# Config file for generating the mara_bench videos
-# Excample run: 
-#    python scripts/local/launch_simp.py -c mara_bench.yaml
+# Legacy monolithic config (approaches + envs in one file)
+# Usage: python scripts/local/launch_simp.py -c predicatorv3/predicator_v3.yaml
 ---
 APPROACHES:
   # human_interaction:
@@ -16,13 +15,12 @@ APPROACHES:
       demonstrator: "oracle_process_planning"
       terminate_on_goal_reached_and_option_terminated: True
       bilevel_plan_without_sim: True
-  # mf_agent:
+  # agent_planner:
   #   NAME: "agent_planner"
   #   FLAGS:
   #     explorer: "agent"
   #     demonstrator: "oracle_process_planning"
   #     terminate_on_goal_reached_and_option_terminated: True
-  #     agent_planner_isolate_test_session: True
   #     # agent_sdk_use_docker_sandbox: True
   #     agent_sdk_use_local_sandbox: True
   #     option_model_terminate_on_repeat: False
@@ -31,14 +29,27 @@ APPROACHES:
   #     agent_planner_use_visualize_state: True
   #     agent_planner_use_annotate_scene: True
   #     option_model_use_gui: True
-  # mf_option_learning_agent:
+  # agent_bilevel:
+  #   NAME: "agent_bilevel"
+  #   FLAGS:
+  #     explorer: "agent"
+  #     demonstrator: "oracle_process_planning"
+  #     terminate_on_goal_reached_and_option_terminated: True
+  #     # agent_sdk_use_docker_sandbox: True
+  #     agent_sdk_use_local_sandbox: True
+  #     option_model_terminate_on_repeat: False
+  #     agent_sdk_max_agent_turns_per_iteration: 50
+  #     agent_planner_use_scratchpad: False
+  #     agent_planner_use_visualize_state: True
+  #     agent_planner_use_annotate_scene: True
+  #     option_model_use_gui: True
+  # agent_option_learning:
   #   NAME: "agent_option_learning"
   #   FLAGS:
   #     explorer: "agent"
   #     option_learner: "agent"
   #     demonstrator: "oracle_process_planning"
   #     terminate_on_goal_reached_and_option_terminated: True
-  #     agent_planner_isolate_test_session: True
   #     # agent_sdk_use_docker_sandbox: True
   #     agent_sdk_use_local_sandbox: True
   #     agent_sdk_max_agent_turns_per_iteration: 50
@@ -54,7 +65,7 @@ APPROACHES:
   #     online_nsrt_learning_requests_per_cycle: 1
 ENVS:
   # grow:
-  #   NAME: "mara_grow"
+  #   NAME: "pybullet_grow"
   #   FLAGS:
   #     excluded_predicates: "HandTilted"
   #     coffee_use_pixelated_jug: True
@@ -99,7 +110,7 @@ ENVS:
   #     horizon: 300
   #     script_option_file_name: "coffee.txt"
   boil:
-    NAME: "mara_boil"
+    NAME: "pybullet_boil"
     FLAGS:
       excluded_objects_in_state_str: "switch"
       max_num_steps_option_rollout: 100
@@ -124,7 +135,7 @@ ENVS:
   #     pybullet_birrt_path_subsample_ratio: 2
 ARGS:
   - "debug"
-  - "use_gui"
+  # - "use_gui"
   - "make_failure_videos"
   - "make_test_videos"
   # - "make_demo_videos"