Mechanical Design Recipe Portfolio#

Source: examples/mechanical_design_recipe_portfolio.py

Introduction#

Build and run a compact portfolio of study recipes around mechanical design trade-offs such as bracket stiffness, truss weight, and manufacturability.

Technical Implementation#

  1. Define a mechanical benchmark bundle plus recipe/config objects for bivariate and architecture-focused studies.

  2. Materialize a compact bivariate study, execute it with deterministic mock agents, and export canonical analysis tables.

  3. Print the generated study IDs and exported analysis artifacts for the mechanical-design portfolio.

  1from __future__ import annotations
  2
  3from pathlib import Path
  4
  5import design_research_experiments as drex
  6
  7
  8def _mechanical_bundle() -> drex.BenchmarkBundle:
  9    """Return a small mechanical-design benchmark bundle."""
 10    return drex.BenchmarkBundle(
 11        bundle_id="mechanical-design",
 12        name="Mechanical Design Bundle",
 13        description=(
 14            "Bracket, truss, and enclosure redesign tasks for structural trade-off studies."
 15        ),
 16        problem_ids=("cantilever-bracket-redesign", "space-truss-joint"),
 17        agent_specs=("baseline-cad-agent", "constraint-aware-agent"),
 18        metadata={"domain": "mechanical-engineering"},
 19    )
 20
 21
 22def _problem_registry(problem_ids: tuple[str, ...]) -> dict[str, drex.ProblemPacket]:
 23    """Return a deterministic problem registry for mechanical design tasks."""
 24
 25    def evaluator(output: dict[str, object]) -> list[dict[str, object]]:
 26        """Score one synthetic mechanical-design response."""
 27        text = str(output.get("text", ""))
 28        mass_score = round(min(0.95, 0.55 + len(text.split()) / 100.0), 4)
 29        return [{"metric_name": "primary_outcome", "metric_value": mass_score}]
 30
 31    registry: dict[str, drex.ProblemPacket] = {}
 32    for problem_id in problem_ids:
 33        registry[problem_id] = drex.ProblemPacket(
 34            problem_id=problem_id,
 35            family="mechanical-design",
 36            brief=f"Design a lighter, stiffer concept for {problem_id}.",
 37            evaluator=evaluator,
 38        )
 39    return registry
 40
 41
 42def _agent_factory(agent_name: str):
 43    """Create a deterministic mechanical-design agent callable."""
 44
 45    def _agent(
 46        *,
 47        problem_packet: drex.ProblemPacket,
 48        run_spec: drex.RunSpec,
 49        condition: drex.Condition,
 50    ) -> dict[str, object]:
 51        """Return one deterministic run payload for the requested condition."""
 52        comparison_arm = str(condition.factor_assignments.get("comparison_arm", "baseline"))
 53        prompt_regime = str(condition.factor_assignments.get("prompt_regime", "standard"))
 54        improvement_bonus = 0.07 if comparison_arm == "treatment" else 0.0
 55        structure_bonus = 0.03 if prompt_regime == "structured" else 0.0
 56        agent_bonus = 0.04 if agent_name == "constraint-aware-agent" else 0.0
 57        primary_outcome = round(0.58 + improvement_bonus + structure_bonus + agent_bonus, 4)
 58        text = (
 59            f"{agent_name} reviewed {problem_packet.problem_id} with "
 60            f"comparison_arm={comparison_arm} prompt_regime={prompt_regime} seed={run_spec.seed}"
 61        )
 62        return {
 63            "output": {"text": text},
 64            "metrics": {"primary_outcome": primary_outcome, "latency_s": 1.2},
 65            "events": [
 66                {"event_type": "assistant_output", "text": text, "actor_id": agent_name},
 67            ],
 68            "metadata": {"model_name": "example-mechanical-model"},
 69        }
 70
 71    return _agent
 72
 73
 74def _portfolio_extensions() -> dict[str, object]:
 75    """Return study-building hooks that extend the mechanical portfolio."""
 76    return {
 77        "agent_architecture_config": drex.AgentArchitectureComparisonConfig,
 78        "benchmark_bundle": drex.BenchmarkBundle,
 79        "bivariate_config": drex.BivariateComparisonConfig,
 80        "block": drex.Block,
 81        "comparison_config": drex.ComparisonStudyConfig,
 82        "constraint": drex.Constraint,
 83        "diversity_config": drex.DiversityAndExplorationConfig,
 84        "export_analysis_tables": drex.export_analysis_tables,
 85        "grammar_scaffold_config": drex.GrammarScaffoldConfig,
 86        "human_vs_agent_config": drex.HumanVsAgentProcessConfig,
 87        "recipe_config": drex.RecipeStudyConfig,
 88        "resume_study": drex.resume_study,
 89        "run_result_type": drex.RunResult,
 90        "univariate_config": drex.UnivariateComparisonConfig,
 91    }
 92
 93
 94def main() -> None:
 95    """Build and execute a compact mechanical-design recipe portfolio."""
 96    mechanical_bundle = _mechanical_bundle()
 97    fabrication_block = drex.Block(name="mechanical_family", levels=("bracket", "truss"))
 98    manufacturability_constraint = drex.Constraint(
 99        constraint_id="structured-treatment-pairing",
100        description="Treatment runs must use a structured prompt regime.",
101        expression='comparison_arm != "treatment" or prompt_regime == "structured"',
102    )
103
104    portfolio_template = drex.RecipeStudyConfig(
105        bundle=mechanical_bundle,
106        output_dir=Path("artifacts") / "mechanical-design-portfolio",
107    )
108    comparison_template = drex.ComparisonStudyConfig(
109        bundle=mechanical_bundle,
110        blocks=(fabrication_block,),
111    )
112    bivariate_study = drex.build_bivariate_comparison_study(
113        drex.BivariateComparisonConfig(
114            bundle=mechanical_bundle,
115            blocks=(fabrication_block,),
116            constraints=(manufacturability_constraint,),
117            run_budget=drex.RunBudget(replicates=1, parallelism=1, max_runs=4),
118            output_dir=Path("artifacts") / "mechanical-bivariate-study",
119        )
120    )
121    architecture_study = drex.build_agent_architecture_comparison_study(
122        drex.AgentArchitectureComparisonConfig(
123            bundle=mechanical_bundle,
124            blocks=(fabrication_block,),
125            output_dir=Path("artifacts") / "mechanical-architecture-study",
126        )
127    )
128    univariate_study = drex.build_univariate_comparison_study(
129        drex.UnivariateComparisonConfig(
130            bundle=mechanical_bundle,
131            blocks=(fabrication_block,),
132        )
133    )
134
135    problem_registry = _problem_registry(bivariate_study.problem_ids)
136    agent_bindings = {
137        agent_name: (lambda _condition, *, _agent_name=agent_name: _agent_factory(_agent_name))
138        for agent_name in bivariate_study.agent_specs
139    }
140    conditions = drex.build_design(bivariate_study)
141    run_results = drex.run_study(
142        bivariate_study,
143        conditions=conditions,
144        agent_bindings=agent_bindings,
145        problem_registry=problem_registry,
146        show_progress=False,
147    )
148    exported_paths = drex.export_analysis_tables(
149        bivariate_study,
150        conditions=conditions,
151        run_results=run_results,
152        output_dir=bivariate_study.output_dir / "analysis",
153    )
154
155    print("Portfolio template output dir:", portfolio_template.output_dir)
156    print("Comparison template type:", type(comparison_template).__name__)
157    print(
158        "Studies:", bivariate_study.study_id, architecture_study.study_id, univariate_study.study_id
159    )
160    print("Completed runs:", len(run_results))
161    print("RunResult objects:", all(isinstance(result, drex.RunResult) for result in run_results))
162    print("Exported analysis tables:", ", ".join(path.name for path in exported_paths.values()))
163    print("Portfolio extension hooks:", ", ".join(sorted(_portfolio_extensions())))
164
165
166if __name__ == "__main__":
167    main()

Expected Results#

Run Command

PYTHONPATH=src python examples/mechanical_design_recipe_portfolio.py

The script prints the recipe study IDs, the number of completed runs, and the artifact filenames exported for downstream analysis.