From 5f32aeaff13ce03b16684824aca7d06744b450a2 Mon Sep 17 00:00:00 2001 From: jerelvelarde Date: Thu, 26 Mar 2026 12:13:16 -0700 Subject: [PATCH] docs: add usage and integration guide Series of markdown files covering setup, architecture, agent state, generative UI, tools, human-in-the-loop, MCP, deployment, and how to bring these patterns into your own application. --- docs/README.md | 25 ++++ docs/agent-state.md | 154 +++++++++++++++++++++++ docs/agent-tools.md | 141 +++++++++++++++++++++ docs/architecture.md | 90 ++++++++++++++ docs/bring-to-your-app.md | 256 ++++++++++++++++++++++++++++++++++++++ docs/deployment.md | 98 +++++++++++++++ docs/generative-ui.md | 169 +++++++++++++++++++++++++ docs/getting-started.md | 71 +++++++++++ docs/human-in-the-loop.md | 104 ++++++++++++++++ docs/mcp-integration.md | 115 +++++++++++++++++ 10 files changed, 1223 insertions(+) create mode 100644 docs/README.md create mode 100644 docs/agent-state.md create mode 100644 docs/agent-tools.md create mode 100644 docs/architecture.md create mode 100644 docs/bring-to-your-app.md create mode 100644 docs/deployment.md create mode 100644 docs/generative-ui.md create mode 100644 docs/getting-started.md create mode 100644 docs/human-in-the-loop.md create mode 100644 docs/mcp-integration.md diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..9663b05 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,25 @@ +# Open Generative UI Documentation + +Open Generative UI is a showcase and template for building AI agents with [CopilotKit](https://copilotkit.ai) and [LangGraph](https://langchain-ai.github.io/langgraph/). It demonstrates agent-driven UI where an AI agent and users collaboratively manipulate shared application state. + +## Prerequisites + +- Node.js 22+ +- Python 3.12+ +- [pnpm](https://pnpm.io/) 9+ +- [uv](https://docs.astral.sh/uv/) (Python package manager) +- An OpenAI API key + +## Documentation + +| Guide | Description | +|-------|-------------| +| [Getting Started](getting-started.md) | Install, configure, and run the project | +| [Architecture](architecture.md) | How the monorepo and request flow are structured | +| [Agent State](agent-state.md) | Bidirectional state sync between agent and frontend | +| [Generative UI](generative-ui.md) | Register React components the agent can render | +| [Agent Tools](agent-tools.md) | Create Python tools that read and update state | +| [Human in the Loop](human-in-the-loop.md) | Pause the agent to collect user input | +| [MCP Integration](mcp-integration.md) | Optional Model Context Protocol server | +| [Deployment](deployment.md) | Deploy to Render or other platforms | +| [Bring to Your App](bring-to-your-app.md) | Adopt these patterns in your own project | diff --git a/docs/agent-state.md b/docs/agent-state.md new file mode 100644 index 0000000..6124bc9 --- /dev/null +++ b/docs/agent-state.md @@ -0,0 +1,154 @@ +# Agent State + +The core pattern in this project is **CopilotKit v2's agent state** — state lives in the LangGraph agent and syncs bidirectionally with the React frontend. Both the user and agent can read and write the same state. + +## Define State in the Agent + +State is defined as a Python `TypedDict` that extends `BaseAgentState`: + +```python +# apps/agent/src/todos.py + +from langchain.agents import AgentState as BaseAgentState +from typing import TypedDict, Literal + +class Todo(TypedDict): + id: str + title: str + description: str + emoji: str + status: Literal["pending", "completed"] + +class AgentState(BaseAgentState): + todos: list[Todo] +``` + +The state schema is passed to the agent via `context_schema`: + +```python +# apps/agent/main.py + +agent = create_deep_agent( + model=ChatOpenAI(model="gpt-5.4-2026-03-05"), + tools=[...], + middleware=[CopilotKitMiddleware()], + context_schema=AgentState, # ← state schema + ... +) +``` + +## Read State in React + +Use the `useAgent()` hook to access agent state: + +```tsx +import { useAgent } from "@copilotkit/react-core/v2"; + +function MyComponent() { + const { agent } = useAgent(); + const todos = agent.state?.todos || []; + + return ( + + ); +} +``` + +## Write State from React + +Call `agent.setState()` to update state from the frontend: + +```tsx +const toggleTodo = (todoId: string) => { + const updated = todos.map(t => + t.id === todoId + ? { ...t, status: t.status === "completed" ? "pending" : "completed" } + : t + ); + agent.setState({ todos: updated }); +}; + +const addTodo = () => { + const newTodo = { + id: crypto.randomUUID(), + title: "New task", + description: "", + emoji: "📝", + status: "pending", + }; + agent.setState({ todos: [...todos, newTodo] }); +}; +``` + +## Write State from Agent Tools + +Tools update state by returning a `Command` with an `update` dict: + +```python +# apps/agent/src/todos.py + +from langgraph.types import Command +from langchain.tools import tool, ToolRuntime +from langchain.messages import ToolMessage + +@tool +def manage_todos(todos: list[Todo], runtime: ToolRuntime) -> Command: + """Manage the current todos.""" + for todo in todos: + if "id" not in todo or not todo["id"]: + todo["id"] = str(uuid.uuid4()) + + return Command(update={ + "todos": todos, + "messages": [ + ToolMessage( + content="Successfully updated todos", + tool_call_id=runtime.tool_call_id + ) + ] + }) +``` + +## Read State in Agent Tools + +Use `runtime.state` to read current state: + +```python +@tool +def get_todos(runtime: ToolRuntime): + """Get the current todos.""" + return runtime.state.get("todos", []) +``` + +## How State Flows + +1. **User edits a todo** → `agent.setState({ todos: [...] })` +2. **CopilotKit syncs** the change to the agent backend +3. **Agent observes** the updated state via `runtime.state` +4. **Agent calls a tool** → `Command(update={ "todos": [...] })` +5. **CopilotKit syncs** the update back to the frontend +6. **React re-renders** because `agent.state.todos` changed + +The key insight: there is no separate frontend state management. State lives in the agent, and CopilotKit handles the sync. + +## Adding New State Fields + +To add a new field to agent state: + +1. Add the field to `AgentState` in Python: + ```python + class AgentState(BaseAgentState): + todos: list[Todo] + tags: list[str] # new field + ``` + +2. Read it in React: + ```tsx + const tags = agent.state?.tags || []; + ``` + +3. Write it from React or tools the same way as above. diff --git a/docs/agent-tools.md b/docs/agent-tools.md new file mode 100644 index 0000000..dbe2217 --- /dev/null +++ b/docs/agent-tools.md @@ -0,0 +1,141 @@ +# Agent Tools + +Tools are Python functions that the LangGraph agent can call. They can read and update agent state, fetch data, and perform any server-side logic. + +## Creating a Tool + +Use the `@tool` decorator from LangChain: + +```python +from langchain.tools import tool, ToolRuntime + +@tool +def my_tool(arg1: str, arg2: int, runtime: ToolRuntime): + """Description of what this tool does. The agent reads this to decide when to call it.""" + return f"Result: {arg1} x {arg2}" +``` + +- The docstring tells the agent when and how to use the tool +- `runtime: ToolRuntime` gives access to agent state (optional parameter) +- Return value is sent back to the agent as the tool result + +## Reading State + +Access current agent state via `runtime.state`: + +```python +# apps/agent/src/todos.py + +@tool +def get_todos(runtime: ToolRuntime): + """Get the current todos.""" + return runtime.state.get("todos", []) +``` + +## Updating State + +Return a `Command` with an `update` dict to modify agent state: + +```python +from langgraph.types import Command +from langchain.messages import ToolMessage + +@tool +def manage_todos(todos: list[Todo], runtime: ToolRuntime) -> Command: + """Manage the current todos.""" + # Ensure all todos have unique IDs + for todo in todos: + if "id" not in todo or not todo["id"]: + todo["id"] = str(uuid.uuid4()) + + return Command(update={ + "todos": todos, + "messages": [ + ToolMessage( + content="Successfully updated todos", + tool_call_id=runtime.tool_call_id + ) + ] + }) +``` + +Key points: +- `Command(update={...})` merges the update into agent state +- Include a `ToolMessage` in the `messages` list to acknowledge the tool call +- Use `runtime.tool_call_id` for the message's `tool_call_id` + +## Returning Data (No State Update) + +For tools that just return data without modifying state, return the value directly: + +```python +# apps/agent/src/query.py + +import csv +from pathlib import Path +from langchain.tools import tool + +# Load data at module init +_data = [] +with open(Path(__file__).parent / "db.csv") as f: + _data = list(csv.DictReader(f)) + +@tool +def query_data(query: str): + """Query the financial transactions database. Call this before creating charts.""" + return _data +``` + +## Registering Tools with the Agent + +Add tools to the agent's `tools` list in `apps/agent/main.py`: + +```python +from src.todos import todo_tools # [manage_todos, get_todos] +from src.query import query_data +from src.plan import plan_visualization + +agent = create_deep_agent( + model=ChatOpenAI(model="gpt-5.4-2026-03-05"), + tools=[query_data, plan_visualization, *todo_tools], + middleware=[CopilotKitMiddleware()], + context_schema=AgentState, + ... +) +``` + +You can pass individual tools or spread a list of tools. + +## Example: Adding a New Tool + +Say you want to add a tool that fetches weather data: + +**1. Create the tool** (`apps/agent/src/weather.py`): + +```python +from langchain.tools import tool + +@tool +def get_weather(city: str): + """Get the current weather for a city.""" + # Your implementation here + return {"city": city, "temp": 72, "condition": "sunny"} +``` + +**2. Register it** in `apps/agent/main.py`: + +```python +from src.weather import get_weather + +agent = create_deep_agent( + tools=[query_data, plan_visualization, *todo_tools, get_weather], + ... +) +``` + +The agent can now call `get_weather` when a user asks about weather. If you want a custom UI for the result, register a `useRenderTool` on the frontend (see [Generative UI](generative-ui.md)). + +## Next Steps + +- [Agent State](agent-state.md) — How state sync works between tools and the frontend +- [Generative UI](generative-ui.md) — Render custom UI for tool results diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..fdcdcde --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,90 @@ +# Architecture + +## Monorepo Structure + +This is a Turborepo monorepo with three apps: + +``` +apps/ +├── app/ Next.js 16 frontend (React 19, TailwindCSS 4) +├── agent/ Python LangGraph agent (FastAPI, OpenAI) +└── mcp/ Model Context Protocol server (optional) +``` + +The frontend and MCP server are managed by pnpm workspaces. The Python agent is managed separately with `uv`. + +## Request Flow + +``` +Browser + │ + ▼ +Next.js App (:3000) + │ + ├── /api/copilotkit ← CopilotKit API route + │ │ + │ ▼ + │ CopilotRuntime + │ │ + │ ▼ + │ LangGraphHttpAgent ──→ FastAPI Agent (:8123) + │ │ + │ ├── LangGraph tools + │ ├── CopilotKitMiddleware + │ └── State management + │ + └── React UI + │ + ├── useAgent() ← Read/write agent state + ├── useComponent() ← Register generative UI + ├── useFrontendTool() ← Agent-callable frontend actions + └── useHumanInTheLoop() ← Interactive prompts +``` + +## Key Files + +### Frontend (`apps/app/`) + +| File | Purpose | +|------|---------| +| `src/app/layout.tsx` | Wraps the app with `` and `` | +| `src/app/page.tsx` | Main page — chat UI, demo gallery, background | +| `src/app/api/copilotkit/route.ts` | Connects CopilotKit runtime to the LangGraph agent | +| `src/hooks/use-generative-ui-examples.tsx` | Registers all generative UI components | +| `src/hooks/use-example-suggestions.tsx` | Chat suggestion prompts | +| `src/components/generative-ui/` | Chart, widget, and interactive components | + +### Agent (`apps/agent/`) + +| File | Purpose | +|------|---------| +| `main.py` | Agent definition, FastAPI app, system prompt | +| `src/todos.py` | `AgentState` schema and todo tools | +| `src/query.py` | Sample data query tool | +| `src/plan.py` | Visualization planning tool | +| `src/form.py` | Form generation tool (AG-UI) | +| `src/bounded_memory_saver.py` | Memory-capped checkpointer | +| `skills/` | Skill documents loaded at startup | + +### MCP (`apps/mcp/`) + +| File | Purpose | +|------|---------| +| `src/server.ts` | MCP resources, prompts, and tools | +| `src/renderer.ts` | HTML document assembly with design system | +| `src/skills.ts` | Skill file loader | +| `src/index.ts` | HTTP server | +| `src/stdio.ts` | Stdio transport for Claude Desktop | + +## State Sync + +State flows bidirectionally between the agent and frontend via CopilotKit: + +``` +Frontend Agent +──────── ───── +agent.state.todos ◄──────── AgentState.todos +agent.setState(...) ────────► Command(update={...}) +``` + +Both the user (via React UI) and the agent (via tools) can modify the same state. CopilotKit handles synchronization automatically. See [Agent State](agent-state.md) for details. diff --git a/docs/bring-to-your-app.md b/docs/bring-to-your-app.md new file mode 100644 index 0000000..e681ec8 --- /dev/null +++ b/docs/bring-to-your-app.md @@ -0,0 +1,256 @@ +# Bring to Your App + +This guide walks through adopting the CopilotKit + LangGraph patterns from this project into your own application. + +## What You Need + +- A React frontend (Next.js recommended) +- A Python backend (or willingness to add one) +- An OpenAI API key (or another LLM provider) + +## Step 1: Install Frontend Packages + +```bash +npm install @copilotkit/react-core @copilotkit/runtime zod +``` + +## Step 2: Add the CopilotKit Provider + +Wrap your app with the `` provider: + +```tsx +// app/layout.tsx (Next.js App Router) +import { CopilotKit } from "@copilotkit/react-core"; + +export default function RootLayout({ children }) { + return ( + + + + {children} + + + + ); +} +``` + +## Step 3: Create the API Route + +```typescript +// app/api/copilotkit/route.ts +import { + CopilotRuntime, + ExperimentalEmptyAdapter, + copilotRuntimeNextJSAppRouterEndpoint, +} from "@copilotkit/runtime"; +import { LangGraphHttpAgent } from "@copilotkit/runtime/langgraph"; +import { NextRequest } from "next/server"; + +const agent = new LangGraphHttpAgent({ + url: process.env.LANGGRAPH_DEPLOYMENT_URL || "http://localhost:8123", +}); + +export const POST = async (req: NextRequest) => { + const { handleRequest } = copilotRuntimeNextJSAppRouterEndpoint({ + endpoint: "/api/copilotkit", + serviceAdapter: new ExperimentalEmptyAdapter(), + runtime: new CopilotRuntime({ + agents: { default: agent }, + }), + }); + return handleRequest(req); +}; +``` + +## Step 4: Set Up the Python Agent + +Install dependencies: + +```bash +pip install copilotkit langgraph langchain langchain-openai fastapi uvicorn ag-ui-langgraph deepagents +``` + +Create the agent: + +```python +# agent/main.py +import os +from dotenv import load_dotenv +from fastapi import FastAPI +from copilotkit import CopilotKitMiddleware, LangGraphAGUIAgent +from ag_ui_langgraph import add_langgraph_fastapi_endpoint +from deepagents import create_deep_agent +from langchain_openai import ChatOpenAI + +load_dotenv() + +agent = create_deep_agent( + model=ChatOpenAI(model=os.environ.get("LLM_MODEL", "gpt-5.4-2026-03-05")), + tools=[], # add your tools here + middleware=[CopilotKitMiddleware()], + system_prompt="You are a helpful assistant.", +) + +app = FastAPI() + +@app.get("/health") +def health(): + return {"status": "ok"} + +add_langgraph_fastapi_endpoint( + app=app, + agent=LangGraphAGUIAgent( + name="my_agent", + description="My agent", + graph=agent, + ), + path="/", +) + +if __name__ == "__main__": + import uvicorn + uvicorn.run("main:app", host="0.0.0.0", port=8123, reload=True) +``` + +## Step 5: Define Your State + +```python +# agent/state.py +from langchain.agents import AgentState as BaseAgentState +from typing import TypedDict + +class Item(TypedDict): + id: str + name: str + done: bool + +class AgentState(BaseAgentState): + items: list[Item] +``` + +Pass it to the agent: + +```python +from state import AgentState + +agent = create_deep_agent( + ... + context_schema=AgentState, +) +``` + +## Step 6: Create Tools + +```python +# agent/tools.py +from langchain.tools import tool, ToolRuntime +from langchain.messages import ToolMessage +from langgraph.types import Command + +@tool +def update_items(items: list, runtime: ToolRuntime) -> Command: + """Update the items list.""" + return Command(update={ + "items": items, + "messages": [ToolMessage(content="Updated", tool_call_id=runtime.tool_call_id)] + }) + +@tool +def get_items(runtime: ToolRuntime): + """Get the current items.""" + return runtime.state.get("items", []) +``` + +Register them: + +```python +from tools import update_items, get_items + +agent = create_deep_agent( + tools=[update_items, get_items], + ... +) +``` + +## Step 7: Use Agent State in React + +```tsx +import { useAgent, CopilotChat } from "@copilotkit/react-core/v2"; + +function MyPage() { + const { agent } = useAgent(); + const items = agent.state?.items || []; + + return ( +
+
    + {items.map(item => ( +
  • + { + const updated = items.map(i => + i.id === item.id ? { ...i, done: !i.done } : i + ); + agent.setState({ items: updated }); + }} + /> + {item.name} +
    • + ))} +
+ +
+ ); +} +``` + +## Step 8: Add Generative UI (Optional) + +Register components the agent can render in the chat: + +```tsx +import { useComponent } from "@copilotkit/react-core/v2"; +import { z } from "zod"; + +useComponent({ + name: "statusCard", + description: "Show a status card with a title and message.", + parameters: z.object({ + title: z.string(), + message: z.string(), + type: z.enum(["info", "success", "error"]), + }), + render: ({ title, message, type }) => ( +
+

{title}

+

{message}

+
+ ), +}); +``` + +## What to Keep vs. What's Demo-Specific + +**Keep (core patterns):** +- CopilotKit provider + API route +- Agent state schema + `CopilotKitMiddleware` +- Tool pattern with `Command(update={...})` +- `useAgent()` for reading/writing state +- `useComponent()` / `useFrontendTool()` / `useHumanInTheLoop()` + +**Demo-specific (replace with your own):** +- Todo state schema and tools +- Demo gallery and explainer cards +- Widget renderer and chart components +- Sample data (`db.csv`) +- Skills documents +- Animated background and glassmorphism styling + +## Next Steps + +- [Agent State](agent-state.md) — Deep dive into state sync +- [Generative UI](generative-ui.md) — All the hooks for rendering UI +- [Agent Tools](agent-tools.md) — Building backend tools diff --git a/docs/deployment.md b/docs/deployment.md new file mode 100644 index 0000000..9d19ea4 --- /dev/null +++ b/docs/deployment.md @@ -0,0 +1,98 @@ +# Deployment + +## Render + +The project includes a `render.yaml` for one-click deployment to [Render](https://render.com/). + +### Services + +**Agent** (Python): +- Runtime: Python 3.12.6 +- Build: `pip install uv && uv sync` +- Start: `uv run uvicorn main:app --host 0.0.0.0 --port $PORT` +- Health check: `GET /health` +- Root directory: `apps/agent` + +**Frontend** (Node): +- Runtime: Node 22 +- Build: `corepack enable && pnpm install --no-frozen-lockfile && pnpm --filter @repo/app build` +- Start: `pnpm --filter @repo/app start` +- Health check: `GET /api/health` +- Root directory: (repo root) + +### Environment Variables + +| Variable | Service | Required | Notes | +|----------|---------|----------|-------| +| `OPENAI_API_KEY` | Agent | Yes | Your OpenAI API key | +| `LLM_MODEL` | Agent | No | Defaults to `gpt-5.4-2026-03-05` | +| `LANGSMITH_API_KEY` | Agent | No | For LangSmith tracing | +| `LANGGRAPH_DEPLOYMENT_URL` | Frontend | Auto | Injected from agent service via `fromService` | +| `SKIP_INSTALL_DEPS` | Frontend | No | Set to `true` to skip redundant installs | + +### Auto-Scaling + +Both services are configured with: +- Min instances: 1 +- Max instances: 3 +- Memory target: 80% +- CPU target: 70% + +### Deploy + +1. Fork the repository +2. Create a new **Blueprint** on Render +3. Connect your forked repo +4. Add `OPENAI_API_KEY` as a secret +5. Deploy + +Render reads `render.yaml` and creates both services. The frontend automatically gets the agent URL via service discovery. + +## General Deployment + +For other platforms, you need to deploy two services: + +### 1. Agent (Python) + +```bash +cd apps/agent +pip install uv +uv sync +uv run uvicorn main:app --host 0.0.0.0 --port 8123 +``` + +Requirements: +- Python 3.12+ +- `OPENAI_API_KEY` environment variable +- Port exposed for the frontend to reach + +### 2. Frontend (Node) + +```bash +# From repo root +corepack enable +pnpm install +pnpm --filter @repo/app build +LANGGRAPH_DEPLOYMENT_URL=http://your-agent-host:8123 pnpm --filter @repo/app start +``` + +Requirements: +- Node 22+ +- `LANGGRAPH_DEPLOYMENT_URL` pointing to the agent service +- Port 3000 exposed + +### Health Checks + +| Service | Endpoint | Expected | +|---------|----------|----------| +| Agent | `GET /health` | `{"status": "ok"}` | +| Frontend | `GET /api/health` | 200 OK | + +## Docker + +A Dockerfile for the frontend is available at `docker/Dockerfile.app`. The agent can be containerized with a standard Python Dockerfile using `uv`. + +## Next Steps + +- [Getting Started](getting-started.md) — Local development setup +- [Architecture](architecture.md) — Understand the service topology diff --git a/docs/generative-ui.md b/docs/generative-ui.md new file mode 100644 index 0000000..5ae2f2f --- /dev/null +++ b/docs/generative-ui.md @@ -0,0 +1,169 @@ +# Generative UI + +Generative UI lets the agent render React components directly in the chat. Instead of responding with text, the agent can produce charts, interactive widgets, visualizations, and custom UI. + +All generative UI in this project is registered in `apps/app/src/hooks/use-generative-ui-examples.tsx`. + +## Hooks Overview + +| Hook | Purpose | +|------|---------| +| `useComponent` | Register a named React component the agent can render with parameters | +| `useFrontendTool` | Register a tool the agent can call that runs in the browser | +| `useRenderTool` | Custom renderer for a specific backend tool | +| `useDefaultRenderTool` | Fallback renderer for any tool without a custom renderer | +| `useHumanInTheLoop` | Interactive component that pauses the agent for user input | + +All hooks are imported from `@copilotkit/react-core/v2`. + +## useComponent — Agent-Rendered Components + +Register a React component with a name, description, Zod schema, and render function. The agent decides when to use it based on the description. + +### Chart Example + +```tsx +import { z } from "zod"; +import { useComponent } from "@copilotkit/react-core/v2"; + +const PieChartProps = z.object({ + title: z.string(), + description: z.string(), + data: z.array(z.object({ + label: z.string(), + value: z.number(), + })), +}); + +useComponent({ + name: "pieChart", + description: "Displays data as a pie chart.", + parameters: PieChartProps, + render: PieChart, // your React component +}); +``` + +### Widget Renderer (Sandboxed HTML) + +The `widgetRenderer` component renders arbitrary HTML/SVG in a sandboxed iframe. This is the most flexible generative UI — the agent writes HTML and it gets rendered with full interactivity. + +```tsx +useComponent({ + name: "widgetRenderer", + description: + "Renders interactive HTML/SVG visualizations in a sandboxed iframe. " + + "Use for algorithm visualizations, diagrams, interactive widgets, " + + "simulations, math plots, and any visual explanation.", + parameters: WidgetRendererProps, // { title, description, html } + render: WidgetRenderer, +}); +``` + +The iframe environment includes: + +**ES Module Libraries** (use `