External Providers¶

External providers are added by implementing the Model protocol and registering the implementation.

You can register a provider in application code:

from uisurf_agent import register_provider


@register_provider("my-provider")
class MyModel:
    ...

Or from an installed package with the uisurf_agent.models entry-point group:

[project.entry-points."uisurf_agent.models"]
my-provider = "my_package.my_model:MyModel"

Custom providers that support tool calling should also read browser, desktop, and mobile tool declarations from tool_registry. See Tool Extensions for the registry API.

Minimal Model¶

This minimal model returns a final answer and never calls tools.

from typing import Any

from uisurf_agent import ModelTurn, Observation, ToolResult, register_provider


@register_provider("echo")
class EchoModel:
    async def encode_observation(self, observation: Observation) -> dict[str, Any]:
        return {
            "task": observation.task,
            "screenshot": observation.screenshot,
            "mime_type": observation.mime_type,
            "metadata": observation.metadata,
        }

    async def generate_response(self, history: list[Any]) -> ModelTurn:
        return ModelTurn(text="I received the UI observation.")

    async def add_response_to_history(
        self,
        response: ModelTurn,
        history: list[Any],
    ) -> None:
        history.append({"role": "assistant", "text": response.text})

    async def encode_action_results(
        self,
        results: list[ToolResult],
    ) -> dict[str, Any]:
        return {"role": "tool", "results": results}

    def prepare_history(
        self,
        history: list[Any],
        max_observation_images: int,
    ) -> list[Any]:
        return history

Use it:

from uisurf_agent import BrowserAgent

agent = BrowserAgent(provider_name="echo")

OpenAI-Compatible Example¶

This example targets OpenAI-compatible /v1/chat/completions servers such as vLLM, LM Studio, or local gateways.

import base64
import json
from typing import Any

from openai import AsyncOpenAI
from uisurf_agent import ModelTurn, Observation, ToolCall, ToolResult, register_provider


@register_provider("openai-compatible")
class OpenAICompatibleModel:
    def __init__(
        self,
        model_id: str,
        base_url: str,
        api_key: str = "not-needed",
        **_: Any,
    ) -> None:
        self.model_id = model_id
        self.client = AsyncOpenAI(base_url=base_url, api_key=api_key)

    async def encode_observation(self, observation: Observation) -> dict[str, Any]:
        image = base64.b64encode(observation.screenshot).decode("utf-8")
        content = []

        if observation.task:
            content.append({"type": "text", "text": observation.task})

        content.append({
            "type": "image_url",
            "image_url": {
                "url": f"data:{observation.mime_type};base64,{image}",
            },
        })

        return {"role": "user", "content": content}

    async def generate_response(self, history: list[Any]) -> ModelTurn:
        response = await self.client.chat.completions.create(
            model=self.model_id,
            messages=[
                {
                    "role": "system",
                    "content": (
                        "You control a UI through tools. Return JSON only: "
                        '{"text": string|null, "tool_calls": [{"name": string, "args": object}]}'
                    ),
                },
                *history,
            ],
            temperature=0,
        )

        raw_text = response.choices[0].message.content or "{}"
        try:
            payload = json.loads(raw_text)
        except json.JSONDecodeError:
            return ModelTurn(text=raw_text, raw=response)

        tool_calls = [
            ToolCall(
                id=None,
                name=item["name"],
                args=item.get("args", {}),
            )
            for item in payload.get("tool_calls", [])
        ]

        return ModelTurn(
            text=payload.get("text"),
            tool_calls=tool_calls,
            raw=response,
        )

    async def add_response_to_history(
        self,
        response: ModelTurn,
        history: list[Any],
    ) -> None:
        if response.tool_calls:
            history.append({
                "role": "assistant",
                "content": json.dumps({
                    "text": response.text,
                    "tool_calls": [
                        {"name": call.name, "args": call.args}
                        for call in response.tool_calls
                    ],
                }),
            })
            return

        if response.text:
            history.append({"role": "assistant", "content": response.text})

    async def encode_action_results(
        self,
        results: list[ToolResult],
    ) -> dict[str, Any]:
        content = []

        for result in results:
            content.append({
                "type": "text",
                "text": json.dumps({
                    "tool": result.name,
                    "result": result.result,
                    "metadata": result.metadata,
                    "safety_acknowledged": result.safety_acknowledged,
                }),
            })

            if result.screenshot:
                image = base64.b64encode(result.screenshot).decode("utf-8")
                content.append({
                    "type": "image_url",
                    "image_url": {
                        "url": f"data:{result.mime_type};base64,{image}",
                    },
                })

        return {"role": "user", "content": content}

    def prepare_history(
        self,
        history: list[Any],
        max_observation_images: int,
    ) -> list[Any]:
        image_indexes = [
            index
            for index, message in enumerate(history)
            if isinstance(message.get("content"), list)
            and any(
                isinstance(part, dict) and part.get("type") == "image_url"
                for part in message["content"]
            )
        ]

        keep = set(image_indexes[-max_observation_images:])
        compacted = []

        for index, message in enumerate(history):
            if index in keep or index not in image_indexes:
                compacted.append(message)
                continue

            content = [
                part
                for part in message.get("content", [])
                if not (isinstance(part, dict) and part.get("type") == "image_url")
            ]
            if content:
                compacted.append({**message, "content": content})

        return compacted

Use it from Python:

from uisurf_agent import BrowserAgent

agent = BrowserAgent(
    provider_name="openai-compatible",
    model_id="Qwen/Qwen2.5-VL-7B-Instruct",
    model_base_url="http://localhost:8000/v1",
)

Use it from the CLI after the package/module containing the registration has been imported or exposed through entry points:

uv run uisurf_agent run browser_agent \
  --provider openai-compatible \
  --model-id Qwen/Qwen2.5-VL-7B-Instruct \
  --model-base-url http://localhost:8000/v1 \
  --task "Open example.com and summarize the page"

Included Browser Model Examples¶

The repository includes two example provider adapters for local visual browser models. Both examples use BrowserAgent observations as input: the agent captures a screenshot, passes the current task and URL to the provider adapter, parses the model response into a browser action, then executes that action through the agent runtime.

Fara-7B¶

Fara-7B is a Microsoft Research computer-use model with 7 billion parameters. Its model card describes it as a multimodal, decoder-only model that takes screenshots plus text context and predicts grounded thoughts and actions for browser automation. The model is designed for web tasks where the model repeatedly observes the page, reasons about the next step, and emits action arguments such as click coordinates or URLs.

Fara-7B can be served through runtimes such as vLLM or SGLang that expose an OpenAI-compatible /v1/chat/completions API. The model card shows this pattern with vllm serve "microsoft/Fara-7B" followed by a request to /v1/chat/completions. In other words, the serving runtime provides the API compatibility; the Fara example consumes that API.

Use examples/fara_example.py when you have a local OpenAI-compatible Fara server running. The example registers a fara provider and points BrowserAgent at that local /v1/chat/completions service:

async with BrowserAgent(
    provider_name="fara",
    model_id="microsoft/Fara-7B",
    model_base_url="http://localhost:5000/v1",
    auto_mode=True,
    max_observation_images=5,
) as agent:
    async for event in agent.run(
        "Go to wikipedia and search for Large Language Models.",
        max_steps=20,
    ):
        print(event)

Run it from the repository root:

uv run python examples/fara_example.py

MolmoWeb¶

MolmoWeb is an Ai2 open visual web agent built on the Molmo 2 multimodal model family. Ai2 describes it as a self-hosted browser agent that observes live webpages through screenshots and predicts the next step, such as clicking, typing, scrolling, or navigating. In this UISurf example, the local /predict endpoint is used only for model inference; it returns text for the adapter to parse, and BrowserAgent performs the actual browser action through Playwright. The public release emphasizes open weights, training data, code, and evaluation tools for building and inspecting web agents.

Use examples/molmoweb_example.py when you have a local MolmoWeb /predict server running. The example registers a molmoweb provider, sends each browser screenshot as image_base64 with a task prompt, and maps the model's response into UISurf ToolCall objects for the agent to execute:

async with BrowserAgent(
    provider_name="molmoweb",
    model_base_url="http://127.0.0.1:8001/predict",
    auto_mode=True,
    max_observation_images=1,
) as agent:
    async for event in agent.run(
        "Go to https://www.ibm.com/think/topics/large-language-models, read the page and tell me whats is the title of the article",
        max_steps=20,
    ):
        print(event)

Run it from the repository root:

uv run python examples/molmoweb_example.py

Implementation Checklist¶

Every custom model must decide how to:

encode an Observation into the provider's user message format
call the provider and normalize the response into ModelTurn
add the provider's assistant message back into history
encode ToolResult objects into the provider's tool-result format
prepare old history for the next model call through prepare_history

For human confirmation, emit a ToolCall with args["safety_decision"]. The agent runtime handles prompting and approval.