kiln_ai.adapters.prompt_builders

  1import json
  2from abc import ABCMeta, abstractmethod
  3from typing import Dict, Union
  4
  5from kiln_ai.datamodel import Task
  6from kiln_ai.utils.formatting import snake_case
  7
  8
  9class BasePromptBuilder(metaclass=ABCMeta):
 10    """Base class for building prompts from tasks.
 11
 12    Provides the core interface and basic functionality for prompt builders.
 13    """
 14
 15    def __init__(self, task: Task):
 16        """Initialize the prompt builder with a task.
 17
 18        Args:
 19            task (Task): The task containing instructions and requirements.
 20        """
 21        self.task = task
 22
 23    @abstractmethod
 24    def build_prompt(self) -> str:
 25        """Build and return the complete prompt string.
 26
 27        Returns:
 28            str: The constructed prompt.
 29        """
 30        pass
 31
 32    @classmethod
 33    def prompt_builder_name(cls) -> str:
 34        """Returns the name of the prompt builder, to be used for persisting into the datastore.
 35
 36        Default implementation gets the name of the prompt builder in snake case. If you change the class name, you should override this so prior saved data is compatible.
 37
 38        Returns:
 39            str: The prompt builder name in snake_case format.
 40        """
 41        return snake_case(cls.__name__)
 42
 43    def build_user_message(self, input: Dict | str) -> str:
 44        """Build a user message from the input.
 45
 46        Args:
 47            input (Union[Dict, str]): The input to format into a message.
 48
 49        Returns:
 50            str: The formatted user message.
 51        """
 52        if isinstance(input, Dict):
 53            return f"The input is:\n{json.dumps(input, indent=2)}"
 54
 55        return f"The input is:\n{input}"
 56
 57
 58class SimplePromptBuilder(BasePromptBuilder):
 59    """A basic prompt builder that combines task instruction with requirements."""
 60
 61    def build_prompt(self) -> str:
 62        """Build a simple prompt with instruction and requirements.
 63
 64        Returns:
 65            str: The constructed prompt string.
 66        """
 67        base_prompt = self.task.instruction
 68
 69        # TODO: this is just a quick version. Formatting and best practices TBD
 70        if len(self.task.requirements) > 0:
 71            base_prompt += (
 72                "\n\nYour response should respect the following requirements:\n"
 73            )
 74            # iterate requirements, formatting them in numbereed list like 1) task.instruction\n2)...
 75            for i, requirement in enumerate(self.task.requirements):
 76                base_prompt += f"{i+1}) {requirement.instruction}\n"
 77
 78        return base_prompt
 79
 80
 81class MultiShotPromptBuilder(BasePromptBuilder):
 82    """A prompt builder that includes multiple examples in the prompt."""
 83
 84    @classmethod
 85    def example_count(cls) -> int:
 86        """Get the maximum number of examples to include in the prompt.
 87
 88        Returns:
 89            int: The maximum number of examples (default 25).
 90        """
 91        return 25
 92
 93    def build_prompt(self) -> str:
 94        """Build a prompt with instruction, requirements, and multiple examples.
 95
 96        Returns:
 97            str: The constructed prompt string with examples.
 98        """
 99        base_prompt = f"# Instruction\n\n{ self.task.instruction }\n\n"
100
101        if len(self.task.requirements) > 0:
102            base_prompt += "# Requirements\n\nYour response should respect the following requirements:\n"
103            for i, requirement in enumerate(self.task.requirements):
104                base_prompt += f"{i+1}) {requirement.instruction}\n"
105            base_prompt += "\n"
106
107        valid_examples: list[tuple[str, str]] = []
108        runs = self.task.runs()
109
110        # first pass, we look for repaired outputs. These are the best examples.
111        for run in runs:
112            if len(valid_examples) >= self.__class__.example_count():
113                break
114            if run.repaired_output is not None:
115                valid_examples.append((run.input, run.repaired_output.output))
116
117        # second pass, we look for high quality outputs (rating based)
118        # Minimum is "high_quality" (4 star in star rating scale), then sort by rating
119        # exclude repaired outputs as they were used above
120        runs_with_rating = [
121            run
122            for run in runs
123            if run.output.rating is not None
124            and run.output.rating.value is not None
125            and run.output.rating.is_high_quality()
126            and run.repaired_output is None
127        ]
128        runs_with_rating.sort(
129            key=lambda x: (x.output.rating and x.output.rating.value) or 0, reverse=True
130        )
131        for run in runs_with_rating:
132            if len(valid_examples) >= self.__class__.example_count():
133                break
134            valid_examples.append((run.input, run.output.output))
135
136        if len(valid_examples) > 0:
137            base_prompt += "# Example Outputs\n\n"
138            for i, example in enumerate(valid_examples):
139                base_prompt += (
140                    f"## Example {i+1}\n\nInput: {example[0]}\nOutput: {example[1]}\n\n"
141                )
142
143        return base_prompt
144
145
146class FewShotPromptBuilder(MultiShotPromptBuilder):
147    """A prompt builder that includes a small number of examples in the prompt."""
148
149    @classmethod
150    def example_count(cls) -> int:
151        """Get the maximum number of examples to include in the prompt.
152
153        Returns:
154            int: The maximum number of examples (4).
155        """
156        return 4
157
158
159prompt_builder_registry = {
160    "simple_prompt_builder": SimplePromptBuilder,
161    "multi_shot_prompt_builder": MultiShotPromptBuilder,
162    "few_shot_prompt_builder": FewShotPromptBuilder,
163}
164
165
166# Our UI has some names that are not the same as the class names, which also hint parameters.
167def prompt_builder_from_ui_name(ui_name: str) -> type[BasePromptBuilder]:
168    """Convert a name used in the UI to the corresponding prompt builder class.
169
170    Args:
171        ui_name (str): The UI name for the prompt builder type.
172
173    Returns:
174        type[BasePromptBuilder]: The corresponding prompt builder class.
175
176    Raises:
177        ValueError: If the UI name is not recognized.
178    """
179    match ui_name:
180        case "basic":
181            return SimplePromptBuilder
182        case "few_shot":
183            return FewShotPromptBuilder
184        case "many_shot":
185            return MultiShotPromptBuilder
186        case _:
187            raise ValueError(f"Unknown prompt builder: {ui_name}")
class BasePromptBuilder:
10class BasePromptBuilder(metaclass=ABCMeta):
11    """Base class for building prompts from tasks.
12
13    Provides the core interface and basic functionality for prompt builders.
14    """
15
16    def __init__(self, task: Task):
17        """Initialize the prompt builder with a task.
18
19        Args:
20            task (Task): The task containing instructions and requirements.
21        """
22        self.task = task
23
24    @abstractmethod
25    def build_prompt(self) -> str:
26        """Build and return the complete prompt string.
27
28        Returns:
29            str: The constructed prompt.
30        """
31        pass
32
33    @classmethod
34    def prompt_builder_name(cls) -> str:
35        """Returns the name of the prompt builder, to be used for persisting into the datastore.
36
37        Default implementation gets the name of the prompt builder in snake case. If you change the class name, you should override this so prior saved data is compatible.
38
39        Returns:
40            str: The prompt builder name in snake_case format.
41        """
42        return snake_case(cls.__name__)
43
44    def build_user_message(self, input: Dict | str) -> str:
45        """Build a user message from the input.
46
47        Args:
48            input (Union[Dict, str]): The input to format into a message.
49
50        Returns:
51            str: The formatted user message.
52        """
53        if isinstance(input, Dict):
54            return f"The input is:\n{json.dumps(input, indent=2)}"
55
56        return f"The input is:\n{input}"

Base class for building prompts from tasks.

Provides the core interface and basic functionality for prompt builders.

BasePromptBuilder(task: kiln_ai.datamodel.Task)
16    def __init__(self, task: Task):
17        """Initialize the prompt builder with a task.
18
19        Args:
20            task (Task): The task containing instructions and requirements.
21        """
22        self.task = task

Initialize the prompt builder with a task.

Args: task (Task): The task containing instructions and requirements.

task
@abstractmethod
def build_prompt(self) -> str:
24    @abstractmethod
25    def build_prompt(self) -> str:
26        """Build and return the complete prompt string.
27
28        Returns:
29            str: The constructed prompt.
30        """
31        pass

Build and return the complete prompt string.

Returns: str: The constructed prompt.

@classmethod
def prompt_builder_name(cls) -> str:
33    @classmethod
34    def prompt_builder_name(cls) -> str:
35        """Returns the name of the prompt builder, to be used for persisting into the datastore.
36
37        Default implementation gets the name of the prompt builder in snake case. If you change the class name, you should override this so prior saved data is compatible.
38
39        Returns:
40            str: The prompt builder name in snake_case format.
41        """
42        return snake_case(cls.__name__)

Returns the name of the prompt builder, to be used for persisting into the datastore.

Default implementation gets the name of the prompt builder in snake case. If you change the class name, you should override this so prior saved data is compatible.

Returns: str: The prompt builder name in snake_case format.

def build_user_message(self, input: Union[Dict, str]) -> str:
44    def build_user_message(self, input: Dict | str) -> str:
45        """Build a user message from the input.
46
47        Args:
48            input (Union[Dict, str]): The input to format into a message.
49
50        Returns:
51            str: The formatted user message.
52        """
53        if isinstance(input, Dict):
54            return f"The input is:\n{json.dumps(input, indent=2)}"
55
56        return f"The input is:\n{input}"

Build a user message from the input.

Args: input (Union[Dict, str]): The input to format into a message.

Returns: str: The formatted user message.

class SimplePromptBuilder(BasePromptBuilder):
59class SimplePromptBuilder(BasePromptBuilder):
60    """A basic prompt builder that combines task instruction with requirements."""
61
62    def build_prompt(self) -> str:
63        """Build a simple prompt with instruction and requirements.
64
65        Returns:
66            str: The constructed prompt string.
67        """
68        base_prompt = self.task.instruction
69
70        # TODO: this is just a quick version. Formatting and best practices TBD
71        if len(self.task.requirements) > 0:
72            base_prompt += (
73                "\n\nYour response should respect the following requirements:\n"
74            )
75            # iterate requirements, formatting them in numbereed list like 1) task.instruction\n2)...
76            for i, requirement in enumerate(self.task.requirements):
77                base_prompt += f"{i+1}) {requirement.instruction}\n"
78
79        return base_prompt

A basic prompt builder that combines task instruction with requirements.

def build_prompt(self) -> str:
62    def build_prompt(self) -> str:
63        """Build a simple prompt with instruction and requirements.
64
65        Returns:
66            str: The constructed prompt string.
67        """
68        base_prompt = self.task.instruction
69
70        # TODO: this is just a quick version. Formatting and best practices TBD
71        if len(self.task.requirements) > 0:
72            base_prompt += (
73                "\n\nYour response should respect the following requirements:\n"
74            )
75            # iterate requirements, formatting them in numbereed list like 1) task.instruction\n2)...
76            for i, requirement in enumerate(self.task.requirements):
77                base_prompt += f"{i+1}) {requirement.instruction}\n"
78
79        return base_prompt

Build a simple prompt with instruction and requirements.

Returns: str: The constructed prompt string.

class MultiShotPromptBuilder(BasePromptBuilder):
 82class MultiShotPromptBuilder(BasePromptBuilder):
 83    """A prompt builder that includes multiple examples in the prompt."""
 84
 85    @classmethod
 86    def example_count(cls) -> int:
 87        """Get the maximum number of examples to include in the prompt.
 88
 89        Returns:
 90            int: The maximum number of examples (default 25).
 91        """
 92        return 25
 93
 94    def build_prompt(self) -> str:
 95        """Build a prompt with instruction, requirements, and multiple examples.
 96
 97        Returns:
 98            str: The constructed prompt string with examples.
 99        """
100        base_prompt = f"# Instruction\n\n{ self.task.instruction }\n\n"
101
102        if len(self.task.requirements) > 0:
103            base_prompt += "# Requirements\n\nYour response should respect the following requirements:\n"
104            for i, requirement in enumerate(self.task.requirements):
105                base_prompt += f"{i+1}) {requirement.instruction}\n"
106            base_prompt += "\n"
107
108        valid_examples: list[tuple[str, str]] = []
109        runs = self.task.runs()
110
111        # first pass, we look for repaired outputs. These are the best examples.
112        for run in runs:
113            if len(valid_examples) >= self.__class__.example_count():
114                break
115            if run.repaired_output is not None:
116                valid_examples.append((run.input, run.repaired_output.output))
117
118        # second pass, we look for high quality outputs (rating based)
119        # Minimum is "high_quality" (4 star in star rating scale), then sort by rating
120        # exclude repaired outputs as they were used above
121        runs_with_rating = [
122            run
123            for run in runs
124            if run.output.rating is not None
125            and run.output.rating.value is not None
126            and run.output.rating.is_high_quality()
127            and run.repaired_output is None
128        ]
129        runs_with_rating.sort(
130            key=lambda x: (x.output.rating and x.output.rating.value) or 0, reverse=True
131        )
132        for run in runs_with_rating:
133            if len(valid_examples) >= self.__class__.example_count():
134                break
135            valid_examples.append((run.input, run.output.output))
136
137        if len(valid_examples) > 0:
138            base_prompt += "# Example Outputs\n\n"
139            for i, example in enumerate(valid_examples):
140                base_prompt += (
141                    f"## Example {i+1}\n\nInput: {example[0]}\nOutput: {example[1]}\n\n"
142                )
143
144        return base_prompt

A prompt builder that includes multiple examples in the prompt.

@classmethod
def example_count(cls) -> int:
85    @classmethod
86    def example_count(cls) -> int:
87        """Get the maximum number of examples to include in the prompt.
88
89        Returns:
90            int: The maximum number of examples (default 25).
91        """
92        return 25

Get the maximum number of examples to include in the prompt.

Returns: int: The maximum number of examples (default 25).

def build_prompt(self) -> str:
 94    def build_prompt(self) -> str:
 95        """Build a prompt with instruction, requirements, and multiple examples.
 96
 97        Returns:
 98            str: The constructed prompt string with examples.
 99        """
100        base_prompt = f"# Instruction\n\n{ self.task.instruction }\n\n"
101
102        if len(self.task.requirements) > 0:
103            base_prompt += "# Requirements\n\nYour response should respect the following requirements:\n"
104            for i, requirement in enumerate(self.task.requirements):
105                base_prompt += f"{i+1}) {requirement.instruction}\n"
106            base_prompt += "\n"
107
108        valid_examples: list[tuple[str, str]] = []
109        runs = self.task.runs()
110
111        # first pass, we look for repaired outputs. These are the best examples.
112        for run in runs:
113            if len(valid_examples) >= self.__class__.example_count():
114                break
115            if run.repaired_output is not None:
116                valid_examples.append((run.input, run.repaired_output.output))
117
118        # second pass, we look for high quality outputs (rating based)
119        # Minimum is "high_quality" (4 star in star rating scale), then sort by rating
120        # exclude repaired outputs as they were used above
121        runs_with_rating = [
122            run
123            for run in runs
124            if run.output.rating is not None
125            and run.output.rating.value is not None
126            and run.output.rating.is_high_quality()
127            and run.repaired_output is None
128        ]
129        runs_with_rating.sort(
130            key=lambda x: (x.output.rating and x.output.rating.value) or 0, reverse=True
131        )
132        for run in runs_with_rating:
133            if len(valid_examples) >= self.__class__.example_count():
134                break
135            valid_examples.append((run.input, run.output.output))
136
137        if len(valid_examples) > 0:
138            base_prompt += "# Example Outputs\n\n"
139            for i, example in enumerate(valid_examples):
140                base_prompt += (
141                    f"## Example {i+1}\n\nInput: {example[0]}\nOutput: {example[1]}\n\n"
142                )
143
144        return base_prompt

Build a prompt with instruction, requirements, and multiple examples.

Returns: str: The constructed prompt string with examples.

class FewShotPromptBuilder(MultiShotPromptBuilder):
147class FewShotPromptBuilder(MultiShotPromptBuilder):
148    """A prompt builder that includes a small number of examples in the prompt."""
149
150    @classmethod
151    def example_count(cls) -> int:
152        """Get the maximum number of examples to include in the prompt.
153
154        Returns:
155            int: The maximum number of examples (4).
156        """
157        return 4

A prompt builder that includes a small number of examples in the prompt.

@classmethod
def example_count(cls) -> int:
150    @classmethod
151    def example_count(cls) -> int:
152        """Get the maximum number of examples to include in the prompt.
153
154        Returns:
155            int: The maximum number of examples (4).
156        """
157        return 4

Get the maximum number of examples to include in the prompt.

Returns: int: The maximum number of examples (4).

prompt_builder_registry = {'simple_prompt_builder': <class 'SimplePromptBuilder'>, 'multi_shot_prompt_builder': <class 'MultiShotPromptBuilder'>, 'few_shot_prompt_builder': <class 'FewShotPromptBuilder'>}
def prompt_builder_from_ui_name(ui_name: str) -> type[BasePromptBuilder]:
168def prompt_builder_from_ui_name(ui_name: str) -> type[BasePromptBuilder]:
169    """Convert a name used in the UI to the corresponding prompt builder class.
170
171    Args:
172        ui_name (str): The UI name for the prompt builder type.
173
174    Returns:
175        type[BasePromptBuilder]: The corresponding prompt builder class.
176
177    Raises:
178        ValueError: If the UI name is not recognized.
179    """
180    match ui_name:
181        case "basic":
182            return SimplePromptBuilder
183        case "few_shot":
184            return FewShotPromptBuilder
185        case "many_shot":
186            return MultiShotPromptBuilder
187        case _:
188            raise ValueError(f"Unknown prompt builder: {ui_name}")

Convert a name used in the UI to the corresponding prompt builder class.

Args: ui_name (str): The UI name for the prompt builder type.

Returns: type[BasePromptBuilder]: The corresponding prompt builder class.

Raises: ValueError: If the UI name is not recognized.