kiln_ai.adapters.prompt_builders
1import json 2from abc import ABCMeta, abstractmethod 3from typing import Dict, Union 4 5from kiln_ai.datamodel import Task 6from kiln_ai.utils.formatting import snake_case 7 8 9class BasePromptBuilder(metaclass=ABCMeta): 10 """Base class for building prompts from tasks. 11 12 Provides the core interface and basic functionality for prompt builders. 13 """ 14 15 def __init__(self, task: Task): 16 """Initialize the prompt builder with a task. 17 18 Args: 19 task (Task): The task containing instructions and requirements. 20 """ 21 self.task = task 22 23 @abstractmethod 24 def build_prompt(self) -> str: 25 """Build and return the complete prompt string. 26 27 Returns: 28 str: The constructed prompt. 29 """ 30 pass 31 32 @classmethod 33 def prompt_builder_name(cls) -> str: 34 """Returns the name of the prompt builder, to be used for persisting into the datastore. 35 36 Default implementation gets the name of the prompt builder in snake case. If you change the class name, you should override this so prior saved data is compatible. 37 38 Returns: 39 str: The prompt builder name in snake_case format. 40 """ 41 return snake_case(cls.__name__) 42 43 def build_user_message(self, input: Dict | str) -> str: 44 """Build a user message from the input. 45 46 Args: 47 input (Union[Dict, str]): The input to format into a message. 48 49 Returns: 50 str: The formatted user message. 51 """ 52 if isinstance(input, Dict): 53 return f"The input is:\n{json.dumps(input, indent=2)}" 54 55 return f"The input is:\n{input}" 56 57 58class SimplePromptBuilder(BasePromptBuilder): 59 """A basic prompt builder that combines task instruction with requirements.""" 60 61 def build_prompt(self) -> str: 62 """Build a simple prompt with instruction and requirements. 63 64 Returns: 65 str: The constructed prompt string. 66 """ 67 base_prompt = self.task.instruction 68 69 # TODO: this is just a quick version. Formatting and best practices TBD 70 if len(self.task.requirements) > 0: 71 base_prompt += ( 72 "\n\nYour response should respect the following requirements:\n" 73 ) 74 # iterate requirements, formatting them in numbereed list like 1) task.instruction\n2)... 75 for i, requirement in enumerate(self.task.requirements): 76 base_prompt += f"{i+1}) {requirement.instruction}\n" 77 78 return base_prompt 79 80 81class MultiShotPromptBuilder(BasePromptBuilder): 82 """A prompt builder that includes multiple examples in the prompt.""" 83 84 @classmethod 85 def example_count(cls) -> int: 86 """Get the maximum number of examples to include in the prompt. 87 88 Returns: 89 int: The maximum number of examples (default 25). 90 """ 91 return 25 92 93 def build_prompt(self) -> str: 94 """Build a prompt with instruction, requirements, and multiple examples. 95 96 Returns: 97 str: The constructed prompt string with examples. 98 """ 99 base_prompt = f"# Instruction\n\n{ self.task.instruction }\n\n" 100 101 if len(self.task.requirements) > 0: 102 base_prompt += "# Requirements\n\nYour response should respect the following requirements:\n" 103 for i, requirement in enumerate(self.task.requirements): 104 base_prompt += f"{i+1}) {requirement.instruction}\n" 105 base_prompt += "\n" 106 107 valid_examples: list[tuple[str, str]] = [] 108 runs = self.task.runs() 109 110 # first pass, we look for repaired outputs. These are the best examples. 111 for run in runs: 112 if len(valid_examples) >= self.__class__.example_count(): 113 break 114 if run.repaired_output is not None: 115 valid_examples.append((run.input, run.repaired_output.output)) 116 117 # second pass, we look for high quality outputs (rating based) 118 # Minimum is "high_quality" (4 star in star rating scale), then sort by rating 119 # exclude repaired outputs as they were used above 120 runs_with_rating = [ 121 run 122 for run in runs 123 if run.output.rating is not None 124 and run.output.rating.value is not None 125 and run.output.rating.is_high_quality() 126 and run.repaired_output is None 127 ] 128 runs_with_rating.sort( 129 key=lambda x: (x.output.rating and x.output.rating.value) or 0, reverse=True 130 ) 131 for run in runs_with_rating: 132 if len(valid_examples) >= self.__class__.example_count(): 133 break 134 valid_examples.append((run.input, run.output.output)) 135 136 if len(valid_examples) > 0: 137 base_prompt += "# Example Outputs\n\n" 138 for i, example in enumerate(valid_examples): 139 base_prompt += ( 140 f"## Example {i+1}\n\nInput: {example[0]}\nOutput: {example[1]}\n\n" 141 ) 142 143 return base_prompt 144 145 146class FewShotPromptBuilder(MultiShotPromptBuilder): 147 """A prompt builder that includes a small number of examples in the prompt.""" 148 149 @classmethod 150 def example_count(cls) -> int: 151 """Get the maximum number of examples to include in the prompt. 152 153 Returns: 154 int: The maximum number of examples (4). 155 """ 156 return 4 157 158 159prompt_builder_registry = { 160 "simple_prompt_builder": SimplePromptBuilder, 161 "multi_shot_prompt_builder": MultiShotPromptBuilder, 162 "few_shot_prompt_builder": FewShotPromptBuilder, 163} 164 165 166# Our UI has some names that are not the same as the class names, which also hint parameters. 167def prompt_builder_from_ui_name(ui_name: str) -> type[BasePromptBuilder]: 168 """Convert a name used in the UI to the corresponding prompt builder class. 169 170 Args: 171 ui_name (str): The UI name for the prompt builder type. 172 173 Returns: 174 type[BasePromptBuilder]: The corresponding prompt builder class. 175 176 Raises: 177 ValueError: If the UI name is not recognized. 178 """ 179 match ui_name: 180 case "basic": 181 return SimplePromptBuilder 182 case "few_shot": 183 return FewShotPromptBuilder 184 case "many_shot": 185 return MultiShotPromptBuilder 186 case _: 187 raise ValueError(f"Unknown prompt builder: {ui_name}")
10class BasePromptBuilder(metaclass=ABCMeta): 11 """Base class for building prompts from tasks. 12 13 Provides the core interface and basic functionality for prompt builders. 14 """ 15 16 def __init__(self, task: Task): 17 """Initialize the prompt builder with a task. 18 19 Args: 20 task (Task): The task containing instructions and requirements. 21 """ 22 self.task = task 23 24 @abstractmethod 25 def build_prompt(self) -> str: 26 """Build and return the complete prompt string. 27 28 Returns: 29 str: The constructed prompt. 30 """ 31 pass 32 33 @classmethod 34 def prompt_builder_name(cls) -> str: 35 """Returns the name of the prompt builder, to be used for persisting into the datastore. 36 37 Default implementation gets the name of the prompt builder in snake case. If you change the class name, you should override this so prior saved data is compatible. 38 39 Returns: 40 str: The prompt builder name in snake_case format. 41 """ 42 return snake_case(cls.__name__) 43 44 def build_user_message(self, input: Dict | str) -> str: 45 """Build a user message from the input. 46 47 Args: 48 input (Union[Dict, str]): The input to format into a message. 49 50 Returns: 51 str: The formatted user message. 52 """ 53 if isinstance(input, Dict): 54 return f"The input is:\n{json.dumps(input, indent=2)}" 55 56 return f"The input is:\n{input}"
Base class for building prompts from tasks.
Provides the core interface and basic functionality for prompt builders.
16 def __init__(self, task: Task): 17 """Initialize the prompt builder with a task. 18 19 Args: 20 task (Task): The task containing instructions and requirements. 21 """ 22 self.task = task
Initialize the prompt builder with a task.
Args: task (Task): The task containing instructions and requirements.
24 @abstractmethod 25 def build_prompt(self) -> str: 26 """Build and return the complete prompt string. 27 28 Returns: 29 str: The constructed prompt. 30 """ 31 pass
Build and return the complete prompt string.
Returns: str: The constructed prompt.
33 @classmethod 34 def prompt_builder_name(cls) -> str: 35 """Returns the name of the prompt builder, to be used for persisting into the datastore. 36 37 Default implementation gets the name of the prompt builder in snake case. If you change the class name, you should override this so prior saved data is compatible. 38 39 Returns: 40 str: The prompt builder name in snake_case format. 41 """ 42 return snake_case(cls.__name__)
Returns the name of the prompt builder, to be used for persisting into the datastore.
Default implementation gets the name of the prompt builder in snake case. If you change the class name, you should override this so prior saved data is compatible.
Returns: str: The prompt builder name in snake_case format.
44 def build_user_message(self, input: Dict | str) -> str: 45 """Build a user message from the input. 46 47 Args: 48 input (Union[Dict, str]): The input to format into a message. 49 50 Returns: 51 str: The formatted user message. 52 """ 53 if isinstance(input, Dict): 54 return f"The input is:\n{json.dumps(input, indent=2)}" 55 56 return f"The input is:\n{input}"
Build a user message from the input.
Args: input (Union[Dict, str]): The input to format into a message.
Returns: str: The formatted user message.
59class SimplePromptBuilder(BasePromptBuilder): 60 """A basic prompt builder that combines task instruction with requirements.""" 61 62 def build_prompt(self) -> str: 63 """Build a simple prompt with instruction and requirements. 64 65 Returns: 66 str: The constructed prompt string. 67 """ 68 base_prompt = self.task.instruction 69 70 # TODO: this is just a quick version. Formatting and best practices TBD 71 if len(self.task.requirements) > 0: 72 base_prompt += ( 73 "\n\nYour response should respect the following requirements:\n" 74 ) 75 # iterate requirements, formatting them in numbereed list like 1) task.instruction\n2)... 76 for i, requirement in enumerate(self.task.requirements): 77 base_prompt += f"{i+1}) {requirement.instruction}\n" 78 79 return base_prompt
A basic prompt builder that combines task instruction with requirements.
62 def build_prompt(self) -> str: 63 """Build a simple prompt with instruction and requirements. 64 65 Returns: 66 str: The constructed prompt string. 67 """ 68 base_prompt = self.task.instruction 69 70 # TODO: this is just a quick version. Formatting and best practices TBD 71 if len(self.task.requirements) > 0: 72 base_prompt += ( 73 "\n\nYour response should respect the following requirements:\n" 74 ) 75 # iterate requirements, formatting them in numbereed list like 1) task.instruction\n2)... 76 for i, requirement in enumerate(self.task.requirements): 77 base_prompt += f"{i+1}) {requirement.instruction}\n" 78 79 return base_prompt
Build a simple prompt with instruction and requirements.
Returns: str: The constructed prompt string.
Inherited Members
82class MultiShotPromptBuilder(BasePromptBuilder): 83 """A prompt builder that includes multiple examples in the prompt.""" 84 85 @classmethod 86 def example_count(cls) -> int: 87 """Get the maximum number of examples to include in the prompt. 88 89 Returns: 90 int: The maximum number of examples (default 25). 91 """ 92 return 25 93 94 def build_prompt(self) -> str: 95 """Build a prompt with instruction, requirements, and multiple examples. 96 97 Returns: 98 str: The constructed prompt string with examples. 99 """ 100 base_prompt = f"# Instruction\n\n{ self.task.instruction }\n\n" 101 102 if len(self.task.requirements) > 0: 103 base_prompt += "# Requirements\n\nYour response should respect the following requirements:\n" 104 for i, requirement in enumerate(self.task.requirements): 105 base_prompt += f"{i+1}) {requirement.instruction}\n" 106 base_prompt += "\n" 107 108 valid_examples: list[tuple[str, str]] = [] 109 runs = self.task.runs() 110 111 # first pass, we look for repaired outputs. These are the best examples. 112 for run in runs: 113 if len(valid_examples) >= self.__class__.example_count(): 114 break 115 if run.repaired_output is not None: 116 valid_examples.append((run.input, run.repaired_output.output)) 117 118 # second pass, we look for high quality outputs (rating based) 119 # Minimum is "high_quality" (4 star in star rating scale), then sort by rating 120 # exclude repaired outputs as they were used above 121 runs_with_rating = [ 122 run 123 for run in runs 124 if run.output.rating is not None 125 and run.output.rating.value is not None 126 and run.output.rating.is_high_quality() 127 and run.repaired_output is None 128 ] 129 runs_with_rating.sort( 130 key=lambda x: (x.output.rating and x.output.rating.value) or 0, reverse=True 131 ) 132 for run in runs_with_rating: 133 if len(valid_examples) >= self.__class__.example_count(): 134 break 135 valid_examples.append((run.input, run.output.output)) 136 137 if len(valid_examples) > 0: 138 base_prompt += "# Example Outputs\n\n" 139 for i, example in enumerate(valid_examples): 140 base_prompt += ( 141 f"## Example {i+1}\n\nInput: {example[0]}\nOutput: {example[1]}\n\n" 142 ) 143 144 return base_prompt
A prompt builder that includes multiple examples in the prompt.
85 @classmethod 86 def example_count(cls) -> int: 87 """Get the maximum number of examples to include in the prompt. 88 89 Returns: 90 int: The maximum number of examples (default 25). 91 """ 92 return 25
Get the maximum number of examples to include in the prompt.
Returns: int: The maximum number of examples (default 25).
94 def build_prompt(self) -> str: 95 """Build a prompt with instruction, requirements, and multiple examples. 96 97 Returns: 98 str: The constructed prompt string with examples. 99 """ 100 base_prompt = f"# Instruction\n\n{ self.task.instruction }\n\n" 101 102 if len(self.task.requirements) > 0: 103 base_prompt += "# Requirements\n\nYour response should respect the following requirements:\n" 104 for i, requirement in enumerate(self.task.requirements): 105 base_prompt += f"{i+1}) {requirement.instruction}\n" 106 base_prompt += "\n" 107 108 valid_examples: list[tuple[str, str]] = [] 109 runs = self.task.runs() 110 111 # first pass, we look for repaired outputs. These are the best examples. 112 for run in runs: 113 if len(valid_examples) >= self.__class__.example_count(): 114 break 115 if run.repaired_output is not None: 116 valid_examples.append((run.input, run.repaired_output.output)) 117 118 # second pass, we look for high quality outputs (rating based) 119 # Minimum is "high_quality" (4 star in star rating scale), then sort by rating 120 # exclude repaired outputs as they were used above 121 runs_with_rating = [ 122 run 123 for run in runs 124 if run.output.rating is not None 125 and run.output.rating.value is not None 126 and run.output.rating.is_high_quality() 127 and run.repaired_output is None 128 ] 129 runs_with_rating.sort( 130 key=lambda x: (x.output.rating and x.output.rating.value) or 0, reverse=True 131 ) 132 for run in runs_with_rating: 133 if len(valid_examples) >= self.__class__.example_count(): 134 break 135 valid_examples.append((run.input, run.output.output)) 136 137 if len(valid_examples) > 0: 138 base_prompt += "# Example Outputs\n\n" 139 for i, example in enumerate(valid_examples): 140 base_prompt += ( 141 f"## Example {i+1}\n\nInput: {example[0]}\nOutput: {example[1]}\n\n" 142 ) 143 144 return base_prompt
Build a prompt with instruction, requirements, and multiple examples.
Returns: str: The constructed prompt string with examples.
Inherited Members
147class FewShotPromptBuilder(MultiShotPromptBuilder): 148 """A prompt builder that includes a small number of examples in the prompt.""" 149 150 @classmethod 151 def example_count(cls) -> int: 152 """Get the maximum number of examples to include in the prompt. 153 154 Returns: 155 int: The maximum number of examples (4). 156 """ 157 return 4
A prompt builder that includes a small number of examples in the prompt.
150 @classmethod 151 def example_count(cls) -> int: 152 """Get the maximum number of examples to include in the prompt. 153 154 Returns: 155 int: The maximum number of examples (4). 156 """ 157 return 4
Get the maximum number of examples to include in the prompt.
Returns: int: The maximum number of examples (4).
168def prompt_builder_from_ui_name(ui_name: str) -> type[BasePromptBuilder]: 169 """Convert a name used in the UI to the corresponding prompt builder class. 170 171 Args: 172 ui_name (str): The UI name for the prompt builder type. 173 174 Returns: 175 type[BasePromptBuilder]: The corresponding prompt builder class. 176 177 Raises: 178 ValueError: If the UI name is not recognized. 179 """ 180 match ui_name: 181 case "basic": 182 return SimplePromptBuilder 183 case "few_shot": 184 return FewShotPromptBuilder 185 case "many_shot": 186 return MultiShotPromptBuilder 187 case _: 188 raise ValueError(f"Unknown prompt builder: {ui_name}")
Convert a name used in the UI to the corresponding prompt builder class.
Args: ui_name (str): The UI name for the prompt builder type.
Returns: type[BasePromptBuilder]: The corresponding prompt builder class.
Raises: ValueError: If the UI name is not recognized.