| # Specify one or more prompt refinement technique to be used. If you specify more than one prompt refinement techniques, | |
| # all these technique would run on same seed data. Result, iterations needed & cost incurred for each of these | |
| # technique would be logged. And winning technique for each data instance and overall would be logged. | |
| # Supported prompt refinement techniques: Basic, RecursiveEval, MedPrompt | |
| # Uncomment techniques that you want to use | |
| ############################ Critique Task Description Start ############################ | |
| prompt_technique_name: "critique_n_refine" | |
| # unique_model_id of model defined in llm_config.yaml | |
| unique_model_id: gpt-4o | |
| # Number of iterations for conducting <mutation_rounds> rounds of mutation of task description | |
| # followed by refinement of instructions | |
| mutate_refine_iterations: 3 | |
| # Number of rounds of mutation to be performed when generating different styles | |
| mutation_rounds: 3 | |
| # Refine instruction post mutation | |
| refine_instruction: true | |
| # Number of iterations for refining task description and in context examples for few-shot | |
| refine_task_eg_iterations: 3 | |
| # Number of variations of prompts to generate in given iteration | |
| style_variation: 5 | |
| # Number of questions to be asked to LLM in a single batch, during training step | |
| questions_batch_size: 1 | |
| # Number of batches of questions to correctly answered, for a prompt to be considered as performing good | |
| min_correct_count: 3 | |
| # Max number of mini-batches on which we should evaluate our prompt | |
| max_eval_batches: 6 | |
| # Number of top best performing prompts to be considered for next iterations | |
| top_n: 1 | |
| # Description of task. This will be fed to prompt | |
| task_description : 'Extract the second letter from the input word.' | |
| # Base instruction, in line with your dataset. This will be fed to prompt | |
| base_instruction : 'Output the second letter. Think step by step to arrive at the solution.' | |
| # Instruction for specifying answer format | |
| answer_format : 'For each input word, present the reasoning followed by the extracted letter (only single letter) between <ANS_START> and <ANS_END> tags' | |
| # Number of samples from dataset, set aside as training data. In every iteration we would be drawing | |
| # `questions_batch_size` examples from training data with replacement. | |
| seen_set_size: 25 | |
| # Number of examples to be given for few shots | |
| few_shot_count: 5 | |
| # Number of synthetic training examples to be generated | |
| num_train_examples: 20 | |
| # Generate synthetic reasoning | |
| generate_reasoning: true | |
| # Generate description of an expert which can solve the task at hand | |
| generate_expert_identity: true | |
| # Generate keywords that describe the intent of the task | |
| generate_intent_keywords: false | |
| ############################ Critique Task Description End ############################ | |