Add io.yaml files for answer relevance intrinsics

answer_relevance_classifier/alora/granite-3.3-2b-instruct/io.yaml

@@ -0,0 +1,58 @@
+# Model name string, or null to use whatever is provided in the chat completion request
+model: ~
+# JSON schema of the model's output
+response_format:  |
+  {
+    "properties": {
+      "answer_relevance_analysis": {
+        "title": "Answer Relevance Analysis",
+        "type": "string"
+      },
+      "answer_relevance_category": {
+        "title": "Answer Relevance Category",
+        "type": "string",
+        "enum": [
+          "Pertinent", 
+          "Pertinent with relevant extra",
+          "Excessive unnecessary information",
+          "Unduly restrictive",
+          "Too vague or generic",
+          "Contextual misalignment",
+          "Misinterpreted inquiry",
+          "No attempt"
+        ]
+      },
+      "answer_relevance_judgment": {
+        "title": "Answer Relevance Judgment",
+        "type": "boolean"
+      }
+    },
+    "required": [
+      "answer_relevance_analysis",
+      "answer_relevance_category",
+      "answer_relevance_judgment"
+    ],
+    "title": "AnswerRelevanceRawOutput",
+    "type": "object"
+  }
+# Additional turn of instructions to add to the chat
+instruction: "answer_relevance"
+# Data transformations to perform during post-processing
+transformations:
+  # Convert categorical answer to continuous value by decoding logprobs
+  - type: likelihood
+    categories_to_values:
+      true: 1.0
+      false: 0.0
+    input_path: ["answer_relevance_judgment"]
+  # Rename answer_relevance_judgment column to reflect likelihood transformation
+  - type: project
+    input_path: []
+    retained_fields:
+      "answer_relevance_analysis": "answer_relevance_analysis"
+      "answer_relevance_category": "answer_relevance_category"
+      "answer_relevance_judgment": "answer_relevance_likelihood"
+parameters:
+  # Current LoRA can be quite verbose in its explanations.
+  max_completion_tokens: 1024
+sentence_boundaries: ~

answer_relevance_classifier/alora/granite-3.3-8b-instruct/io.yaml

@@ -0,0 +1,58 @@
+# Model name string, or null to use whatever is provided in the chat completion request
+model: ~
+# JSON schema of the model's output
+response_format:  |
+  {
+    "properties": {
+      "answer_relevance_analysis": {
+        "title": "Answer Relevance Analysis",
+        "type": "string"
+      },
+      "answer_relevance_category": {
+        "title": "Answer Relevance Category",
+        "type": "string",
+        "enum": [
+          "Pertinent", 
+          "Pertinent with relevant extra",
+          "Excessive unnecessary information",
+          "Unduly restrictive",
+          "Too vague or generic",
+          "Contextual misalignment",
+          "Misinterpreted inquiry",
+          "No attempt"
+        ]
+      },
+      "answer_relevance_judgment": {
+        "title": "Answer Relevance Judgment",
+        "type": "boolean"
+      }
+    },
+    "required": [
+      "answer_relevance_analysis",
+      "answer_relevance_category",
+      "answer_relevance_judgment"
+    ],
+    "title": "AnswerRelevanceRawOutput",
+    "type": "object"
+  }
+# Additional turn of instructions to add to the chat
+instruction: "answer_relevance"
+# Data transformations to perform during post-processing
+transformations:
+  # Convert categorical answer to continuous value by decoding logprobs
+  - type: likelihood
+    categories_to_values:
+      true: 1.0
+      false: 0.0
+    input_path: ["answer_relevance_judgment"]
+  # Rename answer_relevance_judgment column to reflect likelihood transformation
+  - type: project
+    input_path: []
+    retained_fields:
+      "answer_relevance_analysis": "answer_relevance_analysis"
+      "answer_relevance_category": "answer_relevance_category"
+      "answer_relevance_judgment": "answer_relevance_likelihood"
+parameters:
+  # Current LoRA can be quite verbose in its explanations.
+  max_completion_tokens: 1024
+sentence_boundaries: ~

answer_relevance_classifier/lora/granite-3.3-2b-instruct/io.yaml

@@ -0,0 +1,58 @@
+# Model name string, or null to use whatever is provided in the chat completion request
+model: ~
+# JSON schema of the model's output
+response_format:  |
+  {
+    "properties": {
+      "answer_relevance_analysis": {
+        "title": "Answer Relevance Analysis",
+        "type": "string"
+      },
+      "answer_relevance_category": {
+        "title": "Answer Relevance Category",
+        "type": "string",
+        "enum": [
+          "Pertinent", 
+          "Pertinent with relevant extra",
+          "Excessive unnecessary information",
+          "Unduly restrictive",
+          "Too vague or generic",
+          "Contextual misalignment",
+          "Misinterpreted inquiry",
+          "No attempt"
+        ]
+      },
+      "answer_relevance_judgment": {
+        "title": "Answer Relevance Judgment",
+        "type": "boolean"
+      }
+    },
+    "required": [
+      "answer_relevance_analysis",
+      "answer_relevance_category",
+      "answer_relevance_judgment"
+    ],
+    "title": "AnswerRelevanceRawOutput",
+    "type": "object"
+  }
+# Additional turn of instructions to add to the chat
+instruction: "answer_relevance"
+# Data transformations to perform during post-processing
+transformations:
+  # Convert categorical answer to continuous value by decoding logprobs
+  - type: likelihood
+    categories_to_values:
+      true: 1.0
+      false: 0.0
+    input_path: ["answer_relevance_judgment"]
+  # Rename answer_relevance_judgment column to reflect likelihood transformation
+  - type: project
+    input_path: []
+    retained_fields:
+      "answer_relevance_analysis": "answer_relevance_analysis"
+      "answer_relevance_category": "answer_relevance_category"
+      "answer_relevance_judgment": "answer_relevance_likelihood"
+parameters:
+  # Current LoRA can be quite verbose in its explanations.
+  max_completion_tokens: 1024
+sentence_boundaries: ~

answer_relevance_classifier/lora/granite-3.3-8b-instruct/io.yaml

@@ -0,0 +1,58 @@
+# Model name string, or null to use whatever is provided in the chat completion request
+model: ~
+# JSON schema of the model's output
+response_format:  |
+  {
+    "properties": {
+      "answer_relevance_analysis": {
+        "title": "Answer Relevance Analysis",
+        "type": "string"
+      },
+      "answer_relevance_category": {
+        "title": "Answer Relevance Category",
+        "type": "string",
+        "enum": [
+          "Pertinent", 
+          "Pertinent with relevant extra",
+          "Excessive unnecessary information",
+          "Unduly restrictive",
+          "Too vague or generic",
+          "Contextual misalignment",
+          "Misinterpreted inquiry",
+          "No attempt"
+        ]
+      },
+      "answer_relevance_judgment": {
+        "title": "Answer Relevance Judgment",
+        "type": "boolean"
+      }
+    },
+    "required": [
+      "answer_relevance_analysis",
+      "answer_relevance_category",
+      "answer_relevance_judgment"
+    ],
+    "title": "AnswerRelevanceRawOutput",
+    "type": "object"
+  }
+# Additional turn of instructions to add to the chat
+instruction: "answer_relevance"
+# Data transformations to perform during post-processing
+transformations:
+  # Convert categorical answer to continuous value by decoding logprobs
+  - type: likelihood
+    categories_to_values:
+      true: 1.0
+      false: 0.0
+    input_path: ["answer_relevance_judgment"]
+  # Rename answer_relevance_judgment column to reflect likelihood transformation
+  - type: project
+    input_path: []
+    retained_fields:
+      "answer_relevance_analysis": "answer_relevance_analysis"
+      "answer_relevance_category": "answer_relevance_category"
+      "answer_relevance_judgment": "answer_relevance_likelihood"
+parameters:
+  # Current LoRA can be quite verbose in its explanations.
+  max_completion_tokens: 1024
+sentence_boundaries: ~

answer_relevance_rewriter/alora/granite-3.3-2b-instruct/io.yaml

@@ -0,0 +1,29 @@
+# Model name string, or null to use whatever is provided in the chat completion request
+model: ~
+# JSON schema of the model's output, or null if constrained decoding is not needed
+response_format: |
+  {
+    "properties": {
+      "answer_relevance_rewrite": {
+        "title": "Rewritten answer",
+        "type": "string"
+      }
+    },
+    "required": [
+      "answer_relevance_rewrite"
+    ],
+    "type": "object"
+  }
+# Additional turn of instructions to add to the chat
+instruction: |
+  Rewrite the response for relevance.
+  The last assistant response is considered not fully relevant to the last user inquiry due to {answer_relevance_category}: {answer_relevance_analysis}
+  Decide if you agree with this assessment, then act according to the following instructions: 
+  If you disagree with the assessment, provide a verbatim copy of the original response.  DO NOT attempt to correct any other perceived defects in the response. 
+  If you agree with the assessment, provide an updated response that no longer fit the label {answer_relevance_category}, by {correction_method}.  Your response should be entirely based on the provided documents and should not rely on other prior knowledge. Your response should be suitable to be directly provided to the user.  It should NOT contain meta information regarding the original response, its assessment, or this instruction.  The user does not see any of these.  Your response is the only response they will see to their inquiry, in place of the original response.
+# Data transformations to perform during post-processing
+transformations: ~
+parameters:
+  # Rewritten response could be long.
+  max_completion_tokens: 1024
+sentence_boundaries: ~

answer_relevance_rewriter/alora/granite-3.3-8b-instruct/io.yaml

@@ -0,0 +1,29 @@
+# Model name string, or null to use whatever is provided in the chat completion request
+model: ~
+# JSON schema of the model's output, or null if constrained decoding is not needed
+response_format: |
+  {
+    "properties": {
+      "answer_relevance_rewrite": {
+        "title": "Rewritten answer",
+        "type": "string"
+      }
+    },
+    "required": [
+      "answer_relevance_rewrite"
+    ],
+    "type": "object"
+  }
+# Additional turn of instructions to add to the chat
+instruction: |
+  Rewrite the response for relevance.
+  The last assistant response is considered not fully relevant to the last user inquiry due to {answer_relevance_category}: {answer_relevance_analysis}
+  Decide if you agree with this assessment, then act according to the following instructions: 
+  If you disagree with the assessment, provide a verbatim copy of the original response.  DO NOT attempt to correct any other perceived defects in the response. 
+  If you agree with the assessment, provide an updated response that no longer fit the label {answer_relevance_category}, by {correction_method}.  Your response should be entirely based on the provided documents and should not rely on other prior knowledge. Your response should be suitable to be directly provided to the user.  It should NOT contain meta information regarding the original response, its assessment, or this instruction.  The user does not see any of these.  Your response is the only response they will see to their inquiry, in place of the original response.
+# Data transformations to perform during post-processing
+transformations: ~
+parameters:
+  # Rewritten response could be long.
+  max_completion_tokens: 1024
+sentence_boundaries: ~

answer_relevance_rewriter/lora/granite-3.3-2b-instruct/io.yaml

@@ -0,0 +1,29 @@
+# Model name string, or null to use whatever is provided in the chat completion request
+model: ~
+# JSON schema of the model's output, or null if constrained decoding is not needed
+response_format: |
+  {
+    "properties": {
+      "answer_relevance_rewrite": {
+        "title": "Rewritten answer",
+        "type": "string"
+      }
+    },
+    "required": [
+      "answer_relevance_rewrite"
+    ],
+    "type": "object"
+  }
+# Additional turn of instructions to add to the chat
+instruction: |
+  Rewrite the response for relevance.
+  The last assistant response is considered not fully relevant to the last user inquiry due to {answer_relevance_category}: {answer_relevance_analysis}
+  Decide if you agree with this assessment, then act according to the following instructions: 
+  If you disagree with the assessment, provide a verbatim copy of the original response.  DO NOT attempt to correct any other perceived defects in the response. 
+  If you agree with the assessment, provide an updated response that no longer fit the label {answer_relevance_category}, by {correction_method}.  Your response should be entirely based on the provided documents and should not rely on other prior knowledge. Your response should be suitable to be directly provided to the user.  It should NOT contain meta information regarding the original response, its assessment, or this instruction.  The user does not see any of these.  Your response is the only response they will see to their inquiry, in place of the original response.
+# Data transformations to perform during post-processing
+transformations: ~
+parameters:
+  # Rewritten response could be long.
+  max_completion_tokens: 1024
+sentence_boundaries: ~

answer_relevance_rewriter/lora/granite-3.3-8b-instruct/io.yaml

@@ -0,0 +1,29 @@
+# Model name string, or null to use whatever is provided in the chat completion request
+model: ~
+# JSON schema of the model's output, or null if constrained decoding is not needed
+response_format: |
+  {
+    "properties": {
+      "answer_relevance_rewrite": {
+        "title": "Rewritten answer",
+        "type": "string"
+      }
+    },
+    "required": [
+      "answer_relevance_rewrite"
+    ],
+    "type": "object"
+  }
+# Additional turn of instructions to add to the chat
+instruction: |
+  Rewrite the response for relevance.
+  The last assistant response is considered not fully relevant to the last user inquiry due to {answer_relevance_category}: {answer_relevance_analysis}
+  Decide if you agree with this assessment, then act according to the following instructions: 
+  If you disagree with the assessment, provide a verbatim copy of the original response.  DO NOT attempt to correct any other perceived defects in the response. 
+  If you agree with the assessment, provide an updated response that no longer fit the label {answer_relevance_category}, by {correction_method}.  Your response should be entirely based on the provided documents and should not rely on other prior knowledge. Your response should be suitable to be directly provided to the user.  It should NOT contain meta information regarding the original response, its assessment, or this instruction.  The user does not see any of these.  Your response is the only response they will see to their inquiry, in place of the original response.
+# Data transformations to perform during post-processing
+transformations: ~
+parameters:
+  # Rewritten response could be long.
+  max_completion_tokens: 1024
+sentence_boundaries: ~