Skip to content

Add new evaluation #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JayaSurya-27 merged 4 commits into from
May 6, 2025
Merged

Add new evaluation #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
9b96b9e
Add new evaluation
JayaSurya-27 Apr 15, 2025
ebb92f6
changes to README files
JayaSurya-27 Apr 20, 2025
9b6e376
Merge branch 'dev' into feature/evals
JayaSurya-27 May 6, 2025
0c05f95
Update version numbers for multiple frameworks to 0.1.4 and 0.1.1, en…
JayaSurya-27 May 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 1 addition & 3 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,9 +59,7 @@ from fi_instrumentation.fi_types import ProjectType

trace_provider = register(
project_type=ProjectType.OBSERVE,
project_name="openai_app",
project_version_name="v1",
session_name="chat-bot"
project_name="openai_app"
)
```

Expand Down
42 changes: 35 additions & 7 deletions python/fi_instrumentation/fi_types.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -477,6 +477,10 @@ class EvalName(Enum):
AUDIO_TRANSCRIPTION = "Audio Transcription"
EVAL_AUDIO_DESCRIPTION = "Eval Audio Description"
AUDIO_QUALITY = "Audio Quality"
JSON_SCHEMA_VALIDATION = "Json Scheme Validation"
CHUNK_ATTRIBUTION = "Chunk Attribution"
CHUNK_UTILIZATION = "Chunk Utilization"
EVAL_RANKING = "Eval Ranking"


@dataclass
Expand Down Expand Up @@ -587,9 +591,7 @@ def get_config_for_eval(eval_name: EvalName) -> Dict[str, Dict[str, Any]]:
"case_sensitive": ConfigField(type=bool, default=True),
"keywords": ConfigField(type=list, default=[]),
},
EvalName.GROUNDEDNESS: {
"model": ConfigField(type=str, default="gpt-4o-mini")
},
EvalName.GROUNDEDNESS: {},
EvalName.ANSWER_SIMILARITY: {
"comparator": ConfigField(type=str, default="CosineSimilarity"),
"failure_threshold": ConfigField(type=float, default=0.5),
Expand All @@ -607,9 +609,6 @@ def get_config_for_eval(eval_name: EvalName) -> Dict[str, Dict[str, Any]]:
default="Evaluate if the context is relevant and sufficient to support the output.",
)
},
# EvalName.EVAL_RANKING: {
# "criteria": ConfigField(type=str, default="Check if the summary concisely captures the main points while maintaining accuracy and relevance to the original content.")
# },
EvalName.EVAL_IMAGE_INSTRUCTION: {
"criteria": ConfigField(
type=str,
Expand Down Expand Up @@ -683,6 +682,17 @@ def get_config_for_eval(eval_name: EvalName) -> Dict[str, Dict[str, Any]]:
),
"model": ConfigField(type=str, default="gemini-2.0-flash"),
},
EvalName.JSON_SCHEMA_VALIDATION: {
"validations": ConfigField(type=list, default=[]),
},
EvalName.CHUNK_ATTRIBUTION: {},
EvalName.CHUNK_UTILIZATION: {},
EvalName.EVAL_RANKING: {
"criteria": ConfigField(
type=str,
default="Check if the summary concisely captures the main points while maintaining accuracy and relevance to the original content."
),
},
}

# Convert ConfigField objects to dictionary format
Expand Down Expand Up @@ -772,7 +782,7 @@ def get_mapping_for_eval(eval_name: EvalName) -> Dict[str, Dict[str, Any]]:
EvalName.CONTAINS: {"text": ConfigField(type=str, required=True)},
EvalName.CONTAINS_ANY: {"text": ConfigField(type=str, required=True)},
EvalName.GROUNDEDNESS: {
"context": ConfigField(type=str, required=True),
"output": ConfigField(type=str, required=True),
"input": ConfigField(type=str, required=True),
},
EvalName.ANSWER_SIMILARITY: {
Expand Down Expand Up @@ -829,6 +839,24 @@ def get_mapping_for_eval(eval_name: EvalName) -> Dict[str, Dict[str, Any]]:
EvalName.AUDIO_QUALITY: {
"input audio": ConfigField(type=str, required=True)
},
EvalName.JSON_SCHEMA_VALIDATION: {
"actual_json": ConfigField(type=dict, required=True),
"expected_json": ConfigField(type=str, required=True),
},
EvalName.CHUNK_ATTRIBUTION: {
"input": ConfigField(type=str),
"output": ConfigField(type=str, required=True),
"context": ConfigField(type=str, required=True),
},
EvalName.CHUNK_UTILIZATION: {
"input": ConfigField(type=str),
"output": ConfigField(type=str, required=True),
"context": ConfigField(type=str, required=True),
},
EvalName.EVAL_RANKING: {
"input": ConfigField(type=str, required=True),
"context" : ConfigField(type=str, required=True),
},
}

# Convert ConfigField objects to dictionary format
Expand Down
4 changes: 1 addition & 3 deletions python/frameworks/anthropic/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,7 @@ from fi_instrumentation.fi_types import ProjectType

trace_provider = register(
project_type=ProjectType.OBSERVE,
project_name="anthropic_app",
project_version_name="v1",
session_name="chat-bot"
project_name="anthropic_app"
)
```

Expand Down
4 changes: 2 additions & 2 deletions python/frameworks/anthropic/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[tool.poetry]
name = "traceAI-anthropic"
version = "0.1.3"
version = "0.1.4"
description = "OpenTelemetry instrumentation for Anthropic"
authors = ["Future AGI <no-reply@futureagi.com>"]
readme = "README.md"
Expand All @@ -12,7 +12,7 @@ packages = [
[tool.poetry.dependencies]
python = ">=3.9, <3.13"
anthropic = ">=0.30.0,<1.0.0"
fi-instrumentation-otel = "0.1.3"
fi-instrumentation-otel = "0.1.4"


[build-system]
Expand Down
4 changes: 1 addition & 3 deletions python/frameworks/autogen/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,7 @@ from fi_instrumentation.fi_types import ProjectType

trace_provider = register(
project_type=ProjectType.OBSERVE,
project_name="autogen_app",
project_version_name="v1",
session_name="chat-bot"
project_name="autogen_app"
)
```

Expand Down
4 changes: 2 additions & 2 deletions python/frameworks/autogen/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[tool.poetry]
name = "traceAI-autogen"
version = "0.1.0"
version = "0.1.1"
description = "OpenTelemetry instrumentation for Autogen"
authors = ["Future AGI <no-reply@futureagi.com>"]
readme = "README.md"
Expand All @@ -12,7 +12,7 @@ packages = [
[tool.poetry.dependencies]
python = ">=3.9, <3.14"
autogen = ">=0.5.0"
fi-instrumentation-otel = "0.1.3"
fi-instrumentation-otel = "0.1.4"


[build-system]
Expand Down
4 changes: 1 addition & 3 deletions python/frameworks/bedrock/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,7 @@ from fi_instrumentation.fi_types import ProjectType

trace_provider = register(
project_type=ProjectType.OBSERVE,
project_name="bedrock_app",
project_version_name="v1",
session_name="s1"
project_name="bedrock_app"
)
```

Expand Down
4 changes: 2 additions & 2 deletions python/frameworks/bedrock/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[tool.poetry]
name = "traceAI-bedrock"
version = "0.1.3"
version = "0.1.4"
description = "OpenTelemetry instrumentation for Bedrock"
authors = ["Future AGI <no-reply@futureagi.com>"]
readme = "README.md"
Expand All @@ -12,7 +12,7 @@ packages = [
[tool.poetry.dependencies]
python = ">3.10, <3.13"
dacite = "^1.9.1"
fi-instrumentation-otel = "0.1.3"
fi-instrumentation-otel = "0.1.4"
botocore = ">=1.34.116"
boto3 = ">=1.28.57"

Expand Down
4 changes: 1 addition & 3 deletions python/frameworks/crewai/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,7 @@ from fi_instrumentation.fi_types import ProjectType

trace_provider = register(
project_type=ProjectType.OBSERVE,
project_name="crewai_app",
project_version_name="v1",
session_name="chat-bot"
project_name="crewai_app"
)
```

Expand Down
4 changes: 2 additions & 2 deletions python/frameworks/crewai/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[tool.poetry]
name = "traceAI-crewai"
version = "0.1.3"
version = "0.1.4"
description = "OpenTelemetry instrumentation for Crewai"
authors = ["Future AGI <no-reply@futureagi.com>"]
readme = "README.md"
Expand All @@ -13,7 +13,7 @@ packages = [
python = ">=3.10, <3.13"
wrapt = "^1.15.0"
crewai = ">0.19.0"
fi-instrumentation-otel = "0.1.3"
fi-instrumentation-otel = "0.1.4"


[build-system]
Expand Down
4 changes: 1 addition & 3 deletions python/frameworks/dspy/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,7 @@ from fi_instrumentation.fi_types import ProjectType

trace_provider = register(
project_type=ProjectType.OBSERVE,
project_name="dspy_app",
project_version_name="v1",
session_name="chat-bot"
project_name="dspy_app"
)
```

Expand Down
4 changes: 2 additions & 2 deletions python/frameworks/dspy/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[tool.poetry]
name = "traceAI-DSPy"
version = "0.1.3"
version = "0.1.4"
description = "OpenTelemetry instrumentation for DSPy"
authors = ["Future AGI <no-reply@futureagi.com>"]
readme = "README.md"
Expand All @@ -12,7 +12,7 @@ packages = [
[tool.poetry.dependencies]
python = ">=3.10, <3.13"
dspy = ">=2.5.0"
fi-instrumentation-otel = "0.1.3"
fi-instrumentation-otel = "0.1.4"


[build-system]
Expand Down
4 changes: 1 addition & 3 deletions python/frameworks/groq/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,7 @@ from fi_instrumentation.fi_types import ProjectType

trace_provider = register(
project_type=ProjectType.OBSERVE,
project_name="groq_app",
project_version_name="v1",
session_name="chat-bot"
project_name="groq_app"
)
```

Expand Down
4 changes: 2 additions & 2 deletions python/frameworks/groq/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[tool.poetry]
name = "traceAI-groq"
version = "0.1.3"
version = "0.1.4"
description = "OpenTelemetry instrumentation for Groq"
authors = ["Future AGI <no-reply@futureagi.com>"]
readme = "README.md"
Expand All @@ -12,7 +12,7 @@ packages = [
[tool.poetry.dependencies]
python = ">=3.10, <3.13"
groq = ">=0.9.0"
fi-instrumentation-otel = "0.1.3"
fi-instrumentation-otel = "0.1.4"


[build-system]
Expand Down
4 changes: 1 addition & 3 deletions python/frameworks/guardrails/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,7 @@ from fi_instrumentation.fi_types import ProjectType

trace_provider = register(
project_type=ProjectType.OBSERVE,
project_name="guardrails_app",
project_version_name="v1",
session_name="chat-bot"
project_name="guardrails_app"
)
```

Expand Down
4 changes: 2 additions & 2 deletions python/frameworks/guardrails/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[tool.poetry]
name = "traceAI-guardrails"
version = "0.1.0"
version = "0.1.1"
description = "OpenTelemetry instrumentation for Guardrails"
authors = ["Future AGI <no-reply@futureagi.com>"]
readme = "README.md"
Expand All @@ -13,7 +13,7 @@ packages = [
python = ">=3.9, <3.13"
packaging = "*"
guardrails-ai = ">=0.4.5,<0.5.1"
fi-instrumentation-otel = "0.1.3"
fi-instrumentation-otel = "0.1.4"


[build-system]
Expand Down
4 changes: 1 addition & 3 deletions python/frameworks/haystack/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,7 @@ from fi_instrumentation.fi_types import ProjectType

trace_provider = register(
project_type=ProjectType.OBSERVE,
project_name="haystack_app",
project_version_name="v1",
session_name="chat-bot"
project_name="haystack_app"
)
```

Expand Down
4 changes: 2 additions & 2 deletions python/frameworks/haystack/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[tool.poetry]
name = "traceAI-haystack"
version = "0.1.3"
version = "0.1.4"
description = "OpenTelemetry instrumentation for Haystack"
authors = ["Future AGI <no-reply@futureagi.com>"]
readme = "README.md"
Expand All @@ -12,7 +12,7 @@ packages = [
[tool.poetry.dependencies]
python = ">3.9, <3.13"
haystack-ai = ">=2.10.0"
fi-instrumentation-otel = "0.1.3"
fi-instrumentation-otel = "0.1.4"


[build-system]
Expand Down
4 changes: 1 addition & 3 deletions python/frameworks/instructor/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,7 @@ from fi_instrumentation.fi_types import ProjectType

trace_provider = register(
project_type=ProjectType.OBSERVE,
project_name="instructor_app",
project_version_name="v1",
session_name="chat-bot"
project_name="instructor_app"
)
```

Expand Down
4 changes: 2 additions & 2 deletions python/frameworks/instructor/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[tool.poetry]
name = "traceAI-instructor"
version = "0.1.3"
version = "0.1.4"
description = "OpenTelemetry instrumentation for Instructor"
authors = ["Future AGI <no-reply@futureagi.com>"]
readme = "README.md"
Expand All @@ -11,7 +11,7 @@ packages = [

[tool.poetry.dependencies]
python = ">3.9, <4.0"
fi-instrumentation-otel = "0.1.3"
fi-instrumentation-otel = "0.1.4"
instructor = ">=0.0.1"


Expand Down
4 changes: 2 additions & 2 deletions python/frameworks/langchain/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[tool.poetry]
name = "traceAI-langchain"
version = "0.1.4"
version = "0.1.5"
description = "OpenTelemetry instrumentation for Langchain"
authors = ["Future AGI <no-reply@futureagi.com>"]
readme = "README.md"
Expand All @@ -11,7 +11,7 @@ packages = [

[tool.poetry.dependencies]
python = ">3.9, <4.0"
fi-instrumentation-otel = "0.1.3"
fi-instrumentation-otel = "0.1.4"
langchain = {version = "^0.3.9", extras = ["all"]}
langchain-community = {version = "^0.3.9", extras = ["all"]}
langchain-core = ">=0.2.43"
Expand Down
4 changes: 1 addition & 3 deletions python/frameworks/litellm/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,7 @@ from fi_instrumentation.fi_types import ProjectType

trace_provider = register(
project_type=ProjectType.OBSERVE,
project_name="litellm_app",
project_version_name="v1",
session_name="chat-bot"
project_name="litellm_app"
)
```

Expand Down
4 changes: 2 additions & 2 deletions python/frameworks/litellm/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[tool.poetry]
name = "traceAI-litellm"
version = "0.1.3"
version = "0.1.4"
description = "OpenTelemetry instrumentation for LiteLLM"
authors = ["Future AGI <no-reply@futureagi.com>"]
readme = "README.md"
Expand All @@ -12,7 +12,7 @@ packages = [
[tool.poetry.dependencies]
python = ">=3.9, <4.0"
litellm = ">=1.43.0"
fi-instrumentation-otel = "0.1.3"
fi-instrumentation-otel = "0.1.4"



Expand Down
Loading