diff --git a/typescript/packages/fi-core/src/fi_types.ts b/typescript/packages/fi-core/src/fi_types.ts index 8da3dd1..be67660 100644 --- a/typescript/packages/fi-core/src/fi_types.ts +++ b/typescript/packages/fi-core/src/fi_types.ts @@ -538,17 +538,17 @@ enum ProjectType { throw new Error(`type must be an EvalTagType enum, got ${typeof this.type}`); } - - if (this.model && !Object.values(ModelChoices).includes(this.model)) { - throw new Error(`model must be a valid model name, got ${this.model}. Expected values are: ${Object.values(ModelChoices).join(', ')}`); - } - const customEvalTemplate: CheckCustomEvalTemplateExistsResponse = await checkCustomEvalTemplateExists(this.eval_name as string); - if (!customEvalTemplate.result?.is_user_eval_template) { + + if (!customEvalTemplate.result?.isUserEvalTemplate) { if (!Object.values(EvalName).includes(this.eval_name as EvalName)) { throw new Error(`eval_name must be a valid EvalName enum if its not a custom eval template, got ${this.eval_name}. Expected values are: ${Object.values(EvalName).join(', ')}`); } + if (!this.model || !Object.values(ModelChoices).includes(this.model)) { + throw new Error(`Model must be provied in case of fagi evals. Model must be a valid model name, got ${this.model}. Expected values are: ${Object.values(ModelChoices).join(', ')}`); + } + // Get expected config for this eval type const expectedConfig = getConfigForEval(this.eval_name as EvalName); @@ -575,12 +575,13 @@ enum ProjectType { // Get expected mapping for this eval type - const expectedMapping = getMappingForEval(this.eval_name as EvalName); + let expectedMapping = null; let requiredKeys: string[] = []; - if (customEvalTemplate.result?.is_user_eval_template) { - requiredKeys = customEvalTemplate.result?.eval_template?.config?.requiredKeys ?? []; + if (customEvalTemplate.result?.isUserEvalTemplate) { + requiredKeys = customEvalTemplate.result?.evalTemplate?.config?.requiredKeys ?? []; } else { + expectedMapping = getMappingForEval(this.eval_name as EvalName); for (const [key, fieldConfig] of Object.entries(expectedMapping)) { if (fieldConfig.required) { requiredKeys.push(key); @@ -597,7 +598,7 @@ enum ProjectType { // Check for unexpected mapping fields for (const key in this.mapping) { - if (!customEvalTemplate.result?.is_user_eval_template && !(key in expectedMapping)) { + if (!customEvalTemplate.result?.isUserEvalTemplate && expectedMapping && !(key in expectedMapping)) { throw new Error(`Unexpected mapping field '${key}' for ${this.eval_name}. Allowed fields are: ${Object.keys(expectedMapping).join(', ')}`); } if (typeof key !== 'string') { @@ -647,5 +648,6 @@ enum ProjectType { prepareEvalTags, // Export both the interface and the class IEvalTag, - EvalTag + EvalTag, + ModelChoices }; \ No newline at end of file diff --git a/typescript/packages/fi-core/src/otel.ts b/typescript/packages/fi-core/src/otel.ts index 8e886c0..4cf200d 100644 --- a/typescript/packages/fi-core/src/otel.ts +++ b/typescript/packages/fi-core/src/otel.ts @@ -517,8 +517,8 @@ interface CheckExistsResponse { export interface CheckCustomEvalTemplateExistsResponse { result?: { - is_user_eval_template?: boolean; - eval_template?: any + isUserEvalTemplate?: boolean; + evalTemplate?: any }; // Add other fields if the API returns more } @@ -606,8 +606,8 @@ export async function checkCustomEvalTemplateExists( if (!eval_template_name || eval_template_name.length === 0) { const response: CheckCustomEvalTemplateExistsResponse = { result: { - is_user_eval_template: false, - eval_template: null + isUserEvalTemplate: false, + evalTemplate: null } } return response; @@ -665,8 +665,8 @@ export async function checkCustomEvalTemplateExists( ); return { result: { - is_user_eval_template: false, - eval_template: null + isUserEvalTemplate: false, + evalTemplate: null } }; } @@ -680,8 +680,8 @@ export async function checkCustomEvalTemplateExists( diag.error(`checkCustomEvalTemplateExists: Error checking custom eval template: ${error}`); return { result: { - is_user_eval_template: false, - eval_template: null + isUserEvalTemplate: false, + evalTemplate: null } }; } diff --git a/typescript/packages/traceai_openai/examples/basic-otel-test.ts b/typescript/packages/traceai_openai/examples/basic-otel-test.ts index e17357b..5c2b2c7 100644 --- a/typescript/packages/traceai_openai/examples/basic-otel-test.ts +++ b/typescript/packages/traceai_openai/examples/basic-otel-test.ts @@ -1,6 +1,8 @@ -import { register, ProjectType, EvalSpanKind, EvalName, EvalTag, EvalTagType } from "@traceai/fi-core"; +import { register, ProjectType, EvalSpanKind, EvalName, EvalTag, EvalTagType, ModelChoices } from "@traceai/fi-core"; import { OpenAIInstrumentation } from "@traceai/openai"; import { registerInstrumentations } from "@opentelemetry/instrumentation"; +import { DiagConsoleLogger, DiagLogLevel } from "@opentelemetry/api"; +import { diag } from "@opentelemetry/api"; // Enable OpenTelemetry internal diagnostics diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.DEBUG); @@ -31,8 +33,9 @@ async function main() { // 1. Register FI Core TracerProvider (sets up exporter) const tracerProvider = register({ - projectName: "ts-observability-suite-v3", + projectName: "ts-observability-suite-v5", projectType: ProjectType.EXPERIMENT, + projectVersionName: "sarthak_f1", // sessionName: "basic-otel-test-session-" + Date.now(), // OBSERVE only evalTags: [ new EvalTag({ @@ -40,23 +43,61 @@ async function main() { value: EvalSpanKind.LLM, eval_name: EvalName.CHUNK_ATTRIBUTION, config: {}, - custom_eval_name: "Chunk_Attribution", + custom_eval_name: "Chunk_Attribution_3", mapping: { "context": "raw.input", "output": "raw.output" - } + }, + model: ModelChoices.TURING_SMALL }), - new EvalTag({ - type: EvalTagType.OBSERVATION_SPAN, - value: EvalSpanKind.LLM, - eval_name: EvalName.SUMMARY_QUALITY, - config: {}, - custom_eval_name: "Summary_Quality", - mapping: { - "context": "raw.input", - "output": "raw.output" + new EvalTag( + { + type: EvalTagType.OBSERVATION_SPAN, + value: EvalSpanKind.LLM, + eval_name: "toxic_nature", + custom_eval_name: "toxic_nature_custom_eval_config_2", + mapping: { + "output": "raw.output" + } + } + ), + new EvalTag( + { + type: EvalTagType.OBSERVATION_SPAN, + value: EvalSpanKind.LLM, + eval_name: "custom-eval-1", + custom_eval_name: "custom-eval-1-config_eval_2", + mapping: { + "output": "raw.output", + "input": "raw.input" + } + } + ), + new EvalTag( + { + type: EvalTagType.OBSERVATION_SPAN, + value: EvalSpanKind.LLM, + eval_name: "detereministic_custom_eval_template", + custom_eval_name: "detereministic_custom_eval_template_2", + mapping: { + "output": "raw.output", + "query": "raw.input", + "input": "raw.input" + } } - }) + ) + // , + // new EvalTag({ + // type: EvalTagType.OBSERVATION_SPAN, + // value: EvalSpanKind.LLM, + // eval_name: EvalName.SUMMARY_QUALITY, + // config: {}, + // custom_eval_name: "Summary_Quality_1", + // mapping: { + // "context": "raw.input", + // "output": "raw.output" + // } + // }) ] });