1import { ScalarTypeEnum, SchemaModel } from '@lssm-tech/lib.schema';
2import { docId } from '../../docs/registry';
3import type { DocBlock } from '../../docs/types';
4import { defineCommand } from '../../operations';
5import {
6	PROVIDER_RANKING_DOMAIN,
7	PROVIDER_RANKING_OWNERS,
8	PROVIDER_RANKING_STABILITY,
9	PROVIDER_RANKING_TAGS,
10} from '../constants';
11import { BenchmarkCustomCompletedEvent } from '../events/benchmarkCustomCompleted.event';
12
13export const BenchmarkRunCustomCommand = defineCommand({
14	meta: {
15		key: 'provider-ranking.benchmark.run-custom',
16		title: 'Run Custom Benchmark',
17		version: '1.0.0',
18		description:
19			'Launch a custom benchmark evaluation against a specific model.',
20		goal: 'Evaluate model performance using internal eval suites.',
21		context:
22			'Used by operators to run proprietary benchmarks and compare models.',
23		domain: PROVIDER_RANKING_DOMAIN,
24		owners: PROVIDER_RANKING_OWNERS,
25		tags: [...PROVIDER_RANKING_TAGS, 'custom', 'eval'],
26		stability: PROVIDER_RANKING_STABILITY,
27		docId: [docId('docs.tech.provider-ranking.benchmark.run-custom')],
28	},
29	capability: {
30		key: 'provider-ranking.system',
31		version: '1.0.0',
32	},
33	io: {
34		input: BenchmarkRunCustomInput,
35		output: BenchmarkRunCustomOutput,
36	},
37	policy: {
38		auth: 'user',
39		pii: [],
40	},
41	sideEffects: {
42		emits: [
43			{
44				ref: BenchmarkCustomCompletedEvent.meta,
45				when: 'Custom benchmark evaluation finishes execution.',
46			},
47		],
48	},
49});