Prompt-Based Visibility Testing

1// Prompt-Based Visibility Testing Implementation
2 
3interface PromptDefinition {
4  id: string;
5  prompt: string;
6  category: string;
7  intent: 'informational' | 'commercial' | 'navigational' | 'comparison';
8  expectedOutcome: 'should-appear' | 'should-not-appear' | 'competitive';
9  competitors?: string[];
10  variations?: string[];
11  priority: 'critical' | 'high' | 'medium' | 'low';
12}
13 
14interface TestResult {
15  promptId: string;
16  prompt: string;
17  platform: string;
18  modelVersion: string;
19  response: string;
20  brandDetected: boolean;
21  brandPosition: 'featured' | 'recommended' | 'listed' | 'mentioned' | 'absent';
22  sentiment: 'positive' | 'neutral' | 'negative' | 'mixed';
23  accuracy: 'accurate' | 'partial' | 'inaccurate' | 'outdated';
24  competitorsDetected: string[];
25  competitivePosition: 'winning' | 'parity' | 'losing' | 'not-competitive';
26  timestamp: Date;
27  responseLatency: number;
28}
29 
30interface TestingProtocol {
31  promptLibrary: PromptDefinition[];
32  platforms: string[];
33  repetitionsPerPrompt: number;
34  testingFrequency: 'daily' | 'weekly' | 'monthly';
35  statisticalThreshold: number; // confidence level required
36}
37 
38// Execute systematic prompt testing
39async function executePromptTest(
40  protocol: TestingProtocol,
41  brand: string,
42  brandVariations: string[]
43): Promise<TestResult[]> {
44  const results: TestResult[] = [];
45  
46  for (const platform of protocol.platforms) {
47    for (const promptDef of protocol.promptLibrary) {
48      // Test main prompt
49      for (let i = 0; i < protocol.repetitionsPerPrompt; i++) {
50        const result = await testSinglePrompt(
51          promptDef, 
52          platform, 
53          brand, 
54          brandVariations
55        );
56        results.push(result);
57      }
58      
59      // Test variations if defined
60      if (promptDef.variations) {
61        for (const variation of promptDef.variations) {
62          const variationDef = { ...promptDef, prompt: variation };
63          const result = await testSinglePrompt(
64            variationDef, 
65            platform, 
66            brand, 
67            brandVariations
68          );
69          results.push(result);
70        }
71      }
72    }
73  }
74  
75  return results;
76}
77 
78async function testSinglePrompt(
79  promptDef: PromptDefinition,
80  platform: string,
81  brand: string,
82  brandVariations: string[]
83): Promise<TestResult> {
84  const startTime = Date.now();
85  const response = await queryPlatform(platform, promptDef.prompt);
86  const latency = Date.now() - startTime;
87  
88  const brandAnalysis = analyzeBrandPresence(response, brand, brandVariations);
89  const competitorAnalysis = promptDef.competitors 
90    ? analyzeCompetitorPresence(response, promptDef.competitors)
91    : { detected: [], position: 'not-competitive' as const };
92  
93  return {
94    promptId: promptDef.id,
95    prompt: promptDef.prompt,
96    platform,
97    modelVersion: await getModelVersion(platform),
98    response,
99    brandDetected: brandAnalysis.detected,
100    brandPosition: brandAnalysis.position,
101    sentiment: brandAnalysis.sentiment,
102    accuracy: brandAnalysis.accuracy,
103    competitorsDetected: competitorAnalysis.detected,
104    competitivePosition: determineCompetitivePosition(
105      brandAnalysis, 
106      competitorAnalysis
107    ),
108    timestamp: new Date(),
109    responseLatency: latency
110  };
111}
112 
113function analyzeBrandPresence(
114  response: string,
115  brand: string,
116  variations: string[]
117): {
118  detected: boolean;
119  position: string;
120  sentiment: string;
121  accuracy: string;
122} {
123  const responseLower = response.toLowerCase();
124  const allBrandTerms = [brand, ...variations].map(b => b.toLowerCase());
125  
126  // Check for brand presence
127  const detected = allBrandTerms.some(term => responseLower.includes(term));
128  
129  if (!detected) {
130    return {
131      detected: false,
132      position: 'absent',
133      sentiment: 'neutral',
134      accuracy: 'accurate' // N/A when not mentioned
135    };
136  }
137  
138  // Analyze position
139  const position = determinePosition(response, brand);
140  const sentiment = analyzeSentiment(response, brand);
141  const accuracy = 'accurate'; // Would need fact-checking logic
142  
143  return { detected, position, sentiment, accuracy };
144}
145 
146function determinePosition(response: string, brand: string): string {
147  const responseLower = response.toLowerCase();
148  const brandLower = brand.toLowerCase();
149  
150  // Check for featured/recommended positioning
151  const featuredIndicators = [
152    `${brandLower} is the best`,
153    `recommend ${brandLower}`,
154    `${brandLower} stands out`,
155    `top choice.*${brandLower}`
156  ];
157  
158  for (const indicator of featuredIndicators) {
159    if (new RegExp(indicator).test(responseLower)) {
160      return 'featured';
161    }
162  }
163  
164  // Check if recommended
165  if (responseLower.includes('recommend') && 
166      responseLower.indexOf(brandLower) < responseLower.indexOf('recommend') + 100) {
167    return 'recommended';
168  }
169  
170  // Check if in a list context
171  if (responseLower.includes('include') || 
172      responseLower.includes('such as') ||
173      responseLower.includes('options')) {
174    return 'listed';
175  }
176  
177  return 'mentioned';
178}
179 
180// Generate visibility report from test results
181function generateVisibilityReport(
182  results: TestResult[],
183  protocol: TestingProtocol
184): VisibilityReport {
185  // Overall metrics
186  const totalTests = results.length;
187  const detections = results.filter(r => r.brandDetected).length;
188  const promptSuccessRate = (detections / totalTests) * 100;
189  
190  // By platform
191  const platforms = [...new Set(results.map(r => r.platform))];
192  const byPlatform: Record<string, PlatformMetrics> = {};
193  
194  platforms.forEach(platform => {
195    const platformResults = results.filter(r => r.platform === platform);
196    const platformDetections = platformResults.filter(r => r.brandDetected).length;
197    byPlatform[platform] = {
198      successRate: (platformDetections / platformResults.length) * 100,
199      averagePosition: calculateAveragePosition(platformResults),
200      sentimentBreakdown: calculateSentimentBreakdown(platformResults),
201      competitiveWinRate: calculateCompetitiveWinRate(platformResults)
202    };
203  });
204  
205  // By category
206  const categories = [...new Set(protocol.promptLibrary.map(p => p.category))];
207  const byCategory: Record<string, number> = {};
208  
209  categories.forEach(category => {
210    const categoryPrompts = protocol.promptLibrary
211      .filter(p => p.category === category)
212      .map(p => p.id);
213    const categoryResults = results.filter(r => categoryPrompts.includes(r.promptId));
214    const categoryDetections = categoryResults.filter(r => r.brandDetected).length;
215    byCategory[category] = categoryResults.length > 0
216      ? (categoryDetections / categoryResults.length) * 100
217      : 0;
218  });
219  
220  // Consistency analysis
221  const consistencyScore = calculateConsistencyScore(results, protocol);
222  
223  // Statistical confidence
224  const changeDetectionConfidence = calculateStatisticalConfidence(results);
225  
226  return {
227    testDate: new Date(),
228    totalPromptsTested: protocol.promptLibrary.length,
229    totalTestsRun: totalTests,
230    overallSuccessRate: promptSuccessRate,
231    platformMetrics: byPlatform,
232    categoryMetrics: byCategory,
233    consistencyScore,
234    statisticalConfidence: changeDetectionConfidence,
235    recommendations: generateRecommendations(results, protocol)
236  };
237}
238 
239function calculateConsistencyScore(
240  results: TestResult[],
241  protocol: TestingProtocol
242): number {
243  // Group results by prompt
244  const promptGroups = new Map<string, TestResult[]>();
245  results.forEach(r => {
246    const existing = promptGroups.get(r.promptId) || [];
247    existing.push(r);
248    promptGroups.set(r.promptId, existing);
249  });
250  
251  // Calculate variance in detection across repetitions
252  let totalVariance = 0;
253  let promptCount = 0;
254  
255  promptGroups.forEach((group, promptId) => {
256    if (group.length > 1) {
257      const detectionRate = group.filter(r => r.brandDetected).length / group.length;
258      // Variance from either 0 or 1 (consistent outcome)
259      const variance = Math.min(detectionRate, 1 - detectionRate);
260      totalVariance += variance;
261      promptCount++;
262    }
263  });
264  
265  // Convert to 0-100 score where 100 is perfectly consistent
266  return promptCount > 0 
267    ? (1 - (totalVariance / promptCount)) * 100 
268    : 100;
269}