Source Reliability Scoring (LLMs)

1// Analyzing and optimizing for source reliability scoring
2 
3// 1. Reliability signal audit
4interface ReliabilityAudit {
5  domain: string;
6  scores: {
7    authorCredentials: number;
8    citationQuality: number;
9    contentConsistency: number;
10    temporalSignals: number;
11    institutionalAssociation: number;
12    networkAuthority: number;
13    claimCalibration: number;
14    negativePatterns: number;
15  };
16  overall: number;
17  recommendations: string[];
18}
19 
20function auditReliabilitySignals(content: Content): ReliabilityAudit {
21  const scores = {
22    // Author credential verification
23    authorCredentials: assessAuthorCredentials(content.author),
24    
25    // Citation chain quality
26    citationQuality: assessCitationQuality(content.citations),
27    
28    // Internal consistency check
29    contentConsistency: assessConsistency(content),
30    
31    // Freshness and update patterns
32    temporalSignals: assessTemporalSignals(content),
33    
34    // Institutional connections
35    institutionalAssociation: assessInstitutionalLinks(content),
36    
37    // Inbound authority signals
38    networkAuthority: assessNetworkAuthority(content.domain),
39    
40    // Claim vs evidence calibration
41    claimCalibration: assessClaimCalibration(content),
42    
43    // Negative reliability patterns
44    negativePatterns: detectNegativePatterns(content)
45  };
46  
47  const overall = calculateOverallReliability(scores);
48  const recommendations = generateRecommendations(scores);
49  
50  return {
51    domain: content.domain,
52    scores,
53    overall,
54    recommendations
55  };
56}
57 
58// 2. Citation quality assessment
59function assessCitationQuality(citations: Citation[]): number {
60  if (!citations || citations.length === 0) return 0.2;
61  
62  let score = 0;
63  
64  for (const citation of citations) {
65    // Primary source bonus
66    if (citation.isPrimarySource) score += 0.3;
67    
68    // Verifiable link bonus
69    if (citation.url && isAccessible(citation.url)) score += 0.1;
70    
71    // Authority source bonus
72    if (isKnownAuthority(citation.source)) score += 0.2;
73    
74    // Academic/institutional source bonus
75    if (isAcademicSource(citation.source)) score += 0.15;
76    
77    // Recency bonus for time-sensitive topics
78    if (citation.date && isRecent(citation.date)) score += 0.1;
79  }
80  
81  // Normalize by citation count with diminishing returns
82  return Math.min(1, score / (1 + Math.log(citations.length)));
83}
84 
85// 3. Claim calibration assessment
86function assessClaimCalibration(content: Content): number {
87  const claims = extractClaims(content.text);
88  let calibrationScore = 0;
89  
90  for (const claim of claims) {
91    const claimStrength = assessClaimStrength(claim);
92    const evidenceStrength = assessEvidenceStrength(claim);
93    
94    // Well-calibrated: claim strength matches evidence
95    if (claimStrength <= evidenceStrength) {
96      calibrationScore += 1;
97    } 
98    // Overclaimed: definitive claim, weak evidence
99    else if (claimStrength - evidenceStrength > 0.5) {
100      calibrationScore -= 0.5;
101    }
102    // Slight overclaim
103    else {
104      calibrationScore += 0.5;
105    }
106  }
107  
108  return Math.max(0, Math.min(1, calibrationScore / claims.length));
109}
110 
111// 4. Negative pattern detection
112function detectNegativePatterns(content: Content): number {
113  let penalty = 0;
114  
115  // Check for clickbait patterns
116  if (hasClickbaitPatterns(content.headline)) penalty += 0.2;
117  
118  // Check for broken citations
119  const brokenCitations = content.citations?.filter(c => !isAccessible(c.url));
120  penalty += (brokenCitations?.length || 0) * 0.05;
121  
122  // Check for outdated statistics
123  const outdatedStats = findOutdatedStatistics(content.text);
124  penalty += outdatedStats.length * 0.1;
125  
126  // Check for unattributed claims
127  const unattributedClaims = findUnattributedClaims(content.text);
128  penalty += unattributedClaims.length * 0.03;
129  
130  // Check for factual errors against known facts
131  const factualErrors = checkAgainstKnownFacts(content.text);
132  penalty += factualErrors.length * 0.15;
133  
134  // Return inverted score (higher = better = fewer negative patterns)
135  return Math.max(0, 1 - penalty);
136}
137 
138// 5. Structured reliability schema
139const reliabilitySchema = {
140  "@context": "https://schema.org",
141  "@type": "WebPage",
142  "mainEntity": {
143    "@type": "Article",
144    "reliability": {
145      "@type": "Rating",
146      "ratingValue": 4.5,
147      "bestRating": 5,
148      "worstRating": 1,
149      "ratingExplanation": "Expert-authored, peer-reviewed, fully cited"
150    },
151    "reviewProcess": {
152      "@type": "Review",
153      "reviewAspect": "Factual accuracy",
154      "reviewRating": {
155        "@type": "Rating",
156        "ratingValue": 5,
157        "bestRating": 5
158      }
159    },
160    "correctionPolicy": "https://example.com/corrections",
161    "accuracyCommitment": {
162      "@type": "CreativeWork",
163      "name": "Editorial Standards",
164      "url": "https://example.com/standards"
165    }
166  }
167};