79 lines
2.1 KiB
Python
79 lines
2.1 KiB
Python
"""Label Quality Scorer Router"""
|
|
from fastapi import APIRouter, UploadFile, File
|
|
from pydantic import BaseModel
|
|
from typing import Optional
|
|
|
|
router = APIRouter()
|
|
|
|
|
|
class AgreementMetrics(BaseModel):
|
|
cohens_kappa: Optional[float] = None
|
|
fleiss_kappa: Optional[float] = None
|
|
krippendorff_alpha: Optional[float] = None
|
|
percent_agreement: float
|
|
interpretation: str # poor, fair, moderate, good, excellent
|
|
|
|
|
|
class DisagreementSample(BaseModel):
|
|
sample_id: str
|
|
labels: dict # annotator -> label
|
|
majority_label: Optional[str] = None
|
|
|
|
|
|
class QualityReport(BaseModel):
|
|
total_samples: int
|
|
total_annotators: int
|
|
metrics: AgreementMetrics
|
|
disagreements: list[DisagreementSample]
|
|
recommendations: list[str]
|
|
|
|
|
|
@router.post("/analyze", response_model=QualityReport)
|
|
async def analyze_labels(
|
|
file: UploadFile = File(...),
|
|
sample_id_column: str = "id",
|
|
annotator_columns: Optional[list[str]] = None
|
|
):
|
|
"""Analyze labeling quality from annotations file"""
|
|
# TODO: Implement label quality analysis
|
|
return QualityReport(
|
|
total_samples=0,
|
|
total_annotators=0,
|
|
metrics=AgreementMetrics(
|
|
percent_agreement=0.0,
|
|
interpretation="unknown"
|
|
),
|
|
disagreements=[],
|
|
recommendations=[]
|
|
)
|
|
|
|
|
|
@router.post("/pairwise")
|
|
async def pairwise_agreement(
|
|
file: UploadFile = File(...),
|
|
annotator1: str = None,
|
|
annotator2: str = None
|
|
):
|
|
"""Calculate pairwise agreement between two annotators"""
|
|
# TODO: Implement pairwise analysis
|
|
return {
|
|
"annotator1": annotator1,
|
|
"annotator2": annotator2,
|
|
"agreement": 0.0,
|
|
"kappa": 0.0
|
|
}
|
|
|
|
|
|
@router.get("/thresholds")
|
|
async def get_quality_thresholds():
|
|
"""Get interpretation thresholds for agreement metrics"""
|
|
return {
|
|
"kappa_interpretation": {
|
|
"poor": "< 0.00",
|
|
"slight": "0.00 - 0.20",
|
|
"fair": "0.21 - 0.40",
|
|
"moderate": "0.41 - 0.60",
|
|
"substantial": "0.61 - 0.80",
|
|
"almost_perfect": "0.81 - 1.00"
|
|
}
|
|
}
|