"""Privacy Scanner Router - PII Detection and Redaction"""
from fastapi import APIRouter, UploadFile, File, HTTPException, Form
from pydantic import BaseModel
from typing import Optional
import re
import io
import tempfile
import os
import duckdb
import json
import base64
import unicodedata
import html

router = APIRouter()

# Valid TLDs for email detection - prevents false positives on "at 7pm", "at cvs", etc.
VALID_TLDS = {
    'com', 'org', 'net', 'edu', 'gov', 'io', 'co', 'ai', 'dev',
    'app', 'xyz', 'info', 'biz', 'me', 'uk', 'de', 'fr', 'ca',
    'au', 'in', 'jp', 'ru', 'br', 'it', 'nl', 'es', 'ch', 'se',
    'no', 'fi', 'dk', 'at', 'be', 'pl', 'pt', 'ie', 'nz', 'sg',
    'hk', 'kr', 'mx', 'ar', 'cl', 'za', 'us', 'mil', 'int', 'eu',
    'asia', 'mobi', 'name', 'pro', 'aero', 'coop', 'museum', 'jobs',
    'travel', 'xxx', 'cat', 'tel', 'post', 'club', 'online', 'site',
    'tech', 'store', 'blog', 'shop', 'live', 'cloud', 'news', 'email'
}

# Words that commonly follow "at" but are NOT domains
FALSE_TRIGGER_WORDS = {
    'the', 'a', 'an', 'my', 'your', 'his', 'her', 'our', 'their', 'ur',
    'home', 'work', 'office', 'school', 'noon', 'night', 'midnight',
    'dawn', 'dusk', 'once', 'least', 'most', 'first', 'last', 'all',
    'gate', 'terminal', 'platform', 'station', 'airport', 'store',
    'mall', 'gym', 'park', 'beach', 'bar', 'restaurant', 'hotel',
    'clinic', 'hospital', 'bank', 'church', 'cvs', 'target', 'walmart'
}


# PII Detection Patterns
PII_PATTERNS = {
    "EMAIL": {
        # Negative lookbehind (?<!:) excludes connection strings like password@host
        "pattern": r'(?<!:)\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b',
        "description": "Email addresses",
        "category": "pii"
    },
    "EMAIL_OBFUSCATED": {
        # Obfuscated emails: t-e-s-t [at] example [dot] com, test(at)example(dot)com
        # IMPORTANT: Do NOT match plain " at " - only match bracketed [at], (at), or literal @
        # Plain " at " causes massive false positives ("meet at 5pm", "close at 9am", etc.)
        "pattern": r'[A-Za-z0-9](?:[-\s]*[A-Za-z0-9]){2,}\s*(?:\[at\]|\(at\))\s*[A-Za-z0-9](?:[-\s]*[A-Za-z0-9]){2,}\s*(?:\[dot\]|\(dot\)|\s+dot\s+)\s*[A-Za-z]{2,}',
        "description": "Obfuscated email addresses",
        "category": "pii"
    },
    "PHONE_US": {
        # US phone numbers: (555) 123-4567, 555-123-4567, +1 555 123 4567
        "pattern": r'\b(?:\+?1[-.\s]?)?\(?[0-9]{3}\)?[-.\s]?[0-9]{3}[-.\s]?[0-9]{4}\b',
        "description": "US Phone numbers",
        "category": "pii"
    },
    "PHONE_INTL": {
        # International phone numbers: +49 30 1234567, +44 20 7946 0958, +55 11 98765-4321
        # EU: 49(DE), 44(UK), 33(FR), 39(IT), 34(ES), 31(NL), 32(BE), 43(AT), 41(CH), 48(PL), etc.
        # LATAM: 55(BR), 52(MX), 54(AR), 56(CL), 57(CO), 51(PE)
        # APAC: 81(JP), 82(KR), 86(CN), 91(IN), 61(AU), 64(NZ), 65(SG), 852(HK)
        "pattern": r'\+(?:49|44|33|39|34|31|32|43|41|48|351|353|358|47|46|45|420|36|40|359|385|386|421|370|371|372|352|356|357|30|55|52|54|56|57|51|81|82|86|91|61|64|65|852)\s?[0-9]{1,4}[\s-]?[0-9]{3,4}[\s-]?[0-9]{3,6}\b',
        "description": "International Phone numbers (Global)",
        "category": "pii"
    },
    "SSN": {
        # US SSN: Requires separators (dashes, dots, spaces, or underscores)
        # Matches: 123-45-6789, 123.45.6789, 123 45 6789, 123_45_6789
        "pattern": r'\b\d{3}[-.\s_]\d{2}[-.\s_]\d{4}\b',
        "description": "Social Security Numbers (US)",
        "category": "pii"
    },
    "MEDICARE_ID": {
        # US Medicare Beneficiary Identifier (MBI): 11 characters, alphanumeric
        # Format: 1A12-B12-CD12 or 1-A-1-2-B-1-2-C-D-1-2 (with extra dashes)
        "pattern": r'\b[1-9][-]?[A-Z][-]?(?:[A-Z0-9][-]?){2}[A-Z][-]?(?:[A-Z0-9][-]?){2}[A-Z][-]?[A-Z][-]?[0-9][-]?[0-9]\b',
        "description": "Medicare Beneficiary Identifier (MBI)",
        "category": "pii"
    },
    "GERMAN_ID": {
        # German Personalausweisnummer (ID card): 10 characters
        # Format: T220001293 or similar
        "pattern": r'\b[A-Z0-9]{10}\b(?=.*(?:ausweis|personalausweis|id\s*card|national\s*id))',
        "description": "German National ID (Personalausweis)",
        "category": "pii"
    },
    "CREDIT_CARD": {
        # Matches Visa, Mastercard, Amex, Discover with optional spaces/dashes
        "pattern": r'\b(?:4[0-9]{3}[-\s]?[0-9]{4}[-\s]?[0-9]{4}[-\s]?[0-9]{4}|4[0-9]{12}(?:[0-9]{3})?|5[1-5][0-9]{2}[-\s]?[0-9]{4}[-\s]?[0-9]{4}[-\s]?[0-9]{4}|5[1-5][0-9]{14}|3[47][0-9]{2}[-\s]?[0-9]{6}[-\s]?[0-9]{5}|3[47][0-9]{13}|6(?:011|5[0-9]{2})[-\s]?[0-9]{4}[-\s]?[0-9]{4}[-\s]?[0-9]{4}|6(?:011|5[0-9]{2})[0-9]{12})\b',
        "description": "Credit card numbers",
        "category": "financial"
    },
    "IP_ADDRESS": {
        "pattern": r'\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b',
        "description": "IP addresses",
        "category": "pii"
    },
    "IP_DEFANGED": {
        # Defanged IPs: 192[.]168[.]1[.]1, 192[dot]168[dot]1[dot]1, 192(.)168(.)1(.)1
        "pattern": r'\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(?:\[(?:\.|\s*dot\s*)\]|\((?:\.|\s*dot\s*)\)|\[\.\])){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b',
        "description": "Defanged IP addresses",
        "category": "pii"
    },
    "DATE_OF_BIRTH": {
        # Matches multiple date formats:
        # MM/DD/YYYY, MM-DD-YYYY, DD/MM/YYYY, YYYY-MM-DD, Month DD, YYYY
        "pattern": r'\b(?:(?:0?[1-9]|1[0-2])[/-](?:0?[1-9]|[12][0-9]|3[01])[/-](?:19|20)\d{2}|(?:0?[1-9]|[12][0-9]|3[01])[/-](?:0?[1-9]|1[0-2])[/-](?:19|20)\d{2}|(?:19|20)\d{2}[-/](?:0?[1-9]|1[0-2])[-/](?:0?[1-9]|[12][0-9]|3[01])|(?:January|February|March|April|May|June|July|August|September|October|November|December|Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[.,]?\s+(?:0?[1-9]|[12][0-9]|3[01])(?:st|nd|rd|th)?[.,]?\s+(?:19|20)\d{2})\b',
        "description": "Dates of birth (multiple formats)",
        "category": "pii"
    },
    "IBAN": {
        # International Bank Account Numbers - very flexible pattern
        # Format: 2 letters (country) + 2 digits (check) + up to 30 alphanumeric (BBAN)
        # Supports with/without spaces, common country formats
        "pattern": r'\b(?:DE|GB|FR|ES|IT|NL|BE|AT|CH|PL|PT|IE|FI|NO|SE|DK|CZ|HU|RO|BG|HR|SI|SK|LT|LV|EE|LU|MT|CY|GR)\d{2}[\s]?[A-Z0-9]{4}[\s]?[A-Z0-9]{4}[\s]?[A-Z0-9]{4}[\s]?[A-Z0-9]{0,18}\b',
        "description": "International Bank Account Numbers",
        "category": "financial"
    },
    "BANK_ACCOUNT": {
        # Generic bank account patterns (routing + account, with labels)
        "pattern": r'\b(?:account|acct|a/c|acc)[:\s#]*[0-9]{8,17}\b',
        "description": "Bank Account Numbers",
        "category": "financial"
    },
    "ROUTING_NUMBER": {
        # US Bank Routing/ABA numbers (9 digits)
        "pattern": r'\b(?:routing|aba|rtn)[:\s#]*[0-9]{9}\b',
        "description": "Bank Routing/ABA Numbers",
        "category": "financial"
    },
    "SWIFT_BIC": {
        # SWIFT/BIC codes (8 or 11 chars) - requires colon separator to avoid false positives
        # Format: 4 letters (bank) + 2 letters (country ISO) + 2 alphanum (location) + optional 3 alphanum (branch)
        # Pattern requires colon or "code" keyword to distinguish from "SWIFT transfer" etc.
        "pattern": r'(?:swift|bic|swift/bic)(?::|(?:\s+code\s*))[:\s]*([A-Z]{4}[A-Z]{2}[A-Z0-9]{2}(?:[A-Z0-9]{3})?)\b',
        "description": "SWIFT/BIC Codes",
        "category": "financial"
    },
    "ZIP_CODE": {
        "pattern": r'\b\d{5}(?:[-\s]\d{4})?\b',
        "description": "US ZIP codes",
        "category": "pii"
    },
    "DRIVERS_LICENSE": {
        # US Driver's License - requires context to avoid false positives
        # Most states use letter + 7-12 digits, but pattern too generic without context
        "pattern": r'\b[A-Z][0-9]{7,12}\b(?=.*(?:license|licence|driver|dmv|dl[:#\s]|driving))',
        "description": "Driver's license numbers",
        "category": "pii"
    },
    # Physical Address patterns
    "US_ADDRESS": {
        "pattern": r'\b\d{1,5}\s+(?:[A-Za-z]+\s+){1,4}(?:Street|St|Avenue|Ave|Road|Rd|Boulevard|Blvd|Drive|Dr|Lane|Ln|Court|Ct|Way|Place|Pl|Circle|Cir|Highway|Hwy)\.?(?:\s+(?:Apt|Suite|Ste|Unit|#)\s*[A-Za-z0-9-]+)?(?:,\s*[A-Za-z\s]+,\s*[A-Z]{2}\s+\d{5}(?:-\d{4})?)?\b',
        "description": "US physical addresses",
        "category": "pii"
    },
    # Secrets and API Keys
    "AWS_ACCESS_KEY": {
        "pattern": r'\b(?:AKIA|ABIA|ACCA|ASIA)[A-Z0-9]{16}\b',
        "description": "AWS Access Key IDs",
        "category": "secret"
    },
    "AWS_SECRET_KEY": {
        "pattern": r'\b[A-Za-z0-9/+=]{40}\b',
        "description": "AWS Secret Access Keys (40 char base64)",
        "category": "secret"
    },
    "GITHUB_TOKEN": {
        "pattern": r'\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{36,}\b',
        "description": "GitHub Personal Access Tokens",
        "category": "secret"
    },
    "GITHUB_TOKEN_CLASSIC": {
        "pattern": r'\bgh[pousr]_[A-Za-z0-9]{36}\b',
        "description": "GitHub Classic Tokens",
        "category": "secret"
    },
    "SLACK_TOKEN": {
        "pattern": r'\bxox[baprs]-[0-9]{10,13}-[0-9]{10,13}[a-zA-Z0-9-]*\b',
        "description": "Slack API Tokens",
        "category": "secret"
    },
    "GENERIC_API_KEY": {
        "pattern": r'\b(?:api[_-]?key|apikey|api[_-]?secret|secret[_-]?key)["\']?\s*[:=]\s*["\']?([A-Za-z0-9_\-]{20,64})["\']?\b',
        "description": "Generic API keys in key=value format",
        "category": "secret"
    },
    "PASSWORD_IN_URL": {
        # Passwords with = or : or "to" keyword
        # Matches: password=secret, pwd: MyPass123, changed pwd to P@ssw0rd123!
        "pattern": r'(?:password|passwd|pwd|pass|secret|credential)["\']?\s*(?:[:=]|(?:\s+(?:is|to|as)\s+))\s*["\']?([^\s"\'&,]{6,})["\']?',
        "description": "Passwords in plaintext",
        "category": "secret"
    },
    "PRIVATE_KEY": {
        "pattern": r'-----BEGIN\s+(?:RSA\s+)?PRIVATE\s+KEY-----',
        "description": "Private key headers",
        "category": "secret"
    },
    "JWT_TOKEN": {
        "pattern": r'\beyJ[A-Za-z0-9_-]*\.eyJ[A-Za-z0-9_-]*\.[A-Za-z0-9_-]*\b',
        "description": "JSON Web Tokens",
        "category": "secret"
    },
    "STRIPE_KEY": {
        # Stripe keys: sk_live_xxx, pk_test_xxx - min 8 chars after prefix to catch truncated
        "pattern": r'\b(?:sk|pk)_(?:test|live)_[A-Za-z0-9]{8,}\b',
        "description": "Stripe API Keys",
        "category": "secret"
    },
    "API_KEY_IN_URL": {
        # API keys embedded in URLs: /key/xxx, /api_key/xxx, /token/xxx
        "pattern": r'(?:/(?:key|api[_-]?key|token|secret|auth)[/=])([A-Za-z0-9_-]{16,})',
        "description": "API Keys in URLs",
        "category": "secret"
    },
    "AUTH_CODE": {
        # Authorization codes: auth_code: 9921, authorization: ABC123
        "pattern": r'(?:auth[_-]?code|authorization|auth[_-]?token|otp|verification[_-]?code)[:\s]+[A-Za-z0-9]{4,12}\b',
        "description": "Authorization/Verification Codes",
        "category": "secret"
    },
    "UUID": {
        # UUIDs/GUIDs: 550e8400-e29b-41d4-a716-446655440000
        "pattern": r'\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b',
        "description": "UUIDs/Trace IDs",
        "category": "pii"
    },
    "GEO_COORDINATES": {
        # Geo coordinates: 40.7128° N, 74.0060° W or 40.7128, -74.0060
        "pattern": r'\b-?\d{1,3}\.\d{4,}°?\s*[NS]?\s*,?\s*-?\d{1,3}\.\d{4,}°?\s*[EW]?\b',
        "description": "Geographic Coordinates",
        "category": "pii"
    },
    "GOOGLE_API_KEY": {
        "pattern": r'\bAIza[A-Za-z0-9_-]{35}\b',
        "description": "Google API Keys",
        "category": "secret"
    },
    # GCP and Azure Cloud Keys
    "GCP_SERVICE_ACCOUNT": {
        "pattern": r'\b[a-z0-9-]+@[a-z0-9-]+\.iam\.gserviceaccount\.com\b',
        "description": "GCP Service Account Emails",
        "category": "secret"
    },
    "GCP_PRIVATE_KEY_ID": {
        "pattern": r'"private_key_id"\s*:\s*"([a-f0-9]{40})"',
        "description": "GCP Private Key IDs in JSON",
        "category": "secret"
    },
    "AZURE_CLIENT_SECRET": {
        "pattern": r'\b[a-zA-Z0-9~._-]{34,40}\b(?=.*(?:azure|client[_-]?secret|tenant))',
        "description": "Azure Client Secrets",
        "category": "secret"
    },
    "AZURE_CONNECTION_STRING": {
        # Azure connection strings - match AccountKey=xxx pattern
        "pattern": r'AccountKey=[A-Za-z0-9+/=]{10,}',
        "description": "Azure Storage Connection Strings",
        "category": "secret"
    },
    "AZURE_SAS_TOKEN": {
        "pattern": r'\?sv=\d{4}-\d{2}-\d{2}&[^"\s]+sig=[A-Za-z0-9%]+',
        "description": "Azure SAS Tokens",
        "category": "secret"
    },
    # International Address Patterns
    "UK_POSTCODE": {
        # Valid UK postcode formats: A9 9AA, A99 9AA, A9A 9AA, AA9 9AA, AA99 9AA, AA9A 9AA
        # Must have space between outward and inward codes, and be word-bounded
        "pattern": r'\b(?:[A-Z]{1,2}[0-9][0-9A-Z]?\s+[0-9][A-Z]{2})\b',
        "description": "UK Postcodes",
        "category": "pii"
    },
    "UK_ADDRESS": {
        "pattern": r'\b\d{1,5}\s+[A-Za-z]+(?:\s+[A-Za-z]+)*\s*,\s*[A-Za-z]+(?:\s+[A-Za-z]+)*\s*,?\s*[A-Z]{1,2}[0-9][0-9A-Z]?\s*[0-9][A-Z]{2}\b',
        "description": "UK Addresses with Postcode",
        "category": "pii"
    },
    "EU_ADDRESS": {
        # European addresses: German (strasse 42), French (25 rue xxx, 75016 Paris), etc
        "pattern": r'\b(?:[A-Za-z]+(?:strasse|straße|street|straat|calle|via|strada|gasse|weg|platz|plein|place|plaza)\s+\d{1,5}|\d{1,5}\s+(?:rue|av\.?|avenue|boulevard|blvd\.?|chemin|allée|impasse|passage)\s+[A-Za-z][A-Za-z\s]{2,25})[,\s]+\d{4,5}\s+[A-Za-z]+',
        "description": "European Addresses (DE/FR/NL/IT/ES)",
        "category": "pii"
    },
    "INTERNATIONAL_ADDRESS": {
        # Generic pattern for addresses with postal codes
        "pattern": r'\b\d{1,5}\s+[A-Za-z][A-Za-z\s]{2,30},\s*[A-Za-z][A-Za-z\s]{2,20},?\s*(?:[A-Z]{2}\s*)?\d{4,6}(?:-\d{4})?\b',
        "description": "International Addresses with Postal Codes",
        "category": "pii"
    },
    # Medical IDs (distinct from passports)
    "MEDICAL_RECORD_NUMBER": {
        "pattern": r'\b(?:MRN|MR#|Medical\s*Record|Patient\s*ID)[:\s#]*[A-Z0-9]{6,12}\b',
        "description": "Medical Record Numbers",
        "category": "pii"
    },
    "NPI_NUMBER": {
        "pattern": r'\b(?:NPI)[:\s#]*[0-9]{10}\b',
        "description": "National Provider Identifier (NPI)",
        "category": "pii"
    },
    "DEA_NUMBER": {
        "pattern": r'\b[A-Z][A-Z9][0-9]{7}\b',
        "description": "DEA Registration Numbers",
        "category": "pii"
    },
    # EU VAT Numbers (detected directly via dedicated function for better accuracy)
    "EU_VAT_NUMBER": {
        "pattern": r'\b(?:ATU\d{8}|BE0?\d{9,10}|BG\d{9,10}|CY\d{8}[A-Z]|CZ\d{8,10}|DE\d{9}|DK\d{8}|EE\d{9}|EL\d{9}|ES[A-Z0-9]\d{7}[A-Z0-9]|FI\d{8}|FR[A-Z0-9]{2}\d{9}|HR\d{11}|HU\d{8}|IE\d[A-Z0-9+*]\d{5}[A-Z]{1,2}|IT\d{11}|LT\d{9,12}|LU\d{8}|LV\d{11}|MT\d{8}|NL\d{9}B\d{2}|PL\d{10}|PT\d{9}|RO\d{2,10}|SE\d{12}|SI\d{8}|SK\d{10}|GB\d{9,12})\b',
        "description": "European Union VAT Numbers",
        "category": "financial"
    },
    # Spelled-out SSN pattern (detected via NLP layer)
    "SSN_SPELLED": {
        "pattern": r'\b(?:S\.?S\.?N\.?|social\s*security)\s+(?:is\s+)?(?:(?:zero|one|two|three|four|five|six|seven|eight|nine|oh|o)\s+)+(?:dash|hyphen)\s+(?:(?:zero|one|two|three|four|five|six|seven|eight|nine|oh|o)\s+)+(?:dash|hyphen)\s+(?:(?:zero|one|two|three|four|five|six|seven|eight|nine|oh|o)\s*)+',
        "description": "Spelled-out Social Security Numbers",
        "category": "pii"
    },
    # Base64 encoded secrets (marker for decoded content)
    "BASE64_SECRET": {
        "pattern": r'PLACEHOLDER_FOR_DECODED_BASE64',  # Detected via decode layer
        "description": "Base64 Encoded Secrets",
        "category": "secret"
    },
    # JSON embedded PII (marker for nested content)
    "JSON_EMBEDDED_PII": {
        "pattern": r'PLACEHOLDER_FOR_JSON_PII',  # Detected via JSON extraction layer
        "description": "PII Found in JSON Blobs",
        "category": "pii"
    },
    # =========================================================================
    # INTERNATIONAL ID PATTERNS (v1.1)
    # =========================================================================
    "UK_NATIONAL_INSURANCE": {
        # UK National Insurance Number: AB123456C
        "pattern": r'\b[A-CEGHJ-PR-TW-Z]{2}\s?\d{2}\s?\d{2}\s?\d{2}\s?[A-D]\b',
        "description": "UK National Insurance Number",
        "category": "pii"
    },
    "CANADIAN_SIN": {
        # Canadian Social Insurance Number: 123-456-789 or 123 456 789
        "pattern": r'\b\d{3}[-\s]\d{3}[-\s]\d{3}\b',
        "description": "Canadian Social Insurance Number",
        "category": "pii"
    },
    "INDIA_AADHAAR": {
        # India Aadhaar: 1234 5678 9012 (12 digits with spaces)
        "pattern": r'\b\d{4}\s\d{4}\s\d{4}\b',
        "description": "India Aadhaar Number",
        "category": "pii"
    },
    "INDIA_PAN": {
        # India PAN: ABCDE1234F (5 letters, 4 digits, 1 letter)
        "pattern": r'\b[A-Z]{5}\d{4}[A-Z]\b',
        "description": "India PAN Card Number",
        "category": "pii"
    },
    "AUSTRALIA_TFN": {
        # Australian Tax File Number: 123 456 789 (9 digits)
        "pattern": r'\b\d{3}\s\d{3}\s\d{3}\b',
        "description": "Australian Tax File Number",
        "category": "pii"
    },
    "BRAZIL_CPF": {
        # Brazil CPF: 123.456.789-00
        "pattern": r'\b\d{3}\.\d{3}\.\d{3}-\d{2}\b',
        "description": "Brazil CPF Number",
        "category": "pii"
    },
    "MEXICO_CURP": {
        # Mexico CURP: 18 alphanumeric
        "pattern": r'\b[A-Z]{4}\d{6}[HM][A-Z]{5}[A-Z\d]{2}\b',
        "description": "Mexico CURP",
        "category": "pii"
    },
    "SOUTH_AFRICA_ID": {
        # South Africa ID: 13 digits YYMMDDGGGSCAZ
        "pattern": r'\b\d{2}(?:0[1-9]|1[0-2])(?:0[1-9]|[12]\d|3[01])\d{4}[01]\d{2}\b',
        "description": "South Africa ID Number",
        "category": "pii"
    },
    # =========================================================================
    # ADDITIONAL CLOUD TOKENS (v1.1)
    # =========================================================================
    "DISCORD_TOKEN": {
        # Discord bot/user tokens: base64.base64.base64 format
        "pattern": r'\b[MN][A-Za-z\d]{23,}\.[\w-]{6}\.[\w-]{27}\b',
        "description": "Discord Bot/User Tokens",
        "category": "secret"
    },
    "DISCORD_WEBHOOK": {
        "pattern": r'https://discord(?:app)?\.com/api/webhooks/\d+/[\w-]+',
        "description": "Discord Webhook URLs",
        "category": "secret"
    },
    "TWILIO_API_KEY": {
        "pattern": r'\bSK[a-f0-9]{32}\b',
        "description": "Twilio API Keys",
        "category": "secret"
    },
    "TWILIO_AUTH_TOKEN": {
        "pattern": r'\b[a-f0-9]{32}\b(?=.*(?:twilio|auth_token))',
        "description": "Twilio Auth Tokens",
        "category": "secret"
    },
    "SENDGRID_API_KEY": {
        "pattern": r'\bSG\.[A-Za-z0-9_-]{22}\.[A-Za-z0-9_-]{43}\b',
        "description": "SendGrid API Keys",
        "category": "secret"
    },
    "OPENAI_API_KEY": {
        "pattern": r'\bsk-[A-Za-z0-9]{48}\b',
        "description": "OpenAI API Keys",
        "category": "secret"
    },
    "ANTHROPIC_API_KEY": {
        "pattern": r'\bsk-ant-[A-Za-z0-9_-]{40,}\b',
        "description": "Anthropic API Keys",
        "category": "secret"
    },
    "MAILCHIMP_API_KEY": {
        "pattern": r'\b[a-f0-9]{32}-us\d{1,2}\b',
        "description": "Mailchimp API Keys",
        "category": "secret"
    },
    "MAILGUN_API_KEY": {
        "pattern": r'\bkey-[a-f0-9]{32}\b',
        "description": "Mailgun API Keys",
        "category": "secret"
    },
    "HEROKU_API_KEY": {
        "pattern": r'\b[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}\b',
        "description": "Heroku API Keys",
        "category": "secret"
    },
    "SHOPIFY_ACCESS_TOKEN": {
        "pattern": r'\bshpat_[a-fA-F0-9]{32}\b',
        "description": "Shopify Access Tokens",
        "category": "secret"
    },
    "SHOPIFY_SHARED_SECRET": {
        "pattern": r'\bshpss_[a-fA-F0-9]{32}\b',
        "description": "Shopify Shared Secrets",
        "category": "secret"
    },
    "NPM_TOKEN": {
        "pattern": r'\bnpm_[A-Za-z0-9]{36}\b',
        "description": "NPM Access Tokens",
        "category": "secret"
    },
    "PYPI_TOKEN": {
        "pattern": r'\bpypi-[A-Za-z0-9_-]{50,}\b',
        "description": "PyPI API Tokens",
        "category": "secret"
    },
    "DOCKER_AUTH": {
        "pattern": r'"auth"\s*:\s*"[A-Za-z0-9+/=]{20,}"',
        "description": "Docker Registry Auth",
        "category": "secret"
    },
    # =========================================================================
    # CRYPTO ADDRESSES (v1.1)
    # =========================================================================
    "BITCOIN_ADDRESS": {
        # Bitcoin: 1xxx, 3xxx (P2SH), bc1xxx (Bech32)
        "pattern": r'\b(?:[13][a-km-zA-HJ-NP-Z1-9]{25,34}|bc1[a-zA-HJ-NP-Z0-9]{39,59})\b',
        "description": "Bitcoin Wallet Addresses",
        "category": "financial"
    },
    "ETHEREUM_ADDRESS": {
        # Ethereum: 0x followed by 40 hex chars
        "pattern": r'\b0x[a-fA-F0-9]{40}\b',
        "description": "Ethereum Wallet Addresses",
        "category": "financial"
    },
    "MONERO_ADDRESS": {
        # Monero: 4 or 8 followed by 93 or 103 chars
        "pattern": r'\b[48][0-9AB][1-9A-HJ-NP-Za-km-z]{93}\b',
        "description": "Monero Wallet Addresses",
        "category": "financial"
    },
    # =========================================================================
    # FINANCIAL IDENTIFIERS (v1.1)
    # =========================================================================
    "CUSIP": {
        # CUSIP: 9 alphanumeric chars (US/Canada securities)
        "pattern": r'\b[A-Z0-9]{9}\b(?=.*(?:cusip|security|stock|bond))',
        "description": "CUSIP Security Identifiers",
        "category": "financial"
    },
    "ISIN": {
        # ISIN: Valid country code + 9 alphanumeric (NSIN) + 1 check digit
        # Requires financial context to reduce false positives
        "pattern": r'\b(?:US|GB|DE|FR|CH|JP|CA|AU|NL|BE|IT|ES|AT|SE|NO|DK|FI|IE|PT|LU|HK|SG|KR|TW|IN|BR|ZA|MX)[A-Z0-9]{9}\d\b(?=.*(?:isin|security|stock|bond|share|equity|fund))',
        "description": "ISIN Security Identifiers",
        "category": "financial"
    },
    "SEDOL": {
        # SEDOL: 7 alphanumeric (UK securities)
        "pattern": r'\b[B-DF-HJ-NP-TV-Z0-9]{7}\b(?=.*(?:sedol|london|lse))',
        "description": "SEDOL Security Identifiers",
        "category": "financial"
    }
}


class PIIEntity(BaseModel):
    type: str
    value: str
    start: int
    end: int
    confidence: float
    masked_value: str


class ScanResult(BaseModel):
    total_entities: int
    entities_by_type: dict
    entities: list[PIIEntity]
    risk_level: str
    risk_score: int
    redacted_preview: Optional[str] = None


class ScanConfig(BaseModel):
    detect_emails: bool = True
    detect_phones: bool = True
    detect_ssn: bool = True
    detect_credit_cards: bool = True
    detect_ip_addresses: bool = True
    detect_dates: bool = True
    detect_zip_codes: bool = False
    detect_addresses: bool = True
    detect_iban: bool = True
    detect_secrets: bool = True  # AWS keys, GitHub tokens, passwords, etc.


# ============================================================================
# FALSE POSITIVE EXCLUSION PATTERNS (v1.1)
# ============================================================================

# Patterns to EXCLUDE from detection (known false positives)
FALSE_POSITIVE_PATTERNS = {
    # Git commit SHAs (40 hex chars) - not AWS secrets
    "GIT_SHA": r'\b[a-f0-9]{40}\b',
    # Unix timestamps (13 digits) - not phone numbers
    "UNIX_TIMESTAMP_MS": r'\b1[0-9]{12}\b',
    # Unix timestamps (10 digits) - not phone numbers
    "UNIX_TIMESTAMP": r'\b1[0-9]{9}\b',
    # Slack webhook path identifiers (T/B followed by alphanumeric)
    "SLACK_WEBHOOK_PATH": r'T[A-Z0-9]{8,}/B[A-Z0-9]{8,}',
    # Common test/example patterns
    "TEST_PATTERN": r'\b(?:test|example|sample|demo|fake|dummy|xxx+|000[-\s]?000[-\s]?0000)\b',
    # Version numbers (could look like dates)
    "VERSION_NUMBER": r'\bv?\d+\.\d+\.\d+(?:\.\d+)?\b',
    # File hashes (MD5, SHA variants)
    "FILE_HASH_MD5": r'\b[a-f0-9]{32}\b(?!.*(?:twilio|auth_token))',
    "FILE_HASH_SHA256": r'\b[a-f0-9]{64}\b',
}


# ============================================================================
# LAYER 2.0: Pre-Processing - Unicode & HTML Normalization (v1.1)
# ============================================================================

def normalize_unicode(text: str) -> str:
    """
    Layer 2.0a: Apply Unicode NFKC normalization to catch:
    - Fullwidth characters: ｊｏｈｎ@ｅｘａｍｐｌｅ.ｃｏｍ → john@example.com
    - Unicode dots: john․doe@example․com → john.doe@example.com
    - Other homoglyphs and compatibility characters
    """
    return unicodedata.normalize('NFKC', text)


def decode_html_entities(text: str) -> str:
    """
    Layer 2.0b: Decode HTML entities to catch:
    - john&at;example&dot;com → john@example.com
    - john&#64;example&#46;com → john@example.com
    - &amp; &lt; &gt; etc.
    """
    # First pass: standard HTML entities
    decoded = html.unescape(text)

    # Second pass: custom obfuscation patterns
    custom_entities = [
        (r'&at;', '@'),
        (r'&dot;', '.'),
        (r'&#64;', '@'),  # @ in decimal
        (r'&#46;', '.'),  # . in decimal
        (r'&#x40;', '@'), # @ in hex
        (r'&#x2e;', '.'), # . in hex
    ]
    for pattern, replacement in custom_entities:
        decoded = re.sub(pattern, replacement, decoded, flags=re.IGNORECASE)

    return decoded


def preprocess_text(text: str) -> tuple[str, list[dict]]:
    """
    Layer 2.0: Full pre-processing pipeline combining Unicode and HTML normalization.
    Returns preprocessed text and a log of transformations applied.
    """
    transformations = []
    processed = text

    # Step 1: Unicode NFKC normalization
    unicode_normalized = normalize_unicode(processed)
    if unicode_normalized != processed:
        transformations.append({
            "layer": "2.0a",
            "type": "unicode_nfkc",
            "changes": len(processed) - len(unicode_normalized)
        })
        processed = unicode_normalized

    # Step 2: HTML entity decoding
    html_decoded = decode_html_entities(processed)
    if html_decoded != processed:
        transformations.append({
            "layer": "2.0b",
            "type": "html_entity_decode",
            "changes": len(processed) - len(html_decoded)
        })
        processed = html_decoded

    return processed, transformations


def is_false_positive(value: str, pii_type: str, context: str = "") -> bool:
    """
    Check if a detected value is a known false positive pattern.
    Returns True if the value should be excluded.
    """
    value_lower = value.lower()
    context_lower = context.lower() if context else ""

    # Git SHA check (for AWS_SECRET_KEY false positives)
    if pii_type == "AWS_SECRET_KEY":
        # If it's all lowercase hex and 40 chars, it's probably a git SHA
        if re.match(r'^[a-f0-9]{40}$', value):
            return True
        # Also check if "commit" or "sha" in context
        if any(kw in context_lower for kw in ["commit", "sha", "hash", "git", "rev"]):
            return True

    # Timestamp check (for PHONE_US false positives)
    if pii_type in ["PHONE_US", "PHONE_US_NORMALIZED"]:
        # Check if it looks like a timestamp (13 digits starting with 1)
        digits_only = re.sub(r'\D', '', value)
        if len(digits_only) == 13 and digits_only.startswith('1'):
            return True
        if len(digits_only) == 10 and digits_only.startswith('1'):
            return True

    # Slack webhook path check (for DRIVERS_LICENSE false positives)
    if pii_type == "DRIVERS_LICENSE":
        if re.match(r'^T[A-Z0-9]{8,}$', value) or re.match(r'^B[A-Z0-9]{8,}$', value):
            if "slack" in context_lower or "webhook" in context_lower:
                return True
        # B followed by 7 digits could be HK ID or other codes, need context
        if re.match(r'^B\d{7}$', value):
            # Only valid if context suggests driver's license
            if not any(kw in context_lower for kw in ["license", "licence", "driver", "dmv", "dl#", "dl:"]):
                return True

    # Date check for log timestamps (for DATE_OF_BIRTH false positives)
    if pii_type == "DATE_OF_BIRTH":
        # Log timestamps often have time component nearby
        if any(kw in context_lower for kw in ["log", "timestamp", "created", "updated", "time:", "at ", ":"]):
            if re.search(r'\d{2}:\d{2}:\d{2}', context):  # Has time component
                return True

    # Version number check
    if pii_type != "IP_ADDRESS" and re.match(r'^v?\d+\.\d+\.\d+', value):
        return True

    # EMAIL and EMAIL_OBFUSCATED false positives - connection strings and URIs
    if pii_type in ["EMAIL", "EMAIL_OBFUSCATED"]:
        # Connection string pattern: ://user:password@host
        # This catches mongodb+srv://user:pass@cluster, redis://default:pass@host, etc.
        if re.search(r'://[^@]*:[^@]*@', context):
            return True
        # MongoDB/database connection strings: password@cluster.mongodb.net
        if any(db in value_lower for db in [".mongodb.net", ".mongodb.com", "cluster0", "cluster1"]):
            return True
        # Redis connection strings
        if any(db in value_lower for db in [".redis", "redis-", "redislabs"]):
            return True
        # Other database connection patterns
        if any(db in context_lower for db in ["mongodb://", "mongodb+srv://", "redis://", "postgres://", "mysql://", "amqp://", "connection_string", "conn_str"]):
            return True
        # Part of URL (e.g., user:pass@host pattern)
        if re.search(r'://[^@]+' + re.escape(value), context):
            return True

    # UK_POSTCODE false positives - partial passwords, codes
    if pii_type == "UK_POSTCODE":
        # Too short or looks like part of a password/hash
        if len(value) < 6:
            return True
        # Check if it's surrounded by alphanumeric chars (part of longer string)
        value_pos = context.find(value)
        if value_pos > 0:
            char_before = context[value_pos - 1] if value_pos > 0 else ' '
            char_after = context[value_pos + len(value)] if value_pos + len(value) < len(context) else ' '
            if char_before.isalnum() or char_after.isalnum():
                return True

    # ISIN false positives - needs context validation
    if pii_type == "ISIN":
        # ISIN should have country code followed by alphanumeric
        # Exclude if it looks like a passport number or other ID
        if any(kw in context_lower for kw in ["passport", "travel", "visa", "id card", "identity"]):
            return True
        # Exclude random-looking alphanumeric strings without financial context
        if not any(kw in context_lower for kw in ["isin", "security", "stock", "bond", "cusip", "sedol", "share", "equity", "fund"]):
            return True

    # BANK_ACCOUNT false positives - exclude credit card numbers
    if pii_type == "BANK_ACCOUNT":
        digits_only = re.sub(r'\D', '', value)
        # If it's 15-16 digits and passes Luhn, it's a credit card, not bank account
        if len(digits_only) in [15, 16]:
            # Luhn check inline
            digits = [int(d) for d in digits_only]
            odd_digits = digits[-1::-2]
            even_digits = digits[-2::-2]
            total = sum(odd_digits)
            for d in even_digits:
                total += sum(divmod(d * 2, 10))
            if total % 10 == 0:
                return True  # It's a credit card, not a bank account

    # INDIA_AADHAAR false positives - exclude IBAN fragments
    if pii_type == "INDIA_AADHAAR":
        # If context suggests IBAN or European bank context, it's likely an IBAN fragment
        if any(kw in context_lower for kw in ["iban", "bic", "swift", "sepa", "bank", "de", "gb", "fr", "nl", "at", "ch"]):
            return True
        # Check if preceded by 2-letter country code + 2 digits (IBAN prefix pattern)
        # Pattern: XX## followed by the detected value
        iban_prefix_pattern = r'[A-Z]{2}\d{2}\s*' + re.escape(value.replace(' ', r'\s*'))
        if re.search(iban_prefix_pattern, context, re.IGNORECASE):
            return True

    # CANADIAN_SIN false positives - exclude Australian ABN format
    if pii_type == "CANADIAN_SIN":
        # ABN format is similar (9 digits in 3-3-3) but in Australian context
        if any(kw in context_lower for kw in ["abn", "australia", "australian", "business number", "gst"]):
            return True

    # CREDIT_CARD false positives - exclude order/confirmation/reference numbers
    if pii_type in ["CREDIT_CARD", "POSSIBLE_CARD_PATTERN"]:
        # Check for common prefixes that indicate non-card numbers
        order_prefixes = ["order", "order #", "order#", "order:", "order number",
                          "conf", "confirmation", "conf #", "conf#", "confirmation #",
                          "ref", "reference", "ref #", "ref#", "reference #",
                          "invoice", "invoice #", "inv #", "inv#",
                          "tracking", "tracking #", "track #",
                          "ticket", "ticket #", "case #", "case#",
                          "transaction id", "trans id", "txn"]
        if any(prefix in context_lower for prefix in order_prefixes):
            return True
        # Check for pattern: "# followed immediately by the number"
        if re.search(r'#\s*' + re.escape(value), context):
            return True

    return False


# ============================================================================
# LAYER 2: Text Normalization Functions
# ============================================================================

def normalize_text(text: str) -> tuple[str, dict]:
    """
    Layer 2: Normalize text to reveal hidden PII.
    Returns normalized text and a mapping of original positions.

    Handles:
    - [dot] / (dot) / [.] → .
    - [at] / (at) → @
    - Dashes/underscores in numbers → removed
    - Spaced out characters → joined
    """
    normalized = text
    transformations = []

    # Track transformations for position mapping
    # 1. Convert defanged dots: [dot], (dot), [.], (.), [ dot ], etc.
    # Also handle with surrounding spaces: " [dot] " → "."
    dot_patterns = [
        (r'\s*\[\s*dot\s*\]\s*', '.'),  # [dot], [ dot ], etc with optional spaces
        (r'\s*\(\s*dot\s*\)\s*', '.'),  # (dot), ( dot ), etc with optional spaces
        (r'\s*\[\.\]\s*', '.'),         # [.]
        (r'\s*\(\.\)\s*', '.'),         # (.)
        (r'\s+dot\s+', '.'),            # " dot " (word boundary)
    ]
    for pattern, replacement in dot_patterns:
        normalized = re.sub(pattern, replacement, normalized, flags=re.IGNORECASE)

    # 2. Convert defanged at: [at], (at), [ at ], etc.
    # IMPORTANT: Bracketed forms [at] and (at) are always intentional obfuscation
    # But plain " at " must be validated to avoid "meet at 5pm" → "meet@5pm"

    # 2a. Bracketed forms - always convert (these are intentional obfuscation)
    bracketed_at_patterns = [
        (r'\s*\[\s*at\s*\]\s*', '@'),   # [at], [ at ], etc with optional spaces
        (r'\s*\(\s*at\s*\)\s*', '@'),   # (at), ( at ), etc with optional spaces
    ]
    for pattern, replacement in bracketed_at_patterns:
        normalized = re.sub(pattern, replacement, normalized, flags=re.IGNORECASE)

    # 2b. Plain " at " - ONLY convert if it looks like an email pattern:
    # - Preceded by username-like chars (alphanumeric, dots, underscores)
    # - Followed by domain-like pattern (word + dot/space-dot + TLD)
    # Valid: "john at gmail dot com", "mike at company.org"
    # Invalid: "meet at 5pm", "look at this", "ready at cvs", "at gate b7"
    def should_normalize_at(text: str, at_position: int) -> bool:
        """
        Robust validation for ' at ' to '@' conversion.
        Returns True only if this looks like a genuine obfuscated email.

        Validates:
        - Username pattern before "at"
        - Rejects false trigger words after "at" (the, my, home, cvs, etc.)
        - Rejects time patterns ("at 7pm", "at 2:30")
        - Domain pattern with valid TLD after "at"
        """
        # Get context around the "at"
        before_text = text[:at_position]
        after_text = text[at_position + 4:]  # Skip " at "

        # 1. Check username pattern (must end with valid email username chars)
        username_match = re.search(r'([a-zA-Z0-9][a-zA-Z0-9._%+-]{0,63})$', before_text)
        if not username_match:
            return False

        # 2. Get the first word after "at" and check for false triggers
        first_word_match = re.match(r'([a-zA-Z0-9]+)', after_text)
        if not first_word_match:
            return False

        first_word = first_word_match.group(1).lower()

        # Reject if first word is a common false trigger
        if first_word in FALSE_TRIGGER_WORDS:
            return False

        # 3. Reject time patterns: "at 7pm", "at 2:30", "at noon"
        time_pattern = r'^[0-9]{1,2}(?::[0-9]{2})?(?:\s*(?:am|pm|AM|PM))?\s*(?:$|[.,!?\s])'
        if re.match(time_pattern, after_text):
            return False

        # 4. Check for valid domain pattern with TLD
        # Pattern: domain + (dot or " dot ") + valid TLD
        domain_pattern = r'^([a-zA-Z0-9][a-zA-Z0-9-]{0,61}[a-zA-Z0-9]?)(?:\.|(?:\s+dot\s+))([a-zA-Z]{2,10})(?:\s|$|[.,!?])'
        domain_match = re.match(domain_pattern, after_text, re.IGNORECASE)

        if domain_match:
            potential_tld = domain_match.group(2).lower()
            if potential_tld in VALID_TLDS:
                return True

        return False

    def smart_at_replacement(match):
        """Wrapper function for re.sub that uses should_normalize_at()"""
        full_match = match.group(0)
        before = match.group(1)  # Username-like part
        after = match.group(2)   # Domain-like part

        # Find the position of " at " in the original normalized text
        # The match.start() gives us where the full match begins
        # We need to check from where "at" actually appears
        at_pos_in_match = len(before)  # " at " starts right after the username
        at_pos_in_text = match.start() + at_pos_in_match

        if should_normalize_at(normalized, at_pos_in_text):
            return before + '@' + after
        return full_match  # Return unchanged

    # Pattern: (username-like) + " at " + (potential domain)
    normalized = re.sub(
        r'([a-zA-Z0-9][a-zA-Z0-9._%+-]*)\s+at\s+([a-zA-Z0-9][a-zA-Z0-9.\s]*)',
        smart_at_replacement,
        normalized,
        flags=re.IGNORECASE
    )

    # 3. Remove separators from potential numbers (SSN, CC, phone)
    # Only in numeric contexts: sequences that look like numbers with separators
    # Pattern: digit, separator(s), digit, separator(s), digit...
    def clean_numeric_separators(match):
        """Remove dashes, underscores, spaces from numeric sequences"""
        return re.sub(r'[-_\s]', '', match.group())

    # Match sequences that look like formatted numbers (3+ digit groups)
    normalized = re.sub(
        r'\b(\d{2,4})[-_\s]+(\d{2,4})[-_\s]+(\d{2,4})(?:[-_\s]+(\d{2,4}))?\b',
        clean_numeric_separators,
        normalized
    )

    # 4. Join spaced-out characters (t-e-s-t → test, t e s t → test)
    # Only for sequences that look intentionally obfuscated (alternating char-separator)
    def join_spaced_chars(match):
        """Join characters separated by dashes or spaces"""
        chars = re.findall(r'[A-Za-z0-9]', match.group())
        return ''.join(chars)

    # Match spaced-out patterns: a-b-c-d or a b c d (3+ chars)
    normalized = re.sub(
        r'\b([A-Za-z0-9])(?:[-\s]([A-Za-z0-9])){3,}\b',
        join_spaced_chars,
        normalized
    )

    return normalized, {"original_length": len(text), "normalized_length": len(normalized)}


# ============================================================================
# LAYER 2.5: JSON Blob Extraction & Deep Scanning
# ============================================================================

def extract_json_strings(text: str) -> list[tuple[str, int, int]]:
    """
    Extract JSON objects embedded in text for deep scanning.
    Returns list of (json_string, start_pos, end_pos)
    """
    json_strings = []

    # Pattern to find JSON-like structures: {...} or escaped JSON strings
    # Look for JSON objects that might be embedded as string values
    json_patterns = [
        # Standard JSON objects
        r'(\{[^{}]*(?:\{[^{}]*\}[^{}]*)*\})',
        # Escaped JSON in string values (e.g., "{\"key\": \"value\"}")
        r'"(\{(?:\\"|[^"])*\})"',
    ]

    for pattern in json_patterns:
        for match in re.finditer(pattern, text):
            json_str = match.group(1) if match.lastindex else match.group()
            # Unescape if it was escaped JSON
            if json_str.startswith('{') and '\\"' in json_str:
                json_str = json_str.replace('\\"', '"')
            json_strings.append((json_str, match.start(), match.end()))

    return json_strings


def deep_scan_json(json_str: str) -> list[str]:
    """
    Recursively extract all string values from a JSON structure.
    Returns list of string values to scan.
    """
    extracted_values = []

    def extract_strings(obj):
        if isinstance(obj, str):
            extracted_values.append(obj)
        elif isinstance(obj, dict):
            for key, value in obj.items():
                # Also check keys for PII (e.g., "ssn", "email")
                extracted_values.append(f"{key}: {value}" if isinstance(value, str) else key)
                extract_strings(value)
        elif isinstance(obj, list):
            for item in obj:
                extract_strings(item)

    try:
        parsed = json.loads(json_str)
        extract_strings(parsed)
    except json.JSONDecodeError:
        # If not valid JSON, try to extract key-value pairs with regex
        # Pattern for "key": "value" pairs
        kv_pattern = r'"([^"]+)"\s*:\s*"([^"]*)"'
        for match in re.finditer(kv_pattern, json_str):
            extracted_values.append(f"{match.group(1)}: {match.group(2)}")

    return extracted_values


# ============================================================================
# LAYER 2.6: Base64 Auto-Detection & Decoding
# ============================================================================

def is_valid_base64(s: str) -> bool:
    """Check if a string is valid base64 encoded."""
    # Must be at least 8 chars and multiple of 4 (with padding)
    if len(s) < 8:
        return False

    # Check for valid base64 characters
    base64_pattern = r'^[A-Za-z0-9+/]+=*$'
    if not re.match(base64_pattern, s):
        return False

    # Length check (with padding, should be multiple of 4)
    if len(s) % 4 != 0:
        return False

    return True


def decode_base64_strings(text: str, max_depth: int = 2) -> list[tuple[str, str, int, int, int]]:
    """
    Find and decode base64 strings in text with recursive decoding support (v1.1).
    Returns list of (original_b64, decoded_text, start_pos, end_pos, decode_depth)

    Now supports:
    - Recursive decoding (double-encoded base64)
    - JSON extraction from decoded content
    """
    decoded_strings = []

    # Pattern to find potential base64 strings (min 16 chars to avoid false positives)
    # Must contain at least one letter and one number to reduce false positives
    b64_pattern = r'\b([A-Za-z0-9+/]{16,}={0,2})\b'

    def recursive_decode(b64_str: str, start: int, end: int, depth: int = 1):
        """Recursively decode base64 strings up to max_depth"""
        if depth > max_depth:
            return

        if not is_valid_base64(b64_str):
            return

        try:
            decoded = base64.b64decode(b64_str).decode('utf-8', errors='ignore')

            # Only keep if decoded text is printable and looks like real text
            if decoded and len(decoded) >= 4:
                printable_ratio = sum(1 for c in decoded if c.isprintable()) / len(decoded)
                letter_ratio = sum(1 for c in decoded if c.isalpha()) / len(decoded)

                if printable_ratio > 0.7 and letter_ratio > 0.2:
                    decoded_strings.append((b64_str, decoded, start, end, depth))

                    # Recursive: Check if decoded content contains more base64
                    if depth < max_depth:
                        for inner_match in re.finditer(b64_pattern, decoded):
                            recursive_decode(
                                inner_match.group(1),
                                start,  # Keep original position
                                end,
                                depth + 1
                            )

                    # Also check if decoded content is JSON and scan it
                    if decoded.strip().startswith('{') or decoded.strip().startswith('['):
                        try:
                            json.loads(decoded)
                            # Valid JSON - will be picked up by JSON layer
                        except json.JSONDecodeError:
                            pass

        except Exception:
            pass

    for match in re.finditer(b64_pattern, text):
        b64_str = match.group(1)
        recursive_decode(b64_str, match.start(), match.end())

    return decoded_strings


# ============================================================================
# LAYER 2.7: Spelled-Out Number Detection (NLP-lite)
# ============================================================================

# Word to digit mapping
WORD_TO_DIGIT = {
    'zero': '0', 'one': '1', 'two': '2', 'three': '3', 'four': '4',
    'five': '5', 'six': '6', 'seven': '7', 'eight': '8', 'nine': '9',
    'oh': '0', 'o': '0',  # "oh" is often used for zero
}

def convert_spelled_numbers(text: str) -> tuple[str, list[tuple[str, str, int, int]]]:
    """
    Convert spelled-out numbers to digits.
    Returns (converted_text, list of (original, converted, start, end))

    Examples:
    - "nine zero zero dash twelve dash eight eight two one" → "900-12-8821"
    - "four one five five five one two one two" → "415-555-1212"
    """
    conversions = []
    result = text

    # Build pattern for number words
    number_words = '|'.join(WORD_TO_DIGIT.keys())

    # Pattern: sequence of number words separated by spaces, possibly with "dash" or "hyphen"
    # Must have at least 3 number words to be considered a spelled-out number
    spelled_pattern = rf'\b((?:(?:{number_words})\s*)+(?:(?:dash|hyphen)\s*(?:(?:{number_words})\s*)+)+)\b'

    matches = list(re.finditer(spelled_pattern, text, re.IGNORECASE))

    for match in reversed(matches):  # Process in reverse to maintain positions
        original = match.group(1)

        # Convert words to digits
        converted_parts = []
        current_number = ''

        words = re.split(r'\s+', original.lower())
        for word in words:
            word = word.strip()
            if word in WORD_TO_DIGIT:
                current_number += WORD_TO_DIGIT[word]
            elif word in ['dash', 'hyphen']:
                if current_number:
                    converted_parts.append(current_number)
                    current_number = ''

        if current_number:
            converted_parts.append(current_number)

        if converted_parts:
            converted = '-'.join(converted_parts)
            conversions.append((original, converted, match.start(), match.end()))
            result = result[:match.start()] + converted + result[match.end():]

    return result, conversions


# ============================================================================
# LAYER 2.8: Non-Latin Character Support & International Patterns
# ============================================================================

# European VAT Number patterns by country
EU_VAT_PATTERNS = {
    "AT": r'ATU\d{8}',  # Austria
    "BE": r'BE0?\d{9,10}',  # Belgium
    "BG": r'BG\d{9,10}',  # Bulgaria
    "CY": r'CY\d{8}[A-Z]',  # Cyprus
    "CZ": r'CZ\d{8,10}',  # Czech Republic
    "DE": r'DE\d{9}',  # Germany
    "DK": r'DK\d{8}',  # Denmark
    "EE": r'EE\d{9}',  # Estonia
    "EL": r'EL\d{9}',  # Greece (uses EL not GR)
    "ES": r'ES[A-Z0-9]\d{7}[A-Z0-9]',  # Spain
    "FI": r'FI\d{8}',  # Finland
    "FR": r'FR[A-Z0-9]{2}\d{9}',  # France
    "HR": r'HR\d{11}',  # Croatia
    "HU": r'HU\d{8}',  # Hungary
    "IE": r'IE\d[A-Z0-9+*]\d{5}[A-Z]{1,2}',  # Ireland
    "IT": r'IT\d{11}',  # Italy
    "LT": r'LT\d{9,12}',  # Lithuania
    "LU": r'LU\d{8}',  # Luxembourg
    "LV": r'LV\d{11}',  # Latvia
    "MT": r'MT\d{8}',  # Malta
    "NL": r'NL\d{9}B\d{2}',  # Netherlands
    "PL": r'PL\d{10}',  # Poland
    "PT": r'PT\d{9}',  # Portugal
    "RO": r'RO\d{2,10}',  # Romania
    "SE": r'SE\d{12}',  # Sweden
    "SI": r'SI\d{8}',  # Slovenia
    "SK": r'SK\d{10}',  # Slovakia
    "GB": r'GB\d{9,12}',  # UK (pre-Brexit, still used)
}

# Combined EU VAT pattern
EU_VAT_COMBINED = '|'.join(f'({pattern})' for pattern in EU_VAT_PATTERNS.values())

# Greek character transliteration (for detecting Greek names/text)
GREEK_TO_LATIN = {
    'α': 'a', 'β': 'b', 'γ': 'g', 'δ': 'd', 'ε': 'e', 'ζ': 'z', 'η': 'h',
    'θ': 'th', 'ι': 'i', 'κ': 'k', 'λ': 'l', 'μ': 'm', 'ν': 'n', 'ξ': 'x',
    'ο': 'o', 'π': 'p', 'ρ': 'r', 'σ': 's', 'ς': 's', 'τ': 't', 'υ': 'y',
    'φ': 'f', 'χ': 'ch', 'ψ': 'ps', 'ω': 'o',
    'Α': 'A', 'Β': 'B', 'Γ': 'G', 'Δ': 'D', 'Ε': 'E', 'Ζ': 'Z', 'Η': 'H',
    'Θ': 'Th', 'Ι': 'I', 'Κ': 'K', 'Λ': 'L', 'Μ': 'M', 'Ν': 'N', 'Ξ': 'X',
    'Ο': 'O', 'Π': 'P', 'Ρ': 'R', 'Σ': 'S', 'Τ': 'T', 'Υ': 'Y',
    'Φ': 'F', 'Χ': 'Ch', 'Ψ': 'Ps', 'Ω': 'O',
    # Accented variants
    'ά': 'a', 'έ': 'e', 'ή': 'h', 'ί': 'i', 'ό': 'o', 'ύ': 'y', 'ώ': 'o',
    'Ά': 'A', 'Έ': 'E', 'Ή': 'H', 'Ί': 'I', 'Ό': 'O', 'Ύ': 'Y', 'Ώ': 'O',
}

# Cyrillic character transliteration
CYRILLIC_TO_LATIN = {
    'а': 'a', 'б': 'b', 'в': 'v', 'г': 'g', 'д': 'd', 'е': 'e', 'ё': 'yo',
    'ж': 'zh', 'з': 'z', 'и': 'i', 'й': 'y', 'к': 'k', 'л': 'l', 'м': 'm',
    'н': 'n', 'о': 'o', 'п': 'p', 'р': 'r', 'с': 's', 'т': 't', 'у': 'u',
    'ф': 'f', 'х': 'kh', 'ц': 'ts', 'ч': 'ch', 'ш': 'sh', 'щ': 'shch',
    'ъ': '', 'ы': 'y', 'ь': '', 'э': 'e', 'ю': 'yu', 'я': 'ya',
    'А': 'A', 'Б': 'B', 'В': 'V', 'Г': 'G', 'Д': 'D', 'Е': 'E', 'Ё': 'Yo',
    'Ж': 'Zh', 'З': 'Z', 'И': 'I', 'Й': 'Y', 'К': 'K', 'Л': 'L', 'М': 'M',
    'Н': 'N', 'О': 'O', 'П': 'P', 'Р': 'R', 'С': 'S', 'Т': 'T', 'У': 'U',
    'Ф': 'F', 'Х': 'Kh', 'Ц': 'Ts', 'Ч': 'Ch', 'Ш': 'Sh', 'Щ': 'Shch',
    'Ъ': '', 'Ы': 'Y', 'Ь': '', 'Э': 'E', 'Ю': 'Yu', 'Я': 'Ya',
}


def transliterate_text(text: str) -> str:
    """Transliterate Greek and Cyrillic characters to Latin."""
    result = text

    # Apply Greek transliteration
    for greek, latin in GREEK_TO_LATIN.items():
        result = result.replace(greek, latin)

    # Apply Cyrillic transliteration
    for cyrillic, latin in CYRILLIC_TO_LATIN.items():
        result = result.replace(cyrillic, latin)

    return result


def detect_eu_vat_numbers(text: str) -> list[tuple[str, str, int, int]]:
    """
    Detect EU VAT numbers in text.
    Returns list of (vat_number, country_code, start, end)
    """
    vat_matches = []

    for country, pattern in EU_VAT_PATTERNS.items():
        for match in re.finditer(pattern, text):
            vat_matches.append((match.group(), country, match.start(), match.end()))

    return vat_matches


def has_non_latin_chars(text: str) -> bool:
    """Check if text contains non-Latin characters (Greek, Cyrillic, etc.)"""
    # Check for Greek (U+0370–U+03FF) or Cyrillic (U+0400–U+04FF)
    return bool(re.search(r'[\u0370-\u03FF\u0400-\u04FF]', text))


# ============================================================================
# LAYER 3: Context-Based Confidence Scoring
# ============================================================================

# Context keywords that boost confidence when found near PII
CONTEXT_KEYWORDS = {
    "SSN": ["ssn", "social security", "social sec", "ss#", "ss #", "ss number"],
    "CREDIT_CARD": ["card", "credit", "debit", "visa", "mastercard", "amex", "payment", "cc#", "card number"],
    "EMAIL": ["email", "e-mail", "mail", "contact", "reach", "@"],
    "PHONE_US": ["phone", "tel", "telephone", "mobile", "cell", "call", "fax", "contact"],
    "PHONE_INTL": ["phone", "tel", "telephone", "mobile", "cell", "call", "fax", "contact"],
    "IP_ADDRESS": ["ip", "address", "server", "host", "network"],
    "IP_DEFANGED": ["ip", "address", "server", "host", "network", "indicator", "ioc"],
    "PASSWORD_IN_URL": ["password", "pwd", "pass", "secret", "credential", "login", "auth"],
    "IBAN": ["iban", "bank", "account", "transfer", "payment", "wire"],
    "BANK_ACCOUNT": ["account", "bank", "routing", "aba", "deposit"],
    "AWS_ACCESS_KEY": ["aws", "amazon", "access", "key", "iam", "credential"],
    "GITHUB_TOKEN": ["github", "token", "api", "pat", "access"],
    "STRIPE_KEY": ["stripe", "payment", "api", "key", "publishable", "secret"],
    "MEDICARE_ID": ["medicare", "mbi", "beneficiary", "cms", "health"],
    "DATE_OF_BIRTH": ["dob", "birth", "born", "birthday", "age"],
    "GEO_COORDINATES": ["location", "coordinates", "lat", "lng", "latitude", "longitude", "gps"],
    "AUTH_CODE": ["auth", "code", "otp", "verification", "2fa", "mfa"],
}

# Anti-context: keywords that reduce confidence (false positive indicators)
ANTI_CONTEXT_KEYWORDS = {
    "CREDIT_CARD": ["test", "example", "sample", "demo", "fake", "dummy", "xxxx", "part", "sku", "isbn", "serial", "order", "invoice", "ref", "batch", "processing"],
    "SSN": ["test", "example", "sample", "demo", "fake", "dummy", "000-00-0000", "123-45-6789", "part", "sku", "serial", "order", "ref", "batch", "processing", "validation"],
    "PHONE_US": ["test", "example", "555-", "000-000", "sample", "demo", "fake"],
    "IP_ADDRESS": ["example", "test", "localhost", "127.0.0.1", "0.0.0.0"],
    "EMAIL": ["example.com", "test.com", "example.org", "noreply", "sample", "demo"],
    "DATE_OF_BIRTH": ["batch", "created", "updated", "modified", "expires", "valid", "effective", "due", "ship", "order", "processing"],
    "IBAN": ["test", "example", "sample", "demo", "fake", "dummy", "validation"],
}


def is_future_date(date_str: str) -> bool:
    """Check if a date string represents a future date (not a valid DOB)."""
    from datetime import datetime, date
    current_year = datetime.now().year
    current_date = date.today()

    # Try to extract year from common date formats
    full_year_match = re.search(r'((?:19|20)\d{2})', date_str)
    if full_year_match:
        year = int(full_year_match.group(1))

        # Future year = definitely not a birth date
        if year > current_year:
            return True

        # Current year - check if the full date is in the future
        if year == current_year:
            # Try to parse the full date
            # Common formats: MM-DD-YYYY, MM/DD/YYYY, YYYY-MM-DD
            date_patterns = [
                (r'(\d{1,2})[-/](\d{1,2})[-/](\d{4})', 'MDY'),  # MM-DD-YYYY
                (r'(\d{4})[-/](\d{1,2})[-/](\d{1,2})', 'YMD'),  # YYYY-MM-DD
            ]
            for pattern, fmt in date_patterns:
                match = re.search(pattern, date_str)
                if match:
                    try:
                        if fmt == 'MDY':
                            month, day, _ = int(match.group(1)), int(match.group(2)), int(match.group(3))
                        else:  # YMD
                            _, month, day = int(match.group(1)), int(match.group(2)), int(match.group(3))
                        parsed_date = date(year, month, day)
                        if parsed_date > current_date:
                            return True
                    except ValueError:
                        pass  # Invalid date

        # Very old date (before 1900) = probably not a birth date
        if year < 1900:
            return True

    return False


def apply_context_scoring(text: str, entities: list[PIIEntity], window: int = 100) -> list[PIIEntity]:
    """
    Layer 3: Adjust confidence scores based on surrounding context.

    Args:
        text: Original text
        entities: List of detected PII entities
        window: Number of characters to look around each entity

    Returns:
        Entities with adjusted confidence scores
    """
    text_lower = text.lower()
    adjusted_entities = []

    for entity in entities:
        # Get context window around the entity
        start = max(0, entity.start - window)
        end = min(len(text), entity.end + window)
        context = text_lower[start:end]

        confidence_adjustment = 0.0

        # Check for boosting keywords
        if entity.type in CONTEXT_KEYWORDS:
            for keyword in CONTEXT_KEYWORDS[entity.type]:
                if keyword in context:
                    confidence_adjustment += 0.10  # Boost 10% per matching keyword (increased)
                    break  # Only apply once per type

        # Check for anti-context (false positive indicators)
        if entity.type in ANTI_CONTEXT_KEYWORDS:
            for keyword in ANTI_CONTEXT_KEYWORDS[entity.type]:
                if keyword in context:
                    confidence_adjustment -= 0.30  # Reduce 30% for test/example data (increased penalty)
                    break

        # LOGIC GATE: Date validation - future dates cannot be birth dates
        if entity.type == "DATE_OF_BIRTH":
            if is_future_date(entity.value):
                confidence_adjustment -= 0.50  # Heavily penalize future dates

        # Apply adjustment (cap between 0.3 and 0.99)
        new_confidence = max(0.3, min(0.99, entity.confidence + confidence_adjustment))

        adjusted_entities.append(PIIEntity(
            type=entity.type,
            value=entity.value,
            start=entity.start,
            end=entity.end,
            confidence=new_confidence,
            masked_value=entity.masked_value
        ))

    return adjusted_entities


# ============================================================================
# LAYER 4: Checksum Validation Functions
# ============================================================================

def validate_iban(iban: str) -> bool:
    """
    Validate IBAN using MOD-97 checksum algorithm (ISO 7064).
    Returns True if valid, False otherwise.
    """
    # Remove spaces and convert to uppercase
    iban = re.sub(r'\s', '', iban).upper()

    # Check minimum length
    if len(iban) < 15:
        return False

    # Move first 4 chars to end
    rearranged = iban[4:] + iban[:4]

    # Convert letters to numbers (A=10, B=11, ..., Z=35)
    numeric = ''
    for char in rearranged:
        if char.isdigit():
            numeric += char
        elif char.isalpha():
            numeric += str(ord(char) - ord('A') + 10)
        else:
            return False  # Invalid character

    # Perform MOD-97 check
    try:
        return int(numeric) % 97 == 1
    except ValueError:
        return False


def validate_luhn(number: str) -> bool:
    """Validate number using Luhn algorithm (credit cards, etc.)"""
    digits = [int(d) for d in re.sub(r'\D', '', number)]
    if len(digits) < 8:
        return False

    checksum = 0
    for i, d in enumerate(reversed(digits)):
        if i % 2 == 1:
            d *= 2
            if d > 9:
                d -= 9
        checksum += d
    return checksum % 10 == 0


# ============================================================================
# Multi-Layer Detection Pipeline
# ============================================================================

def detect_pii_multilayer(text: str, config: Optional[ScanConfig] = None, coordinates_only: bool = False) -> list[PIIEntity]:
    """
    Multi-layer PII detection pipeline (v1.1):
    - Layer 2.0: Pre-processing (Unicode NFKC + HTML entity decode)
    - Layer 1: Standard regex matching (high-speed DuckDB-compatible)
    - Layer 2: Text normalization to catch obfuscated PII
    - Layer 2.5: JSON blob extraction and deep scanning
    - Layer 2.6: Base64 auto-decoding (with recursive decode + JSON scan)
    - Layer 2.7: Spelled-out number detection (NLP-lite)
    - Layer 2.8: Non-Latin character support (Greek, Cyrillic)
    - Layer 3: Context-based confidence adjustment
    - Layer 4: Checksum verification (Luhn, IBAN MOD-97)
    - Layer 5: False positive filtering
    """
    if config is None:
        config = ScanConfig()

    # Layer 2.0: Pre-processing - Unicode NFKC + HTML entity decode (v1.1)
    preprocessed_text, preprocessing_log = preprocess_text(text)
    
    # For coordinates_only mode: if preprocessing changed text length, 
    # we need to use original text for detection to get accurate coordinates
    if coordinates_only and len(preprocessed_text) != len(text):
        # Text length changed during preprocessing - use original text for accurate coords
        detection_text = text
    else:
        detection_text = preprocessed_text

    # Layer 1: Standard detection
    entities = detect_pii_standard(detection_text, config)

    # Layer 2: Normalize text and re-scan for hidden PII
    normalized_text, transform_info = normalize_text(text)
    if normalized_text != text:
        # Only re-scan if normalization changed something
        normalized_entities = detect_pii_standard(normalized_text, config)

        # Mark normalized entities and add unique ones
        existing_values = {e.value.lower() for e in entities}
        for entity in normalized_entities:
            if entity.value.lower() not in existing_values:
                # This is a newly discovered entity from normalization
                # Reduce confidence slightly as it required normalization
                entity_dict = entity.model_dump()
                entity_dict['confidence'] = entity.confidence * 0.95
                entity_dict['type'] = f"{entity.type}_NORMALIZED"
                entities.append(PIIEntity(**entity_dict))
                existing_values.add(entity.value.lower())

    # Layer 2.5: JSON Blob Extraction & Deep Scanning
    existing_values = {e.value.lower() for e in entities}
    json_blobs = extract_json_strings(text)
    for json_str, start_pos, end_pos in json_blobs:
        extracted_values = deep_scan_json(json_str)
        for extracted in extracted_values:
            # Scan extracted JSON values for PII
            json_entities = detect_pii_standard(extracted, config)
            for entity in json_entities:
                if entity.value.lower() not in existing_values:
                    # Mark as found within JSON, adjust positions to original text
                    entities.append(PIIEntity(
                        type=f"{entity.type}_IN_JSON",
                        value=entity.value,
                        start=start_pos,  # Use JSON blob position
                        end=end_pos,
                        confidence=entity.confidence * 0.90,  # Slight reduction for nested detection
                        masked_value=entity.masked_value
                    ))
                    existing_values.add(entity.value.lower())

    # Layer 2.6: Base64 Auto-Decoding (v1.1 - recursive + JSON scan)
    decoded_b64_strings = decode_base64_strings(preprocessed_text)
    for original_b64, decoded_text, start_pos, end_pos, depth in decoded_b64_strings:
        # Scan decoded base64 content for PII/secrets
        b64_entities = detect_pii_standard(decoded_text, config)

        # Also scan decoded JSON content if present
        if decoded_text.strip().startswith('{') or decoded_text.strip().startswith('['):
            try:
                json_values = deep_scan_json(decoded_text)
                for json_val in json_values:
                    json_entities = detect_pii_standard(json_val, config)
                    for je in json_entities:
                        if je.value.lower() not in existing_values:
                            entities.append(PIIEntity(
                                type=f"{je.type}_BASE64_JSON",
                                value=je.value,
                                start=start_pos,
                                end=end_pos,
                                confidence=je.confidence * 0.80,  # Lower for double-nested
                                masked_value=je.masked_value
                            ))
                            existing_values.add(je.value.lower())
            except Exception:
                pass

        for entity in b64_entities:
            if entity.value.lower() not in existing_values:
                # Mark as found in base64 encoded content
                depth_suffix = f"_DEPTH{depth}" if depth > 1 else ""
                entities.append(PIIEntity(
                    type=f"{entity.type}_BASE64{depth_suffix}",
                    value=entity.value,
                    start=start_pos,
                    end=end_pos,
                    confidence=entity.confidence * (0.85 ** depth),  # Reduce more for deeper encoding
                    masked_value=entity.masked_value
                ))
                existing_values.add(entity.value.lower())

        # Also flag the base64 string itself if it decoded to sensitive content
        if b64_entities:
            entities.append(PIIEntity(
                type="BASE64_ENCODED_SECRET",
                value=original_b64[:50] + "..." if len(original_b64) > 50 else original_b64,
                start=start_pos,
                end=end_pos,
                confidence=0.85,
                masked_value="[BASE64 ENCODED CONTENT REDACTED]"
            ))

    # Layer 2.7: Spelled-Out Number Detection
    converted_text, conversions = convert_spelled_numbers(text)
    if conversions:
        # Scan the converted text for SSN patterns
        for original, converted, start_pos, end_pos in conversions:
            # Check if converted looks like an SSN (XXX-XX-XXXX format)
            ssn_match = re.match(r'^(\d{3})-(\d{2})-(\d{4})$', converted)
            if ssn_match:
                entities.append(PIIEntity(
                    type="SSN_SPELLED_OUT",
                    value=converted,
                    start=start_pos,
                    end=end_pos,
                    confidence=0.90,  # High confidence for spelled-out SSN
                    masked_value=f"***-**-{ssn_match.group(3)}"
                ))

    # Layer 2.8: Non-Latin Character Support
    if has_non_latin_chars(text):
        # Transliterate and re-scan
        transliterated = transliterate_text(text)
        if transliterated != text:
            trans_entities = detect_pii_standard(transliterated, config)
            for entity in trans_entities:
                if entity.value.lower() not in existing_values:
                    entities.append(PIIEntity(
                        type=f"{entity.type}_TRANSLITERATED",
                        value=entity.value,
                        start=entity.start,
                        end=entity.end,
                        confidence=entity.confidence * 0.90,
                        masked_value=entity.masked_value
                    ))
                    existing_values.add(entity.value.lower())

    # Layer 3: Apply context-based scoring
    entities = apply_context_scoring(text, entities)

    # Layer 4: Apply checksum verification for applicable types
    verified_entities = []
    for entity in entities:
        if entity.type == "IBAN":
            if validate_iban(entity.value):
                # Valid IBAN - boost confidence
                verified_entities.append(PIIEntity(
                    type=entity.type,
                    value=entity.value,
                    start=entity.start,
                    end=entity.end,
                    confidence=min(0.99, entity.confidence + 0.10),
                    masked_value=entity.masked_value
                ))
            else:
                # Invalid checksum - cap at 30% ceiling (consistent across all types)
                verified_entities.append(PIIEntity(
                    type="POSSIBLE_IBAN_PATTERN",
                    value=entity.value,
                    start=entity.start,
                    end=entity.end,
                    confidence=0.30,  # Consistent 30% ceiling for invalid checksums
                    masked_value=entity.masked_value
                ))
        elif entity.type == "UUID":
            # UUIDs are typically trace IDs, not PII - cap at 40% max confidence
            verified_entities.append(PIIEntity(
                type=entity.type,
                value=entity.value,
                start=entity.start,
                end=entity.end,
                confidence=min(0.40, entity.confidence),  # Cap UUID confidence at 40%
                masked_value=entity.masked_value
            ))
        elif entity.type == "CREDIT_CARD" or entity.type == "CREDIT_CARD_NORMALIZED":
            # LOGIC GATE: Apply Luhn validation - reclassify invalid credit cards
            if validate_luhn(entity.value):
                # Valid credit card - boost confidence
                verified_entities.append(PIIEntity(
                    type=entity.type,
                    value=entity.value,
                    start=entity.start,
                    end=entity.end,
                    confidence=min(0.99, entity.confidence + 0.10),
                    masked_value=entity.masked_value
                ))
            else:
                # Invalid Luhn checksum - reclassify as POSSIBLE_CARD_PATTERN
                # Check if context suggests it's test/example data - if so, discard
                context_start = max(0, entity.start - 50)
                context_end = min(len(text), entity.end + 50)
                context = text[context_start:context_end].lower()

                # Discard if context clearly indicates test/example data
                if any(kw in context for kw in ["test", "example", "sample", "demo", "fake", "dummy"]):
                    # Skip this entity entirely - it's test data
                    continue

                # Reclassify as POSSIBLE_CARD_PATTERN with 30% ceiling (consistent)
                verified_entities.append(PIIEntity(
                    type="POSSIBLE_CARD_PATTERN",
                    value=entity.value,
                    start=entity.start,
                    end=entity.end,
                    confidence=0.30,  # Consistent 30% ceiling for invalid checksums
                    masked_value=mask_value(entity.value, "CREDIT_CARD")
                ))
        else:
            verified_entities.append(entity)

    # Deduplicate by position AND by overlapping normalized variants
    # Step 1: Group by exact position
    deduplicated = {}
    for entity in verified_entities:
        key = (entity.start, entity.end)
        if key not in deduplicated or entity.confidence > deduplicated[key].confidence:
            deduplicated[key] = entity

    # Step 2: Merge overlapping entities of same base type (e.g., PHONE_US vs PHONE_US_NORMALIZED)
    # This prevents showing both "PHONE_US" and "PHONE_US_NORMALIZED" for the same number
    def get_base_type(pii_type: str) -> str:
        """Get base type without suffixes like _NORMALIZED, _IN_JSON, _BASE64, etc."""
        suffixes = ['_NORMALIZED', '_IN_JSON', '_BASE64', '_TRANSLITERATED', '_SPELLED_OUT']
        for suffix in suffixes:
            if pii_type.endswith(suffix):
                return pii_type[:-len(suffix)]
        return pii_type

    def normalize_value_for_comparison(value: str, pii_type: str) -> str:
        """Normalize a value for duplicate comparison (strip formatting)."""
        base = get_base_type(pii_type)
        if base in ['PHONE_US', 'PHONE_INTL', 'SSN', 'CREDIT_CARD']:
            # For these types, compare just the digits
            return re.sub(r'\D', '', value)
        return value.lower().strip()

    def overlaps(e1, e2, threshold: int = 5) -> bool:
        """Check if two entities overlap or are within threshold characters."""
        return not (e1.end + threshold < e2.start or e2.end + threshold < e1.start)

    def same_normalized_value(e1, e2) -> bool:
        """Check if two entities represent the same underlying value."""
        return normalize_value_for_comparison(e1.value, e1.type) == normalize_value_for_comparison(e2.value, e2.type)

    # Sort by start position for overlap detection
    sorted_entities = sorted(deduplicated.values(), key=lambda x: x.start)
    merged = []

    for entity in sorted_entities:
        base_type = get_base_type(entity.type)
        merged_with_existing = False

        # Check if this overlaps with any existing entity of same base type
        # OR has the same normalized value (handles PHONE_US vs PHONE_US_NORMALIZED with different positions)
        for i, existing in enumerate(merged):
            existing_base = get_base_type(existing.type)

            if base_type == existing_base:
                # Merge if overlapping OR same value (for _NORMALIZED variants)
                if overlaps(entity, existing) or same_normalized_value(entity, existing):
                    # Keep the one with higher confidence
                    if entity.confidence > existing.confidence:
                        merged[i] = entity
                    merged_with_existing = True
                    break

        if not merged_with_existing:
            merged.append(entity)

    deduplicated = {(e.start, e.end): e for e in merged}

    # Layer 5: False positive filtering (v1.1)
    filtered_entities = []
    for entity in deduplicated.values():
        # Get context for false positive check
        context_start = max(0, entity.start - 50)
        context_end = min(len(detection_text), entity.end + 50)
        context = detection_text[context_start:context_end]

        # Skip if detected as false positive
        if is_false_positive(entity.value, entity.type, context):
            continue

        filtered_entities.append(entity)

    # Layer 6: Cross-type overlap resolution
    # Sort by confidence (desc), then by length (desc) to prioritize better matches
    sorted_by_priority = sorted(filtered_entities, key=lambda x: (-x.confidence, -(x.end - x.start)))
    
    final_entities = []
    for entity in sorted_by_priority:
        # Check if this entity overlaps with any already accepted entity
        overlaps_existing = False
        for existing in final_entities:
            # Check for any overlap
            if not (entity.end <= existing.start or entity.start >= existing.end):
                overlaps_existing = True
                break
        if not overlaps_existing:
            final_entities.append(entity)
    
    # Sort by position for output
    result = sorted(final_entities, key=lambda x: x.start)

    return result


def detect_pii_standard(text: str, config: Optional[ScanConfig] = None) -> list[PIIEntity]:
    """Layer 1: Standard regex-based PII detection"""
    if config is None:
        config = ScanConfig()

    entities = []

    # Map config to pattern types
    type_mapping = {
        "EMAIL": config.detect_emails,
        # Phone numbers (US and International)
        "PHONE_US": config.detect_phones,
        "PHONE_INTL": config.detect_phones,
        # Identity documents
        "SSN": config.detect_ssn,
        "MEDICARE_ID": config.detect_ssn,  # Group with SSN as it's US health ID
        "GERMAN_ID": True,  # Always detect national IDs
        "CREDIT_CARD": config.detect_credit_cards,
        "IP_ADDRESS": config.detect_ip_addresses,
        "DATE_OF_BIRTH": config.detect_dates,
        "ZIP_CODE": config.detect_zip_codes,
        # Addresses (US and International)
        "US_ADDRESS": config.detect_addresses,
        "UK_POSTCODE": config.detect_addresses,
        "UK_ADDRESS": config.detect_addresses,
        "EU_ADDRESS": config.detect_addresses,
        "INTERNATIONAL_ADDRESS": config.detect_addresses,
        # Financial
        "IBAN": config.detect_iban,
        "BANK_ACCOUNT": config.detect_iban,
        "ROUTING_NUMBER": config.detect_iban,
        "SWIFT_BIC": config.detect_iban,
        # Medical IDs (always on - high sensitivity)
        "MEDICAL_RECORD_NUMBER": True,
        "NPI_NUMBER": True,
        "DEA_NUMBER": True,
        # Secrets - AWS
        "AWS_ACCESS_KEY": config.detect_secrets,
        "AWS_SECRET_KEY": config.detect_secrets,
        # Secrets - GitHub
        "GITHUB_TOKEN": config.detect_secrets,
        "GITHUB_TOKEN_CLASSIC": config.detect_secrets,
        # Secrets - GCP/Azure
        "GCP_SERVICE_ACCOUNT": config.detect_secrets,
        "GCP_PRIVATE_KEY_ID": config.detect_secrets,
        "AZURE_CLIENT_SECRET": config.detect_secrets,
        "AZURE_CONNECTION_STRING": config.detect_secrets,
        "AZURE_SAS_TOKEN": config.detect_secrets,
        # Secrets - Other
        "SLACK_TOKEN": config.detect_secrets,
        "GENERIC_API_KEY": config.detect_secrets,
        "PASSWORD_IN_URL": config.detect_secrets,
        "PRIVATE_KEY": config.detect_secrets,
        "JWT_TOKEN": config.detect_secrets,
        "STRIPE_KEY": config.detect_secrets,
        "GOOGLE_API_KEY": config.detect_secrets,
        "API_KEY_IN_URL": config.detect_secrets,
        "AUTH_CODE": config.detect_secrets,
        # Evasion-resistant patterns
        "EMAIL_OBFUSCATED": config.detect_emails,
        "IP_DEFANGED": config.detect_ip_addresses,
        # Identifiers
        "UUID": True,  # Always detect UUIDs/trace IDs
        "GEO_COORDINATES": config.detect_addresses,  # Group with addresses
        # =========================================================================
        # NEW v1.1 PATTERNS
        # =========================================================================
        # International IDs
        "UK_NATIONAL_INSURANCE": True,
        "CANADIAN_SIN": True,
        "INDIA_AADHAAR": True,
        "INDIA_PAN": True,
        "AUSTRALIA_TFN": True,
        "BRAZIL_CPF": True,
        "MEXICO_CURP": True,
        "SOUTH_AFRICA_ID": True,
        # Additional cloud tokens
        "DISCORD_TOKEN": config.detect_secrets,
        "DISCORD_WEBHOOK": config.detect_secrets,
        "TWILIO_API_KEY": config.detect_secrets,
        "TWILIO_AUTH_TOKEN": config.detect_secrets,
        "SENDGRID_API_KEY": config.detect_secrets,
        "OPENAI_API_KEY": config.detect_secrets,
        "ANTHROPIC_API_KEY": config.detect_secrets,
        "MAILCHIMP_API_KEY": config.detect_secrets,
        "MAILGUN_API_KEY": config.detect_secrets,
        "HEROKU_API_KEY": config.detect_secrets,
        "SHOPIFY_ACCESS_TOKEN": config.detect_secrets,
        "SHOPIFY_SHARED_SECRET": config.detect_secrets,
        "NPM_TOKEN": config.detect_secrets,
        "PYPI_TOKEN": config.detect_secrets,
        "DOCKER_AUTH": config.detect_secrets,
        # Crypto addresses
        "BITCOIN_ADDRESS": config.detect_iban,  # Group with financial
        "ETHEREUM_ADDRESS": config.detect_iban,
        "MONERO_ADDRESS": config.detect_iban,
        # Financial identifiers
        "CUSIP": config.detect_iban,
        "ISIN": config.detect_iban,
        "SEDOL": config.detect_iban,
    }

    for pii_type, pattern_info in PII_PATTERNS.items():
        # Check if this type should be detected
        if pii_type in type_mapping and not type_mapping[pii_type]:
            continue

        pattern = pattern_info["pattern"]

        for match in re.finditer(pattern, text, re.IGNORECASE):
            value = match.group()

            # Calculate confidence based on pattern specificity
            confidence = 0.85
            if pii_type == "EMAIL":
                confidence = 0.95
            elif pii_type == "SSN":
                # Pattern now requires dashes, so high confidence
                confidence = 0.95
            elif pii_type in ["PHONE_US", "PHONE_INTL"]:
                confidence = 0.85
            elif pii_type == "MEDICARE_ID":
                confidence = 0.90
            elif pii_type == "GERMAN_ID":
                confidence = 0.85  # Requires context (ausweis keyword)
            elif pii_type == "CREDIT_CARD":
                # Validate with Luhn algorithm
                if validate_credit_card(value):
                    confidence = 0.95
                else:
                    confidence = 0.60
            elif pii_type == "ZIP_CODE":
                confidence = 0.70  # Could be any 5-digit number
            elif pii_type in ["US_ADDRESS", "UK_ADDRESS", "EU_ADDRESS", "INTERNATIONAL_ADDRESS"]:
                confidence = 0.80
            elif pii_type == "UK_POSTCODE":
                confidence = 0.85
            elif pii_type == "IBAN":
                # Validate IBAN length and format
                clean_iban = re.sub(r'\s', '', value)
                if len(clean_iban) >= 15 and len(clean_iban) <= 34:
                    confidence = 0.95
                else:
                    confidence = 0.70
            elif pii_type in ["BANK_ACCOUNT", "ROUTING_NUMBER"]:
                confidence = 0.85
            elif pii_type == "SWIFT_BIC":
                confidence = 0.80  # Could be other 8/11 char codes
            # Cloud provider keys
            elif pii_type in ["AWS_ACCESS_KEY", "GITHUB_TOKEN", "GITHUB_TOKEN_CLASSIC", "STRIPE_KEY", "GOOGLE_API_KEY"]:
                confidence = 0.95  # Very specific patterns
            elif pii_type in ["GCP_SERVICE_ACCOUNT", "GCP_PRIVATE_KEY_ID"]:
                confidence = 0.95
            elif pii_type in ["AZURE_CONNECTION_STRING", "AZURE_SAS_TOKEN"]:
                confidence = 0.95
            elif pii_type == "AZURE_CLIENT_SECRET":
                confidence = 0.75  # Less specific pattern
            elif pii_type in ["AWS_SECRET_KEY", "JWT_TOKEN"]:
                confidence = 0.85
            elif pii_type in ["PASSWORD_IN_URL", "GENERIC_API_KEY"]:
                confidence = 0.80
            elif pii_type == "PRIVATE_KEY":
                confidence = 0.99
            # Medical IDs
            elif pii_type in ["MEDICAL_RECORD_NUMBER", "NPI_NUMBER"]:
                confidence = 0.90
            elif pii_type == "DEA_NUMBER":
                confidence = 0.85
            # Evasion-resistant patterns
            elif pii_type == "EMAIL_OBFUSCATED":
                confidence = 0.90  # High - intentional obfuscation is suspicious
            elif pii_type == "IP_DEFANGED":
                confidence = 0.95  # Very high - defanging is deliberate
            elif pii_type == "API_KEY_IN_URL":
                confidence = 0.85
            elif pii_type == "AUTH_CODE":
                confidence = 0.80
            elif pii_type == "UUID":
                confidence = 0.70  # Could be any GUID
            elif pii_type == "GEO_COORDINATES":
                confidence = 0.85

            # Create masked value
            masked_value = mask_value(value, pii_type)

            entities.append(PIIEntity(
                type=pii_type,
                value=value,
                start=match.start(),
                end=match.end(),
                confidence=confidence,
                masked_value=masked_value
            ))

    # Sort by position
    entities.sort(key=lambda x: x.start)

    return entities


def validate_credit_card(number: str) -> bool:
    """Validate credit card using Luhn algorithm"""
    digits = [int(d) for d in re.sub(r'\D', '', number)]
    if len(digits) < 13 or len(digits) > 19:
        return False

    checksum = 0
    for i, d in enumerate(reversed(digits)):
        if i % 2 == 1:
            d *= 2
            if d > 9:
                d -= 9
        checksum += d
    return checksum % 10 == 0


def mask_value(value: str, pii_type: str) -> str:
    """Mask a PII value based on its type"""
    # Handle various suffixes by using base type for masking
    base_type = pii_type
    for suffix in ["_NORMALIZED", "_IN_JSON", "_BASE64", "_TRANSLITERATED", "_SPELLED_OUT"]:
        base_type = base_type.replace(suffix, "")

    if base_type == "EMAIL":
        parts = value.split('@')
        if len(parts) == 2:
            return parts[0][:2] + '***@' + parts[1]
        return '***@***'
    elif base_type in ["PHONE_US", "PHONE_INTL"]:
        clean = re.sub(r'\D', '', value)
        if len(clean) >= 4:
            return '***-***-' + clean[-4:]
        return '***-***-****'
    elif base_type == "MEDICARE_ID":
        return '[MEDICARE ID REDACTED]'
    elif base_type == "GERMAN_ID":
        return '[GERMAN ID REDACTED]'
    elif base_type == "SSN":
        return '***-**-' + value[-4:] if len(value) >= 4 else '***-**-****'
    elif base_type == "CREDIT_CARD":
        clean = re.sub(r'\D', '', value)
        return '****-****-****-' + clean[-4:] if len(clean) >= 4 else '****-****-****-****'
    elif base_type == "IP_ADDRESS":
        parts = value.split('.')
        if len(parts) == 4:
            return f'{parts[0]}.***.***.*'
        return '***.***.***.***'
    elif base_type == "DATE_OF_BIRTH":
        return '**/**/' + value[-4:] if len(value) >= 4 else '**/**/****'
    elif base_type == "IBAN":
        # Show country code and last 4 chars
        if len(value) >= 6:
            return value[:2] + '**' + '*' * (len(value) - 6) + value[-4:]
        return '*' * len(value)
    elif base_type in ["US_ADDRESS", "UK_ADDRESS", "EU_ADDRESS", "INTERNATIONAL_ADDRESS"]:
        # Mask the street number and name
        return '[ADDRESS REDACTED]'
    elif base_type == "UK_POSTCODE":
        return value[:2] + '** ***' if len(value) >= 2 else '[POSTCODE REDACTED]'
    elif base_type in ["BANK_ACCOUNT", "ROUTING_NUMBER"]:
        # Show last 4 digits
        clean = re.sub(r'\D', '', value)
        return '******' + clean[-4:] if len(clean) >= 4 else '*' * len(value)
    elif base_type == "SWIFT_BIC":
        return value[:4] + '****' if len(value) >= 4 else '*' * len(value)
    elif base_type in ["AWS_ACCESS_KEY", "AWS_SECRET_KEY"]:
        return value[:4] + '*' * (len(value) - 8) + value[-4:] if len(value) > 8 else '*' * len(value)
    elif base_type in ["GITHUB_TOKEN", "GITHUB_TOKEN_CLASSIC"]:
        return value[:4] + '*' * (len(value) - 8) + value[-4:] if len(value) > 8 else '*' * len(value)
    elif base_type in ["STRIPE_KEY", "GOOGLE_API_KEY"]:
        return value[:7] + '*' * (len(value) - 11) + value[-4:] if len(value) > 11 else '*' * len(value)
    elif base_type == "PASSWORD_IN_URL":
        return '[PASSWORD REDACTED]'
    elif base_type == "PRIVATE_KEY":
        return '[PRIVATE KEY REDACTED]'
    elif base_type == "JWT_TOKEN":
        return value[:10] + '...[JWT REDACTED]...' + value[-10:] if len(value) > 20 else '[JWT REDACTED]'
    elif base_type in ["SLACK_TOKEN", "GENERIC_API_KEY"]:
        if len(value) > 8:
            return value[:4] + '*' * (len(value) - 8) + value[-4:]
        return '*' * len(value)
    # GCP/Azure cloud credentials
    elif base_type == "GCP_SERVICE_ACCOUNT":
        parts = value.split('@')
        if len(parts) == 2:
            return parts[0][:3] + '***@' + parts[1]
        return '[GCP SERVICE ACCOUNT REDACTED]'
    elif base_type == "GCP_PRIVATE_KEY_ID":
        return '[GCP PRIVATE KEY ID REDACTED]'
    elif base_type in ["AZURE_CLIENT_SECRET", "AZURE_CONNECTION_STRING", "AZURE_SAS_TOKEN"]:
        return '[AZURE CREDENTIAL REDACTED]'
    # Medical IDs
    elif base_type in ["MEDICAL_RECORD_NUMBER", "NPI_NUMBER", "DEA_NUMBER"]:
        return '[MEDICAL ID REDACTED]'
    # Evasion-resistant patterns
    elif base_type == "EMAIL_OBFUSCATED":
        return '[OBFUSCATED EMAIL REDACTED]'
    elif base_type == "IP_DEFANGED":
        return '[DEFANGED IP REDACTED]'
    elif base_type == "API_KEY_IN_URL":
        return '[API KEY IN URL REDACTED]'
    elif base_type == "AUTH_CODE":
        return '[AUTH CODE REDACTED]'
    elif base_type == "UUID":
        return value[:8] + '-****-****-****-' + value[-12:] if len(value) >= 36 else '[UUID REDACTED]'
    elif base_type == "GEO_COORDINATES":
        return '[COORDINATES REDACTED]'
    # New advanced detection types
    elif base_type == "EU_VAT_NUMBER":
        return value[:2] + '***' + value[-4:] if len(value) > 6 else '[VAT REDACTED]'
    elif base_type == "SSN_SPELLED_OUT":
        return '***-**-' + value[-4:] if len(value) >= 4 else '***-**-****'
    elif base_type == "BASE64_ENCODED_SECRET":
        return '[BASE64 SECRET REDACTED]'
    else:
        # Generic masking - show first and last char
        if len(value) > 2:
            return value[0] + '*' * (len(value) - 2) + value[-1]
        return '*' * len(value)


def calculate_risk_level(entities: list[PIIEntity]) -> tuple[str, int]:
    """Calculate risk level based on PII found"""
    if not entities:
        return "LOW", 0

    # Weight by sensitivity
    sensitivity_weights = {
        # High sensitivity - identity theft risk
        "SSN": 100,
        "CREDIT_CARD": 95,
        "DRIVERS_LICENSE": 85,
        "MEDICARE_ID": 90,  # Health insurance ID
        "GERMAN_ID": 90,  # National ID
        # Financial
        "IBAN": 85,
        "BANK_ACCOUNT": 80,
        "ROUTING_NUMBER": 75,
        "SWIFT_BIC": 60,
        # Secrets - security breach risk (AWS)
        "AWS_ACCESS_KEY": 100,
        "AWS_SECRET_KEY": 100,
        "PRIVATE_KEY": 100,
        # Secrets - GitHub
        "GITHUB_TOKEN": 95,
        "GITHUB_TOKEN_CLASSIC": 95,
        # Secrets - GCP/Azure
        "GCP_SERVICE_ACCOUNT": 95,
        "GCP_PRIVATE_KEY_ID": 100,
        "AZURE_CLIENT_SECRET": 95,
        "AZURE_CONNECTION_STRING": 100,
        "AZURE_SAS_TOKEN": 90,
        # Secrets - Other
        "STRIPE_KEY": 95,
        "SLACK_TOKEN": 90,
        "GOOGLE_API_KEY": 85,
        "JWT_TOKEN": 85,
        "PASSWORD_IN_URL": 90,
        "GENERIC_API_KEY": 80,
        # Medical IDs - HIPAA compliance
        "MEDICAL_RECORD_NUMBER": 90,
        "NPI_NUMBER": 85,
        "DEA_NUMBER": 80,
        # Phone numbers
        "PHONE_US": 35,
        "PHONE_INTL": 35,
        # Medium sensitivity - Addresses
        "DATE_OF_BIRTH": 50,
        "US_ADDRESS": 55,
        "UK_ADDRESS": 55,
        "EU_ADDRESS": 55,
        "INTERNATIONAL_ADDRESS": 50,
        "UK_POSTCODE": 40,
        # Lower sensitivity
        "EMAIL": 40,
        "IP_ADDRESS": 30,
        "ZIP_CODE": 20,
        # Possible patterns (failed validation but still flagged)
        "POSSIBLE_CARD_PATTERN": 25,  # Low - failed Luhn, likely not a real card
        # Advanced detection types
        "EU_VAT_NUMBER": 70,
        "SSN_SPELLED_OUT": 95,  # High - intentional evasion
        "BASE64_ENCODED_SECRET": 90,  # High - deliberately hidden
        # JSON embedded types inherit from base type + 5
        # Base64 decoded types inherit from base type + 5
        # Transliterated types inherit from base type
    }

    total_score = 0
    for entity in entities:
        # Handle various suffixes by looking up base type
        base_type = entity.type
        for suffix in ["_NORMALIZED", "_IN_JSON", "_BASE64", "_TRANSLITERATED", "_SPELLED_OUT"]:
            base_type = base_type.replace(suffix, "")

        # Get weight - add bonus for encoded/hidden PII (evasion attempts)
        weight = sensitivity_weights.get(base_type, sensitivity_weights.get(entity.type, 25))

        # Bonus for evasion techniques (intentionally hidden PII is more suspicious)
        if "_IN_JSON" in entity.type or "_BASE64" in entity.type:
            weight = min(100, weight + 10)  # Cap at 100
        total_score += weight * entity.confidence

    # Normalize score (0-100)
    risk_score = min(100, int(total_score / max(1, len(entities)) + len(entities) * 5))

    if risk_score >= 70:
        return "CRITICAL", risk_score
    elif risk_score >= 50:
        return "HIGH", risk_score
    elif risk_score >= 30:
        return "MEDIUM", risk_score
    else:
        return "LOW", risk_score


def redact_text(text: str, entities: list[PIIEntity], mode: str = "mask") -> str:
    """Redact PII from text"""
    if not entities:
        return text

    # Sort by position in reverse to not mess up indices
    sorted_entities = sorted(entities, key=lambda x: x.start, reverse=True)

    result = text
    for entity in sorted_entities:
        if mode == "mask":
            replacement = entity.masked_value
        elif mode == "remove":
            replacement = "[REDACTED]"
        elif mode == "type":
            replacement = f"[{entity.type}]"
        else:
            replacement = entity.masked_value

        result = result[:entity.start] + replacement + result[entity.end:]

    return result


# Alias for backwards compatibility
detect_pii = detect_pii_multilayer


@router.post("/scan-text")
async def scan_text(
    text: str = Form(...),
    detect_emails: bool = Form(True),
    detect_phones: bool = Form(True),
    detect_ssn: bool = Form(True),
    detect_credit_cards: bool = Form(True),
    detect_ip_addresses: bool = Form(True),
    detect_dates: bool = Form(True),
    detect_addresses: bool = Form(True),
    detect_iban: bool = Form(True),
    detect_secrets: bool = Form(True),
    coordinates_only: bool = Form(False)
):
    """
    Scan text for PII and secrets using multi-layer detection.

    Security Options:
    - coordinates_only: If True, returns only PII positions (start, end, type, confidence)
      without the actual values. The frontend can then perform client-side masking
      using these coordinates, ensuring the backend never "sees" the raw PII.
      This is useful for ultra-sensitive data where even the backend shouldn't
      have access to actual PII values.
    """
    # Normalize CRLF to LF when returning coordinates for client-side redaction
    # Browser FormData converts LF->CRLF per RFC 7578, but frontend uses LF text
    if coordinates_only:
        text = text.replace("\r\n", "\n")

    config = ScanConfig(
        detect_emails=detect_emails,
        detect_phones=detect_phones,
        detect_ssn=detect_ssn,
        detect_credit_cards=detect_credit_cards,
        detect_ip_addresses=detect_ip_addresses,
        detect_dates=detect_dates,
        detect_addresses=detect_addresses,
        detect_iban=detect_iban,
        detect_secrets=detect_secrets
    )

    entities = detect_pii_multilayer(text, config, coordinates_only=coordinates_only)

    # Group by type
    entities_by_type = {}
    for entity in entities:
        if entity.type not in entities_by_type:
            entities_by_type[entity.type] = 0
        entities_by_type[entity.type] += 1

    risk_level, risk_score = calculate_risk_level(entities)

    # Coordinates-only mode: Strip actual PII values for client-side redaction
    if coordinates_only:
        # Return only positions and types - no actual PII values
        # Frontend will use these coordinates to mask text client-side
        entities_coords = [
            {
                "type": e.type,
                "start": e.start,
                "end": e.end,
                "confidence": e.confidence,
                "length": e.end - e.start  # So frontend knows how many chars to mask
            }
            for e in entities
        ]
        return {
            "total_entities": len(entities),
            "entities_by_type": entities_by_type,
            "entities": entities_coords,  # Coordinates only - no values!
            "risk_level": risk_level,
            "risk_score": risk_score,
            "redacted_preview": None,  # Frontend handles redaction
            "coordinates_only": True
        }

    # Standard mode: Return full entity details including values
    redacted_preview = redact_text(text, entities, "mask")

    return {
        "total_entities": len(entities),
        "entities_by_type": entities_by_type,
        "entities": [e.model_dump() for e in entities],
        "risk_level": risk_level,
        "risk_score": risk_score,
        "redacted_preview": redacted_preview,
        "coordinates_only": False
    }


@router.post("/scan-file")
async def scan_file(file: UploadFile = File(...)):
    """Scan a file for PII (CSV, TXT, JSON)"""
    try:
        content = await file.read()
        filename = file.filename.lower()

        if filename.endswith('.csv'):
            # Use duckdb to read CSV and extract all text
            with tempfile.NamedTemporaryFile(delete=False, suffix='.csv') as tmp:
                tmp.write(content)
                tmp_path = tmp.name
            try:
                con = duckdb.connect(':memory:')
                result = con.execute(f"SELECT * FROM read_csv_auto('{tmp_path}')").fetchall()
                columns = con.execute(f"DESCRIBE SELECT * FROM read_csv_auto('{tmp_path}')").fetchall()
                all_text = ' '.join(str(cell) for row in result for cell in row if cell is not None)
                con.close()
            finally:
                os.unlink(tmp_path)
        elif filename.endswith('.json'):
            # Use duckdb to read JSON
            with tempfile.NamedTemporaryFile(delete=False, suffix='.json') as tmp:
                tmp.write(content)
                tmp_path = tmp.name
            try:
                con = duckdb.connect(':memory:')
                result = con.execute(f"SELECT * FROM read_json_auto('{tmp_path}')").fetchall()
                all_text = ' '.join(str(cell) for row in result for cell in row if cell is not None)
                con.close()
            finally:
                os.unlink(tmp_path)
        else:
            # Treat as text file
            all_text = content.decode('utf-8', errors='ignore')

        entities = detect_pii(all_text)

        # Group by type
        entities_by_type = {}
        for entity in entities:
            if entity.type not in entities_by_type:
                entities_by_type[entity.type] = 0
            entities_by_type[entity.type] += 1

        risk_level, risk_score = calculate_risk_level(entities)

        # Limit preview length
        preview_text = all_text[:2000] if len(all_text) > 2000 else all_text
        redacted_preview = redact_text(preview_text, [e for e in entities if e.end <= 2000], "mask")

        return {
            "filename": file.filename,
            "total_entities": len(entities),
            "entities_by_type": entities_by_type,
            "entities": [e.model_dump() for e in entities[:100]],  # Limit to first 100
            "risk_level": risk_level,
            "risk_score": risk_score,
            "redacted_preview": redacted_preview
        }
    except Exception as e:
        raise HTTPException(status_code=400, detail=f"Could not process file: {str(e)}")


@router.post("/scan-dataframe")
async def scan_dataframe(file: UploadFile = File(...)):
    """Scan a CSV/Excel file and analyze each column for PII"""
    try:
        content = await file.read()
        filename = file.filename.lower()

        # Determine file extension and create temp file
        if filename.endswith('.csv'):
            suffix = '.csv'
        elif filename.endswith(('.xls', '.xlsx')):
            suffix = '.xlsx' if filename.endswith('.xlsx') else '.xls'
        else:
            raise HTTPException(status_code=400, detail="Unsupported file format. Use CSV or Excel.")

        with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp:
            tmp.write(content)
            tmp_path = tmp.name

        try:
            con = duckdb.connect(':memory:')

            # Read file based on type
            if suffix == '.csv':
                # Get column names and data
                schema = con.execute(f"DESCRIBE SELECT * FROM read_csv_auto('{tmp_path}')").fetchall()
                columns = [col[0] for col in schema]
                data = con.execute(f"SELECT * FROM read_csv_auto('{tmp_path}')").fetchall()
            else:
                # For Excel files, duckdb needs the spatial extension or we use a workaround
                # DuckDB can read xlsx via read_xlsx (requires extension) - fall back to CSV-like approach
                try:
                    schema = con.execute(f"DESCRIBE SELECT * FROM st_read('{tmp_path}')").fetchall()
                    columns = [col[0] for col in schema]
                    data = con.execute(f"SELECT * FROM st_read('{tmp_path}')").fetchall()
                except:
                    # If spatial extension not available, try xlsx reader
                    schema = con.execute(f"DESCRIBE SELECT * FROM read_xlsx('{tmp_path}')").fetchall()
                    columns = [col[0] for col in schema]
                    data = con.execute(f"SELECT * FROM read_xlsx('{tmp_path}')").fetchall()

            total_rows = len(data)
            total_columns = len(columns)

            column_results = []
            total_pii = 0

            for col_idx, col in enumerate(columns):
                # Extract all values for this column
                col_values = [str(row[col_idx]) for row in data if row[col_idx] is not None]
                col_text = ' '.join(col_values)
                entities = detect_pii(col_text)

                # Group by type
                entities_by_type = {}
                for entity in entities:
                    if entity.type not in entities_by_type:
                        entities_by_type[entity.type] = 0
                    entities_by_type[entity.type] += 1

                pii_count = len(entities)
                total_pii += pii_count

                column_results.append({
                    "column": col,
                    "pii_count": pii_count,
                    "pii_types": entities_by_type,
                    "sample_pii": [e.model_dump() for e in entities[:3]] if entities else []
                })

            con.close()
        finally:
            os.unlink(tmp_path)

        # Sort by PII count descending
        column_results.sort(key=lambda x: x["pii_count"], reverse=True)

        risk_level = "CRITICAL" if total_pii > 50 else "HIGH" if total_pii > 20 else "MEDIUM" if total_pii > 5 else "LOW"

        return {
            "filename": file.filename,
            "total_rows": total_rows,
            "total_columns": total_columns,
            "total_pii_found": total_pii,
            "risk_level": risk_level,
            "columns_with_pii": len([c for c in column_results if c["pii_count"] > 0]),
            "column_analysis": column_results
        }
    except Exception as e:
        raise HTTPException(status_code=400, detail=f"Could not process file: {str(e)}")


@router.post("/redact")
async def redact_text_endpoint(
    text: str = Form(...),
    mode: str = Form("mask")
):
    """Redact PII from text"""
    entities = detect_pii(text)
    redacted = redact_text(text, entities, mode)

    return {
        "original_length": len(text),
        "redacted_length": len(redacted),
        "entities_redacted": len(entities),
        "redacted_text": redacted
    }


@router.get("/entity-types")
async def list_entity_types():
    """List supported PII entity types"""
    return {
        "entity_types": [
            {"type": key, "description": value["description"]}
            for key, value in PII_PATTERNS.items()
        ]
    }
