Grundlagen der Sprachverarbeitung (NLP) für Webentwickler

# ❌ Naive split breaks on common edge cases
text = "I can't believe it's $9.99! Ship to New York."
tokens = text.split(" ")
# ['I', "can't", 'believe', "it's", '$9.99!', 'Ship', 'to', 'New', 'York.']
# "can't", "$9.99!", "York." — punctuation attached to words

# ✅ Proper tokenization with spaCy
import spacy
 
nlp = spacy.load("en_core_web_sm")
doc = nlp("I can't believe it's $9.99! Ship to New York.")
 
tokens = [token.text for token in doc]
# ['I', 'ca', "n't", 'believe', 'it', "'s", '$', '9.99', '!',
#  'Ship', 'to', 'New', 'York', '.']
# Contractions split correctly, punctuation separated, prices preserved

from transformers import pipeline
 
# Pre-trained sentiment model — downloads on first run (~250MB)
sentiment = pipeline("sentiment-analysis")
 
reviews = [
    "The battery life is incredible, lasts two full days.",
    "Terrible customer support. Waited 3 hours on hold.",
    "It works fine. Nothing special but does the job.",
]
 
for review in reviews:
    result = sentiment(review)[0]
    print(f"{result['label']:>8} ({result['score']:.3f}): {review[:50]}")
 
# Output:
# POSITIVE (0.999): The battery life is incredible, lasts two full da
# NEGATIVE (0.999): Terrible customer support. Waited 3 hours on hold
# POSITIVE (0.876): It works fine. Nothing special but does the job.

import spacy
 
nlp = spacy.load("en_core_web_sm")
 
text = """
Apple announced a $3 billion investment in its Austin, Texas facility
on March 15, 2021. CEO Tim Cook said the expansion would create
5,000 new jobs by 2024.
"""
 
doc = nlp(text)
 
for ent in doc.ents:
    print(f"{ent.label_:>10}: {ent.text}")
 
# Output:
#        ORG: Apple
#      MONEY: $3 billion
#        GPE: Austin
#        GPE: Texas
#       DATE: March 15, 2021
#     PERSON: Tim Cook
#   CARDINAL: 5,000
#       DATE: 2024

// Using NER results in a web application
interface ExtractedEntity {
  text: string;
  label: string;
  start: number;
  end: number;
}
 
interface TicketAnalysis {
  entities: ExtractedEntity[];
  vendor: string | null;
  amount: number | null;
  priority: 'low' | 'medium' | 'high';
}
 
function analyzeTicket(nerResults: ExtractedEntity[]): TicketAnalysis {
  const vendor = nerResults.find(e => e.label === 'ORG')?.text ?? null;
  const moneyEntity = nerResults.find(e => e.label === 'MONEY');
 
  const amount = moneyEntity
    ? parseFloat(moneyEntity.text.replace(/[^0-9.]/g, ''))
    : null;
 
  const priority = amount && amount > 1000 ? 'high'
    : amount && amount > 100 ? 'medium'
    : 'low';
 
  return { entities: nerResults, vendor, amount, priority };
}

from sentence_transformers import SentenceTransformer
import numpy as np
 
model = SentenceTransformer('all-MiniLM-L6-v2')
 
# Knowledge base articles
articles = [
    "How to reset your password and recover account access",
    "Setting up two-factor authentication for your account",
    "Understanding your monthly billing statement",
    "Upgrading your subscription from free to premium",
    "Troubleshooting login errors and account lockouts",
]
 
# Compute embeddings (do this once, store in database)
article_embeddings = model.encode(articles)
 
# User query
query = "I forgot my login credentials"
query_embedding = model.encode([query])
 
# Cosine similarity
similarities = np.dot(article_embeddings, query_embedding.T).flatten()
 
# Rank by relevance
ranked = sorted(enumerate(similarities), key=lambda x: x[1], reverse=True)
 
for idx, score in ranked[:3]:
    print(f"{score:.3f}: {articles[idx]}")
 
# Output:
# 0.687: How to reset your password and recover account access
# 0.542: Troubleshooting login errors and account lockouts
# 0.213: Setting up two-factor authentication for your account

// ❌ Sending unbounded user text without validation
async function analyzeText(userInput: string) {
  const response = await fetch('https://api.example.com/nlp/analyze', {
    method: 'POST',
    headers: { 'Authorization': `Bearer ${process.env.NLP_API_KEY}` },
    body: JSON.stringify({ text: userInput }),
  });
  return response.json();
}

// ✅ Validate and truncate input before sending to the API
const MAX_TEXT_LENGTH = 5000;
 
async function analyzeText(userInput: string): Promise<NLPResult> {
  const sanitized = userInput.trim();
 
  if (sanitized.length === 0) {
    return { entities: [], sentiment: null, error: 'Empty input' };
  }
 
  // Truncate to API limit — most NLP APIs have token limits
  const text = sanitized.slice(0, MAX_TEXT_LENGTH);
 
  const controller = new AbortController();
  const timeout = setTimeout(() => controller.abort(), 10_000);
 
  try {
    const response = await fetch('https://api.example.com/nlp/analyze', {
      method: 'POST',
      headers: {
        'Authorization': `Bearer ${process.env.NLP_API_KEY}`,
        'Content-Type': 'application/json',
      },
      body: JSON.stringify({ text }),
      signal: controller.signal,
    });
 
    if (!response.ok) {
      throw new Error(`NLP API error: ${response.status}`);
    }
 
    return await response.json();
  } finally {
    clearTimeout(timeout);
  }
}

import re
import unicodedata
 
def preprocess_text(text: str) -> str:
    # Normalize unicode (é → e, ñ → n for matching purposes)
    text = unicodedata.normalize('NFKD', text)
 
    # Remove URLs
    text = re.sub(r'https?://\S+', '', text)
 
    # Remove excessive whitespace
    text = re.sub(r'\s+', ' ', text).strip()
 
    # Lowercase for comparison (keep original for display)
    return text.lower()
 
# Example:
raw = "Check out  https://example.com — it's AMAZING!!!   🚀🚀🚀"
clean = preprocess_text(raw)
# "check out — it's amazing!!! 🚀🚀🚀"