IPFS-portal-deploy/src/lib/search-index.ts

/* ── Search Index — Client-side full-text search voor IPFS ──
 *
 * Bewaart een miniatuur search index in localStorage.
 * Periodiek te updaten door gepinde files te crawlen en te indexeren.
 *
 * Beperking: alleen tekstuele content (.txt, .md, .json, .html)
 */

'use client';

/* ════════════════════════════ Types ════════════════════════════ */

export interface IndexedEntry {
  cid: string;
  name: string;
  type: 'file' | 'dir';
  text: string;          // geëxtraheerde tekst (max 2048 chars)
  size: number;
  indexedAt: number;     // timestamp
}

export interface SearchResult {
  entry: IndexedEntry;
  score: number;         // relevance (higher = better)
  matchField: 'name' | 'text' | 'cid';
}

/* ════════════════════════════ Storage ════════════════════════════ */

const STORAGE_KEY = 'ipfs-search-index';

function loadIndex(): IndexedEntry[] {
  if (typeof window === 'undefined') return [];
  try {
    const raw = localStorage.getItem(STORAGE_KEY);
    return raw ? JSON.parse(raw) : [];
  } catch {
    return [];
  }
}

function saveIndex(entries: IndexedEntry[]) {
  if (typeof window === 'undefined') return;
  try {
    localStorage.setItem(STORAGE_KEY, JSON.stringify(entries));
  } catch {
    // localStorage vol of blocked — silently fail
  }
}

/* ════════════════════════════ Tekst extractie ════════════════════════════ */

const TEXT_EXTENSIONS = new Set(['.txt', '.md', '.json', '.html', '.htm', '.csv', '.log', '.xml', '.yaml', '.yml', '.toml', '.ini', '.cfg']);

export function isTextFile(name: string): boolean {
  const ext = name.toLowerCase().slice(name.lastIndexOf('.'));
  return TEXT_EXTENSIONS.has(ext);
}

export function extractText(raw: string): string {
  return raw
    .replace(/<[^>]*>/g, ' ')     // strip HTML tags
    .replace(/\s+/g, ' ')          // collapse whitespace
    .replace(/[{}\[\]]/g, ' ')     // strip JSON braces
    .trim()
    .slice(0, 2048);
}

/* ════════════════════════════ Index beheer ════════════════════════════ */

export function addToIndex(entry: IndexedEntry) {
  const index = loadIndex();
  // Remove oud voor deze CID
  const filtered = index.filter((e) => e.cid !== entry.cid);
  filtered.push(entry);
  saveIndex(filtered);
}

export function removeFromIndex(cid: string) {
  const index = loadIndex().filter((e) => e.cid !== cid);
  saveIndex(index);
}

export function clearIndex() {
  if (typeof window !== 'undefined') {
    localStorage.removeItem(STORAGE_KEY);
  }
}

export function getIndexStats() {
  const index = loadIndex();
  const totalSize = index.reduce((sum, e) => sum + e.text.length, 0);
  return {
    entries: index.length,
    totalChars: totalSize,
    lastIndexed: index.length > 0 ? Math.max(...index.map((e) => e.indexedAt)) : 0,
  };
}

/* ════════════════════════════ Zoeken ════════════════════════════ */

const STOP_WORDS = new Set([
  'de', 'het', 'een', 'van', 'en', 'in', 'is', 'te', 'met', 'op',
  'voor', 'aan', 'dat', 'die', 'door', 'bij', 'ook', 'maar', 'uit',
  'naar', 'om', 'al', 'als', 'nog', 'dan', 'of', 'er', 'niet', 'dit',
  'the', 'a', 'an', 'and', 'or', 'in', 'on', 'at', 'to', 'for',
  'of', 'is', 'it', 'be', 'by', 'as', 'are', 'was', 'were', 'been',
]);

function tokenize(text: string): string[] {
  return text
    .toLowerCase()
    .split(/[^a-z0-9]+/)
    .filter((t) => t.length > 1 && !STOP_WORDS.has(t));
}

function scoreEntry(queryTokens: string[], entry: IndexedEntry): number {
  let score = 0;
  const nameLower = entry.name.toLowerCase();
  const textLower = entry.text.toLowerCase();
  const cidLower = entry.cid.toLowerCase();

  for (const token of queryTokens) {
    // Exact match in name = hoogste score
    if (nameLower.includes(token)) {
      score += token.length * 10;
      if (nameLower === token) score += 20; // exact match bonus
    }
    // Match in text
    if (textLower.includes(token)) {
      score += token.length * 3;
    }
    // Match in CID (prefix)
    if (cidLower.startsWith(token)) {
      score += token.length * 8;
    }
    // Frequentie in text
    const count = (textLower.match(new RegExp(token.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'), 'g')) || []).length;
    score += count;
  }

  return score;
}

export function searchIndex(query: string, maxResults = 20): SearchResult[] {
  if (!query || query.trim().length < 2) return [];

  const queryTokens = tokenize(query);
  if (queryTokens.length === 0) return [];

  const index = loadIndex();
  const scored: SearchResult[] = [];

  for (const entry of index) {
    const score = scoreEntry(queryTokens, entry);
    if (score > 0) {
      // Bepaal het beste match veld
      const nameLower = entry.name.toLowerCase();
      const cidLower = entry.cid.toLowerCase();
      let matchField: 'name' | 'text' | 'cid' = 'text';
      if (queryTokens.some((t) => nameLower.includes(t))) matchField = 'name';
      else if (queryTokens.some((t) => cidLower.startsWith(t))) matchField = 'cid';

      scored.push({ entry, score, matchField });
    }
  }

  return scored.sort((a, b) => b.score - a.score).slice(0, maxResults);
}

/* ════════════════════════════ Crawler ════════════════════════════ */

export async function rebuildIndex(
  cids: { cid: string; name: string }[],
  fetchContent: (cid: string) => Promise<string>,
  onProgress?: (done: number, total: number) => void,
): Promise<{ indexed: number; failed: number }> {
  let indexed = 0;
  let failed = 0;

  // Filter op text files
  const textFiles = cids.filter((c) => isTextFile(c.name));

  for (let i = 0; i < textFiles.length; i++) {
    const file = textFiles[i];
    try {
      const raw = await fetchContent(file.cid);
      const text = extractText(raw);
      if (text.length > 0) {
        addToIndex({
          cid: file.cid,
          name: file.name,
          type: 'file',
          text,
          size: raw.length,
          indexedAt: Date.now(),
        });
        indexed++;
      }
    } catch {
      failed++;
    }
    onProgress?.(i + 1, textFiles.length);
  }

  return { indexed, failed };
}