Full-Text Search

import {
  IndexedLWWMap,
  simpleAttribute,
  HLC
} from '@topgunbuild/core';

interface Product {
  id: string;
  name: string;
  description: string;
  tags: string[];
}

const hlc = new HLC('node-1');
const products = new IndexedLWWMap<string, Product>(hlc);

// Add inverted index on text fields
const nameAttr = simpleAttribute<Product, string>('name', p => p.name);
products.addInvertedIndex(nameAttr);

// Add some products
products.set('p1', {
  id: 'p1',
  name: 'Wireless Bluetooth Mouse',
  description: 'Ergonomic design with 6 buttons',
  tags: ['electronics', 'wireless', 'mouse']
});

products.set('p2', {
  id: 'p2',
  name: 'USB-C Wireless Keyboard',
  description: 'Mechanical switches with RGB',
  tags: ['electronics', 'wireless', 'keyboard']
});

// Search for products containing "wireless"
const results = products.queryValues({
  type: 'contains',
  attribute: 'name',
  value: 'wireless'
});
// Returns both products (both names contain "wireless")

// 1. contains - All tokens must match (AND semantics)
const wireless = products.queryValues({
  type: 'contains',
  attribute: 'name',
  value: 'wireless mouse'  // Matches: "wireless" AND "mouse"
});
// Returns: [{ name: 'Wireless Bluetooth Mouse', ... }]

// 2. containsAll - All specified values must match
const withTags = products.queryValues({
  type: 'containsAll',
  attribute: 'name',
  values: ['wireless', 'bluetooth']
});
// Returns: [{ name: 'Wireless Bluetooth Mouse', ... }]

// 3. containsAny - Any token matches (OR semantics)
const anyMatch = products.queryValues({
  type: 'containsAny',
  attribute: 'name',
  values: ['keyboard', 'mouse']
});
// Returns both products

import {
  TokenizationPipeline,
  WordBoundaryTokenizer,
  LowercaseFilter,
  MinLengthFilter,
  StopWordFilter
} from '@topgunbuild/core';

// Simple pipeline (default)
const simple = TokenizationPipeline.simple();
simple.process("Hello World!");
// → ["hello", "world"]

// Search pipeline (with stop words removed)
const search = TokenizationPipeline.search();
search.process("The quick brown fox");
// → ["quick", "brown", "fox"]  ("the" removed as stop word)

// Custom pipeline
const custom = TokenizationPipeline.custom(
  new WordBoundaryTokenizer(),
  [
    new LowercaseFilter(),
    new MinLengthFilter(3),     // Min 3 characters
    new StopWordFilter()        // Remove common words
  ]
);
custom.process("I have a wireless mouse");
// → ["wireless", "mouse"]  ("i", "have", "a" removed)

// Available tokenizers:
// - WhitespaceTokenizer: splits on whitespace
// - WordBoundaryTokenizer: splits on word boundaries (default)
// - NGramTokenizer: generates n-grams for substring matching

// Available filters:
// - LowercaseFilter: converts to lowercase
// - MinLengthFilter(n): removes tokens shorter than n
// - MaxLengthFilter(n): removes tokens longer than n
// - StopWordFilter: removes common words (the, a, is, etc.)
// - TrimFilter: trims whitespace
// - UniqueFilter: removes duplicate tokens

import {
  IndexedLWWMap,
  simpleAttribute,
  TokenizationPipeline,
  NGramTokenizer,
  LowercaseFilter,
  HLC
} from '@topgunbuild/core';

// N-gram pipeline for substring matching
const ngramPipeline = TokenizationPipeline.custom(
  new NGramTokenizer(3),  // 3-character n-grams
  [new LowercaseFilter()]
);

const products = new IndexedLWWMap<string, Product>(hlc);

// Use custom pipeline for description field
const descAttr = simpleAttribute<Product, string>('description', p => p.description);
products.addInvertedIndex(descAttr, ngramPipeline);

// Now substring matches work
products.set('p1', { id: 'p1', description: 'Ergonomic mouse' });
const results = products.queryValues({
  type: 'contains',
  attribute: 'description',
  value: 'rgo'  // Matches "Ergonomic" via n-gram
});

import { multiAttribute } from '@topgunbuild/core';

// Index an array field (tags)
const tagsAttr = multiAttribute<Product, string>('tags', p => p.tags);
products.addInvertedIndex(tagsAttr);

// Each tag becomes searchable
const wirelessProducts = products.queryValues({
  type: 'contains',
  attribute: 'tags',
  value: 'wireless'
});

// Combine text search with other predicates
const results = products.queryValues({
  type: 'and',
  children: [
    { type: 'contains', attribute: 'name', value: 'wireless' },
    { type: 'eq', attribute: 'status', value: 'active' },
    { type: 'between', attribute: 'price', from: 10, to: 100 }
  ]
});

// Query optimizer will use:
// - InvertedIndex for "contains"
// - HashIndex for "eq" (if indexed)
// - NavigableIndex for "between" (if indexed)

import { IndexedORMap, HLC } from '@topgunbuild/core';

interface Article {
  title: string;
  body: string;
  author: string;
}

const hlc = new HLC('node-1');
const articles = new IndexedORMap<string, Article>(hlc);

// Enable BM25 full-text search on title and body fields
articles.enableFullTextSearch({
  fields: ['title', 'body']
});

// Add some articles
articles.add('a1', {
  title: 'Introduction to Machine Learning',
  body: 'Machine learning is a subset of artificial intelligence...',
  author: 'Alice'
});

articles.add('a2', {
  title: 'Deep Learning Tutorial',
  body: 'Deep learning uses neural networks with many layers...',
  author: 'Bob'
});

articles.add('a3', {
  title: 'Getting Started with AI',
  body: 'Artificial intelligence is transforming industries...',
  author: 'Charlie'
});

// Search with BM25 relevance ranking
const results = articles.search('machine learning');
// Results sorted by relevance score:
// [
//   { key: 'a1', score: 2.34, matchedTerms: ['machin', 'learn'], value: {...} },
//   { key: 'a2', score: 0.89, matchedTerms: ['learn'], value: {...} }
// ]

// Search with options
const results = articles.search('artificial intelligence', {
  limit: 10,           // Maximum results to return
  minScore: 0.5,       // Minimum relevance score threshold
  boost: {             // Field boosting weights
    title: 2.0,        // Title matches worth 2x
    body: 1.0          // Body matches worth 1x (default)
  }
});

// Results are ranked by weighted BM25 scores
for (const result of results) {
  console.log(`[${result.score.toFixed(2)}] ${result.value.title}`);
  console.log(`  Matched terms: ${result.matchedTerms.join(', ')}`);
}

// Configure tokenizer and BM25 parameters
articles.enableFullTextSearch({
  fields: ['title', 'body', 'tags'],

  // Tokenizer options
  tokenizer: {
    minLength: 2,        // Minimum token length (default: 2)
    maxLength: 50,       // Maximum token length (default: 50)
    lowercase: true,     // Convert to lowercase (default: true)
    // Uses Porter stemmer and 174 English stopwords by default
  },

  // BM25 scoring parameters
  bm25: {
    k1: 1.2,  // Term frequency saturation (default: 1.2)
              // Higher = more weight to term frequency
    b: 0.75   // Document length normalization (default: 0.75)
              // 0 = no normalization, 1 = full normalization
  }
});

// Two approaches to full-text search:

// 1. InvertedIndex - Boolean matching (fast, no ranking)
const nameAttr = simpleAttribute<Product, string>('name', p => p.name);
products.addInvertedIndex(nameAttr);
const matches = products.queryValues({
  type: 'contains',
  attribute: 'name',
  value: 'wireless mouse'
});
// Returns all products containing both "wireless" AND "mouse"
// No relevance score, order is arbitrary

// 2. BM25 Search - Relevance ranking (search engine style)
products.enableFullTextSearch({ fields: ['name', 'description'] });
const ranked = products.search('wireless mouse');
// Returns products sorted by relevance
// "Wireless Bluetooth Mouse" ranks higher than "Mouse pad for wireless setup"
// Each result includes: score, matchedTerms

// Serialize index for persistence
const ftsIndex = articles.getFullTextIndex();
const serialized = ftsIndex.serialize();

// Save to storage (IndexedDB, localStorage, file, etc.)
localStorage.setItem('fts-index', JSON.stringify(serialized));

// Later: restore from storage
const saved = JSON.parse(localStorage.getItem('fts-index'));
ftsIndex.load(saved);

// Index is ready to use immediately
const results = articles.search('machine learning');

import { ServerCoordinator } from '@topgunbuild/server';

const server = new ServerCoordinator({
  port: 8080,
  // Enable full-text search for specific maps
  fullTextSearch: {
    articles: {
      fields: ['title', 'body'],
      tokenizer: { minLength: 2 },
      bm25: { k1: 1.2, b: 0.75 }
    },
    products: {
      fields: ['name', 'description', 'tags']
    }
  }
});

await server.start();
// Server now maintains FTS indexes for 'articles' and 'products' maps
// Indexes are automatically updated when data changes
// Indexes are backfilled from storage on startup

import { TopGunClient } from '@topgunbuild/client';

const client = new TopGunClient({
  serverUrl: 'ws://localhost:8080'
});

await client.authenticate({ token: 'user-token' });

// Search articles on the server
const results = await client.search<Article>('articles', 'machine learning', {
  limit: 20,
  minScore: 0.5,
  boost: { title: 2.0, body: 1.0 }
});

// Results are sorted by relevance
for (const result of results) {
  console.log(`[${result.score.toFixed(2)}] ${result.key}: ${result.value.title}`);
  console.log(`  Matched: ${result.matchedTerms.join(', ')}`);
}

// SearchResult<T> interface
interface SearchResult<T> {
  key: string;        // Document key
  value: T;           // Full document value
  score: number;      // BM25 relevance score
  matchedTerms: string[];  // Stemmed terms that matched
}

// Example result
const result: SearchResult<Article> = {
  key: 'a1',
  value: { title: 'Introduction to ML', body: '...' },
  score: 2.34,
  matchedTerms: ['machin', 'learn']  // Stemmed
};

// Server security configuration
const server = new ServerCoordinator({
  port: 8080,
  fullTextSearch: {
    articles: { fields: ['title', 'body'] }
  },
  security: {
    permissions: {
      // Users need READ permission to search a map
      articles: {
        read: ['user', 'admin'],
        write: ['admin']
      }
    }
  }
});

// Client-side: search requires READ permission
try {
  const results = await client.search('articles', 'query');
} catch (error) {
  // Error: Permission denied for map: articles
}

import { TopGunClient } from '@topgunbuild/client';

const client = new TopGunClient({
  serverUrl: 'ws://localhost:8080'
});

// Create a live search subscription
const handle = client.searchSubscribe<Article>('articles', 'machine learning', {
  limit: 20,
  minScore: 0.5,
  boost: { title: 2.0, body: 1.0 }
});

// Subscribe to result changes (includes initial results + delta updates)
const unsubscribe = handle.subscribe((results) => {
  console.log('Search results updated:', results.length);
  for (const result of results) {
    console.log(`[${result.score.toFixed(2)}] ${result.key}: ${result.value.title}`);
  }
});

// Get current results snapshot at any time
const snapshot = handle.getResults();

// Update query dynamically (re-subscribes automatically)
handle.setQuery('deep learning');

// Cleanup when done
handle.dispose();

// Delta update types:
// - ENTER: Document now matches the query (was added or score increased above minScore)
// - UPDATE: Document still matches but score/value changed
// - LEAVE: Document no longer matches (was removed or score dropped below minScore)

// The SearchHandle maintains a sorted result set internally.
// Your subscribe callback receives the full sorted array on each change.

// Example: Building a real-time search UI
const handle = client.searchSubscribe<Product>('products', 'wireless');

handle.subscribe((results) => {
  // Results are always sorted by score (highest first)
  renderSearchResults(results);
});

// When a product matching "wireless" is added to the 'products' map,
// or an existing product's text is updated to include "wireless",
// your callback fires with the updated results array.

import { useSearch } from '@topgunbuild/react';

function SearchResults() {
  const [searchTerm, setSearchTerm] = useState('');

  const { results, loading, error } = useSearch<Article>('articles', searchTerm, {
    limit: 20,
    boost: { title: 2.0 }
  });

  if (loading) return <Spinner />;
  if (error) return <div>Error: {error.message}</div>;

  return (
    <ul>
      {results.map(r => (
        <li key={r.key}>
          [{r.score.toFixed(2)}] {r.value.title}
          <small>Matched: {r.matchedTerms.join(', ')}</small>
        </li>
      ))}
    </ul>
  );
}

import { useSearch } from '@topgunbuild/react';
import { useState } from 'react';

function SearchInput() {
  const [input, setInput] = useState('');

  // Debounce search queries by 300ms to avoid excessive server requests
  const { results, loading, error } = useSearch<Product>('products', input, {
    debounceMs: 300,  // Wait 300ms after user stops typing
    limit: 10,
    minScore: 0.5
  });

  return (
    <div>
      <input
        value={input}
        onChange={(e) => setInput(e.target.value)}
        placeholder="Search products..."
      />
      {loading && <span>Searching...</span>}
      {error && <span className="error">{error.message}</span>}
      <ul>
        {results.map(r => (
          <li key={r.key}>{r.value.name} - ${r.value.price}</li>
        ))}
      </ul>
    </div>
  );
}

import { useHybridQuery } from '@topgunbuild/react';
import { Predicates } from '@topgunbuild/core';

function TechArticles() {
  // Combine FTS with traditional filters
  const { results, loading, error } = useHybridQuery<Article>('articles', {
    predicate: Predicates.and(
      Predicates.match('body', 'machine learning'),  // FTS predicate
      Predicates.equal('category', 'tech')           // Filter predicate
    ),
    sort: { _score: 'desc' },  // Sort by relevance
    limit: 20
  });

  if (loading) return <Spinner />;
  if (error) return <div>Error: {error.message}</div>;

  return (
    <ul>
      {results.map(r => (
        <li key={r._key}>
          [{r._score?.toFixed(2)}] {r.value.title}
          <small>Matched: {r._matchedTerms?.join(', ')}</small>
        </li>
      ))}
    </ul>
  );
}

import { useHybridQuery } from '@topgunbuild/react';
import { Predicates } from '@topgunbuild/core';
import { useState, useMemo } from 'react';

function SearchWithFilters() {
  const [searchTerm, setSearchTerm] = useState('');
  const [category, setCategory] = useState('all');
  const [priceMax, setPriceMax] = useState(1000);

  // Build predicate dynamically
  const filter = useMemo(() => {
    const conditions = [];

    // Add FTS if search term exists
    if (searchTerm.trim()) {
      conditions.push(Predicates.match('description', searchTerm));
    }

    // Add category filter
    if (category !== 'all') {
      conditions.push(Predicates.equal('category', category));
    }

    // Add price filter
    conditions.push(Predicates.lessThanOrEqual('price', priceMax));

    return {
      predicate: conditions.length > 1
        ? Predicates.and(...conditions)
        : conditions[0],
      sort: searchTerm ? { _score: 'desc' } : { createdAt: 'desc' },
      limit: 20
    };
  }, [searchTerm, category, priceMax]);

  const { results, loading } = useHybridQuery<Product>('products', filter);

  return (
    <div>
      <input
        value={searchTerm}
        onChange={(e) => setSearchTerm(e.target.value)}
        placeholder="Search products..."
      />
      <select value={category} onChange={(e) => setCategory(e.target.value)}>
        <option value="all">All Categories</option>
        <option value="electronics">Electronics</option>
        <option value="clothing">Clothing</option>
      </select>
      <input
        type="range"
        value={priceMax}
        onChange={(e) => setPriceMax(Number(e.target.value))}
        min={0}
        max={1000}
      />
      {loading && <span>Loading...</span>}
      <ul>
        {results.map(r => (
          <li key={r._key}>
            {r.value.name} - ${r.value.price}
            {r._score && <span> (score: {r._score.toFixed(2)})</span>}
          </li>
        ))}
      </ul>
    </div>
  );
}

import { TopGunClient } from '@topgunbuild/client';
import { Predicates } from '@topgunbuild/core';

const client = new TopGunClient({
  serverUrl: 'ws://localhost:8080'
});

// Create a hybrid query handle
const handle = client.hybridQuery<Article>('articles', {
  predicate: Predicates.and(
    Predicates.match('body', 'artificial intelligence'),
    Predicates.equal('status', 'published'),
    Predicates.greaterThan('views', 100)
  ),
  sort: { _score: 'desc' },
  limit: 50
});

// Subscribe to results
const unsubscribe = handle.subscribe((results) => {
  for (const r of results) {
    console.log(`[${r._score?.toFixed(2)}] ${r._key}: ${r.value.title}`);
  }
});

// Cleanup
unsubscribe();

// Get extended statistics for inverted index
const index = products.getIndexes().find(i => i.type === 'inverted');
if (index) {
  const stats = index.getExtendedStats();
  console.log(`Unique tokens: ${stats.totalTokens}`);
  console.log(`Documents indexed: ${stats.totalEntries}`);
  console.log(`Avg tokens/doc: ${stats.avgTokensPerDocument.toFixed(1)}`);
  console.log(`Max docs/token: ${stats.maxDocumentsPerToken}`);
}