Cluster Replication

import { ServerCoordinator } from '@topgunbuild/server';

// Node 1 (seed node)
const node1 = new ServerCoordinator({
  port: 8080,
  nodeId: 'node-1',
  host: 'localhost',
  clusterPort: 9000,
  peers: [],  // First node has no peers
  jwtSecret: process.env.JWT_SECRET,
  replicationEnabled: true,
  defaultConsistency: 'EVENTUAL',
});

await node1.ready();

// Node 2 (joins via seed)
const node2 = new ServerCoordinator({
  port: 8081,
  nodeId: 'node-2',
  host: 'localhost',
  clusterPort: 9001,
  peers: ['localhost:9000'],  // Connect to node-1
  jwtSecret: process.env.JWT_SECRET,
  replicationEnabled: true,
  defaultConsistency: 'EVENTUAL',
});

await node2.ready();

// Cluster membership uses gossip protocol
// Nodes automatically discover each other

const server = new ServerCoordinator({
  clusterPort: 9000,
  peers: ['node1:9000'],  // Just one seed is enough
});

// When node2 joins:
// 1. Connects to seed node (node1)
// 2. Sends HELLO message with its info
// 3. Receives MEMBER_LIST with all known members
// 4. Connects to discovered members
// 5. Broadcasts updated member list

// Result: All nodes learn about all others automatically

// EVENTUAL - Fire-and-forget replication (default)
// Best for: High throughput, tolerance for brief inconsistency
const eventualServer = new ServerCoordinator({
  defaultConsistency: 'EVENTUAL',
  replicationConfig: {
    batchIntervalMs: 50,   // Batch replication every 50ms
    batchSize: 100,        // Max operations per batch
    queueSizeLimit: 10000, // Max queued operations
  },
});

// QUORUM - Wait for majority of replicas
// Best for: Balance of consistency and availability
const quorumServer = new ServerCoordinator({
  defaultConsistency: 'QUORUM',
  replicationConfig: {
    ackTimeoutMs: 5000,  // Wait up to 5s for acks
  },
});

// STRONG - Wait for all replicas
// Best for: Critical data requiring full consistency
const strongServer = new ServerCoordinator({
  defaultConsistency: 'STRONG',
  replicationConfig: {
    ackTimeoutMs: 10000, // Longer timeout for all replicas
  },
});

// TopGun uses consistent hashing with 271 partitions
// Each partition has an owner and backup nodes

// Data is automatically distributed:
// - Key is hashed to partition ID (0-270)
// - Partition owner stores the primary copy
// - Backup nodes store replica copies

// Example: With 3 nodes and BACKUP_COUNT=1
// - Node A owns ~90 partitions, backs up ~90 others
// - Node B owns ~90 partitions, backs up ~90 others
// - Node C owns ~90 partitions, backs up ~90 others

// Total: Each piece of data exists on 2 nodes (owner + 1 backup)

// Anti-entropy repair runs automatically
const server = new ServerCoordinator({
  replicationEnabled: true,
  // RepairScheduler configuration (defaults shown)
  repairConfig: {
    enabled: true,
    scanIntervalMs: 300000,    // Full scan every 5 minutes
    repairBatchSize: 1000,     // Keys per repair batch
    maxConcurrentRepairs: 2,   // Parallel repair streams
    throttleMs: 100,           // Delay between batches
    prioritizeRecent: true,    // Repair recent data first
  },
});

// Repair process:
// 1. MerkleTreeManager builds hash tree of partition data
// 2. RepairScheduler compares Merkle roots with backup nodes
// 3. Differences trigger targeted data sync
// 4. Uses efficient bucket-level comparison before key-level

// Automatic failover on node failure
// FailureDetector uses Phi Accrual algorithm

const server = new ServerCoordinator({
  // FailureDetector config (defaults shown)
  failureDetector: {
    heartbeatIntervalMs: 1000,
    suspicionTimeoutMs: 5000,
    confirmationTimeoutMs: 10000,
    phiThreshold: 8,
  },
});

// Failover process:
// 1. Heartbeats stop arriving from failed node
// 2. Phi value exceeds threshold -> node marked SUSPECT
// 3. After confirmationTimeout -> node marked FAILED
// 4. PartitionService rebalances:
//    - Backup nodes become new owners
//    - Remaining nodes become new backups
// 5. Clients receive updated partition map

// Read from replicas for lower latency
const server = new ServerCoordinator({
  replicationEnabled: true,
  // ReadReplicaHandler is automatically enabled
});

// Read routing options:
// - Primary: Always read from partition owner
// - Replica: Read from nearest replica (lower latency)
// - Any: Read from any node that has the data

// Clients can specify read preference per query

version: '3.8'

services:
  node1:
    image: topgun:latest
    ports:
      - "8080:8080"
      - "9000:9000"
    environment:
      TOPGUN_NODE_ID: node-1
      TOPGUN_PORT: 8080
      TOPGUN_CLUSTER_PORT: 9000
      TOPGUN_PEERS: ""
      TOPGUN_REPLICATION: "true"
      TOPGUN_CONSISTENCY: EVENTUAL

  node2:
    image: topgun:latest
    ports:
      - "8081:8080"
      - "9001:9000"
    environment:
      TOPGUN_NODE_ID: node-2
      TOPGUN_PORT: 8080
      TOPGUN_CLUSTER_PORT: 9000
      TOPGUN_PEERS: node1:9000
      TOPGUN_REPLICATION: "true"
      TOPGUN_CONSISTENCY: EVENTUAL
    depends_on:
      - node1

  node3:
    image: topgun:latest
    ports:
      - "8082:8080"
      - "9002:9000"
    environment:
      TOPGUN_NODE_ID: node-3
      TOPGUN_PORT: 8080
      TOPGUN_CLUSTER_PORT: 9000
      TOPGUN_PEERS: node1:9000
      TOPGUN_REPLICATION: "true"
      TOPGUN_CONSISTENCY: EVENTUAL
    depends_on:
      - node1

docker-compose up -d

// Replication metrics available via Prometheus
// GET /metrics on metricsPort

# Replication queue size per node
topgun_replication_queue_size{node="node-2"} 15

# Pending synchronous acknowledgments
topgun_replication_pending_acks 3

# Replication lag in milliseconds
topgun_replication_lag_ms{node="node-2"} 42

# Health status
topgun_replication_healthy 1
topgun_replication_unhealthy_nodes 0