Add background IP intel and restore dashboard stats

2026-03-12 11:10:59 +01:00
parent 14a711038b
commit 1822e2148a
9 changed files with 506 additions and 31 deletions
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -56,11 +56,15 @@ type StorageConfig struct {
 }

 type InvestigationConfig struct {
-	Enabled         bool     `yaml:"enabled"`
-	RefreshAfter    Duration `yaml:"refresh_after"`
-	Timeout         Duration `yaml:"timeout"`
-	UserAgent       string   `yaml:"user_agent"`
-	SpamhausEnabled bool     `yaml:"spamhaus_enabled"`
+	Enabled                bool     `yaml:"enabled"`
+	RefreshAfter           Duration `yaml:"refresh_after"`
+	Timeout                Duration `yaml:"timeout"`
+	UserAgent              string   `yaml:"user_agent"`
+	SpamhausEnabled        bool     `yaml:"spamhaus_enabled"`
+	BackgroundWorkers      int      `yaml:"background_workers"`
+	BackgroundPollInterval Duration `yaml:"background_poll_interval"`
+	BackgroundLookback     Duration `yaml:"background_lookback"`
+	BackgroundBatchSize    int      `yaml:"background_batch_size"`
 }

 type OPNsenseConfig struct {
@@ -181,6 +185,18 @@ func (c *Config) applyDefaults() error {
 	if !c.Investigation.SpamhausEnabled {
 		c.Investigation.SpamhausEnabled = true
 	}
+	if c.Investigation.BackgroundWorkers == 0 {
+		c.Investigation.BackgroundWorkers = 2
+	}
+	if c.Investigation.BackgroundPollInterval.Duration == 0 {
+		c.Investigation.BackgroundPollInterval.Duration = 30 * time.Second
+	}
+	if c.Investigation.BackgroundLookback.Duration == 0 {
+		c.Investigation.BackgroundLookback.Duration = 24 * time.Hour
+	}
+	if c.Investigation.BackgroundBatchSize == 0 {
+		c.Investigation.BackgroundBatchSize = 256
+	}

 	if c.OPNsense.Timeout.Duration == 0 {
 		c.OPNsense.Timeout.Duration = 8 * time.Second
--- a/internal/model/types.go
+++ b/internal/model/types.go
@@ -174,6 +174,7 @@ type RecentIPRow struct {
 	LastSeenAt     time.Time          `json:"last_seen_at"`
 	Reason         string             `json:"reason"`
 	ManualOverride ManualOverride     `json:"manual_override"`
+	Bot            *BotMatch          `json:"bot,omitempty"`
 	Actions        ActionAvailability `json:"actions"`
 }

--- a/internal/service/service.go
+++ b/internal/service/service.go
@@ -29,6 +29,10 @@ type Service struct {
 	blocker      opnsense.AliasClient
 	investigator Investigator
 	logger       *log.Logger
+
+	investigationQueueMu sync.Mutex
+	investigationQueued  map[string]struct{}
+	investigationQueue   chan string
 }

 type Investigator interface {
@@ -39,7 +43,7 @@ func New(cfg *config.Config, db *store.Store, blocker opnsense.AliasClient, inve
 	if logger == nil {
 		logger = log.New(io.Discard, "", 0)
 	}
-	return &Service{
+	service := &Service{
 		cfg:          cfg,
 		store:        db,
 		evaluator:    engine.NewEvaluator(),
@@ -47,10 +51,33 @@ func New(cfg *config.Config, db *store.Store, blocker opnsense.AliasClient, inve
 		investigator: investigator,
 		logger:       logger,
 	}
+	if investigator != nil && cfg.Investigation.BackgroundWorkers > 0 {
+		queueSize := cfg.Investigation.BackgroundBatchSize
+		if queueSize < 64 {
+			queueSize = 64
+		}
+		service.investigationQueue = make(chan string, queueSize)
+		service.investigationQueued = make(map[string]struct{}, queueSize)
+	}
+	return service
 }

 func (s *Service) Run(ctx context.Context) error {
 	var wg sync.WaitGroup
+	if s.investigationQueue != nil {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			s.runInvestigationScheduler(ctx)
+		}()
+		for workerIndex := 0; workerIndex < s.cfg.Investigation.BackgroundWorkers; workerIndex++ {
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				s.runInvestigationWorker(ctx)
+			}()
+		}
+	}
 	for _, source := range s.cfg.Sources {
 		source := source
 		wg.Add(1)
@@ -81,12 +108,25 @@ func (s *Service) ListRecentIPs(ctx context.Context, since time.Time, limit int)
 	if err != nil {
 		return nil, err
 	}
+	investigations, err := s.store.GetInvestigationsForIPs(ctx, recentRowIPs(items))
+	if err != nil {
+		return nil, err
+	}
+	staleSince := time.Now().UTC().Add(-s.cfg.Investigation.RefreshAfter.Duration)
 	for index := range items {
 		state := model.IPState{
 			IP:             items[index].IP,
 			State:          items[index].State,
 			ManualOverride: items[index].ManualOverride,
 		}
+		if investigation, ok := investigations[items[index].IP]; ok {
+			items[index].Bot = investigation.Bot
+			if investigation.UpdatedAt.Before(staleSince) {
+				s.enqueueInvestigation(items[index].IP)
+			}
+		} else {
+			s.enqueueInvestigation(items[index].IP)
+		}
 		backend := s.resolveOPNsenseStatus(ctx, state)
 		items[index].Actions = actionAvailability(state, backend)
 	}
@@ -114,22 +154,12 @@ func (s *Service) InvestigateIP(ctx context.Context, ip string) (model.IPDetails
 	if err != nil {
 		return model.IPDetails{}, err
 	}
-	if s.investigator != nil {
-		investigation, found, err := s.store.GetInvestigation(ctx, normalized)
-		if err != nil {
-			return model.IPDetails{}, err
-		}
-		shouldRefresh := !found || time.Since(investigation.UpdatedAt) >= s.cfg.Investigation.RefreshAfter.Duration
-		if shouldRefresh {
-			fresh, err := s.investigator.Investigate(ctx, normalized, collectUserAgents(details.RecentEvents))
-			if err != nil {
-				return model.IPDetails{}, err
-			}
-			if err := s.store.SaveInvestigation(ctx, fresh); err != nil {
-				return model.IPDetails{}, err
-			}
-			details.Investigation = &fresh
-		}
+	fresh, err := s.refreshInvestigation(ctx, normalized, true)
+	if err != nil {
+		return model.IPDetails{}, err
+	}
+	if fresh != nil {
+		details.Investigation = fresh
 	}
 	return s.decorateDetails(ctx, details)
 }
@@ -281,9 +311,125 @@ func (s *Service) processRecord(ctx context.Context, source config.SourceConfig,
 			return err
 		}
 	}
+	s.enqueueInvestigation(record.ClientIP)
 	return nil
 }

+func (s *Service) runInvestigationScheduler(ctx context.Context) {
+	s.enqueueRecentInvestigations(ctx)
+	ticker := time.NewTicker(s.cfg.Investigation.BackgroundPollInterval.Duration)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			s.enqueueRecentInvestigations(ctx)
+		}
+	}
+}
+
+func (s *Service) runInvestigationWorker(ctx context.Context) {
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case ip := <-s.investigationQueue:
+			func() {
+				defer s.markInvestigationDone(ip)
+				workerCtx, cancel := context.WithTimeout(ctx, s.cfg.Investigation.Timeout.Duration)
+				_, err := s.refreshInvestigation(workerCtx, ip, false)
+				cancel()
+				if err != nil && !errors.Is(err, context.Canceled) {
+					s.logger.Printf("investigation %s: %v", ip, err)
+				}
+			}()
+		}
+	}
+}
+
+func (s *Service) enqueueRecentInvestigations(ctx context.Context) {
+	if s.investigationQueue == nil {
+		return
+	}
+	since := time.Now().UTC().Add(-s.cfg.Investigation.BackgroundLookback.Duration)
+	items, err := s.store.ListRecentIPRows(ctx, since, s.cfg.Investigation.BackgroundBatchSize)
+	if err != nil {
+		s.logger.Printf("list recent IPs for investigation: %v", err)
+		return
+	}
+	investigations, err := s.store.GetInvestigationsForIPs(ctx, recentRowIPs(items))
+	if err != nil {
+		s.logger.Printf("list investigations for recent IPs: %v", err)
+		return
+	}
+	staleSince := time.Now().UTC().Add(-s.cfg.Investigation.RefreshAfter.Duration)
+	for _, item := range items {
+		investigation, found := investigations[item.IP]
+		if !found || investigation.UpdatedAt.Before(staleSince) {
+			s.enqueueInvestigation(item.IP)
+		}
+	}
+}
+
+func (s *Service) enqueueInvestigation(ip string) {
+	if s.investigationQueue == nil {
+		return
+	}
+	normalized, err := normalizeIP(ip)
+	if err != nil {
+		return
+	}
+	s.investigationQueueMu.Lock()
+	if _, ok := s.investigationQueued[normalized]; ok {
+		s.investigationQueueMu.Unlock()
+		return
+	}
+	s.investigationQueued[normalized] = struct{}{}
+	s.investigationQueueMu.Unlock()
+	select {
+	case s.investigationQueue <- normalized:
+	default:
+		s.markInvestigationDone(normalized)
+	}
+}
+
+func (s *Service) markInvestigationDone(ip string) {
+	s.investigationQueueMu.Lock()
+	defer s.investigationQueueMu.Unlock()
+	delete(s.investigationQueued, ip)
+}
+
+func (s *Service) refreshInvestigation(ctx context.Context, ip string, force bool) (*model.IPInvestigation, error) {
+	if s.investigator == nil {
+		return nil, nil
+	}
+	normalized, err := normalizeIP(ip)
+	if err != nil {
+		return nil, err
+	}
+	investigation, found, err := s.store.GetInvestigation(ctx, normalized)
+	if err != nil {
+		return nil, err
+	}
+	shouldRefresh := force || !found || time.Since(investigation.UpdatedAt) >= s.cfg.Investigation.RefreshAfter.Duration
+	if !shouldRefresh {
+		return &investigation, nil
+	}
+	userAgents, err := s.store.ListRecentUserAgentsForIP(ctx, normalized, 12)
+	if err != nil {
+		return nil, err
+	}
+	fresh, err := s.investigator.Investigate(ctx, normalized, userAgents)
+	if err != nil {
+		return nil, err
+	}
+	if err := s.store.SaveInvestigation(ctx, fresh); err != nil {
+		return nil, err
+	}
+	return &fresh, nil
+}
+
 func (s *Service) readNewLines(ctx context.Context, source config.SourceConfig) ([]string, error) {
 	info, err := os.Stat(source.Path)
 	if err != nil {
@@ -521,3 +667,11 @@ func collectUserAgents(events []model.Event) []string {
 	}
 	return items
 }
+
+func recentRowIPs(items []model.RecentIPRow) []string {
+	result := make([]string, 0, len(items))
+	for _, item := range items {
+		result = append(result, item.IP)
+	}
+	return result
+}
--- a/internal/service/service_test.go
+++ b/internal/service/service_test.go
@@ -159,6 +159,87 @@ sources:
 	}
 }

+func TestServiceBackgroundInvestigationEnrichesRecentIPs(t *testing.T) {
+	t.Parallel()
+
+	tempDir := t.TempDir()
+	logPath := filepath.Join(tempDir, "access.log")
+	if err := os.WriteFile(logPath, nil, 0o600); err != nil {
+		t.Fatalf("create log: %v", err)
+	}
+
+	configPath := filepath.Join(tempDir, "config.yaml")
+	payload := fmt.Sprintf(`storage:
+  path: %s/blocker.db
+investigation:
+  enabled: true
+  refresh_after: 24h
+  timeout: 500ms
+  background_workers: 1
+  background_poll_interval: 50ms
+  background_lookback: 24h
+  background_batch_size: 32
+profiles:
+  main:
+    auto_block: false
+    block_unexpected_posts: true
+    suspicious_path_prefixes:
+      - /wp-login.php
+sources:
+  - name: main
+    path: %s
+    profile: main
+    initial_position: beginning
+    poll_interval: 20ms
+    batch_size: 128
+`, tempDir, logPath)
+	if err := os.WriteFile(configPath, []byte(payload), 0o600); err != nil {
+		t.Fatalf("write config: %v", err)
+	}
+
+	cfg, err := config.Load(configPath)
+	if err != nil {
+		t.Fatalf("load config: %v", err)
+	}
+	database, err := store.Open(cfg.Storage.Path)
+	if err != nil {
+		t.Fatalf("open store: %v", err)
+	}
+	defer database.Close()
+	investigator := &fakeInvestigator{}
+	svc := New(cfg, database, nil, investigator, log.New(os.Stderr, "", 0))
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	go func() { _ = svc.Run(ctx) }()
+
+	appendLine(t, logPath, caddyJSONLine("203.0.113.33", "198.51.100.33", "example.test", "GET", "/wp-login.php", 404, "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", time.Now().UTC()))
+
+	waitFor(t, 3*time.Second, func() bool {
+		recentRows, err := svc.ListRecentIPs(context.Background(), time.Now().UTC().Add(-time.Hour), 10)
+		if err != nil {
+			return false
+		}
+		row, found := findRecentIPRow(recentRows, "203.0.113.33")
+		return found && row.Bot != nil && row.Bot.Name == "Googlebot"
+	})
+
+	recentRows, err := svc.ListRecentIPs(context.Background(), time.Now().UTC().Add(-time.Hour), 10)
+	if err != nil {
+		t.Fatalf("list recent ips: %v", err)
+	}
+	row, found := findRecentIPRow(recentRows, "203.0.113.33")
+	if !found {
+		t.Fatalf("expected recent row for investigated ip: %+v", recentRows)
+	}
+	if row.Bot == nil || row.Bot.Name != "Googlebot" {
+		t.Fatalf("expected background investigation bot on recent row, got %+v", row)
+	}
+	if investigator.callCount() == 0 {
+		t.Fatalf("expected background investigator to be called")
+	}
+}
+
 type fakeOPNsenseServer struct {
 	*httptest.Server
 	mu          sync.Mutex
@@ -269,3 +350,30 @@ func findRecentIPRow(items []model.RecentIPRow, ip string) (model.RecentIPRow, b
 	}
 	return model.RecentIPRow{}, false
 }
+
+type fakeInvestigator struct {
+	mu    sync.Mutex
+	count int
+}
+
+func (f *fakeInvestigator) Investigate(_ context.Context, ip string, _ []string) (model.IPInvestigation, error) {
+	f.mu.Lock()
+	f.count++
+	f.mu.Unlock()
+	return model.IPInvestigation{
+		IP:        ip,
+		UpdatedAt: time.Now().UTC(),
+		Bot: &model.BotMatch{
+			ProviderID: "google_official",
+			Name:       "Googlebot",
+			Method:     "test",
+			Verified:   true,
+		},
+	}, nil
+}
+
+func (f *fakeInvestigator) callCount() int {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.count
+}
--- a/internal/store/store.go
+++ b/internal/store/store.go
@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 	"time"

@@ -591,6 +592,51 @@ func (s *Store) GetInvestigation(ctx context.Context, ip string) (model.IPInvest
 	return item, true, nil
 }

+func (s *Store) GetInvestigationsForIPs(ctx context.Context, ips []string) (map[string]model.IPInvestigation, error) {
+	unique := uniqueNonEmptyStrings(ips)
+	if len(unique) == 0 {
+		return map[string]model.IPInvestigation{}, nil
+	}
+	placeholders := make([]string, len(unique))
+	args := make([]any, 0, len(unique))
+	for index, ip := range unique {
+		placeholders[index] = "?"
+		args = append(args, ip)
+	}
+	rows, err := s.db.QueryContext(ctx, fmt.Sprintf(
+		`SELECT ip, payload_json, updated_at FROM ip_investigations WHERE ip IN (%s)`,
+		strings.Join(placeholders, ", "),
+	), args...)
+	if err != nil {
+		return nil, fmt.Errorf("query ip investigations: %w", err)
+	}
+	defer rows.Close()
+
+	items := make(map[string]model.IPInvestigation, len(unique))
+	for rows.Next() {
+		var ip string
+		var payload string
+		var updatedAt string
+		if err := rows.Scan(&ip, &payload, &updatedAt); err != nil {
+			return nil, fmt.Errorf("scan ip investigation: %w", err)
+		}
+		var item model.IPInvestigation
+		if err := json.Unmarshal([]byte(payload), &item); err != nil {
+			return nil, fmt.Errorf("decode ip investigation %q: %w", ip, err)
+		}
+		parsed, err := parseTime(updatedAt)
+		if err != nil {
+			return nil, fmt.Errorf("parse ip investigation updated_at for %q: %w", ip, err)
+		}
+		item.UpdatedAt = parsed
+		items[ip] = item
+	}
+	if err := rows.Err(); err != nil {
+		return nil, fmt.Errorf("iterate ip investigations: %w", err)
+	}
+	return items, nil
+}
+
 func (s *Store) SaveInvestigation(ctx context.Context, item model.IPInvestigation) error {
 	if item.UpdatedAt.IsZero() {
 		item.UpdatedAt = time.Now().UTC()
@@ -616,6 +662,39 @@ func (s *Store) SaveInvestigation(ctx context.Context, item model.IPInvestigatio
 	return nil
 }

+func (s *Store) ListRecentUserAgentsForIP(ctx context.Context, ip string, limit int) ([]string, error) {
+	if limit <= 0 {
+		limit = 10
+	}
+	rows, err := s.db.QueryContext(ctx, `
+		SELECT user_agent
+		FROM events
+		WHERE client_ip = ? AND TRIM(user_agent) <> ''
+		GROUP BY user_agent
+		ORDER BY MAX(occurred_at) DESC, user_agent ASC
+		LIMIT ?`,
+		ip,
+		limit,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("list recent user agents for ip %q: %w", ip, err)
+	}
+	defer rows.Close()
+
+	items := make([]string, 0, limit)
+	for rows.Next() {
+		var userAgent string
+		if err := rows.Scan(&userAgent); err != nil {
+			return nil, fmt.Errorf("scan recent user agent for ip %q: %w", ip, err)
+		}
+		items = append(items, userAgent)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, fmt.Errorf("iterate recent user agents for ip %q: %w", ip, err)
+	}
+	return items, nil
+}
+
 func (s *Store) GetSourceOffset(ctx context.Context, sourceName string) (model.SourceOffset, bool, error) {
 	row := s.db.QueryRowContext(ctx, `SELECT source_name, path, inode, offset, updated_at FROM source_offsets WHERE source_name = ?`, sourceName)
 	var offset model.SourceOffset
@@ -1086,6 +1165,24 @@ func boolToInt(value bool) int {
 	return 0
 }

+func uniqueNonEmptyStrings(items []string) []string {
+	seen := make(map[string]struct{}, len(items))
+	result := make([]string, 0, len(items))
+	for _, item := range items {
+		trimmed := strings.TrimSpace(item)
+		if trimmed == "" {
+			continue
+		}
+		if _, ok := seen[trimmed]; ok {
+			continue
+		}
+		seen[trimmed] = struct{}{}
+		result = append(result, trimmed)
+	}
+	sort.Strings(result)
+	return result
+}
+
 type queryer interface {
 	QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row
 }
--- a/internal/store/store_test.go
+++ b/internal/store/store_test.go
@@ -139,4 +139,18 @@ func TestStoreRecordsEventsAndState(t *testing.T) {
 	if !found || loadedInvestigation.Bot == nil || loadedInvestigation.Bot.Name != "Googlebot" {
 		t.Fatalf("unexpected investigation payload: found=%v investigation=%+v", found, loadedInvestigation)
 	}
+	investigations, err := db.GetInvestigationsForIPs(ctx, []string{event.ClientIP, "198.51.100.99"})
+	if err != nil {
+		t.Fatalf("get investigations for ips: %v", err)
+	}
+	if len(investigations) != 1 || investigations[event.ClientIP].Bot == nil {
+		t.Fatalf("unexpected investigations map: %+v", investigations)
+	}
+	userAgents, err := db.ListRecentUserAgentsForIP(ctx, event.ClientIP, 10)
+	if err != nil {
+		t.Fatalf("list recent user agents: %v", err)
+	}
+	if len(userAgents) != 1 || userAgents[0] != event.UserAgent {
+		t.Fatalf("unexpected user agents: %#v", userAgents)
+	}
 }
--- a/internal/web/handler.go
+++ b/internal/web/handler.go
@@ -337,13 +337,16 @@ const overviewHTML = `<!doctype html>
    :root { color-scheme: dark; }
    body { font-family: system-ui, sans-serif; margin: 0; background: #0f172a; color: #e2e8f0; }
    header { padding: 1rem 1.5rem; border-bottom: 1px solid #334155; position: sticky; top: 0; background: rgba(15,23,42,.97); }
-    main { padding: 1.5rem; }
+    main { padding: 1.5rem; display: grid; gap: 1.25rem; }
    h1, h2 { margin: 0 0 .75rem 0; }
    table { width: 100%; border-collapse: collapse; }
    th, td { padding: .55rem .65rem; border-bottom: 1px solid #1e293b; text-align: left; vertical-align: top; }
    th { color: #93c5fd; white-space: nowrap; }
    a { color: #93c5fd; text-decoration: none; }
    a:hover { text-decoration: underline; }
+    .stats { display: grid; grid-template-columns: repeat(auto-fit, minmax(140px, 1fr)); gap: .75rem; }
+    .card { background: #111827; border: 1px solid #334155; border-radius: .75rem; padding: .9rem; }
+    .stat-value { font-size: 1.7rem; font-weight: 700; }
    .status { display: inline-block; padding: .15rem .45rem; border-radius: 999px; font-size: .8rem; background: #1e293b; }
    .status.blocked { background: #7f1d1d; }
    .status.review { background: #78350f; }
@@ -362,6 +365,22 @@ const overviewHTML = `<!doctype html>
    button { background: #2563eb; color: white; border: 0; cursor: pointer; }
    button.secondary { background: #475569; }
    button.danger { background: #dc2626; }
+    .ip-cell { display: flex; align-items: center; gap: .45rem; }
+    .bot-chip { display: inline-flex; align-items: center; justify-content: center; width: 1.25rem; height: 1.25rem; border-radius: 999px; border: 1px solid #334155; background: #0f172a; color: #e2e8f0; font-size: .72rem; font-weight: 700; cursor: help; }
+    .bot-chip.verified { border-color: #2563eb; }
+    .bot-chip.hint { border-style: dashed; }
+    .bot-chip.google { background: #2563eb; color: white; }
+    .bot-chip.bing { background: #0284c7; color: white; }
+    .bot-chip.apple { background: #475569; color: white; }
+    .bot-chip.meta { background: #2563eb; color: white; }
+    .bot-chip.duckduckgo { background: #ea580c; color: white; }
+    .bot-chip.openai { background: #059669; color: white; }
+    .bot-chip.anthropic { background: #b45309; color: white; }
+    .bot-chip.perplexity { background: #0f766e; color: white; }
+    .bot-chip.semrush { background: #db2777; color: white; }
+    .bot-chip.yandex { background: #dc2626; color: white; }
+    .bot-chip.baidu { background: #7c3aed; color: white; }
+    .bot-chip.bytespider { background: #111827; color: white; }
  </style>
 </head>
 <body>
@@ -370,6 +389,7 @@ const overviewHTML = `<!doctype html>
    <div class="muted">Local-only review and enforcement console</div>
  </header>
  <main>
+    <section class="stats" id="stats"></section>
    <section class="panel">
      <div class="toolbar">
        <h2>Recent IPs</h2>
@@ -405,6 +425,23 @@ const overviewHTML = `<!doctype html>
    let currentItems = [];
    let currentSort = { key: 'events', direction: 'desc' };

+    function renderStats(data) {
+      const stats = [
+        ['Total events', data.total_events],
+        ['Tracked IPs', data.total_ips],
+        ['Blocked', data.blocked_ips],
+        ['Review', data.review_ips],
+        ['Allowed', data.allowed_ips],
+        ['Observed', data.observed_ips],
+      ];
+      document.getElementById('stats').innerHTML = stats.map(([label, value]) => [
+        '<div class="card">',
+        '  <div class="muted">' + escapeHtml(label) + '</div>',
+        '  <div class="stat-value">' + escapeHtml(value) + '</div>',
+        '</div>'
+      ].join('')).join('');
+    }
+
    function escapeHtml(value) {
      return String(value ?? '').replace(/[&<>"']/g, character => ({'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;',"'":'&#39;'}[character]));
    }
@@ -433,6 +470,41 @@ const overviewHTML = `<!doctype html>
      return leftRank - rightRank;
    }

+    function botVisual(bot) {
+      const candidate = String((bot || {}).provider_id || (bot || {}).name || '').toLowerCase();
+      const catalog = [
+        { match: ['google'], short: 'G', className: 'google' },
+        { match: ['bing', 'microsoft'], short: 'B', className: 'bing' },
+        { match: ['apple'], short: 'A', className: 'apple' },
+        { match: ['facebook', 'meta'], short: 'M', className: 'meta' },
+        { match: ['duckduckgo', 'duckduckbot'], short: 'D', className: 'duckduckgo' },
+        { match: ['gptbot', 'openai'], short: 'O', className: 'openai' },
+        { match: ['claudebot', 'anthropic'], short: 'C', className: 'anthropic' },
+        { match: ['perplexity'], short: 'P', className: 'perplexity' },
+        { match: ['semrush'], short: 'S', className: 'semrush' },
+        { match: ['yandex'], short: 'Y', className: 'yandex' },
+        { match: ['baidu'], short: 'B', className: 'baidu' },
+        { match: ['bytespider', 'tiktok'], short: 'T', className: 'bytespider' },
+      ];
+      for (const entry of catalog) {
+        if (entry.match.some(fragment => candidate.includes(fragment))) {
+          return entry;
+        }
+      }
+      const name = String((bot || {}).name || '').trim();
+      return { short: (name[0] || '?').toUpperCase(), className: 'generic' };
+    }
+
+    function renderBotChip(bot) {
+      if (!bot) {
+        return '';
+      }
+      const visual = botVisual(bot);
+      const statusClass = bot.verified ? 'verified' : 'hint';
+      const title = (bot.name || 'Bot') + (bot.verified ? '' : ' (possible)');
+      return '<span class="bot-chip ' + escapeHtml(visual.className) + ' ' + statusClass + '" title="' + escapeHtml(title) + '">' + escapeHtml(visual.short) + '</span>';
+    }
+
    function updateSortButtons() {
      document.querySelectorAll('button[data-sort]').forEach(button => {
        const key = button.dataset.sort;
@@ -488,7 +560,7 @@ const overviewHTML = `<!doctype html>
    function renderIPs(items) {
      const rows = sortItems(items).map(item => [
        '<tr>',
-        '  <td class="mono"><a href="/ips/' + encodeURIComponent(item.ip) + '">' + escapeHtml(item.ip) + '</a></td>',
+        '  <td class="mono"><div class="ip-cell">' + renderBotChip(item.bot) + '<a href="/ips/' + encodeURIComponent(item.ip) + '">' + escapeHtml(item.ip) + '</a></div></td>',
        '  <td>' + escapeHtml(item.source_name || '—') + '</td>',
        '  <td><span class="status ' + escapeHtml(item.state) + '">' + escapeHtml(item.state) + '</span></td>',
        '  <td>' + escapeHtml(item.events) + '</td>',
@@ -534,14 +606,21 @@ const overviewHTML = `<!doctype html>
    }

    async function refresh() {
-      const response = await fetch('/api/recent-ips?hours=' + recentHours + '&limit=250');
-      const payload = await response.json().catch(() => []);
-      if (!response.ok) {
-        const message = Array.isArray(payload) ? response.statusText : (payload.error || response.statusText);
+      const [overviewResponse, recentResponse] = await Promise.all([
+        fetch('/api/overview?limit=50'),
+        fetch('/api/recent-ips?hours=' + recentHours + '&limit=250')
+      ]);
+      const overviewPayload = await overviewResponse.json().catch(() => ({}));
+      const recentPayload = await recentResponse.json().catch(() => []);
+      if (overviewResponse.ok) {
+        renderStats(overviewPayload || {});
+      }
+      if (!recentResponse.ok) {
+        const message = Array.isArray(recentPayload) ? recentResponse.statusText : (recentPayload.error || recentResponse.statusText);
        document.getElementById('ips-body').innerHTML = '<tr><td colspan="7" class="muted">' + escapeHtml(message) + '</td></tr>';
        return;
      }
-      currentItems = Array.isArray(payload) ? payload : [];
+      currentItems = Array.isArray(recentPayload) ? recentPayload : [];
      render();
    }