From 1822e2148adc1effb6a0494479703d776625bf18 Mon Sep 17 00:00:00 2001 From: "Codex, agent ChatGPT" Date: Thu, 12 Mar 2026 11:10:59 +0100 Subject: [PATCH] Add background IP intel and restore dashboard stats --- README.md | 6 +- config.example.yaml | 4 + internal/config/config.go | 26 ++++- internal/model/types.go | 1 + internal/service/service.go | 188 ++++++++++++++++++++++++++++--- internal/service/service_test.go | 108 ++++++++++++++++++ internal/store/store.go | 97 ++++++++++++++++ internal/store/store_test.go | 14 +++ internal/web/handler.go | 93 +++++++++++++-- 9 files changed, 506 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index e8903e5..344e5b9 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,9 @@ - Real-time ingestion of multiple Caddy JSON log files. - One heuristic profile per log source. - Persistent local state in SQLite. -- Local-only web UI with a sortable “Recent IPs” view for the last 24 hours and a full request history for each selected address. +- Local-only web UI with summary cards, a sortable “Recent IPs” view for the last 24 hours, bot badges, and a full request history for each selected address. - On-demand IP investigation with persistent caching for bot verification, reverse DNS, RDAP, and Spamhaus lookups. +- Background IP investigation workers so cached intelligence appears without blocking page loads. - Manual block, unblock, and clear-override actions with OPNsense-aware UI state. - OPNsense alias backend with automatic alias creation. - Concurrent polling across multiple log files. @@ -31,7 +32,7 @@ This keeps the application usable immediately while leaving room for a more adva - `internal/caddylog`: parses default Caddy JSON access logs - `internal/engine`: evaluates requests against a profile -- `internal/investigation`: performs on-demand bot verification and IP enrichment +- `internal/investigation`: performs bot verification and IP enrichment - `internal/store`: persists events, IP state, manual decisions, backend actions, and source offsets - `internal/opnsense`: manages the target OPNsense alias through its API - `internal/service`: runs concurrent log followers and applies automatic decisions @@ -63,6 +64,7 @@ Important points: - Each source references exactly one profile. - `initial_position: end` means “start following new lines only” on first boot. - The `investigation` section controls how long IP enrichment is cached and whether on-demand Spamhaus lookups are enabled. +- The investigation worker can refresh recent IP intelligence in the background so the dashboard stays fast while bot badges and cached intel keep filling in. - The web UI should stay bound to a local address such as `127.0.0.1:9080`. ## Web UI and API diff --git a/config.example.yaml b/config.example.yaml index 41deaab..e5357a1 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -13,6 +13,10 @@ investigation: timeout: 8s user_agent: caddy-opnsense-blocker/0.2 spamhaus_enabled: true + background_workers: 2 + background_poll_interval: 30s + background_lookback: 24h + background_batch_size: 256 opnsense: enabled: true diff --git a/internal/config/config.go b/internal/config/config.go index f7c0c72..a120d9c 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -56,11 +56,15 @@ type StorageConfig struct { } type InvestigationConfig struct { - Enabled bool `yaml:"enabled"` - RefreshAfter Duration `yaml:"refresh_after"` - Timeout Duration `yaml:"timeout"` - UserAgent string `yaml:"user_agent"` - SpamhausEnabled bool `yaml:"spamhaus_enabled"` + Enabled bool `yaml:"enabled"` + RefreshAfter Duration `yaml:"refresh_after"` + Timeout Duration `yaml:"timeout"` + UserAgent string `yaml:"user_agent"` + SpamhausEnabled bool `yaml:"spamhaus_enabled"` + BackgroundWorkers int `yaml:"background_workers"` + BackgroundPollInterval Duration `yaml:"background_poll_interval"` + BackgroundLookback Duration `yaml:"background_lookback"` + BackgroundBatchSize int `yaml:"background_batch_size"` } type OPNsenseConfig struct { @@ -181,6 +185,18 @@ func (c *Config) applyDefaults() error { if !c.Investigation.SpamhausEnabled { c.Investigation.SpamhausEnabled = true } + if c.Investigation.BackgroundWorkers == 0 { + c.Investigation.BackgroundWorkers = 2 + } + if c.Investigation.BackgroundPollInterval.Duration == 0 { + c.Investigation.BackgroundPollInterval.Duration = 30 * time.Second + } + if c.Investigation.BackgroundLookback.Duration == 0 { + c.Investigation.BackgroundLookback.Duration = 24 * time.Hour + } + if c.Investigation.BackgroundBatchSize == 0 { + c.Investigation.BackgroundBatchSize = 256 + } if c.OPNsense.Timeout.Duration == 0 { c.OPNsense.Timeout.Duration = 8 * time.Second diff --git a/internal/model/types.go b/internal/model/types.go index df39a1c..0ea0b3f 100644 --- a/internal/model/types.go +++ b/internal/model/types.go @@ -174,6 +174,7 @@ type RecentIPRow struct { LastSeenAt time.Time `json:"last_seen_at"` Reason string `json:"reason"` ManualOverride ManualOverride `json:"manual_override"` + Bot *BotMatch `json:"bot,omitempty"` Actions ActionAvailability `json:"actions"` } diff --git a/internal/service/service.go b/internal/service/service.go index a855825..2c67b04 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -29,6 +29,10 @@ type Service struct { blocker opnsense.AliasClient investigator Investigator logger *log.Logger + + investigationQueueMu sync.Mutex + investigationQueued map[string]struct{} + investigationQueue chan string } type Investigator interface { @@ -39,7 +43,7 @@ func New(cfg *config.Config, db *store.Store, blocker opnsense.AliasClient, inve if logger == nil { logger = log.New(io.Discard, "", 0) } - return &Service{ + service := &Service{ cfg: cfg, store: db, evaluator: engine.NewEvaluator(), @@ -47,10 +51,33 @@ func New(cfg *config.Config, db *store.Store, blocker opnsense.AliasClient, inve investigator: investigator, logger: logger, } + if investigator != nil && cfg.Investigation.BackgroundWorkers > 0 { + queueSize := cfg.Investigation.BackgroundBatchSize + if queueSize < 64 { + queueSize = 64 + } + service.investigationQueue = make(chan string, queueSize) + service.investigationQueued = make(map[string]struct{}, queueSize) + } + return service } func (s *Service) Run(ctx context.Context) error { var wg sync.WaitGroup + if s.investigationQueue != nil { + wg.Add(1) + go func() { + defer wg.Done() + s.runInvestigationScheduler(ctx) + }() + for workerIndex := 0; workerIndex < s.cfg.Investigation.BackgroundWorkers; workerIndex++ { + wg.Add(1) + go func() { + defer wg.Done() + s.runInvestigationWorker(ctx) + }() + } + } for _, source := range s.cfg.Sources { source := source wg.Add(1) @@ -81,12 +108,25 @@ func (s *Service) ListRecentIPs(ctx context.Context, since time.Time, limit int) if err != nil { return nil, err } + investigations, err := s.store.GetInvestigationsForIPs(ctx, recentRowIPs(items)) + if err != nil { + return nil, err + } + staleSince := time.Now().UTC().Add(-s.cfg.Investigation.RefreshAfter.Duration) for index := range items { state := model.IPState{ IP: items[index].IP, State: items[index].State, ManualOverride: items[index].ManualOverride, } + if investigation, ok := investigations[items[index].IP]; ok { + items[index].Bot = investigation.Bot + if investigation.UpdatedAt.Before(staleSince) { + s.enqueueInvestigation(items[index].IP) + } + } else { + s.enqueueInvestigation(items[index].IP) + } backend := s.resolveOPNsenseStatus(ctx, state) items[index].Actions = actionAvailability(state, backend) } @@ -114,22 +154,12 @@ func (s *Service) InvestigateIP(ctx context.Context, ip string) (model.IPDetails if err != nil { return model.IPDetails{}, err } - if s.investigator != nil { - investigation, found, err := s.store.GetInvestigation(ctx, normalized) - if err != nil { - return model.IPDetails{}, err - } - shouldRefresh := !found || time.Since(investigation.UpdatedAt) >= s.cfg.Investigation.RefreshAfter.Duration - if shouldRefresh { - fresh, err := s.investigator.Investigate(ctx, normalized, collectUserAgents(details.RecentEvents)) - if err != nil { - return model.IPDetails{}, err - } - if err := s.store.SaveInvestigation(ctx, fresh); err != nil { - return model.IPDetails{}, err - } - details.Investigation = &fresh - } + fresh, err := s.refreshInvestigation(ctx, normalized, true) + if err != nil { + return model.IPDetails{}, err + } + if fresh != nil { + details.Investigation = fresh } return s.decorateDetails(ctx, details) } @@ -281,9 +311,125 @@ func (s *Service) processRecord(ctx context.Context, source config.SourceConfig, return err } } + s.enqueueInvestigation(record.ClientIP) return nil } +func (s *Service) runInvestigationScheduler(ctx context.Context) { + s.enqueueRecentInvestigations(ctx) + ticker := time.NewTicker(s.cfg.Investigation.BackgroundPollInterval.Duration) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + s.enqueueRecentInvestigations(ctx) + } + } +} + +func (s *Service) runInvestigationWorker(ctx context.Context) { + for { + select { + case <-ctx.Done(): + return + case ip := <-s.investigationQueue: + func() { + defer s.markInvestigationDone(ip) + workerCtx, cancel := context.WithTimeout(ctx, s.cfg.Investigation.Timeout.Duration) + _, err := s.refreshInvestigation(workerCtx, ip, false) + cancel() + if err != nil && !errors.Is(err, context.Canceled) { + s.logger.Printf("investigation %s: %v", ip, err) + } + }() + } + } +} + +func (s *Service) enqueueRecentInvestigations(ctx context.Context) { + if s.investigationQueue == nil { + return + } + since := time.Now().UTC().Add(-s.cfg.Investigation.BackgroundLookback.Duration) + items, err := s.store.ListRecentIPRows(ctx, since, s.cfg.Investigation.BackgroundBatchSize) + if err != nil { + s.logger.Printf("list recent IPs for investigation: %v", err) + return + } + investigations, err := s.store.GetInvestigationsForIPs(ctx, recentRowIPs(items)) + if err != nil { + s.logger.Printf("list investigations for recent IPs: %v", err) + return + } + staleSince := time.Now().UTC().Add(-s.cfg.Investigation.RefreshAfter.Duration) + for _, item := range items { + investigation, found := investigations[item.IP] + if !found || investigation.UpdatedAt.Before(staleSince) { + s.enqueueInvestigation(item.IP) + } + } +} + +func (s *Service) enqueueInvestigation(ip string) { + if s.investigationQueue == nil { + return + } + normalized, err := normalizeIP(ip) + if err != nil { + return + } + s.investigationQueueMu.Lock() + if _, ok := s.investigationQueued[normalized]; ok { + s.investigationQueueMu.Unlock() + return + } + s.investigationQueued[normalized] = struct{}{} + s.investigationQueueMu.Unlock() + select { + case s.investigationQueue <- normalized: + default: + s.markInvestigationDone(normalized) + } +} + +func (s *Service) markInvestigationDone(ip string) { + s.investigationQueueMu.Lock() + defer s.investigationQueueMu.Unlock() + delete(s.investigationQueued, ip) +} + +func (s *Service) refreshInvestigation(ctx context.Context, ip string, force bool) (*model.IPInvestigation, error) { + if s.investigator == nil { + return nil, nil + } + normalized, err := normalizeIP(ip) + if err != nil { + return nil, err + } + investigation, found, err := s.store.GetInvestigation(ctx, normalized) + if err != nil { + return nil, err + } + shouldRefresh := force || !found || time.Since(investigation.UpdatedAt) >= s.cfg.Investigation.RefreshAfter.Duration + if !shouldRefresh { + return &investigation, nil + } + userAgents, err := s.store.ListRecentUserAgentsForIP(ctx, normalized, 12) + if err != nil { + return nil, err + } + fresh, err := s.investigator.Investigate(ctx, normalized, userAgents) + if err != nil { + return nil, err + } + if err := s.store.SaveInvestigation(ctx, fresh); err != nil { + return nil, err + } + return &fresh, nil +} + func (s *Service) readNewLines(ctx context.Context, source config.SourceConfig) ([]string, error) { info, err := os.Stat(source.Path) if err != nil { @@ -521,3 +667,11 @@ func collectUserAgents(events []model.Event) []string { } return items } + +func recentRowIPs(items []model.RecentIPRow) []string { + result := make([]string, 0, len(items)) + for _, item := range items { + result = append(result, item.IP) + } + return result +} diff --git a/internal/service/service_test.go b/internal/service/service_test.go index 150c5cc..5e689c6 100644 --- a/internal/service/service_test.go +++ b/internal/service/service_test.go @@ -159,6 +159,87 @@ sources: } } +func TestServiceBackgroundInvestigationEnrichesRecentIPs(t *testing.T) { + t.Parallel() + + tempDir := t.TempDir() + logPath := filepath.Join(tempDir, "access.log") + if err := os.WriteFile(logPath, nil, 0o600); err != nil { + t.Fatalf("create log: %v", err) + } + + configPath := filepath.Join(tempDir, "config.yaml") + payload := fmt.Sprintf(`storage: + path: %s/blocker.db +investigation: + enabled: true + refresh_after: 24h + timeout: 500ms + background_workers: 1 + background_poll_interval: 50ms + background_lookback: 24h + background_batch_size: 32 +profiles: + main: + auto_block: false + block_unexpected_posts: true + suspicious_path_prefixes: + - /wp-login.php +sources: + - name: main + path: %s + profile: main + initial_position: beginning + poll_interval: 20ms + batch_size: 128 +`, tempDir, logPath) + if err := os.WriteFile(configPath, []byte(payload), 0o600); err != nil { + t.Fatalf("write config: %v", err) + } + + cfg, err := config.Load(configPath) + if err != nil { + t.Fatalf("load config: %v", err) + } + database, err := store.Open(cfg.Storage.Path) + if err != nil { + t.Fatalf("open store: %v", err) + } + defer database.Close() + investigator := &fakeInvestigator{} + svc := New(cfg, database, nil, investigator, log.New(os.Stderr, "", 0)) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go func() { _ = svc.Run(ctx) }() + + appendLine(t, logPath, caddyJSONLine("203.0.113.33", "198.51.100.33", "example.test", "GET", "/wp-login.php", 404, "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", time.Now().UTC())) + + waitFor(t, 3*time.Second, func() bool { + recentRows, err := svc.ListRecentIPs(context.Background(), time.Now().UTC().Add(-time.Hour), 10) + if err != nil { + return false + } + row, found := findRecentIPRow(recentRows, "203.0.113.33") + return found && row.Bot != nil && row.Bot.Name == "Googlebot" + }) + + recentRows, err := svc.ListRecentIPs(context.Background(), time.Now().UTC().Add(-time.Hour), 10) + if err != nil { + t.Fatalf("list recent ips: %v", err) + } + row, found := findRecentIPRow(recentRows, "203.0.113.33") + if !found { + t.Fatalf("expected recent row for investigated ip: %+v", recentRows) + } + if row.Bot == nil || row.Bot.Name != "Googlebot" { + t.Fatalf("expected background investigation bot on recent row, got %+v", row) + } + if investigator.callCount() == 0 { + t.Fatalf("expected background investigator to be called") + } +} + type fakeOPNsenseServer struct { *httptest.Server mu sync.Mutex @@ -269,3 +350,30 @@ func findRecentIPRow(items []model.RecentIPRow, ip string) (model.RecentIPRow, b } return model.RecentIPRow{}, false } + +type fakeInvestigator struct { + mu sync.Mutex + count int +} + +func (f *fakeInvestigator) Investigate(_ context.Context, ip string, _ []string) (model.IPInvestigation, error) { + f.mu.Lock() + f.count++ + f.mu.Unlock() + return model.IPInvestigation{ + IP: ip, + UpdatedAt: time.Now().UTC(), + Bot: &model.BotMatch{ + ProviderID: "google_official", + Name: "Googlebot", + Method: "test", + Verified: true, + }, + }, nil +} + +func (f *fakeInvestigator) callCount() int { + f.mu.Lock() + defer f.mu.Unlock() + return f.count +} diff --git a/internal/store/store.go b/internal/store/store.go index 89c2dae..46e2777 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -8,6 +8,7 @@ import ( "fmt" "os" "path/filepath" + "sort" "strings" "time" @@ -591,6 +592,51 @@ func (s *Store) GetInvestigation(ctx context.Context, ip string) (model.IPInvest return item, true, nil } +func (s *Store) GetInvestigationsForIPs(ctx context.Context, ips []string) (map[string]model.IPInvestigation, error) { + unique := uniqueNonEmptyStrings(ips) + if len(unique) == 0 { + return map[string]model.IPInvestigation{}, nil + } + placeholders := make([]string, len(unique)) + args := make([]any, 0, len(unique)) + for index, ip := range unique { + placeholders[index] = "?" + args = append(args, ip) + } + rows, err := s.db.QueryContext(ctx, fmt.Sprintf( + `SELECT ip, payload_json, updated_at FROM ip_investigations WHERE ip IN (%s)`, + strings.Join(placeholders, ", "), + ), args...) + if err != nil { + return nil, fmt.Errorf("query ip investigations: %w", err) + } + defer rows.Close() + + items := make(map[string]model.IPInvestigation, len(unique)) + for rows.Next() { + var ip string + var payload string + var updatedAt string + if err := rows.Scan(&ip, &payload, &updatedAt); err != nil { + return nil, fmt.Errorf("scan ip investigation: %w", err) + } + var item model.IPInvestigation + if err := json.Unmarshal([]byte(payload), &item); err != nil { + return nil, fmt.Errorf("decode ip investigation %q: %w", ip, err) + } + parsed, err := parseTime(updatedAt) + if err != nil { + return nil, fmt.Errorf("parse ip investigation updated_at for %q: %w", ip, err) + } + item.UpdatedAt = parsed + items[ip] = item + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate ip investigations: %w", err) + } + return items, nil +} + func (s *Store) SaveInvestigation(ctx context.Context, item model.IPInvestigation) error { if item.UpdatedAt.IsZero() { item.UpdatedAt = time.Now().UTC() @@ -616,6 +662,39 @@ func (s *Store) SaveInvestigation(ctx context.Context, item model.IPInvestigatio return nil } +func (s *Store) ListRecentUserAgentsForIP(ctx context.Context, ip string, limit int) ([]string, error) { + if limit <= 0 { + limit = 10 + } + rows, err := s.db.QueryContext(ctx, ` + SELECT user_agent + FROM events + WHERE client_ip = ? AND TRIM(user_agent) <> '' + GROUP BY user_agent + ORDER BY MAX(occurred_at) DESC, user_agent ASC + LIMIT ?`, + ip, + limit, + ) + if err != nil { + return nil, fmt.Errorf("list recent user agents for ip %q: %w", ip, err) + } + defer rows.Close() + + items := make([]string, 0, limit) + for rows.Next() { + var userAgent string + if err := rows.Scan(&userAgent); err != nil { + return nil, fmt.Errorf("scan recent user agent for ip %q: %w", ip, err) + } + items = append(items, userAgent) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate recent user agents for ip %q: %w", ip, err) + } + return items, nil +} + func (s *Store) GetSourceOffset(ctx context.Context, sourceName string) (model.SourceOffset, bool, error) { row := s.db.QueryRowContext(ctx, `SELECT source_name, path, inode, offset, updated_at FROM source_offsets WHERE source_name = ?`, sourceName) var offset model.SourceOffset @@ -1086,6 +1165,24 @@ func boolToInt(value bool) int { return 0 } +func uniqueNonEmptyStrings(items []string) []string { + seen := make(map[string]struct{}, len(items)) + result := make([]string, 0, len(items)) + for _, item := range items { + trimmed := strings.TrimSpace(item) + if trimmed == "" { + continue + } + if _, ok := seen[trimmed]; ok { + continue + } + seen[trimmed] = struct{}{} + result = append(result, trimmed) + } + sort.Strings(result) + return result +} + type queryer interface { QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row } diff --git a/internal/store/store_test.go b/internal/store/store_test.go index ca950d6..53c637c 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -139,4 +139,18 @@ func TestStoreRecordsEventsAndState(t *testing.T) { if !found || loadedInvestigation.Bot == nil || loadedInvestigation.Bot.Name != "Googlebot" { t.Fatalf("unexpected investigation payload: found=%v investigation=%+v", found, loadedInvestigation) } + investigations, err := db.GetInvestigationsForIPs(ctx, []string{event.ClientIP, "198.51.100.99"}) + if err != nil { + t.Fatalf("get investigations for ips: %v", err) + } + if len(investigations) != 1 || investigations[event.ClientIP].Bot == nil { + t.Fatalf("unexpected investigations map: %+v", investigations) + } + userAgents, err := db.ListRecentUserAgentsForIP(ctx, event.ClientIP, 10) + if err != nil { + t.Fatalf("list recent user agents: %v", err) + } + if len(userAgents) != 1 || userAgents[0] != event.UserAgent { + t.Fatalf("unexpected user agents: %#v", userAgents) + } } diff --git a/internal/web/handler.go b/internal/web/handler.go index ef2b6a6..d4fc8a9 100644 --- a/internal/web/handler.go +++ b/internal/web/handler.go @@ -337,13 +337,16 @@ const overviewHTML = ` :root { color-scheme: dark; } body { font-family: system-ui, sans-serif; margin: 0; background: #0f172a; color: #e2e8f0; } header { padding: 1rem 1.5rem; border-bottom: 1px solid #334155; position: sticky; top: 0; background: rgba(15,23,42,.97); } - main { padding: 1.5rem; } + main { padding: 1.5rem; display: grid; gap: 1.25rem; } h1, h2 { margin: 0 0 .75rem 0; } table { width: 100%; border-collapse: collapse; } th, td { padding: .55rem .65rem; border-bottom: 1px solid #1e293b; text-align: left; vertical-align: top; } th { color: #93c5fd; white-space: nowrap; } a { color: #93c5fd; text-decoration: none; } a:hover { text-decoration: underline; } + .stats { display: grid; grid-template-columns: repeat(auto-fit, minmax(140px, 1fr)); gap: .75rem; } + .card { background: #111827; border: 1px solid #334155; border-radius: .75rem; padding: .9rem; } + .stat-value { font-size: 1.7rem; font-weight: 700; } .status { display: inline-block; padding: .15rem .45rem; border-radius: 999px; font-size: .8rem; background: #1e293b; } .status.blocked { background: #7f1d1d; } .status.review { background: #78350f; } @@ -362,6 +365,22 @@ const overviewHTML = ` button { background: #2563eb; color: white; border: 0; cursor: pointer; } button.secondary { background: #475569; } button.danger { background: #dc2626; } + .ip-cell { display: flex; align-items: center; gap: .45rem; } + .bot-chip { display: inline-flex; align-items: center; justify-content: center; width: 1.25rem; height: 1.25rem; border-radius: 999px; border: 1px solid #334155; background: #0f172a; color: #e2e8f0; font-size: .72rem; font-weight: 700; cursor: help; } + .bot-chip.verified { border-color: #2563eb; } + .bot-chip.hint { border-style: dashed; } + .bot-chip.google { background: #2563eb; color: white; } + .bot-chip.bing { background: #0284c7; color: white; } + .bot-chip.apple { background: #475569; color: white; } + .bot-chip.meta { background: #2563eb; color: white; } + .bot-chip.duckduckgo { background: #ea580c; color: white; } + .bot-chip.openai { background: #059669; color: white; } + .bot-chip.anthropic { background: #b45309; color: white; } + .bot-chip.perplexity { background: #0f766e; color: white; } + .bot-chip.semrush { background: #db2777; color: white; } + .bot-chip.yandex { background: #dc2626; color: white; } + .bot-chip.baidu { background: #7c3aed; color: white; } + .bot-chip.bytespider { background: #111827; color: white; } @@ -370,6 +389,7 @@ const overviewHTML = `
Local-only review and enforcement console
+

Recent IPs

@@ -405,6 +425,23 @@ const overviewHTML = ` let currentItems = []; let currentSort = { key: 'events', direction: 'desc' }; + function renderStats(data) { + const stats = [ + ['Total events', data.total_events], + ['Tracked IPs', data.total_ips], + ['Blocked', data.blocked_ips], + ['Review', data.review_ips], + ['Allowed', data.allowed_ips], + ['Observed', data.observed_ips], + ]; + document.getElementById('stats').innerHTML = stats.map(([label, value]) => [ + '
', + '
' + escapeHtml(label) + '
', + '
' + escapeHtml(value) + '
', + '
' + ].join('')).join(''); + } + function escapeHtml(value) { return String(value ?? '').replace(/[&<>"']/g, character => ({'&':'&','<':'<','>':'>','"':'"',"'":'''}[character])); } @@ -433,6 +470,41 @@ const overviewHTML = ` return leftRank - rightRank; } + function botVisual(bot) { + const candidate = String((bot || {}).provider_id || (bot || {}).name || '').toLowerCase(); + const catalog = [ + { match: ['google'], short: 'G', className: 'google' }, + { match: ['bing', 'microsoft'], short: 'B', className: 'bing' }, + { match: ['apple'], short: 'A', className: 'apple' }, + { match: ['facebook', 'meta'], short: 'M', className: 'meta' }, + { match: ['duckduckgo', 'duckduckbot'], short: 'D', className: 'duckduckgo' }, + { match: ['gptbot', 'openai'], short: 'O', className: 'openai' }, + { match: ['claudebot', 'anthropic'], short: 'C', className: 'anthropic' }, + { match: ['perplexity'], short: 'P', className: 'perplexity' }, + { match: ['semrush'], short: 'S', className: 'semrush' }, + { match: ['yandex'], short: 'Y', className: 'yandex' }, + { match: ['baidu'], short: 'B', className: 'baidu' }, + { match: ['bytespider', 'tiktok'], short: 'T', className: 'bytespider' }, + ]; + for (const entry of catalog) { + if (entry.match.some(fragment => candidate.includes(fragment))) { + return entry; + } + } + const name = String((bot || {}).name || '').trim(); + return { short: (name[0] || '?').toUpperCase(), className: 'generic' }; + } + + function renderBotChip(bot) { + if (!bot) { + return ''; + } + const visual = botVisual(bot); + const statusClass = bot.verified ? 'verified' : 'hint'; + const title = (bot.name || 'Bot') + (bot.verified ? '' : ' (possible)'); + return '' + escapeHtml(visual.short) + ''; + } + function updateSortButtons() { document.querySelectorAll('button[data-sort]').forEach(button => { const key = button.dataset.sort; @@ -488,7 +560,7 @@ const overviewHTML = ` function renderIPs(items) { const rows = sortItems(items).map(item => [ '', - ' ' + escapeHtml(item.ip) + '', + '
' + renderBotChip(item.bot) + '' + escapeHtml(item.ip) + '
', ' ' + escapeHtml(item.source_name || '—') + '', ' ' + escapeHtml(item.state) + '', ' ' + escapeHtml(item.events) + '', @@ -534,14 +606,21 @@ const overviewHTML = ` } async function refresh() { - const response = await fetch('/api/recent-ips?hours=' + recentHours + '&limit=250'); - const payload = await response.json().catch(() => []); - if (!response.ok) { - const message = Array.isArray(payload) ? response.statusText : (payload.error || response.statusText); + const [overviewResponse, recentResponse] = await Promise.all([ + fetch('/api/overview?limit=50'), + fetch('/api/recent-ips?hours=' + recentHours + '&limit=250') + ]); + const overviewPayload = await overviewResponse.json().catch(() => ({})); + const recentPayload = await recentResponse.json().catch(() => []); + if (overviewResponse.ok) { + renderStats(overviewPayload || {}); + } + if (!recentResponse.ok) { + const message = Array.isArray(recentPayload) ? recentResponse.statusText : (recentPayload.error || recentResponse.statusText); document.getElementById('ips-body').innerHTML = '' + escapeHtml(message) + ''; return; } - currentItems = Array.isArray(payload) ? payload : []; + currentItems = Array.isArray(recentPayload) ? recentPayload : []; render(); }