Cache IP intelligence and add allowed filter

README.md

@@ -9,7 +9,7 @@
 - Persistent local state in SQLite.
 - Local-only web UI with summary cards, a sortable “Recent IPs” view for the last 24 hours, bot badges, and a full request history for each selected address.
 - On-demand IP investigation with persistent caching for bot verification, reverse DNS, RDAP, and Spamhaus lookups.
-- Background IP investigation workers so cached intelligence appears without blocking page loads.
+- Background IP investigation workers so missing cached intelligence appears without blocking page loads.
 - Manual block, unblock, and clear-override actions with OPNsense-aware UI state.
 - OPNsense alias backend with automatic alias creation.
 - Concurrent polling across multiple log files.
@@ -64,7 +64,7 @@ Important points:
 - Each source references exactly one profile.
 - `initial_position: end` means “start following new lines only” on first boot.
 - The `investigation` section controls how long IP enrichment is cached and whether on-demand Spamhaus lookups are enabled.
-- The investigation worker can refresh recent IP intelligence in the background so the dashboard stays fast while bot badges and cached intel keep filling in.
+- The investigation worker fills missing cached intelligence in the background so the dashboard stays fast while bot badges and cached intel keep filling in. Opening an IP details page reuses the cache; the `Refresh investigation` button is the manual override when you explicitly want a new lookup.
 - The web UI should stay bound to a local address such as `127.0.0.1:9080`.
 
 ## Web UI and API

config.example.yaml

@@ -15,7 +15,7 @@ investigation:
   spamhaus_enabled: true
   background_workers: 2
   background_poll_interval: 30s
-  background_lookback: 24h
+  background_lookback: 0s
   background_batch_size: 256
 
 opnsense:

internal/config/config.go

@@ -191,9 +191,6 @@ func (c *Config) applyDefaults() error {
 	if c.Investigation.BackgroundPollInterval.Duration == 0 {
 		c.Investigation.BackgroundPollInterval.Duration = 30 * time.Second
 	}
-	if c.Investigation.BackgroundLookback.Duration == 0 {
-		c.Investigation.BackgroundLookback.Duration = 24 * time.Hour
-	}
 	if c.Investigation.BackgroundBatchSize == 0 {
 		c.Investigation.BackgroundBatchSize = 256
 	}

internal/config/config_test.go

@@ -73,6 +73,9 @@ sources:
 	if got, want := cfg.OPNsense.APISecret, "test-secret"; got != want {
 		t.Fatalf("unexpected api secret: got %q want %q", got, want)
 	}
+	if got := cfg.Investigation.BackgroundLookback.Duration; got != 0 {
+		t.Fatalf("unexpected background lookback default: got %s want 0", got)
+	}
 	profile := cfg.Profiles["main"]
 	if !profile.IsAllowedPostPath("/search") {
 		t.Fatalf("expected /search to be normalized as an allowed POST path")

internal/service/service.go

@@ -112,7 +112,6 @@ func (s *Service) ListRecentIPs(ctx context.Context, since time.Time, limit int)
 	if err != nil {
 		return nil, err
 	}
-	staleSince := time.Now().UTC().Add(-s.cfg.Investigation.RefreshAfter.Duration)
 	for index := range items {
 		state := model.IPState{
 			IP:             items[index].IP,
@@ -121,9 +120,6 @@ func (s *Service) ListRecentIPs(ctx context.Context, since time.Time, limit int)
 		}
 		if investigation, ok := investigations[items[index].IP]; ok {
 			items[index].Bot = investigation.Bot
-			if investigation.UpdatedAt.Before(staleSince) {
-				s.enqueueInvestigation(items[index].IP)
-			}
 		} else {
 			s.enqueueInvestigation(items[index].IP)
 		}
@@ -316,7 +312,7 @@ func (s *Service) processRecord(ctx context.Context, source config.SourceConfig,
 }
 
 func (s *Service) runInvestigationScheduler(ctx context.Context) {
-	s.enqueueRecentInvestigations(ctx)
+	s.enqueueMissingInvestigations(ctx)
 	ticker := time.NewTicker(s.cfg.Investigation.BackgroundPollInterval.Duration)
 	defer ticker.Stop()
 	for {
@@ -324,7 +320,7 @@ func (s *Service) runInvestigationScheduler(ctx context.Context) {
 		case <-ctx.Done():
 			return
 		case <-ticker.C:
-			s.enqueueRecentInvestigations(ctx)
+			s.enqueueMissingInvestigations(ctx)
 		}
 	}
 }
@@ -348,27 +344,21 @@ func (s *Service) runInvestigationWorker(ctx context.Context) {
 	}
 }
 
-func (s *Service) enqueueRecentInvestigations(ctx context.Context) {
+func (s *Service) enqueueMissingInvestigations(ctx context.Context) {
 	if s.investigationQueue == nil {
 		return
 	}
-	since := time.Now().UTC().Add(-s.cfg.Investigation.BackgroundLookback.Duration)
+	since := time.Time{}
-	items, err := s.store.ListRecentIPRows(ctx, since, s.cfg.Investigation.BackgroundBatchSize)
+	if s.cfg.Investigation.BackgroundLookback.Duration > 0 {
+		since = time.Now().UTC().Add(-s.cfg.Investigation.BackgroundLookback.Duration)
+	}
+	items, err := s.store.ListIPsWithoutInvestigation(ctx, since, s.cfg.Investigation.BackgroundBatchSize)
 	if err != nil {
-		s.logger.Printf("list recent IPs for investigation: %v", err)
+		s.logger.Printf("list IPs without investigation: %v", err)
 		return
 	}
-	investigations, err := s.store.GetInvestigationsForIPs(ctx, recentRowIPs(items))
-	if err != nil {
-		s.logger.Printf("list investigations for recent IPs: %v", err)
-		return
-	}
-	staleSince := time.Now().UTC().Add(-s.cfg.Investigation.RefreshAfter.Duration)
 	for _, item := range items {
-		investigation, found := investigations[item.IP]
+		s.enqueueInvestigation(item)
-		if !found || investigation.UpdatedAt.Before(staleSince) {
-			s.enqueueInvestigation(item.IP)
-		}
 	}
 }
 
@@ -412,7 +402,7 @@ func (s *Service) refreshInvestigation(ctx context.Context, ip string, force boo
 	if err != nil {
 		return nil, err
 	}
-	shouldRefresh := force || !found || time.Since(investigation.UpdatedAt) >= s.cfg.Investigation.RefreshAfter.Duration
+	shouldRefresh := force || !found
 	if !shouldRefresh {
 		return &investigation, nil
 	}

internal/service/service_test.go

@@ -240,6 +240,87 @@ sources:
 	}
 }
 
+func TestServiceDoesNotRefreshCachedInvestigationAutomatically(t *testing.T) {
+	t.Parallel()
+
+	tempDir := t.TempDir()
+	configPath := filepath.Join(tempDir, "config.yaml")
+	payload := fmt.Sprintf(`storage:
+  path: %s/blocker.db
+investigation:
+  enabled: true
+  refresh_after: 1ns
+  timeout: 500ms
+  background_workers: 1
+  background_poll_interval: 50ms
+profiles:
+  main:
+    auto_block: false
+sources:
+  - name: main
+    path: %s/access.log
+    profile: main
+`, tempDir, tempDir)
+	if err := os.WriteFile(configPath, []byte(payload), 0o600); err != nil {
+		t.Fatalf("write config: %v", err)
+	}
+
+	cfg, err := config.Load(configPath)
+	if err != nil {
+		t.Fatalf("load config: %v", err)
+	}
+	database, err := store.Open(cfg.Storage.Path)
+	if err != nil {
+		t.Fatalf("open store: %v", err)
+	}
+	defer database.Close()
+	investigator := &fakeInvestigator{}
+	svc := New(cfg, database, nil, investigator, log.New(os.Stderr, "", 0))
+
+	ctx := context.Background()
+	event := &model.Event{
+		SourceName:      "main",
+		ProfileName:     "main",
+		OccurredAt:      time.Now().UTC(),
+		RemoteIP:        "198.51.100.10",
+		ClientIP:        "203.0.113.44",
+		Host:            "example.test",
+		Method:          "GET",
+		URI:             "/cached",
+		Path:            "/cached",
+		Status:          404,
+		UserAgent:       "test-agent/1.0",
+		Decision:        model.DecisionActionReview,
+		DecisionReason:  "review",
+		DecisionReasons: []string{"review"},
+		RawJSON:         `{"status":404}`,
+	}
+	if err := database.RecordEvent(ctx, event); err != nil {
+		t.Fatalf("record event: %v", err)
+	}
+	if err := database.SaveInvestigation(ctx, model.IPInvestigation{
+		IP:        "203.0.113.44",
+		UpdatedAt: time.Now().UTC().Add(-48 * time.Hour),
+		Bot: &model.BotMatch{
+			Name:     "CachedBot",
+			Verified: true,
+		},
+	}); err != nil {
+		t.Fatalf("save investigation: %v", err)
+	}
+
+	loaded, err := svc.refreshInvestigation(ctx, "203.0.113.44", false)
+	if err != nil {
+		t.Fatalf("refresh investigation: %v", err)
+	}
+	if loaded == nil || loaded.Bot == nil || loaded.Bot.Name != "CachedBot" {
+		t.Fatalf("expected cached investigation, got %+v", loaded)
+	}
+	if investigator.callCount() != 0 {
+		t.Fatalf("expected cached investigation to skip external refresh, got %d calls", investigator.callCount())
+	}
+}
+
 type fakeOPNsenseServer struct {
 	*httptest.Server
 	mu          sync.Mutex

internal/store/store.go

@@ -545,6 +545,43 @@ func (s *Store) ListRecentIPRows(ctx context.Context, since time.Time, limit int
 	return items, nil
 }
 
+func (s *Store) ListIPsWithoutInvestigation(ctx context.Context, since time.Time, limit int) ([]string, error) {
+	if limit <= 0 {
+		limit = 200
+	}
+	query := `
+		SELECT s.ip
+		FROM ip_state s
+		LEFT JOIN ip_investigations i ON i.ip = s.ip
+		WHERE i.ip IS NULL`
+	args := make([]any, 0, 2)
+	if !since.IsZero() {
+		query += ` AND s.last_seen_at >= ?`
+		args = append(args, formatTime(since))
+	}
+	query += ` ORDER BY s.last_seen_at DESC, s.ip ASC LIMIT ?`
+	args = append(args, limit)
+
+	rows, err := s.db.QueryContext(ctx, query, args...)
+	if err != nil {
+		return nil, fmt.Errorf("list ips without investigation: %w", err)
+	}
+	defer rows.Close()
+
+	items := make([]string, 0, limit)
+	for rows.Next() {
+		var ip string
+		if err := rows.Scan(&ip); err != nil {
+			return nil, fmt.Errorf("scan ip without investigation: %w", err)
+		}
+		items = append(items, ip)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, fmt.Errorf("iterate ips without investigation: %w", err)
+	}
+	return items, nil
+}
+
 func (s *Store) GetIPDetails(ctx context.Context, ip string, eventLimit, decisionLimit, actionLimit int) (model.IPDetails, error) {
 	state, _, err := s.GetIPState(ctx, ip)
 	if err != nil {

internal/store/store_test.go

@@ -153,4 +153,11 @@ func TestStoreRecordsEventsAndState(t *testing.T) {
 	if len(userAgents) != 1 || userAgents[0] != event.UserAgent {
 		t.Fatalf("unexpected user agents: %#v", userAgents)
 	}
+	missingInvestigationIPs, err := db.ListIPsWithoutInvestigation(ctx, time.Time{}, 10)
+	if err != nil {
+		t.Fatalf("list ips without investigation: %v", err)
+	}
+	if len(missingInvestigationIPs) != 0 {
+		t.Fatalf("expected no IPs without investigation, got %#v", missingInvestigationIPs)
+	}
 }

internal/web/handler.go

@@ -398,6 +398,7 @@ const overviewHTML = `<!doctype html>
         <h2>Recent IPs</h2>
         <div class="toolbar-right">
           <label class="toggle"><input id="show-bots-toggle" type="checkbox" checked onchange="toggleKnownBots()">Show known bots</label>
+          <label class="toggle"><input id="show-allowed-toggle" type="checkbox" checked onchange="toggleAllowed()">Show allowed</label>
           <div class="meta">Last 24 hours · click a column to sort</div>
         </div>
       </div>
@@ -421,6 +422,7 @@ const overviewHTML = `<!doctype html>
     const recentHours = 24;
     const storageKeys = {
       showKnownBots: 'caddy-opnsense-blocker.overview.showKnownBots',
+      showAllowed: 'caddy-opnsense-blocker.overview.showAllowed',
       sortKey: 'caddy-opnsense-blocker.overview.sortKey',
       sortDirection: 'caddy-opnsense-blocker.overview.sortDirection',
     };
@@ -436,6 +438,7 @@ const overviewHTML = `<!doctype html>
     let currentItems = [];
     let currentSort = loadSortPreference();
     let showKnownBots = loadShowKnownBotsPreference();
+    let showAllowed = loadShowAllowedPreference();
 
     function renderStats(data) {
       const stats = [
@@ -477,6 +480,25 @@ const overviewHTML = `<!doctype html>
       }
     }
 
+    function loadShowAllowedPreference() {
+      try {
+        const rawValue = window.localStorage.getItem(storageKeys.showAllowed);
+        if (rawValue === null) {
+          return true;
+        }
+        return rawValue !== 'false';
+      } catch (_) {
+        return true;
+      }
+    }
+
+    function saveShowAllowedPreference(value) {
+      try {
+        window.localStorage.setItem(storageKeys.showAllowed, value ? 'true' : 'false');
+      } catch (_) {
+      }
+    }
+
     function loadSortPreference() {
       const fallback = { key: 'events', direction: 'desc' };
       try {
@@ -563,9 +585,13 @@ const overviewHTML = `<!doctype html>
     }
 
     function updateSortButtons() {
-      const toggle = document.getElementById('show-bots-toggle');
+      const botsToggle = document.getElementById('show-bots-toggle');
-      if (toggle) {
+      if (botsToggle) {
-        toggle.checked = showKnownBots;
+        botsToggle.checked = showKnownBots;
+      }
+      const allowedToggle = document.getElementById('show-allowed-toggle');
+      if (allowedToggle) {
+        allowedToggle.checked = showAllowed;
       }
       document.querySelectorAll('button[data-sort]').forEach(button => {
         const key = button.dataset.sort;
@@ -619,7 +645,7 @@ const overviewHTML = `<!doctype html>
     }
 
     function renderIPs(items) {
-      const filteredItems = items.filter(item => showKnownBots || !item.bot);
+      const filteredItems = items.filter(item => (showKnownBots || !item.bot) && (showAllowed || item.state !== 'allowed'));
       const rows = sortItems(filteredItems).map(item => [
         '<tr>',
         '  <td class="mono"><div class="ip-cell">' + renderBotChip(item.bot) + '<a href="/ips/' + encodeURIComponent(item.ip) + '">' + escapeHtml(item.ip) + '</a></div></td>',
@@ -631,7 +657,14 @@ const overviewHTML = `<!doctype html>
         '  <td>' + renderActions(item) + '</td>',
         '</tr>'
       ].join(''));
-      const emptyMessage = showKnownBots ? 'No IPs seen in the last 24 hours.' : 'No non-bot IPs seen in the last 24 hours.';
+      let emptyMessage = 'No IPs seen in the last 24 hours.';
+      if (!showKnownBots && !showAllowed) {
+        emptyMessage = 'No non-bot, non-allowed IPs seen in the last 24 hours.';
+      } else if (!showKnownBots) {
+        emptyMessage = 'No non-bot IPs seen in the last 24 hours.';
+      } else if (!showAllowed) {
+        emptyMessage = 'No non-allowed IPs seen in the last 24 hours.';
+      }
       document.getElementById('ips-body').innerHTML = rows.length > 0 ? rows.join('') : '<tr><td colspan="7" class="muted">' + escapeHtml(emptyMessage) + '</td></tr>';
     }
 
@@ -658,6 +691,13 @@ const overviewHTML = `<!doctype html>
       render();
     }
 
+    function toggleAllowed() {
+      const toggle = document.getElementById('show-allowed-toggle');
+      showAllowed = !toggle || toggle.checked;
+      saveShowAllowedPreference(showAllowed);
+      render();
+    }
+
     async function sendAction(ip, action, promptLabel) {
       const reason = window.prompt(promptLabel, '');
       if (reason === null) {
@@ -994,7 +1034,7 @@ const ipDetailsHTML = `<!doctype html>
       renderAll(data);
     }
 
-    refresh().then(() => investigate());
+    refresh();
   </script>
 </body>
 </html>`

internal/web/handler_test.go

@@ -80,6 +80,9 @@ func TestHandlerServesOverviewAndManualActions(t *testing.T) {
 	if !strings.Contains(recorder.Body.String(), "Show known bots") {
 		t.Fatalf("overview page should expose the known bots toggle")
 	}
+	if !strings.Contains(recorder.Body.String(), "Show allowed") {
+		t.Fatalf("overview page should expose the allowed toggle")
+	}
 	if !strings.Contains(recorder.Body.String(), "localStorage") {
 		t.Fatalf("overview page should persist preferences in localStorage")
 	}
@@ -97,6 +100,9 @@ func TestHandlerServesOverviewAndManualActions(t *testing.T) {
 	if strings.Contains(body, `const ip = "\"203.0.113.10\"";`) {
 		t.Fatalf("ip details page still renders a doubly quoted IP")
 	}
+	if strings.Contains(body, `refresh().then(() => investigate());`) {
+		t.Fatalf("ip details page should not auto-refresh investigation on load")
+	}
 }
 
 type stubApp struct {