diff --git a/README.md b/README.md
index 344e5b9..1a24c30 100644
--- a/README.md
+++ b/README.md
@@ -9,7 +9,7 @@
- Persistent local state in SQLite.
- Local-only web UI with summary cards, a sortable “Recent IPs” view for the last 24 hours, bot badges, and a full request history for each selected address.
- On-demand IP investigation with persistent caching for bot verification, reverse DNS, RDAP, and Spamhaus lookups.
-- Background IP investigation workers so cached intelligence appears without blocking page loads.
+- Background IP investigation workers so missing cached intelligence appears without blocking page loads.
- Manual block, unblock, and clear-override actions with OPNsense-aware UI state.
- OPNsense alias backend with automatic alias creation.
- Concurrent polling across multiple log files.
@@ -64,7 +64,7 @@ Important points:
- Each source references exactly one profile.
- `initial_position: end` means “start following new lines only” on first boot.
- The `investigation` section controls how long IP enrichment is cached and whether on-demand Spamhaus lookups are enabled.
-- The investigation worker can refresh recent IP intelligence in the background so the dashboard stays fast while bot badges and cached intel keep filling in.
+- The investigation worker fills missing cached intelligence in the background so the dashboard stays fast while bot badges and cached intel keep filling in. Opening an IP details page reuses the cache; the `Refresh investigation` button is the manual override when you explicitly want a new lookup.
- The web UI should stay bound to a local address such as `127.0.0.1:9080`.
## Web UI and API
diff --git a/config.example.yaml b/config.example.yaml
index e5357a1..386b763 100644
--- a/config.example.yaml
+++ b/config.example.yaml
@@ -15,7 +15,7 @@ investigation:
spamhaus_enabled: true
background_workers: 2
background_poll_interval: 30s
- background_lookback: 24h
+ background_lookback: 0s
background_batch_size: 256
opnsense:
diff --git a/internal/config/config.go b/internal/config/config.go
index a120d9c..47d6d6c 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -191,9 +191,6 @@ func (c *Config) applyDefaults() error {
if c.Investigation.BackgroundPollInterval.Duration == 0 {
c.Investigation.BackgroundPollInterval.Duration = 30 * time.Second
}
- if c.Investigation.BackgroundLookback.Duration == 0 {
- c.Investigation.BackgroundLookback.Duration = 24 * time.Hour
- }
if c.Investigation.BackgroundBatchSize == 0 {
c.Investigation.BackgroundBatchSize = 256
}
diff --git a/internal/config/config_test.go b/internal/config/config_test.go
index c094232..a624c5b 100644
--- a/internal/config/config_test.go
+++ b/internal/config/config_test.go
@@ -73,6 +73,9 @@ sources:
if got, want := cfg.OPNsense.APISecret, "test-secret"; got != want {
t.Fatalf("unexpected api secret: got %q want %q", got, want)
}
+ if got := cfg.Investigation.BackgroundLookback.Duration; got != 0 {
+ t.Fatalf("unexpected background lookback default: got %s want 0", got)
+ }
profile := cfg.Profiles["main"]
if !profile.IsAllowedPostPath("/search") {
t.Fatalf("expected /search to be normalized as an allowed POST path")
diff --git a/internal/service/service.go b/internal/service/service.go
index 2c67b04..75b2a8f 100644
--- a/internal/service/service.go
+++ b/internal/service/service.go
@@ -112,7 +112,6 @@ func (s *Service) ListRecentIPs(ctx context.Context, since time.Time, limit int)
if err != nil {
return nil, err
}
- staleSince := time.Now().UTC().Add(-s.cfg.Investigation.RefreshAfter.Duration)
for index := range items {
state := model.IPState{
IP: items[index].IP,
@@ -121,9 +120,6 @@ func (s *Service) ListRecentIPs(ctx context.Context, since time.Time, limit int)
}
if investigation, ok := investigations[items[index].IP]; ok {
items[index].Bot = investigation.Bot
- if investigation.UpdatedAt.Before(staleSince) {
- s.enqueueInvestigation(items[index].IP)
- }
} else {
s.enqueueInvestigation(items[index].IP)
}
@@ -316,7 +312,7 @@ func (s *Service) processRecord(ctx context.Context, source config.SourceConfig,
}
func (s *Service) runInvestigationScheduler(ctx context.Context) {
- s.enqueueRecentInvestigations(ctx)
+ s.enqueueMissingInvestigations(ctx)
ticker := time.NewTicker(s.cfg.Investigation.BackgroundPollInterval.Duration)
defer ticker.Stop()
for {
@@ -324,7 +320,7 @@ func (s *Service) runInvestigationScheduler(ctx context.Context) {
case <-ctx.Done():
return
case <-ticker.C:
- s.enqueueRecentInvestigations(ctx)
+ s.enqueueMissingInvestigations(ctx)
}
}
}
@@ -348,27 +344,21 @@ func (s *Service) runInvestigationWorker(ctx context.Context) {
}
}
-func (s *Service) enqueueRecentInvestigations(ctx context.Context) {
+func (s *Service) enqueueMissingInvestigations(ctx context.Context) {
if s.investigationQueue == nil {
return
}
- since := time.Now().UTC().Add(-s.cfg.Investigation.BackgroundLookback.Duration)
- items, err := s.store.ListRecentIPRows(ctx, since, s.cfg.Investigation.BackgroundBatchSize)
+ since := time.Time{}
+ if s.cfg.Investigation.BackgroundLookback.Duration > 0 {
+ since = time.Now().UTC().Add(-s.cfg.Investigation.BackgroundLookback.Duration)
+ }
+ items, err := s.store.ListIPsWithoutInvestigation(ctx, since, s.cfg.Investigation.BackgroundBatchSize)
if err != nil {
- s.logger.Printf("list recent IPs for investigation: %v", err)
+ s.logger.Printf("list IPs without investigation: %v", err)
return
}
- investigations, err := s.store.GetInvestigationsForIPs(ctx, recentRowIPs(items))
- if err != nil {
- s.logger.Printf("list investigations for recent IPs: %v", err)
- return
- }
- staleSince := time.Now().UTC().Add(-s.cfg.Investigation.RefreshAfter.Duration)
for _, item := range items {
- investigation, found := investigations[item.IP]
- if !found || investigation.UpdatedAt.Before(staleSince) {
- s.enqueueInvestigation(item.IP)
- }
+ s.enqueueInvestigation(item)
}
}
@@ -412,7 +402,7 @@ func (s *Service) refreshInvestigation(ctx context.Context, ip string, force boo
if err != nil {
return nil, err
}
- shouldRefresh := force || !found || time.Since(investigation.UpdatedAt) >= s.cfg.Investigation.RefreshAfter.Duration
+ shouldRefresh := force || !found
if !shouldRefresh {
return &investigation, nil
}
diff --git a/internal/service/service_test.go b/internal/service/service_test.go
index 5e689c6..312174e 100644
--- a/internal/service/service_test.go
+++ b/internal/service/service_test.go
@@ -240,6 +240,87 @@ sources:
}
}
+func TestServiceDoesNotRefreshCachedInvestigationAutomatically(t *testing.T) {
+ t.Parallel()
+
+ tempDir := t.TempDir()
+ configPath := filepath.Join(tempDir, "config.yaml")
+ payload := fmt.Sprintf(`storage:
+ path: %s/blocker.db
+investigation:
+ enabled: true
+ refresh_after: 1ns
+ timeout: 500ms
+ background_workers: 1
+ background_poll_interval: 50ms
+profiles:
+ main:
+ auto_block: false
+sources:
+ - name: main
+ path: %s/access.log
+ profile: main
+`, tempDir, tempDir)
+ if err := os.WriteFile(configPath, []byte(payload), 0o600); err != nil {
+ t.Fatalf("write config: %v", err)
+ }
+
+ cfg, err := config.Load(configPath)
+ if err != nil {
+ t.Fatalf("load config: %v", err)
+ }
+ database, err := store.Open(cfg.Storage.Path)
+ if err != nil {
+ t.Fatalf("open store: %v", err)
+ }
+ defer database.Close()
+ investigator := &fakeInvestigator{}
+ svc := New(cfg, database, nil, investigator, log.New(os.Stderr, "", 0))
+
+ ctx := context.Background()
+ event := &model.Event{
+ SourceName: "main",
+ ProfileName: "main",
+ OccurredAt: time.Now().UTC(),
+ RemoteIP: "198.51.100.10",
+ ClientIP: "203.0.113.44",
+ Host: "example.test",
+ Method: "GET",
+ URI: "/cached",
+ Path: "/cached",
+ Status: 404,
+ UserAgent: "test-agent/1.0",
+ Decision: model.DecisionActionReview,
+ DecisionReason: "review",
+ DecisionReasons: []string{"review"},
+ RawJSON: `{"status":404}`,
+ }
+ if err := database.RecordEvent(ctx, event); err != nil {
+ t.Fatalf("record event: %v", err)
+ }
+ if err := database.SaveInvestigation(ctx, model.IPInvestigation{
+ IP: "203.0.113.44",
+ UpdatedAt: time.Now().UTC().Add(-48 * time.Hour),
+ Bot: &model.BotMatch{
+ Name: "CachedBot",
+ Verified: true,
+ },
+ }); err != nil {
+ t.Fatalf("save investigation: %v", err)
+ }
+
+ loaded, err := svc.refreshInvestigation(ctx, "203.0.113.44", false)
+ if err != nil {
+ t.Fatalf("refresh investigation: %v", err)
+ }
+ if loaded == nil || loaded.Bot == nil || loaded.Bot.Name != "CachedBot" {
+ t.Fatalf("expected cached investigation, got %+v", loaded)
+ }
+ if investigator.callCount() != 0 {
+ t.Fatalf("expected cached investigation to skip external refresh, got %d calls", investigator.callCount())
+ }
+}
+
type fakeOPNsenseServer struct {
*httptest.Server
mu sync.Mutex
diff --git a/internal/store/store.go b/internal/store/store.go
index 46e2777..09e9d56 100644
--- a/internal/store/store.go
+++ b/internal/store/store.go
@@ -545,6 +545,43 @@ func (s *Store) ListRecentIPRows(ctx context.Context, since time.Time, limit int
return items, nil
}
+func (s *Store) ListIPsWithoutInvestigation(ctx context.Context, since time.Time, limit int) ([]string, error) {
+ if limit <= 0 {
+ limit = 200
+ }
+ query := `
+ SELECT s.ip
+ FROM ip_state s
+ LEFT JOIN ip_investigations i ON i.ip = s.ip
+ WHERE i.ip IS NULL`
+ args := make([]any, 0, 2)
+ if !since.IsZero() {
+ query += ` AND s.last_seen_at >= ?`
+ args = append(args, formatTime(since))
+ }
+ query += ` ORDER BY s.last_seen_at DESC, s.ip ASC LIMIT ?`
+ args = append(args, limit)
+
+ rows, err := s.db.QueryContext(ctx, query, args...)
+ if err != nil {
+ return nil, fmt.Errorf("list ips without investigation: %w", err)
+ }
+ defer rows.Close()
+
+ items := make([]string, 0, limit)
+ for rows.Next() {
+ var ip string
+ if err := rows.Scan(&ip); err != nil {
+ return nil, fmt.Errorf("scan ip without investigation: %w", err)
+ }
+ items = append(items, ip)
+ }
+ if err := rows.Err(); err != nil {
+ return nil, fmt.Errorf("iterate ips without investigation: %w", err)
+ }
+ return items, nil
+}
+
func (s *Store) GetIPDetails(ctx context.Context, ip string, eventLimit, decisionLimit, actionLimit int) (model.IPDetails, error) {
state, _, err := s.GetIPState(ctx, ip)
if err != nil {
diff --git a/internal/store/store_test.go b/internal/store/store_test.go
index 53c637c..0ba286a 100644
--- a/internal/store/store_test.go
+++ b/internal/store/store_test.go
@@ -153,4 +153,11 @@ func TestStoreRecordsEventsAndState(t *testing.T) {
if len(userAgents) != 1 || userAgents[0] != event.UserAgent {
t.Fatalf("unexpected user agents: %#v", userAgents)
}
+ missingInvestigationIPs, err := db.ListIPsWithoutInvestigation(ctx, time.Time{}, 10)
+ if err != nil {
+ t.Fatalf("list ips without investigation: %v", err)
+ }
+ if len(missingInvestigationIPs) != 0 {
+ t.Fatalf("expected no IPs without investigation, got %#v", missingInvestigationIPs)
+ }
}
diff --git a/internal/web/handler.go b/internal/web/handler.go
index 6b32e23..9b58575 100644
--- a/internal/web/handler.go
+++ b/internal/web/handler.go
@@ -398,6 +398,7 @@ const overviewHTML = `
Recent IPs
@@ -421,6 +422,7 @@ const overviewHTML = `
const recentHours = 24;
const storageKeys = {
showKnownBots: 'caddy-opnsense-blocker.overview.showKnownBots',
+ showAllowed: 'caddy-opnsense-blocker.overview.showAllowed',
sortKey: 'caddy-opnsense-blocker.overview.sortKey',
sortDirection: 'caddy-opnsense-blocker.overview.sortDirection',
};
@@ -436,6 +438,7 @@ const overviewHTML = `
let currentItems = [];
let currentSort = loadSortPreference();
let showKnownBots = loadShowKnownBotsPreference();
+ let showAllowed = loadShowAllowedPreference();
function renderStats(data) {
const stats = [
@@ -477,6 +480,25 @@ const overviewHTML = `
}
}
+ function loadShowAllowedPreference() {
+ try {
+ const rawValue = window.localStorage.getItem(storageKeys.showAllowed);
+ if (rawValue === null) {
+ return true;
+ }
+ return rawValue !== 'false';
+ } catch (_) {
+ return true;
+ }
+ }
+
+ function saveShowAllowedPreference(value) {
+ try {
+ window.localStorage.setItem(storageKeys.showAllowed, value ? 'true' : 'false');
+ } catch (_) {
+ }
+ }
+
function loadSortPreference() {
const fallback = { key: 'events', direction: 'desc' };
try {
@@ -563,9 +585,13 @@ const overviewHTML = `
}
function updateSortButtons() {
- const toggle = document.getElementById('show-bots-toggle');
- if (toggle) {
- toggle.checked = showKnownBots;
+ const botsToggle = document.getElementById('show-bots-toggle');
+ if (botsToggle) {
+ botsToggle.checked = showKnownBots;
+ }
+ const allowedToggle = document.getElementById('show-allowed-toggle');
+ if (allowedToggle) {
+ allowedToggle.checked = showAllowed;
}
document.querySelectorAll('button[data-sort]').forEach(button => {
const key = button.dataset.sort;
@@ -619,7 +645,7 @@ const overviewHTML = `
}
function renderIPs(items) {
- const filteredItems = items.filter(item => showKnownBots || !item.bot);
+ const filteredItems = items.filter(item => (showKnownBots || !item.bot) && (showAllowed || item.state !== 'allowed'));
const rows = sortItems(filteredItems).map(item => [
'',
' | ',
@@ -631,7 +657,14 @@ const overviewHTML = `
' ' + renderActions(item) + ' | ',
'
'
].join(''));
- const emptyMessage = showKnownBots ? 'No IPs seen in the last 24 hours.' : 'No non-bot IPs seen in the last 24 hours.';
+ let emptyMessage = 'No IPs seen in the last 24 hours.';
+ if (!showKnownBots && !showAllowed) {
+ emptyMessage = 'No non-bot, non-allowed IPs seen in the last 24 hours.';
+ } else if (!showKnownBots) {
+ emptyMessage = 'No non-bot IPs seen in the last 24 hours.';
+ } else if (!showAllowed) {
+ emptyMessage = 'No non-allowed IPs seen in the last 24 hours.';
+ }
document.getElementById('ips-body').innerHTML = rows.length > 0 ? rows.join('') : '| ' + escapeHtml(emptyMessage) + ' |
';
}
@@ -658,6 +691,13 @@ const overviewHTML = `
render();
}
+ function toggleAllowed() {
+ const toggle = document.getElementById('show-allowed-toggle');
+ showAllowed = !toggle || toggle.checked;
+ saveShowAllowedPreference(showAllowed);
+ render();
+ }
+
async function sendAction(ip, action, promptLabel) {
const reason = window.prompt(promptLabel, '');
if (reason === null) {
@@ -994,7 +1034,7 @@ const ipDetailsHTML = `
renderAll(data);
}
- refresh().then(() => investigate());
+ refresh();