infrastructure/caddy-opnsense-blocker
2
Fork 0
Code Issues Activity

Cache IP intelligence and add allowed filter

Browse Source
This commit is contained in:
Codex, agent ChatGPT
2026-03-12 11:48:22 +01:00
parent 61c34699cb
commit 8b744d31f3
10 changed files with 194 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
internal/config/config.go
View File

@@ -191,9 +191,6 @@ func (c *Config) applyDefaults() error {
if c.Investigation.BackgroundPollInterval.Duration == 0 {
c.Investigation.BackgroundPollInterval.Duration = 30 * time.Second
}
if c.Investigation.BackgroundLookback.Duration == 0 {
c.Investigation.BackgroundLookback.Duration = 24 * time.Hour
}
if c.Investigation.BackgroundBatchSize == 0 {
c.Investigation.BackgroundBatchSize = 256
}

3
internal/config/config_test.go
View File

@@ -73,6 +73,9 @@ sources:
if got, want := cfg.OPNsense.APISecret, "test-secret"; got != want {
t.Fatalf("unexpected api secret: got %q want %q", got, want)
}
if got := cfg.Investigation.BackgroundLookback.Duration; got != 0 {
t.Fatalf("unexpected background lookback default: got %s want 0", got)
}
profile := cfg.Profiles["main"]
if !profile.IsAllowedPostPath("/search") {
t.Fatalf("expected /search to be normalized as an allowed POST path")

32
internal/service/service.go
View File

@@ -112,7 +112,6 @@ func (s *Service) ListRecentIPs(ctx context.Context, since time.Time, limit int)
if err != nil {
return nil, err
}
staleSince := time.Now().UTC().Add(-s.cfg.Investigation.RefreshAfter.Duration)
for index := range items {
state := model.IPState{
IP: items[index].IP,
@@ -121,9 +120,6 @@ func (s *Service) ListRecentIPs(ctx context.Context, since time.Time, limit int)
}
if investigation, ok := investigations[items[index].IP]; ok {
items[index].Bot = investigation.Bot
if investigation.UpdatedAt.Before(staleSince) {
s.enqueueInvestigation(items[index].IP)
}
} else {
s.enqueueInvestigation(items[index].IP)
}
@@ -316,7 +312,7 @@ func (s *Service) processRecord(ctx context.Context, source config.SourceConfig,
}
func (s *Service) runInvestigationScheduler(ctx context.Context) {
s.enqueueRecentInvestigations(ctx)
s.enqueueMissingInvestigations(ctx)
ticker := time.NewTicker(s.cfg.Investigation.BackgroundPollInterval.Duration)
defer ticker.Stop()
for {
@@ -324,7 +320,7 @@ func (s *Service) runInvestigationScheduler(ctx context.Context) {
case <-ctx.Done():
return
case <-ticker.C:
s.enqueueRecentInvestigations(ctx)
s.enqueueMissingInvestigations(ctx)
}
}
}
@@ -348,27 +344,21 @@ func (s *Service) runInvestigationWorker(ctx context.Context) {
}
}
func (s *Service) enqueueRecentInvestigations(ctx context.Context) {
func (s *Service) enqueueMissingInvestigations(ctx context.Context) {
if s.investigationQueue == nil {
return
}
since := time.Now().UTC().Add(-s.cfg.Investigation.BackgroundLookback.Duration)
items, err := s.store.ListRecentIPRows(ctx, since, s.cfg.Investigation.BackgroundBatchSize)
since := time.Time{}
if s.cfg.Investigation.BackgroundLookback.Duration > 0 {
since = time.Now().UTC().Add(-s.cfg.Investigation.BackgroundLookback.Duration)
}
items, err := s.store.ListIPsWithoutInvestigation(ctx, since, s.cfg.Investigation.BackgroundBatchSize)
if err != nil {
s.logger.Printf("list recent IPs for investigation: %v", err)
s.logger.Printf("list IPs without investigation: %v", err)
return
}
investigations, err := s.store.GetInvestigationsForIPs(ctx, recentRowIPs(items))
if err != nil {
s.logger.Printf("list investigations for recent IPs: %v", err)
return
}
staleSince := time.Now().UTC().Add(-s.cfg.Investigation.RefreshAfter.Duration)
for _, item := range items {
investigation, found := investigations[item.IP]
if !found || investigation.UpdatedAt.Before(staleSince) {
s.enqueueInvestigation(item.IP)
}
s.enqueueInvestigation(item)
}
}
@@ -412,7 +402,7 @@ func (s *Service) refreshInvestigation(ctx context.Context, ip string, force boo
if err != nil {
return nil, err
}
shouldRefresh := force || !found || time.Since(investigation.UpdatedAt) >= s.cfg.Investigation.RefreshAfter.Duration
shouldRefresh := force || !found
if !shouldRefresh {
return &investigation, nil
}

81
internal/service/service_test.go
View File

@@ -240,6 +240,87 @@ sources:
}
}
func TestServiceDoesNotRefreshCachedInvestigationAutomatically(t *testing.T) {
t.Parallel()
tempDir := t.TempDir()
configPath := filepath.Join(tempDir, "config.yaml")
payload := fmt.Sprintf(`storage:
path: %s/blocker.db
investigation:
enabled: true
refresh_after: 1ns
timeout: 500ms
background_workers: 1
background_poll_interval: 50ms
profiles:
main:
auto_block: false
sources:
- name: main
path: %s/access.log
profile: main
`, tempDir, tempDir)
if err := os.WriteFile(configPath, []byte(payload), 0o600); err != nil {
t.Fatalf("write config: %v", err)
}
cfg, err := config.Load(configPath)
if err != nil {
t.Fatalf("load config: %v", err)
}
database, err := store.Open(cfg.Storage.Path)
if err != nil {
t.Fatalf("open store: %v", err)
}
defer database.Close()
investigator := &fakeInvestigator{}
svc := New(cfg, database, nil, investigator, log.New(os.Stderr, "", 0))
ctx := context.Background()
event := &model.Event{
SourceName: "main",
ProfileName: "main",
OccurredAt: time.Now().UTC(),
RemoteIP: "198.51.100.10",
ClientIP: "203.0.113.44",
Host: "example.test",
Method: "GET",
URI: "/cached",
Path: "/cached",
Status: 404,
UserAgent: "test-agent/1.0",
Decision: model.DecisionActionReview,
DecisionReason: "review",
DecisionReasons: []string{"review"},
RawJSON: `{"status":404}`,
}
if err := database.RecordEvent(ctx, event); err != nil {
t.Fatalf("record event: %v", err)
}
if err := database.SaveInvestigation(ctx, model.IPInvestigation{
IP: "203.0.113.44",
UpdatedAt: time.Now().UTC().Add(-48 * time.Hour),
Bot: &model.BotMatch{
Name: "CachedBot",
Verified: true,
},
}); err != nil {
t.Fatalf("save investigation: %v", err)
}
loaded, err := svc.refreshInvestigation(ctx, "203.0.113.44", false)
if err != nil {
t.Fatalf("refresh investigation: %v", err)
}
if loaded == nil || loaded.Bot == nil || loaded.Bot.Name != "CachedBot" {
t.Fatalf("expected cached investigation, got %+v", loaded)
}
if investigator.callCount() != 0 {
t.Fatalf("expected cached investigation to skip external refresh, got %d calls", investigator.callCount())
}
}
type fakeOPNsenseServer struct {
*httptest.Server
mu sync.Mutex

37
internal/store/store.go
View File

@@ -545,6 +545,43 @@ func (s *Store) ListRecentIPRows(ctx context.Context, since time.Time, limit int
return items, nil
}
func (s *Store) ListIPsWithoutInvestigation(ctx context.Context, since time.Time, limit int) ([]string, error) {
if limit <= 0 {
limit = 200
}
query := `
SELECT s.ip
FROM ip_state s
LEFT JOIN ip_investigations i ON i.ip = s.ip
WHERE i.ip IS NULL`
args := make([]any, 0, 2)
if !since.IsZero() {
query += ` AND s.last_seen_at >= ?`
args = append(args, formatTime(since))
}
query += ` ORDER BY s.last_seen_at DESC, s.ip ASC LIMIT ?`
args = append(args, limit)
rows, err := s.db.QueryContext(ctx, query, args...)
if err != nil {
return nil, fmt.Errorf("list ips without investigation: %w", err)
}
defer rows.Close()
items := make([]string, 0, limit)
for rows.Next() {
var ip string
if err := rows.Scan(&ip); err != nil {
return nil, fmt.Errorf("scan ip without investigation: %w", err)
}
items = append(items, ip)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("iterate ips without investigation: %w", err)
}
return items, nil
}
func (s *Store) GetIPDetails(ctx context.Context, ip string, eventLimit, decisionLimit, actionLimit int) (model.IPDetails, error) {
state, _, err := s.GetIPState(ctx, ip)
if err != nil {

7
internal/store/store_test.go
View File

@@ -153,4 +153,11 @@ func TestStoreRecordsEventsAndState(t *testing.T) {
if len(userAgents) != 1 || userAgents[0] != event.UserAgent {
t.Fatalf("unexpected user agents: %#v", userAgents)
}
missingInvestigationIPs, err := db.ListIPsWithoutInvestigation(ctx, time.Time{}, 10)
if err != nil {
t.Fatalf("list ips without investigation: %v", err)
}
if len(missingInvestigationIPs) != 0 {
t.Fatalf("expected no IPs without investigation, got %#v", missingInvestigationIPs)
}
}

52
internal/web/handler.go
View File

@@ -398,6 +398,7 @@ const overviewHTML = `<!doctype html>
<h2>Recent IPs</h2>
<div class="toolbar-right">
<label class="toggle"><input id="show-bots-toggle" type="checkbox" checked onchange="toggleKnownBots()">Show known bots</label>
<label class="toggle"><input id="show-allowed-toggle" type="checkbox" checked onchange="toggleAllowed()">Show allowed</label>
<div class="meta">Last 24 hours · click a column to sort</div>
</div>
</div>
@@ -421,6 +422,7 @@ const overviewHTML = `<!doctype html>
const recentHours = 24;
const storageKeys = {
showKnownBots: 'caddy-opnsense-blocker.overview.showKnownBots',
showAllowed: 'caddy-opnsense-blocker.overview.showAllowed',
sortKey: 'caddy-opnsense-blocker.overview.sortKey',
sortDirection: 'caddy-opnsense-blocker.overview.sortDirection',
};
@@ -436,6 +438,7 @@ const overviewHTML = `<!doctype html>
let currentItems = [];
let currentSort = loadSortPreference();
let showKnownBots = loadShowKnownBotsPreference();
let showAllowed = loadShowAllowedPreference();
function renderStats(data) {
const stats = [
@@ -477,6 +480,25 @@ const overviewHTML = `<!doctype html>
}
}
function loadShowAllowedPreference() {
try {
const rawValue = window.localStorage.getItem(storageKeys.showAllowed);
if (rawValue === null) {
return true;
}
return rawValue !== 'false';
} catch (_) {
return true;
}
}
function saveShowAllowedPreference(value) {
try {
window.localStorage.setItem(storageKeys.showAllowed, value ? 'true' : 'false');
} catch (_) {
}
}
function loadSortPreference() {
const fallback = { key: 'events', direction: 'desc' };
try {
@@ -563,9 +585,13 @@ const overviewHTML = `<!doctype html>
}
function updateSortButtons() {
const toggle = document.getElementById('show-bots-toggle');
if (toggle) {
toggle.checked = showKnownBots;
const botsToggle = document.getElementById('show-bots-toggle');
if (botsToggle) {
botsToggle.checked = showKnownBots;
}
const allowedToggle = document.getElementById('show-allowed-toggle');
if (allowedToggle) {
allowedToggle.checked = showAllowed;
}
document.querySelectorAll('button[data-sort]').forEach(button => {
const key = button.dataset.sort;
@@ -619,7 +645,7 @@ const overviewHTML = `<!doctype html>
}
function renderIPs(items) {
const filteredItems = items.filter(item => showKnownBots || !item.bot);
const filteredItems = items.filter(item => (showKnownBots || !item.bot) && (showAllowed || item.state !== 'allowed'));
const rows = sortItems(filteredItems).map(item => [
'<tr>',
' <td class="mono"><div class="ip-cell">' + renderBotChip(item.bot) + '<a href="/ips/' + encodeURIComponent(item.ip) + '">' + escapeHtml(item.ip) + '</a></div></td>',
@@ -631,7 +657,14 @@ const overviewHTML = `<!doctype html>
' <td>' + renderActions(item) + '</td>',
'</tr>'
].join(''));
const emptyMessage = showKnownBots ? 'No IPs seen in the last 24 hours.' : 'No non-bot IPs seen in the last 24 hours.';
let emptyMessage = 'No IPs seen in the last 24 hours.';
if (!showKnownBots && !showAllowed) {
emptyMessage = 'No non-bot, non-allowed IPs seen in the last 24 hours.';
} else if (!showKnownBots) {
emptyMessage = 'No non-bot IPs seen in the last 24 hours.';
} else if (!showAllowed) {
emptyMessage = 'No non-allowed IPs seen in the last 24 hours.';
}
document.getElementById('ips-body').innerHTML = rows.length > 0 ? rows.join('') : '<tr><td colspan="7" class="muted">' + escapeHtml(emptyMessage) + '</td></tr>';
}
@@ -658,6 +691,13 @@ const overviewHTML = `<!doctype html>
render();
}
function toggleAllowed() {
const toggle = document.getElementById('show-allowed-toggle');
showAllowed = !toggle || toggle.checked;
saveShowAllowedPreference(showAllowed);
render();
}
async function sendAction(ip, action, promptLabel) {
const reason = window.prompt(promptLabel, '');
if (reason === null) {
@@ -994,7 +1034,7 @@ const ipDetailsHTML = `<!doctype html>
renderAll(data);
}
refresh().then(() => investigate());
refresh();
</script>
</body>
</html>`

6
internal/web/handler_test.go
View File

@@ -80,6 +80,9 @@ func TestHandlerServesOverviewAndManualActions(t *testing.T) {
if !strings.Contains(recorder.Body.String(), "Show known bots") {
t.Fatalf("overview page should expose the known bots toggle")
}
if !strings.Contains(recorder.Body.String(), "Show allowed") {
t.Fatalf("overview page should expose the allowed toggle")
}
if !strings.Contains(recorder.Body.String(), "localStorage") {
t.Fatalf("overview page should persist preferences in localStorage")
}
@@ -97,6 +100,9 @@ func TestHandlerServesOverviewAndManualActions(t *testing.T) {
if strings.Contains(body, `const ip = "\"203.0.113.10\"";`) {
t.Fatalf("ip details page still renders a doubly quoted IP")
}
if strings.Contains(body, `refresh().then(() => investigate());`) {
t.Fatalf("ip details page should not auto-refresh investigation on load")
}
}
type stubApp struct {