diff --git a/README.md b/README.md index 29d14d5..e8903e5 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ - Real-time ingestion of multiple Caddy JSON log files. - One heuristic profile per log source. - Persistent local state in SQLite. -- Local-only web UI for reviewing events, IPs, and the full request history of a selected address. +- Local-only web UI with a sortable “Recent IPs” view for the last 24 hours and a full request history for each selected address. - On-demand IP investigation with persistent caching for bot verification, reverse DNS, RDAP, and Spamhaus lookups. - Manual block, unblock, and clear-override actions with OPNsense-aware UI state. - OPNsense alias backend with automatic alias creation. @@ -74,6 +74,7 @@ It refreshes through lightweight JSON polling and exposes these endpoints: - `GET /api/overview` - `GET /api/events` - `GET /api/ips` +- `GET /api/recent-ips?hours=24` - `GET /api/ips/{ip}` - `POST /api/ips/{ip}/investigate` - `POST /api/ips/{ip}/block` diff --git a/internal/model/types.go b/internal/model/types.go index 5894528..df39a1c 100644 --- a/internal/model/types.go +++ b/internal/model/types.go @@ -166,6 +166,17 @@ type ActionAvailability struct { CanClearOverride bool `json:"can_clear_override"` } +type RecentIPRow struct { + IP string `json:"ip"` + SourceName string `json:"source_name"` + State IPStateStatus `json:"state"` + Events int64 `json:"events"` + LastSeenAt time.Time `json:"last_seen_at"` + Reason string `json:"reason"` + ManualOverride ManualOverride `json:"manual_override"` + Actions ActionAvailability `json:"actions"` +} + type SourceOffset struct { SourceName string Path string diff --git a/internal/service/service.go b/internal/service/service.go index e434fbc..a855825 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -76,6 +76,23 @@ func (s *Service) ListIPs(ctx context.Context, limit int, state string) ([]model return s.store.ListIPStates(ctx, limit, state) } +func (s *Service) ListRecentIPs(ctx context.Context, since time.Time, limit int) ([]model.RecentIPRow, error) { + items, err := s.store.ListRecentIPRows(ctx, since, limit) + if err != nil { + return nil, err + } + for index := range items { + state := model.IPState{ + IP: items[index].IP, + State: items[index].State, + ManualOverride: items[index].ManualOverride, + } + backend := s.resolveOPNsenseStatus(ctx, state) + items[index].Actions = actionAvailability(state, backend) + } + return items, nil +} + func (s *Service) GetIPDetails(ctx context.Context, ip string) (model.IPDetails, error) { normalized, err := normalizeIP(ip) if err != nil { diff --git a/internal/service/service_test.go b/internal/service/service_test.go index 0aeeaa0..150c5cc 100644 --- a/internal/service/service_test.go +++ b/internal/service/service_test.go @@ -126,6 +126,21 @@ sources: t.Fatalf("expected observed state, got %+v", observedState) } + recentRows, err := svc.ListRecentIPs(context.Background(), time.Now().UTC().Add(-time.Hour), 10) + if err != nil { + t.Fatalf("list recent ips: %v", err) + } + blockedRow, found := findRecentIPRow(recentRows, "203.0.113.10") + if !found { + t.Fatalf("expected blocked IP row in recent rows: %+v", recentRows) + } + if blockedRow.SourceName != "main" || blockedRow.Events != 1 { + t.Fatalf("unexpected blocked recent row: %+v", blockedRow) + } + if !blockedRow.Actions.CanUnblock || blockedRow.Actions.CanBlock { + t.Fatalf("unexpected blocked recent row actions: %+v", blockedRow.Actions) + } + if err := svc.ForceAllow(context.Background(), "203.0.113.10", "test", "manual unblock"); err != nil { t.Fatalf("force allow: %v", err) } @@ -245,3 +260,12 @@ func waitFor(t *testing.T, timeout time.Duration, condition func() bool) { } t.Fatalf("condition was not met within %s", timeout) } + +func findRecentIPRow(items []model.RecentIPRow, ip string) (model.RecentIPRow, bool) { + for _, item := range items { + if item.IP == ip { + return item, true + } + } + return model.RecentIPRow{}, false +} diff --git a/internal/store/store.go b/internal/store/store.go index 425c6c5..89c2dae 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -475,6 +475,75 @@ func (s *Store) ListIPStates(ctx context.Context, limit int, stateFilter string) return items, nil } +func (s *Store) ListRecentIPRows(ctx context.Context, since time.Time, limit int) ([]model.RecentIPRow, error) { + if limit <= 0 { + limit = 200 + } + rows, err := s.db.QueryContext(ctx, ` + WITH recent AS ( + SELECT client_ip, COUNT(*) AS event_count, MAX(occurred_at) AS last_seen_at + FROM events + WHERE occurred_at >= ? + GROUP BY client_ip + ) + SELECT s.ip, + COALESCE(( + SELECT e.source_name + FROM events e + WHERE e.client_ip = s.ip AND e.occurred_at >= ? + ORDER BY e.occurred_at DESC, e.id DESC + LIMIT 1 + ), s.last_source_name) AS source_name, + s.state, + recent.event_count, + recent.last_seen_at, + s.state_reason, + s.manual_override + FROM recent + JOIN ip_state s ON s.ip = recent.client_ip + ORDER BY recent.event_count DESC, recent.last_seen_at DESC, s.ip ASC + LIMIT ?`, + formatTime(since), + formatTime(since), + limit, + ) + if err != nil { + return nil, fmt.Errorf("list recent ip rows: %w", err) + } + defer rows.Close() + + items := make([]model.RecentIPRow, 0, limit) + for rows.Next() { + var item model.RecentIPRow + var state string + var lastSeenAt string + var manualOverride string + if err := rows.Scan( + &item.IP, + &item.SourceName, + &state, + &item.Events, + &lastSeenAt, + &item.Reason, + &manualOverride, + ); err != nil { + return nil, fmt.Errorf("scan recent ip row: %w", err) + } + parsedLastSeenAt, err := parseTime(lastSeenAt) + if err != nil { + return nil, fmt.Errorf("parse recent ip row last_seen_at: %w", err) + } + item.State = model.IPStateStatus(state) + item.LastSeenAt = parsedLastSeenAt + item.ManualOverride = model.ManualOverride(manualOverride) + items = append(items, item) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate recent ip rows: %w", err) + } + return items, nil +} + func (s *Store) GetIPDetails(ctx context.Context, ip string, eventLimit, decisionLimit, actionLimit int) (model.IPDetails, error) { state, _, err := s.GetIPState(ctx, ip) if err != nil { diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 5a69f2e..ca950d6 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -106,6 +106,16 @@ func TestStoreRecordsEventsAndState(t *testing.T) { if overview.TotalEvents != 1 || overview.TotalIPs != 1 { t.Fatalf("unexpected overview counters: %+v", overview) } + recentIPs, err := db.ListRecentIPRows(ctx, occurredAt.Add(-time.Hour), 10) + if err != nil { + t.Fatalf("list recent ip rows: %v", err) + } + if len(recentIPs) != 1 { + t.Fatalf("unexpected recent ip rows count: %d", len(recentIPs)) + } + if recentIPs[0].IP != event.ClientIP || recentIPs[0].SourceName != event.SourceName || recentIPs[0].Events != 1 { + t.Fatalf("unexpected recent ip row: %+v", recentIPs[0]) + } details, err := db.GetIPDetails(ctx, event.ClientIP, 10, 10, 10) if err != nil { t.Fatalf("get ip details: %v", err) diff --git a/internal/web/handler.go b/internal/web/handler.go index 3ae9427..4b65fe8 100644 --- a/internal/web/handler.go +++ b/internal/web/handler.go @@ -20,6 +20,7 @@ type App interface { GetOverview(ctx context.Context, limit int) (model.Overview, error) ListEvents(ctx context.Context, limit int) ([]model.Event, error) ListIPs(ctx context.Context, limit int, state string) ([]model.IPState, error) + ListRecentIPs(ctx context.Context, since time.Time, limit int) ([]model.RecentIPRow, error) GetIPDetails(ctx context.Context, ip string) (model.IPDetails, error) InvestigateIP(ctx context.Context, ip string) (model.IPDetails, error) ForceBlock(ctx context.Context, ip string, actor string, reason string) error @@ -57,6 +58,7 @@ func NewHandler(app App) http.Handler { mux.HandleFunc("/api/overview", h.handleAPIOverview) mux.HandleFunc("/api/events", h.handleAPIEvents) mux.HandleFunc("/api/ips", h.handleAPIIPs) + mux.HandleFunc("/api/recent-ips", h.handleAPIRecentIPs) mux.HandleFunc("/api/ips/", h.handleAPIIP) return mux } @@ -141,6 +143,29 @@ func (h *handler) handleAPIIPs(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, items) } +func (h *handler) handleAPIRecentIPs(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/recent-ips" { + http.NotFound(w, r) + return + } + if r.Method != http.MethodGet { + methodNotAllowed(w) + return + } + limit := queryLimit(r, 200) + hours := queryInt(r, "hours", 24) + if hours <= 0 { + hours = 24 + } + since := time.Now().UTC().Add(-time.Duration(hours) * time.Hour) + items, err := h.app.ListRecentIPs(r.Context(), since, limit) + if err != nil { + writeError(w, http.StatusInternalServerError, err) + return + } + writeJSON(w, http.StatusOK, items) +} + func (h *handler) handleAPIIP(w http.ResponseWriter, r *http.Request) { ip, action, ok := extractAPIPath(r.URL.Path) if !ok { @@ -235,6 +260,18 @@ func queryLimit(r *http.Request, fallback int) int { return parsed } +func queryInt(r *http.Request, name string, fallback int) int { + value := strings.TrimSpace(r.URL.Query().Get(name)) + if value == "" { + return fallback + } + parsed, err := strconv.Atoi(value) + if err != nil { + return fallback + } + return parsed +} + func writeJSON(w http.ResponseWriter, status int, payload any) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(status) @@ -300,14 +337,11 @@ const overviewHTML = ` :root { color-scheme: dark; } body { font-family: system-ui, sans-serif; margin: 0; background: #0f172a; color: #e2e8f0; } header { padding: 1rem 1.5rem; border-bottom: 1px solid #334155; position: sticky; top: 0; background: rgba(15,23,42,.97); } - main { padding: 1.5rem; display: grid; gap: 1.5rem; } + main { padding: 1.5rem; } h1, h2 { margin: 0 0 .75rem 0; } - .stats { display: grid; grid-template-columns: repeat(auto-fit, minmax(140px, 1fr)); gap: .75rem; } - .card { background: #111827; border: 1px solid #334155; border-radius: .75rem; padding: .9rem; } - .stat-value { font-size: 1.7rem; font-weight: 700; } table { width: 100%; border-collapse: collapse; } th, td { padding: .55rem .65rem; border-bottom: 1px solid #1e293b; text-align: left; vertical-align: top; } - th { color: #93c5fd; } + th { color: #93c5fd; white-space: nowrap; } a { color: #93c5fd; text-decoration: none; } a:hover { text-decoration: underline; } .status { display: inline-block; padding: .15rem .45rem; border-radius: 999px; font-size: .8rem; background: #1e293b; } @@ -318,6 +352,16 @@ const overviewHTML = ` .muted { color: #94a3b8; } .mono { font-family: ui-monospace, monospace; } .panel { background: #111827; border: 1px solid #334155; border-radius: .75rem; padding: 1rem; overflow: auto; } + .toolbar { display: flex; justify-content: space-between; align-items: baseline; gap: 1rem; margin-bottom: .75rem; } + .toolbar .meta { font-size: .95rem; color: #94a3b8; } + .sort-button { appearance: none; background: transparent; border: 0; color: inherit; cursor: pointer; font: inherit; padding: 0; } + .sort-button[data-active="true"] { color: #dbeafe; } + .actions { display: flex; gap: .35rem; flex-wrap: wrap; } + .action-link, button { display: inline-flex; align-items: center; justify-content: center; gap: .35rem; border-radius: .45rem; padding: .3rem .6rem; font-size: .9rem; } + .action-link { background: #1e293b; color: #e2e8f0; text-decoration: none; } + button { background: #2563eb; color: white; border: 0; cursor: pointer; } + button.secondary { background: #475569; } + button.danger { background: #dc2626; } @@ -326,27 +370,41 @@ const overviewHTML = `
Local-only review and enforcement console
-
-

Recent IPs

+
+

Recent IPs

+
Last 24 hours · click a column to sort
+
- + + + + + + + + +
IPStateOverrideEventsLast seenReason
Actions
-
-

Recent events

- - - - - -
TimeSourceIPHostMethodPathStatusDecision
-
@@ -437,6 +572,22 @@ const ipDetailsHTML = ` .status.observed { background: #1e293b; } .muted { color: #94a3b8; } .badge { display: inline-flex; align-items: center; gap: .35rem; padding: .2rem .55rem; border-radius: 999px; background: #1d4ed8; color: white; font-size: .8rem; } + .bot-badge { background: #0f172a; border: 1px solid #334155; color: #e2e8f0; } + .bot-badge.bot-verified { border-color: #1d4ed8; } + .bot-badge.bot-hint { border-style: dashed; } + .bot-mark { display: inline-flex; align-items: center; justify-content: center; width: 1.15rem; height: 1.15rem; border-radius: 999px; font-size: .72rem; font-weight: 700; color: white; background: #475569; } + .bot-mark.google { background: #2563eb; } + .bot-mark.bing { background: #0284c7; } + .bot-mark.apple { background: #475569; } + .bot-mark.meta { background: #2563eb; } + .bot-mark.duckduckgo { background: #ea580c; } + .bot-mark.openai { background: #059669; } + .bot-mark.anthropic { background: #b45309; } + .bot-mark.perplexity { background: #0f766e; } + .bot-mark.semrush { background: #db2777; } + .bot-mark.yandex { background: #dc2626; } + .bot-mark.baidu { background: #7c3aed; } + .bot-mark.bytespider { background: #111827; } .kv { display: grid; gap: .45rem; } .actions { display: flex; gap: .35rem; flex-wrap: wrap; margin-top: .9rem; } button { background: #2563eb; color: white; border: 0; border-radius: .45rem; padding: .35rem .6rem; cursor: pointer; } @@ -508,6 +659,37 @@ const ipDetailsHTML = ` return new Date(value).toLocaleString(); } + function botVisual(bot) { + const candidate = String((bot || {}).provider_id || (bot || {}).name || '').toLowerCase(); + const catalog = [ + { match: ['google'], short: 'G', className: 'google' }, + { match: ['bing', 'microsoft'], short: 'B', className: 'bing' }, + { match: ['apple'], short: 'A', className: 'apple' }, + { match: ['facebook', 'meta'], short: 'M', className: 'meta' }, + { match: ['duckduckgo', 'duckduckbot'], short: 'D', className: 'duckduckgo' }, + { match: ['gptbot', 'openai'], short: 'O', className: 'openai' }, + { match: ['claudebot', 'anthropic'], short: 'C', className: 'anthropic' }, + { match: ['perplexity'], short: 'P', className: 'perplexity' }, + { match: ['semrush'], short: 'S', className: 'semrush' }, + { match: ['yandex'], short: 'Y', className: 'yandex' }, + { match: ['baidu'], short: 'B', className: 'baidu' }, + { match: ['bytespider', 'tiktok'], short: 'T', className: 'bytespider' }, + ]; + for (const entry of catalog) { + if (entry.match.some(fragment => candidate.includes(fragment))) { + return entry; + } + } + const name = String((bot || {}).name || '').trim(); + return { short: (name[0] || '?').toUpperCase(), className: 'generic' }; + } + + function renderBotBadge(bot) { + const visual = botVisual(bot); + const badgeClass = bot.verified ? 'bot-verified' : 'bot-hint'; + return '' + escapeHtml(visual.short) + '' + escapeHtml(bot.name || 'Bot') + ''; + } + async function sendAction(action, promptLabel) { const reason = window.prompt(promptLabel, ''); if (reason === null) { @@ -579,7 +761,7 @@ const ipDetailsHTML = ` } const rows = []; if (investigation.bot) { - rows.push('
' + (investigation.bot.verified ? 'Bot' : 'Possible bot') + ': ' + escapeHtml(investigation.bot.icon || '🤖') + ' ' + escapeHtml(investigation.bot.name) + ' via ' + escapeHtml(investigation.bot.method) + (investigation.bot.verified ? '' : ' (not verified)') + '
'); + rows.push('
' + (investigation.bot.verified ? 'Bot' : 'Possible bot') + ': ' + renderBotBadge(investigation.bot) + ' via ' + escapeHtml(investigation.bot.method) + (investigation.bot.verified ? '' : ' (not verified)') + '
'); } else { rows.push('
Bot: no verified bot match
'); } diff --git a/internal/web/handler_test.go b/internal/web/handler_test.go index 99bc9a5..d29639c 100644 --- a/internal/web/handler_test.go +++ b/internal/web/handler_test.go @@ -19,7 +19,21 @@ func TestHandlerServesOverviewAndManualActions(t *testing.T) { handler := NewHandler(app) recorder := httptest.NewRecorder() - request := httptest.NewRequest(http.MethodGet, "/api/overview?limit=10", nil) + request := httptest.NewRequest(http.MethodGet, "/api/recent-ips?hours=24&limit=10", nil) + handler.ServeHTTP(recorder, request) + if recorder.Code != http.StatusOK { + t.Fatalf("unexpected recent ip status: %d body=%s", recorder.Code, recorder.Body.String()) + } + var recentIPs []model.RecentIPRow + if err := json.Unmarshal(recorder.Body.Bytes(), &recentIPs); err != nil { + t.Fatalf("decode recent ips payload: %v", err) + } + if len(recentIPs) != 1 || recentIPs[0].IP != "203.0.113.10" { + t.Fatalf("unexpected recent ips payload: %+v", recentIPs) + } + + recorder = httptest.NewRecorder() + request = httptest.NewRequest(http.MethodGet, "/api/overview?limit=10", nil) handler.ServeHTTP(recorder, request) if recorder.Code != http.StatusOK { t.Fatalf("unexpected overview status: %d", recorder.Code) @@ -60,6 +74,9 @@ func TestHandlerServesOverviewAndManualActions(t *testing.T) { if !strings.Contains(recorder.Body.String(), "Local-only review and enforcement console") { t.Fatalf("overview page did not render expected content") } + if strings.Contains(recorder.Body.String(), "Recent events") { + t.Fatalf("overview page should no longer render recent events block") + } } type stubApp struct { @@ -100,6 +117,22 @@ func (s *stubApp) ListIPs(ctx context.Context, limit int, state string) ([]model return overview.RecentIPs, nil } +func (s *stubApp) ListRecentIPs(ctx context.Context, since time.Time, limit int) ([]model.RecentIPRow, error) { + _ = ctx + _ = since + _ = limit + now := time.Now().UTC() + return []model.RecentIPRow{{ + IP: "203.0.113.10", + SourceName: "main", + State: model.IPStateBlocked, + Events: 3, + LastSeenAt: now, + Reason: "php_path", + Actions: model.ActionAvailability{CanUnblock: true}, + }}, nil +} + func (s *stubApp) GetIPDetails(context.Context, string) (model.IPDetails, error) { now := time.Now().UTC() return model.IPDetails{