From 87d2d5f44094f711478714f62bbf7884e1e976d7 Mon Sep 17 00:00:00 2001 From: "Codex, agent ChatGPT" Date: Thu, 12 Mar 2026 16:03:02 +0100 Subject: [PATCH] Refine dashboard leaderboard filters and layout --- docs/api.md | 8 +++ internal/model/types.go | 5 ++ internal/service/service.go | 4 +- internal/service/service_test.go | 2 +- internal/store/store.go | 60 ++++++++++++++++++---- internal/store/store_test.go | 32 +++++++++++- internal/web/handler.go | 87 ++++++++++++++++++++------------ internal/web/handler_test.go | 23 +++++++-- 8 files changed, 169 insertions(+), 52 deletions(-) diff --git a/docs/api.md b/docs/api.md index 7e4f476..9deb5ee 100644 --- a/docs/api.md +++ b/docs/api.md @@ -37,6 +37,14 @@ Query parameters: - optional - default: `24` - used for the top activity leaderboards returned in the same payload +- `show_known_bots` + - optional + - default: `true` + - when `false`, the leaderboards exclude IPs currently identified as known bots +- `show_allowed` + - optional + - default: `true` + - when `false`, the leaderboards exclude IPs whose current state is `allowed` Main response fields: diff --git a/internal/model/types.go b/internal/model/types.go index 726a968..6005f85 100644 --- a/internal/model/types.go +++ b/internal/model/types.go @@ -201,6 +201,11 @@ type TopURLRow struct { LastSeenAt time.Time `json:"last_seen_at"` } +type OverviewOptions struct { + ShowKnownBots bool + ShowAllowed bool +} + type SourceOffset struct { SourceName string Path string diff --git a/internal/service/service.go b/internal/service/service.go index 8f953d0..4e3c814 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -91,8 +91,8 @@ func (s *Service) Run(ctx context.Context) error { return nil } -func (s *Service) GetOverview(ctx context.Context, since time.Time, limit int) (model.Overview, error) { - overview, err := s.store.GetOverview(ctx, since, limit) +func (s *Service) GetOverview(ctx context.Context, since time.Time, limit int, options model.OverviewOptions) (model.Overview, error) { + overview, err := s.store.GetOverview(ctx, since, limit, options) if err != nil { return model.Overview{}, err } diff --git a/internal/service/service_test.go b/internal/service/service_test.go index ac31955..25eadb0 100644 --- a/internal/service/service_test.go +++ b/internal/service/service_test.go @@ -98,7 +98,7 @@ sources: appendLine(t, giteaLogPath, caddyJSONLine("203.0.113.12", "198.51.100.12", "git.example.test", "GET", "/install.php", 404, "curl/8.0", time.Now().UTC())) waitFor(t, 3*time.Second, func() bool { - overview, err := database.GetOverview(context.Background(), time.Now().UTC().Add(-time.Hour), 10) + overview, err := database.GetOverview(context.Background(), time.Now().UTC().Add(-time.Hour), 10, model.OverviewOptions{ShowKnownBots: true, ShowAllowed: true}) return err == nil && overview.TotalEvents == 3 }) diff --git a/internal/store/store.go b/internal/store/store.go index c61008c..b8e2c01 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -225,6 +225,23 @@ func (s *Store) RecordEvent(ctx context.Context, event *model.Event) error { const responseBytesExpression = `CASE WHEN json_valid(e.raw_json) THEN CAST(COALESCE(json_extract(e.raw_json, '$.size'), 0) AS INTEGER) ELSE 0 END` +func overviewFilterQueryParts(options model.OverviewOptions) (joins []string, clauses []string) { + if !options.ShowAllowed { + joins = append(joins, `LEFT JOIN ip_state s ON s.ip = e.client_ip`) + clauses = append(clauses, `COALESCE(s.state, '') <> '`+string(model.IPStateAllowed)+`'`) + } + if !options.ShowKnownBots { + clauses = append(clauses, `NOT EXISTS ( + SELECT 1 + FROM ip_investigations i + WHERE i.ip = e.client_ip + AND json_valid(i.payload_json) + AND json_type(i.payload_json, '$.bot') IS NOT NULL + )`) + } + return joins, clauses +} + func (s *Store) AddDecision(ctx context.Context, decision *model.DecisionRecord) error { if decision == nil { return errors.New("nil decision record") @@ -372,7 +389,7 @@ func (s *Store) ClearManualOverride(ctx context.Context, ip string, reason strin return current, nil } -func (s *Store) GetOverview(ctx context.Context, since time.Time, limit int) (model.Overview, error) { +func (s *Store) GetOverview(ctx context.Context, since time.Time, limit int, options model.OverviewOptions) (model.Overview, error) { if limit <= 0 { limit = 50 } @@ -407,19 +424,19 @@ func (s *Store) GetOverview(ctx context.Context, since time.Time, limit int) (mo if err != nil { return model.Overview{}, err } - topIPsByEvents, err := s.listTopIPRows(ctx, since, limit, "events") + topIPsByEvents, err := s.listTopIPRows(ctx, since, limit, "events", options) if err != nil { return model.Overview{}, err } - topIPsByTraffic, err := s.listTopIPRows(ctx, since, limit, "traffic") + topIPsByTraffic, err := s.listTopIPRows(ctx, since, limit, "traffic", options) if err != nil { return model.Overview{}, err } - topSources, err := s.listTopSourceRows(ctx, since, limit) + topSources, err := s.listTopSourceRows(ctx, since, limit, options) if err != nil { return model.Overview{}, err } - topURLs, err := s.listTopURLRows(ctx, since, limit) + topURLs, err := s.listTopURLRows(ctx, since, limit, options) if err != nil { return model.Overview{}, err } @@ -432,21 +449,28 @@ func (s *Store) GetOverview(ctx context.Context, since time.Time, limit int) (mo return overview, nil } -func (s *Store) listTopIPRows(ctx context.Context, since time.Time, limit int, orderBy string) ([]model.TopIPRow, error) { +func (s *Store) listTopIPRows(ctx context.Context, since time.Time, limit int, orderBy string, options model.OverviewOptions) ([]model.TopIPRow, error) { if limit <= 0 { limit = 10 } + joins, clauses := overviewFilterQueryParts(options) query := fmt.Sprintf(` SELECT e.client_ip, COUNT(*) AS event_count, COALESCE(SUM(%s), 0) AS traffic_bytes, MAX(e.occurred_at) AS last_seen_at FROM events e`, responseBytesExpression) + if len(joins) > 0 { + query += ` ` + strings.Join(joins, ` `) + } args := make([]any, 0, 2) if !since.IsZero() { - query += ` WHERE e.occurred_at >= ?` + clauses = append([]string{`e.occurred_at >= ?`}, clauses...) args = append(args, formatTime(since)) } + if len(clauses) > 0 { + query += ` WHERE ` + strings.Join(clauses, ` AND `) + } query += ` GROUP BY e.client_ip` switch orderBy { case "traffic": @@ -483,21 +507,28 @@ func (s *Store) listTopIPRows(ctx context.Context, since time.Time, limit int, o return items, nil } -func (s *Store) listTopSourceRows(ctx context.Context, since time.Time, limit int) ([]model.TopSourceRow, error) { +func (s *Store) listTopSourceRows(ctx context.Context, since time.Time, limit int, options model.OverviewOptions) ([]model.TopSourceRow, error) { if limit <= 0 { limit = 10 } + joins, clauses := overviewFilterQueryParts(options) query := fmt.Sprintf(` SELECT e.source_name, COUNT(*) AS event_count, COALESCE(SUM(%s), 0) AS traffic_bytes, MAX(e.occurred_at) AS last_seen_at FROM events e`, responseBytesExpression) + if len(joins) > 0 { + query += ` ` + strings.Join(joins, ` `) + } args := make([]any, 0, 2) if !since.IsZero() { - query += ` WHERE e.occurred_at >= ?` + clauses = append([]string{`e.occurred_at >= ?`}, clauses...) args = append(args, formatTime(since)) } + if len(clauses) > 0 { + query += ` WHERE ` + strings.Join(clauses, ` AND `) + } query += ` GROUP BY e.source_name ORDER BY event_count DESC, traffic_bytes DESC, last_seen_at DESC, e.source_name ASC LIMIT ?` args = append(args, limit) @@ -527,10 +558,11 @@ func (s *Store) listTopSourceRows(ctx context.Context, since time.Time, limit in return items, nil } -func (s *Store) listTopURLRows(ctx context.Context, since time.Time, limit int) ([]model.TopURLRow, error) { +func (s *Store) listTopURLRows(ctx context.Context, since time.Time, limit int, options model.OverviewOptions) ([]model.TopURLRow, error) { if limit <= 0 { limit = 10 } + joins, clauses := overviewFilterQueryParts(options) query := fmt.Sprintf(` SELECT e.host, e.uri, @@ -538,11 +570,17 @@ func (s *Store) listTopURLRows(ctx context.Context, since time.Time, limit int) COALESCE(SUM(%s), 0) AS traffic_bytes, MAX(e.occurred_at) AS last_seen_at FROM events e`, responseBytesExpression) + if len(joins) > 0 { + query += ` ` + strings.Join(joins, ` `) + } args := make([]any, 0, 2) if !since.IsZero() { - query += ` WHERE e.occurred_at >= ?` + clauses = append([]string{`e.occurred_at >= ?`}, clauses...) args = append(args, formatTime(since)) } + if len(clauses) > 0 { + query += ` WHERE ` + strings.Join(clauses, ` AND `) + } query += ` GROUP BY e.host, e.uri ORDER BY event_count DESC, traffic_bytes DESC, last_seen_at DESC, e.host ASC, e.uri ASC LIMIT ?` args = append(args, limit) diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 909477c..69d7328 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -99,7 +99,7 @@ func TestStoreRecordsEventsAndState(t *testing.T) { t.Fatalf("unexpected source offset: found=%v offset=%+v", found, offset) } - overview, err := db.GetOverview(ctx, occurredAt.Add(-time.Hour), 10) + overview, err := db.GetOverview(ctx, occurredAt.Add(-time.Hour), 10, model.OverviewOptions{ShowKnownBots: true, ShowAllowed: true}) if err != nil { t.Fatalf("get overview: %v", err) } @@ -242,7 +242,7 @@ func TestStoreOverviewLeaderboardsUseTrafficFromRawJSON(t *testing.T) { } } - overview, err := db.GetOverview(ctx, baseTime.Add(-time.Minute), 10) + overview, err := db.GetOverview(ctx, baseTime.Add(-time.Minute), 10, model.OverviewOptions{ShowKnownBots: true, ShowAllowed: true}) if err != nil { t.Fatalf("get overview: %v", err) } @@ -264,4 +264,32 @@ func TestStoreOverviewLeaderboardsUseTrafficFromRawJSON(t *testing.T) { if len(overview.TopURLs) == 0 || overview.TopURLs[0].URI != "/wp-login.php" || overview.TopURLs[0].Events != 2 { t.Fatalf("unexpected top url rows: %+v", overview.TopURLs) } + + if err := db.SaveInvestigation(ctx, model.IPInvestigation{ + IP: "203.0.113.10", + UpdatedAt: baseTime, + Bot: &model.BotMatch{Name: "Googlebot", ProviderID: "google_official", Verified: true}, + }); err != nil { + t.Fatalf("save top bot investigation: %v", err) + } + if _, err := db.SetManualOverride(ctx, "203.0.113.20", model.ManualOverrideForceAllow, model.IPStateAllowed, "manual allow"); err != nil { + t.Fatalf("set manual override for filter test: %v", err) + } + + filtered, err := db.GetOverview(ctx, baseTime.Add(-time.Minute), 10, model.OverviewOptions{ShowKnownBots: false, ShowAllowed: false}) + if err != nil { + t.Fatalf("get filtered overview: %v", err) + } + if len(filtered.TopIPsByEvents) != 0 { + t.Fatalf("expected filtered top IPs by events to be empty, got %+v", filtered.TopIPsByEvents) + } + if len(filtered.TopIPsByTraffic) != 0 { + t.Fatalf("expected filtered top IPs by traffic to be empty, got %+v", filtered.TopIPsByTraffic) + } + if len(filtered.TopSources) != 0 { + t.Fatalf("expected filtered top sources to be empty, got %+v", filtered.TopSources) + } + if len(filtered.TopURLs) != 0 { + t.Fatalf("expected filtered top urls to be empty, got %+v", filtered.TopURLs) + } } diff --git a/internal/web/handler.go b/internal/web/handler.go index b2ce335..05a7738 100644 --- a/internal/web/handler.go +++ b/internal/web/handler.go @@ -17,7 +17,7 @@ import ( ) type App interface { - GetOverview(ctx context.Context, since time.Time, limit int) (model.Overview, error) + GetOverview(ctx context.Context, since time.Time, limit int, options model.OverviewOptions) (model.Overview, error) ListEvents(ctx context.Context, limit int) ([]model.Event, error) ListIPs(ctx context.Context, limit int, state string) ([]model.IPState, error) ListRecentIPs(ctx context.Context, since time.Time, limit int) ([]model.RecentIPRow, error) @@ -107,7 +107,11 @@ func (h *handler) handleAPIOverview(w http.ResponseWriter, r *http.Request) { hours = 24 } since := time.Now().UTC().Add(-time.Duration(hours) * time.Hour) - overview, err := h.app.GetOverview(r.Context(), since, limit) + options := model.OverviewOptions{ + ShowKnownBots: queryBool(r, "show_known_bots", true), + ShowAllowed: queryBool(r, "show_allowed", true), + } + overview, err := h.app.GetOverview(r.Context(), since, limit, options) if err != nil { writeError(w, http.StatusInternalServerError, err) return @@ -277,6 +281,21 @@ func queryInt(r *http.Request, name string, fallback int) int { return parsed } +func queryBool(r *http.Request, name string, fallback bool) bool { + value := strings.TrimSpace(strings.ToLower(r.URL.Query().Get(name))) + if value == "" { + return fallback + } + switch value { + case "1", "true", "yes", "on": + return true + case "0", "false", "no", "off": + return false + default: + return fallback + } +} + func writeJSON(w http.ResponseWriter, status int, payload any) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(status) @@ -360,7 +379,9 @@ const overviewHTML = ` .muted { color: #94a3b8; } .mono { font-family: ui-monospace, monospace; } .panel { background: #111827; border: 1px solid #334155; border-radius: .75rem; padding: 1rem; overflow: auto; } - .leaders { display: grid; grid-template-columns: repeat(auto-fit, minmax(260px, 1fr)); gap: 1rem; } + .controls { display: flex; justify-content: space-between; align-items: center; gap: 1rem; flex-wrap: wrap; } + .controls-group { display: flex; align-items: center; gap: 1rem; flex-wrap: wrap; } + .leaders { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 1rem; } .leader-card { background: #111827; border: 1px solid #334155; border-radius: .75rem; padding: 1rem; } .leader-card h2 { margin-bottom: .35rem; font-size: 1rem; } .leader-list { list-style: none; margin: .75rem 0 0 0; padding: 0; display: grid; gap: .65rem; } @@ -368,7 +389,7 @@ const overviewHTML = ` .leader-main { display: flex; align-items: center; justify-content: space-between; gap: .75rem; } .leader-main .mono { overflow: hidden; text-overflow: ellipsis; } .leader-value { font-weight: 600; white-space: nowrap; } - .leader-sub { font-size: .87rem; color: #94a3b8; } + @media (max-width: 1100px) { .leaders { grid-template-columns: 1fr; } } .toolbar { display: flex; justify-content: space-between; align-items: baseline; gap: 1rem; margin-bottom: .75rem; } .toolbar .meta { font-size: .95rem; color: #94a3b8; } .toolbar-right { display: flex; align-items: center; gap: 1rem; flex-wrap: wrap; justify-content: flex-end; } @@ -407,13 +428,18 @@ const overviewHTML = `
+
+
+ + +
+
These two filters affect both the leaderboards and the Recent IPs list.
+

Recent IPs

- -
Last 24 hours · click a column to sort
@@ -638,24 +664,20 @@ const overviewHTML = ` } function renderTopIPs(items, primaryMetric) { - const filteredItems = (Array.isArray(items) ? items : []).filter(item => showKnownBots || !item.bot); - if (filteredItems.length === 0) { + const visibleItems = Array.isArray(items) ? items : []; + if (visibleItems.length === 0) { return '
No matching IP activity in the selected window.
'; } - return '
    ' + filteredItems.map(item => { + return '
      ' + visibleItems.map(item => { const primaryValue = primaryMetric === 'traffic' ? formatBytes(item.traffic_bytes) - : String(item.events || 0) + ' event' + (Number(item.events || 0) === 1 ? '' : 's'); - const secondaryValue = primaryMetric === 'traffic' - ? String(item.events || 0) + ' event' + (Number(item.events || 0) === 1 ? '' : 's') - : formatBytes(item.traffic_bytes); + : String(item.events || 0); return [ '
    1. ', '
      ', - '
      ' + renderBotChip(item.bot) + '' + escapeHtml(item.ip) + '
      ', + ' ' + escapeHtml(item.ip) + '', ' ' + escapeHtml(primaryValue) + '', '
      ', - '
      ' + escapeHtml(secondaryValue) + ' · last seen ' + escapeHtml(formatDate(item.last_seen_at)) + '
      ', '
      2. ' ].join(''); }).join('') + '
    '; @@ -669,9 +691,8 @@ const overviewHTML = ` '
  1. ', '
    ', ' ' + escapeHtml(item.source_name || '—') + '', - ' ' + escapeHtml(String(item.events || 0) + ' event' + (Number(item.events || 0) === 1 ? '' : 's')) + '', + ' ' + escapeHtml(String(item.events || 0)) + '', '
    ', - '
    ' + escapeHtml(formatBytes(item.traffic_bytes)) + ' · last seen ' + escapeHtml(formatDate(item.last_seen_at)) + '
    ', '
    2. ' ].join('')).join('') + '
'; } @@ -686,9 +707,8 @@ const overviewHTML = ` '
  • ', '
    ', ' ' + escapeHtml(label) + '', - ' ' + escapeHtml(String(item.events || 0) + ' event' + (Number(item.events || 0) === 1 ? '' : 's')) + '', + ' ' + escapeHtml(String(item.events || 0)) + '', '
    ', - '
    ' + escapeHtml(formatBytes(item.traffic_bytes)) + ' · last seen ' + escapeHtml(formatDate(item.last_seen_at)) + '
    ', '
    • ' ].join(''); }).join('') + ''; @@ -837,8 +857,7 @@ const overviewHTML = ` showKnownBots = !toggle || toggle.checked; saveShowKnownBotsPreference(showKnownBots); render(); - const overviewStats = window.__overviewPayload || {}; - renderLeaderboards(overviewStats); + refreshOverview(); } function toggleAllowed() { @@ -846,6 +865,7 @@ const overviewHTML = ` showAllowed = !toggle || toggle.checked; saveShowAllowedPreference(showAllowed); render(); + refreshOverview(); } function toggleReviewOnly() { @@ -873,18 +893,21 @@ const overviewHTML = ` await refresh(); } - async function refresh() { - const [overviewResponse, recentResponse] = await Promise.all([ - fetch('/api/overview?hours=' + recentHours + '&limit=10'), - fetch('/api/recent-ips?hours=' + recentHours + '&limit=250') - ]); - const overviewPayload = await overviewResponse.json().catch(() => ({})); - const recentPayload = await recentResponse.json().catch(() => []); - if (overviewResponse.ok) { - window.__overviewPayload = overviewPayload || {}; - renderStats(overviewPayload || {}); - renderLeaderboards(overviewPayload || {}); + async function refreshOverview() { + const response = await fetch('/api/overview?hours=' + recentHours + '&limit=10&show_known_bots=' + (showKnownBots ? 'true' : 'false') + '&show_allowed=' + (showAllowed ? 'true' : 'false')); + const payload = await response.json().catch(() => ({})); + if (!response.ok) { + return; } + window.__overviewPayload = payload || {}; + renderStats(payload || {}); + renderLeaderboards(payload || {}); + } + + async function refresh() { + const recentResponse = await fetch('/api/recent-ips?hours=' + recentHours + '&limit=250'); + const recentPayload = await recentResponse.json().catch(() => []); + refreshOverview(); if (!recentResponse.ok) { const message = Array.isArray(recentPayload) ? recentResponse.statusText : (recentPayload.error || recentResponse.statusText); document.getElementById('ips-body').innerHTML = '' + escapeHtml(message) + ''; diff --git a/internal/web/handler_test.go b/internal/web/handler_test.go index 4ad75da..c24d9f2 100644 --- a/internal/web/handler_test.go +++ b/internal/web/handler_test.go @@ -46,6 +46,16 @@ func TestHandlerServesOverviewAndManualActions(t *testing.T) { t.Fatalf("unexpected overview payload: %+v", overview) } + recorder = httptest.NewRecorder() + request = httptest.NewRequest(http.MethodGet, "/api/overview?hours=24&limit=10&show_known_bots=false&show_allowed=false", nil) + handler.ServeHTTP(recorder, request) + if recorder.Code != http.StatusOK { + t.Fatalf("unexpected filtered overview status: %d", recorder.Code) + } + if app.lastOverviewOptions.ShowKnownBots || app.lastOverviewOptions.ShowAllowed { + t.Fatalf("overview filter options were not forwarded correctly: %+v", app.lastOverviewOptions) + } + recorder = httptest.NewRecorder() request = httptest.NewRequest(http.MethodPost, "/api/ips/203.0.113.10/block", strings.NewReader(`{"reason":"test reason","actor":"tester"}`)) request.Header.Set("Content-Type", "application/json") @@ -92,6 +102,9 @@ func TestHandlerServesOverviewAndManualActions(t *testing.T) { if !strings.Contains(recorder.Body.String(), "Top URLs by events") { t.Fatalf("overview page should expose the top URLs block") } + if !strings.Contains(recorder.Body.String(), "These two filters affect both the leaderboards and the Recent IPs list") { + t.Fatalf("overview page should explain the scope of the shared filters") + } if !strings.Contains(recorder.Body.String(), "Show allowed") { t.Fatalf("overview page should expose the allowed toggle") } @@ -130,10 +143,12 @@ func TestHandlerServesOverviewAndManualActions(t *testing.T) { } type stubApp struct { - lastAction string + lastAction string + lastOverviewOptions model.OverviewOptions } -func (s *stubApp) GetOverview(context.Context, time.Time, int) (model.Overview, error) { +func (s *stubApp) GetOverview(_ context.Context, _ time.Time, _ int, options model.OverviewOptions) (model.Overview, error) { + s.lastOverviewOptions = options now := time.Now().UTC() return model.Overview{ TotalEvents: 1, @@ -183,12 +198,12 @@ func (s *stubApp) GetOverview(context.Context, time.Time, int) (model.Overview, } func (s *stubApp) ListEvents(ctx context.Context, limit int) ([]model.Event, error) { - overview, _ := s.GetOverview(ctx, time.Time{}, limit) + overview, _ := s.GetOverview(ctx, time.Time{}, limit, model.OverviewOptions{ShowKnownBots: true, ShowAllowed: true}) return overview.RecentEvents, nil } func (s *stubApp) ListIPs(ctx context.Context, limit int, state string) ([]model.IPState, error) { - overview, _ := s.GetOverview(ctx, time.Time{}, limit) + overview, _ := s.GetOverview(ctx, time.Time{}, limit, model.OverviewOptions{ShowKnownBots: true, ShowAllowed: true}) return overview.RecentIPs, nil }