infrastructure/caddy-opnsense-blocker
2
Fork 0
Code Issues Activity

Refine dashboard leaderboard filters and layout

Browse Source
This commit is contained in:
Codex, agent ChatGPT
2026-03-12 16:03:02 +01:00
parent 49bda65b3b
commit 87d2d5f440
8 changed files with 169 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
docs/api.md
View File

@@ -37,6 +37,14 @@ Query parameters:
- optional
- default: `24`
- used for the top activity leaderboards returned in the same payload
- `show_known_bots`
- optional
- default: `true`
- when `false`, the leaderboards exclude IPs currently identified as known bots
- `show_allowed`
- optional
- default: `true`
- when `false`, the leaderboards exclude IPs whose current state is `allowed`
Main response fields:

5
internal/model/types.go
View File

@@ -201,6 +201,11 @@ type TopURLRow struct {
LastSeenAt time.Time `json:"last_seen_at"`
}
type OverviewOptions struct {
ShowKnownBots bool
ShowAllowed bool
}
type SourceOffset struct {
SourceName string
Path string

4
internal/service/service.go
View File

@@ -91,8 +91,8 @@ func (s *Service) Run(ctx context.Context) error {
return nil
}
func (s *Service) GetOverview(ctx context.Context, since time.Time, limit int) (model.Overview, error) {
overview, err := s.store.GetOverview(ctx, since, limit)
func (s *Service) GetOverview(ctx context.Context, since time.Time, limit int, options model.OverviewOptions) (model.Overview, error) {
overview, err := s.store.GetOverview(ctx, since, limit, options)
if err != nil {
return model.Overview{}, err
}

2
internal/service/service_test.go
View File

@@ -98,7 +98,7 @@ sources:
appendLine(t, giteaLogPath, caddyJSONLine("203.0.113.12", "198.51.100.12", "git.example.test", "GET", "/install.php", 404, "curl/8.0", time.Now().UTC()))
waitFor(t, 3*time.Second, func() bool {
overview, err := database.GetOverview(context.Background(), time.Now().UTC().Add(-time.Hour), 10)
overview, err := database.GetOverview(context.Background(), time.Now().UTC().Add(-time.Hour), 10, model.OverviewOptions{ShowKnownBots: true, ShowAllowed: true})
return err == nil && overview.TotalEvents == 3
})

60
internal/store/store.go
View File

@@ -225,6 +225,23 @@ func (s *Store) RecordEvent(ctx context.Context, event *model.Event) error {
const responseBytesExpression = `CASE WHEN json_valid(e.raw_json) THEN CAST(COALESCE(json_extract(e.raw_json, '$.size'), 0) AS INTEGER) ELSE 0 END`
func overviewFilterQueryParts(options model.OverviewOptions) (joins []string, clauses []string) {
if !options.ShowAllowed {
joins = append(joins, `LEFT JOIN ip_state s ON s.ip = e.client_ip`)
clauses = append(clauses, `COALESCE(s.state, '') <> '`+string(model.IPStateAllowed)+`'`)
}
if !options.ShowKnownBots {
clauses = append(clauses, `NOT EXISTS (
SELECT 1
FROM ip_investigations i
WHERE i.ip = e.client_ip
AND json_valid(i.payload_json)
AND json_type(i.payload_json, '$.bot') IS NOT NULL
)`)
}
return joins, clauses
}
func (s *Store) AddDecision(ctx context.Context, decision *model.DecisionRecord) error {
if decision == nil {
return errors.New("nil decision record")
@@ -372,7 +389,7 @@ func (s *Store) ClearManualOverride(ctx context.Context, ip string, reason strin
return current, nil
}
func (s *Store) GetOverview(ctx context.Context, since time.Time, limit int) (model.Overview, error) {
func (s *Store) GetOverview(ctx context.Context, since time.Time, limit int, options model.OverviewOptions) (model.Overview, error) {
if limit <= 0 {
limit = 50
}
@@ -407,19 +424,19 @@ func (s *Store) GetOverview(ctx context.Context, since time.Time, limit int) (mo
if err != nil {
return model.Overview{}, err
}
topIPsByEvents, err := s.listTopIPRows(ctx, since, limit, "events")
topIPsByEvents, err := s.listTopIPRows(ctx, since, limit, "events", options)
if err != nil {
return model.Overview{}, err
}
topIPsByTraffic, err := s.listTopIPRows(ctx, since, limit, "traffic")
topIPsByTraffic, err := s.listTopIPRows(ctx, since, limit, "traffic", options)
if err != nil {
return model.Overview{}, err
}
topSources, err := s.listTopSourceRows(ctx, since, limit)
topSources, err := s.listTopSourceRows(ctx, since, limit, options)
if err != nil {
return model.Overview{}, err
}
topURLs, err := s.listTopURLRows(ctx, since, limit)
topURLs, err := s.listTopURLRows(ctx, since, limit, options)
if err != nil {
return model.Overview{}, err
}
@@ -432,21 +449,28 @@ func (s *Store) GetOverview(ctx context.Context, since time.Time, limit int) (mo
return overview, nil
}
func (s *Store) listTopIPRows(ctx context.Context, since time.Time, limit int, orderBy string) ([]model.TopIPRow, error) {
func (s *Store) listTopIPRows(ctx context.Context, since time.Time, limit int, orderBy string, options model.OverviewOptions) ([]model.TopIPRow, error) {
if limit <= 0 {
limit = 10
}
joins, clauses := overviewFilterQueryParts(options)
query := fmt.Sprintf(`
SELECT e.client_ip,
COUNT(*) AS event_count,
COALESCE(SUM(%s), 0) AS traffic_bytes,
MAX(e.occurred_at) AS last_seen_at
FROM events e`, responseBytesExpression)
if len(joins) > 0 {
query += ` ` + strings.Join(joins, ` `)
}
args := make([]any, 0, 2)
if !since.IsZero() {
query += ` WHERE e.occurred_at >= ?`
clauses = append([]string{`e.occurred_at >= ?`}, clauses...)
args = append(args, formatTime(since))
}
if len(clauses) > 0 {
query += ` WHERE ` + strings.Join(clauses, ` AND `)
}
query += ` GROUP BY e.client_ip`
switch orderBy {
case "traffic":
@@ -483,21 +507,28 @@ func (s *Store) listTopIPRows(ctx context.Context, since time.Time, limit int, o
return items, nil
}
func (s *Store) listTopSourceRows(ctx context.Context, since time.Time, limit int) ([]model.TopSourceRow, error) {
func (s *Store) listTopSourceRows(ctx context.Context, since time.Time, limit int, options model.OverviewOptions) ([]model.TopSourceRow, error) {
if limit <= 0 {
limit = 10
}
joins, clauses := overviewFilterQueryParts(options)
query := fmt.Sprintf(`
SELECT e.source_name,
COUNT(*) AS event_count,
COALESCE(SUM(%s), 0) AS traffic_bytes,
MAX(e.occurred_at) AS last_seen_at
FROM events e`, responseBytesExpression)
if len(joins) > 0 {
query += ` ` + strings.Join(joins, ` `)
}
args := make([]any, 0, 2)
if !since.IsZero() {
query += ` WHERE e.occurred_at >= ?`
clauses = append([]string{`e.occurred_at >= ?`}, clauses...)
args = append(args, formatTime(since))
}
if len(clauses) > 0 {
query += ` WHERE ` + strings.Join(clauses, ` AND `)
}
query += ` GROUP BY e.source_name ORDER BY event_count DESC, traffic_bytes DESC, last_seen_at DESC, e.source_name ASC LIMIT ?`
args = append(args, limit)
@@ -527,10 +558,11 @@ func (s *Store) listTopSourceRows(ctx context.Context, since time.Time, limit in
return items, nil
}
func (s *Store) listTopURLRows(ctx context.Context, since time.Time, limit int) ([]model.TopURLRow, error) {
func (s *Store) listTopURLRows(ctx context.Context, since time.Time, limit int, options model.OverviewOptions) ([]model.TopURLRow, error) {
if limit <= 0 {
limit = 10
}
joins, clauses := overviewFilterQueryParts(options)
query := fmt.Sprintf(`
SELECT e.host,
e.uri,
@@ -538,11 +570,17 @@ func (s *Store) listTopURLRows(ctx context.Context, since time.Time, limit int)
COALESCE(SUM(%s), 0) AS traffic_bytes,
MAX(e.occurred_at) AS last_seen_at
FROM events e`, responseBytesExpression)
if len(joins) > 0 {
query += ` ` + strings.Join(joins, ` `)
}
args := make([]any, 0, 2)
if !since.IsZero() {
query += ` WHERE e.occurred_at >= ?`
clauses = append([]string{`e.occurred_at >= ?`}, clauses...)
args = append(args, formatTime(since))
}
if len(clauses) > 0 {
query += ` WHERE ` + strings.Join(clauses, ` AND `)
}
query += ` GROUP BY e.host, e.uri ORDER BY event_count DESC, traffic_bytes DESC, last_seen_at DESC, e.host ASC, e.uri ASC LIMIT ?`
args = append(args, limit)

32
internal/store/store_test.go
View File

@@ -99,7 +99,7 @@ func TestStoreRecordsEventsAndState(t *testing.T) {
t.Fatalf("unexpected source offset: found=%v offset=%+v", found, offset)
}
overview, err := db.GetOverview(ctx, occurredAt.Add(-time.Hour), 10)
overview, err := db.GetOverview(ctx, occurredAt.Add(-time.Hour), 10, model.OverviewOptions{ShowKnownBots: true, ShowAllowed: true})
if err != nil {
t.Fatalf("get overview: %v", err)
}
@@ -242,7 +242,7 @@ func TestStoreOverviewLeaderboardsUseTrafficFromRawJSON(t *testing.T) {
}
}
overview, err := db.GetOverview(ctx, baseTime.Add(-time.Minute), 10)
overview, err := db.GetOverview(ctx, baseTime.Add(-time.Minute), 10, model.OverviewOptions{ShowKnownBots: true, ShowAllowed: true})
if err != nil {
t.Fatalf("get overview: %v", err)
}
@@ -264,4 +264,32 @@ func TestStoreOverviewLeaderboardsUseTrafficFromRawJSON(t *testing.T) {
if len(overview.TopURLs) == 0 || overview.TopURLs[0].URI != "/wp-login.php" || overview.TopURLs[0].Events != 2 {
t.Fatalf("unexpected top url rows: %+v", overview.TopURLs)
}
if err := db.SaveInvestigation(ctx, model.IPInvestigation{
IP: "203.0.113.10",
UpdatedAt: baseTime,
Bot: &model.BotMatch{Name: "Googlebot", ProviderID: "google_official", Verified: true},
}); err != nil {
t.Fatalf("save top bot investigation: %v", err)
}
if _, err := db.SetManualOverride(ctx, "203.0.113.20", model.ManualOverrideForceAllow, model.IPStateAllowed, "manual allow"); err != nil {
t.Fatalf("set manual override for filter test: %v", err)
}
filtered, err := db.GetOverview(ctx, baseTime.Add(-time.Minute), 10, model.OverviewOptions{ShowKnownBots: false, ShowAllowed: false})
if err != nil {
t.Fatalf("get filtered overview: %v", err)
}
if len(filtered.TopIPsByEvents) != 0 {
t.Fatalf("expected filtered top IPs by events to be empty, got %+v", filtered.TopIPsByEvents)
}
if len(filtered.TopIPsByTraffic) != 0 {
t.Fatalf("expected filtered top IPs by traffic to be empty, got %+v", filtered.TopIPsByTraffic)
}
if len(filtered.TopSources) != 0 {
t.Fatalf("expected filtered top sources to be empty, got %+v", filtered.TopSources)
}
if len(filtered.TopURLs) != 0 {
t.Fatalf("expected filtered top urls to be empty, got %+v", filtered.TopURLs)
}
}

87
internal/web/handler.go
View File

@@ -17,7 +17,7 @@ import (
)
type App interface {
GetOverview(ctx context.Context, since time.Time, limit int) (model.Overview, error)
GetOverview(ctx context.Context, since time.Time, limit int, options model.OverviewOptions) (model.Overview, error)
ListEvents(ctx context.Context, limit int) ([]model.Event, error)
ListIPs(ctx context.Context, limit int, state string) ([]model.IPState, error)
ListRecentIPs(ctx context.Context, since time.Time, limit int) ([]model.RecentIPRow, error)
@@ -107,7 +107,11 @@ func (h *handler) handleAPIOverview(w http.ResponseWriter, r *http.Request) {
hours = 24
}
since := time.Now().UTC().Add(-time.Duration(hours) * time.Hour)
overview, err := h.app.GetOverview(r.Context(), since, limit)
options := model.OverviewOptions{
ShowKnownBots: queryBool(r, "show_known_bots", true),
ShowAllowed: queryBool(r, "show_allowed", true),
}
overview, err := h.app.GetOverview(r.Context(), since, limit, options)
if err != nil {
writeError(w, http.StatusInternalServerError, err)
return
@@ -277,6 +281,21 @@ func queryInt(r *http.Request, name string, fallback int) int {
return parsed
}
func queryBool(r *http.Request, name string, fallback bool) bool {
value := strings.TrimSpace(strings.ToLower(r.URL.Query().Get(name)))
if value == "" {
return fallback
}
switch value {
case "1", "true", "yes", "on":
return true
case "0", "false", "no", "off":
return false
default:
return fallback
}
}
func writeJSON(w http.ResponseWriter, status int, payload any) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(status)
@@ -360,7 +379,9 @@ const overviewHTML = `<!doctype html>
.muted { color: #94a3b8; }
.mono { font-family: ui-monospace, monospace; }
.panel { background: #111827; border: 1px solid #334155; border-radius: .75rem; padding: 1rem; overflow: auto; }
.leaders { display: grid; grid-template-columns: repeat(auto-fit, minmax(260px, 1fr)); gap: 1rem; }
.controls { display: flex; justify-content: space-between; align-items: center; gap: 1rem; flex-wrap: wrap; }
.controls-group { display: flex; align-items: center; gap: 1rem; flex-wrap: wrap; }
.leaders { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 1rem; }
.leader-card { background: #111827; border: 1px solid #334155; border-radius: .75rem; padding: 1rem; }
.leader-card h2 { margin-bottom: .35rem; font-size: 1rem; }
.leader-list { list-style: none; margin: .75rem 0 0 0; padding: 0; display: grid; gap: .65rem; }
@@ -368,7 +389,7 @@ const overviewHTML = `<!doctype html>
.leader-main { display: flex; align-items: center; justify-content: space-between; gap: .75rem; }
.leader-main .mono { overflow: hidden; text-overflow: ellipsis; }
.leader-value { font-weight: 600; white-space: nowrap; }
.leader-sub { font-size: .87rem; color: #94a3b8; }
@media (max-width: 1100px) { .leaders { grid-template-columns: 1fr; } }
.toolbar { display: flex; justify-content: space-between; align-items: baseline; gap: 1rem; margin-bottom: .75rem; }
.toolbar .meta { font-size: .95rem; color: #94a3b8; }
.toolbar-right { display: flex; align-items: center; gap: 1rem; flex-wrap: wrap; justify-content: flex-end; }
@@ -407,13 +428,18 @@ const overviewHTML = `<!doctype html>
</header>
<main>
<section class="stats" id="stats"></section>
<section class="panel controls">
<div class="controls-group">
<label class="toggle"><input id="show-bots-toggle" type="checkbox" checked onchange="toggleKnownBots()">Show known bots</label>
<label class="toggle"><input id="show-allowed-toggle" type="checkbox" checked onchange="toggleAllowed()">Show allowed</label>
</div>
<div class="muted">These two filters affect both the leaderboards and the Recent IPs list.</div>
</section>
<section class="leaders" id="leaderboards"></section>
<section class="panel">
<div class="toolbar">
<h2>Recent IPs</h2>
<div class="toolbar-right">
<label class="toggle"><input id="show-bots-toggle" type="checkbox" checked onchange="toggleKnownBots()">Show known bots</label>
<label class="toggle"><input id="show-allowed-toggle" type="checkbox" checked onchange="toggleAllowed()">Show allowed</label>
<label class="toggle"><input id="show-review-toggle" type="checkbox" onchange="toggleReviewOnly()">Review only</label>
<div class="meta">Last 24 hours · click a column to sort</div>
</div>
@@ -638,24 +664,20 @@ const overviewHTML = `<!doctype html>
}
function renderTopIPs(items, primaryMetric) {
const filteredItems = (Array.isArray(items) ? items : []).filter(item => showKnownBots || !item.bot);
if (filteredItems.length === 0) {
const visibleItems = Array.isArray(items) ? items : [];
if (visibleItems.length === 0) {
return '<div class="muted">No matching IP activity in the selected window.</div>';
}
return '<ol class="leader-list">' + filteredItems.map(item => {
return '<ol class="leader-list">' + visibleItems.map(item => {
const primaryValue = primaryMetric === 'traffic'
? formatBytes(item.traffic_bytes)
: String(item.events || 0) + ' event' + (Number(item.events || 0) === 1 ? '' : 's');
const secondaryValue = primaryMetric === 'traffic'
? String(item.events || 0) + ' event' + (Number(item.events || 0) === 1 ? '' : 's')
: formatBytes(item.traffic_bytes);
: String(item.events || 0);
return [
'<li class="leader-item">',
' <div class="leader-main">',
' <div class="ip-cell mono">' + renderBotChip(item.bot) + '<a href="/ips/' + encodeURIComponent(item.ip) + '">' + escapeHtml(item.ip) + '</a></div>',
' <a class="mono" href="/ips/' + encodeURIComponent(item.ip) + '">' + escapeHtml(item.ip) + '</a>',
' <span class="leader-value">' + escapeHtml(primaryValue) + '</span>',
' </div>',
' <div class="leader-sub">' + escapeHtml(secondaryValue) + ' · last seen ' + escapeHtml(formatDate(item.last_seen_at)) + '</div>',
'</li>'
].join('');
}).join('') + '</ol>';
@@ -669,9 +691,8 @@ const overviewHTML = `<!doctype html>
'<li class="leader-item">',
' <div class="leader-main">',
' <span class="mono">' + escapeHtml(item.source_name || '—') + '</span>',
' <span class="leader-value">' + escapeHtml(String(item.events || 0) + ' event' + (Number(item.events || 0) === 1 ? '' : 's')) + '</span>',
' <span class="leader-value">' + escapeHtml(String(item.events || 0)) + '</span>',
' </div>',
' <div class="leader-sub">' + escapeHtml(formatBytes(item.traffic_bytes)) + ' · last seen ' + escapeHtml(formatDate(item.last_seen_at)) + '</div>',
'</li>'
].join('')).join('') + '</ol>';
}
@@ -686,9 +707,8 @@ const overviewHTML = `<!doctype html>
'<li class="leader-item">',
' <div class="leader-main">',
' <span class="mono">' + escapeHtml(label) + '</span>',
' <span class="leader-value">' + escapeHtml(String(item.events || 0) + ' event' + (Number(item.events || 0) === 1 ? '' : 's')) + '</span>',
' <span class="leader-value">' + escapeHtml(String(item.events || 0)) + '</span>',
' </div>',
' <div class="leader-sub">' + escapeHtml(formatBytes(item.traffic_bytes)) + ' · last seen ' + escapeHtml(formatDate(item.last_seen_at)) + '</div>',
'</li>'
].join('');
}).join('') + '</ol>';
@@ -837,8 +857,7 @@ const overviewHTML = `<!doctype html>
showKnownBots = !toggle || toggle.checked;
saveShowKnownBotsPreference(showKnownBots);
render();
const overviewStats = window.__overviewPayload || {};
renderLeaderboards(overviewStats);
refreshOverview();
}
function toggleAllowed() {
@@ -846,6 +865,7 @@ const overviewHTML = `<!doctype html>
showAllowed = !toggle || toggle.checked;
saveShowAllowedPreference(showAllowed);
render();
refreshOverview();
}
function toggleReviewOnly() {
@@ -873,18 +893,21 @@ const overviewHTML = `<!doctype html>
await refresh();
}
async function refresh() {
const [overviewResponse, recentResponse] = await Promise.all([
fetch('/api/overview?hours=' + recentHours + '&limit=10'),
fetch('/api/recent-ips?hours=' + recentHours + '&limit=250')
]);
const overviewPayload = await overviewResponse.json().catch(() => ({}));
const recentPayload = await recentResponse.json().catch(() => []);
if (overviewResponse.ok) {
window.__overviewPayload = overviewPayload || {};
renderStats(overviewPayload || {});
renderLeaderboards(overviewPayload || {});
async function refreshOverview() {
const response = await fetch('/api/overview?hours=' + recentHours + '&limit=10&show_known_bots=' + (showKnownBots ? 'true' : 'false') + '&show_allowed=' + (showAllowed ? 'true' : 'false'));
const payload = await response.json().catch(() => ({}));
if (!response.ok) {
return;
}
window.__overviewPayload = payload || {};
renderStats(payload || {});
renderLeaderboards(payload || {});
}
async function refresh() {
const recentResponse = await fetch('/api/recent-ips?hours=' + recentHours + '&limit=250');
const recentPayload = await recentResponse.json().catch(() => []);
refreshOverview();
if (!recentResponse.ok) {
const message = Array.isArray(recentPayload) ? recentResponse.statusText : (recentPayload.error || recentResponse.statusText);
document.getElementById('ips-body').innerHTML = '<tr><td colspan="7" class="muted">' + escapeHtml(message) + '</td></tr>';

23
internal/web/handler_test.go
View File

@@ -46,6 +46,16 @@ func TestHandlerServesOverviewAndManualActions(t *testing.T) {
t.Fatalf("unexpected overview payload: %+v", overview)
}
recorder = httptest.NewRecorder()
request = httptest.NewRequest(http.MethodGet, "/api/overview?hours=24&limit=10&show_known_bots=false&show_allowed=false", nil)
handler.ServeHTTP(recorder, request)
if recorder.Code != http.StatusOK {
t.Fatalf("unexpected filtered overview status: %d", recorder.Code)
}
if app.lastOverviewOptions.ShowKnownBots || app.lastOverviewOptions.ShowAllowed {
t.Fatalf("overview filter options were not forwarded correctly: %+v", app.lastOverviewOptions)
}
recorder = httptest.NewRecorder()
request = httptest.NewRequest(http.MethodPost, "/api/ips/203.0.113.10/block", strings.NewReader(`{"reason":"test reason","actor":"tester"}`))
request.Header.Set("Content-Type", "application/json")
@@ -92,6 +102,9 @@ func TestHandlerServesOverviewAndManualActions(t *testing.T) {
if !strings.Contains(recorder.Body.String(), "Top URLs by events") {
t.Fatalf("overview page should expose the top URLs block")
}
if !strings.Contains(recorder.Body.String(), "These two filters affect both the leaderboards and the Recent IPs list") {
t.Fatalf("overview page should explain the scope of the shared filters")
}
if !strings.Contains(recorder.Body.String(), "Show allowed") {
t.Fatalf("overview page should expose the allowed toggle")
}
@@ -130,10 +143,12 @@ func TestHandlerServesOverviewAndManualActions(t *testing.T) {
}
type stubApp struct {
lastAction string
lastAction string
lastOverviewOptions model.OverviewOptions
}
func (s *stubApp) GetOverview(context.Context, time.Time, int) (model.Overview, error) {
func (s *stubApp) GetOverview(_ context.Context, _ time.Time, _ int, options model.OverviewOptions) (model.Overview, error) {
s.lastOverviewOptions = options
now := time.Now().UTC()
return model.Overview{
TotalEvents: 1,
@@ -183,12 +198,12 @@ func (s *stubApp) GetOverview(context.Context, time.Time, int) (model.Overview,
}
func (s *stubApp) ListEvents(ctx context.Context, limit int) ([]model.Event, error) {
overview, _ := s.GetOverview(ctx, time.Time{}, limit)
overview, _ := s.GetOverview(ctx, time.Time{}, limit, model.OverviewOptions{ShowKnownBots: true, ShowAllowed: true})
return overview.RecentEvents, nil
}
func (s *stubApp) ListIPs(ctx context.Context, limit int, state string) ([]model.IPState, error) {
overview, _ := s.GetOverview(ctx, time.Time{}, limit)
overview, _ := s.GetOverview(ctx, time.Time{}, limit, model.OverviewOptions{ShowKnownBots: true, ShowAllowed: true})
return overview.RecentIPs, nil
}