infrastructure/caddy-opnsense-blocker
2
Fork 0
Code Issues Activity

Refine the dashboard and requests log UX

Browse Source
This commit is contained in:
Codex, agent ChatGPT
2026-03-12 17:40:13 +01:00
parent 0a14dd1df9
commit 0dfa30973e
8 changed files with 423 additions and 180 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
README.md
View File

@@ -129,6 +129,13 @@ The UI is backed by a small JSON API. The main endpoints are:
The legacy `POST /api/ips/{ip}/reset` route is still accepted as a backwards-compatible alias for `clear-override`.
The web UI itself exposes two main pages:
- `GET /` for the dashboard
- `GET /requests` for the paginated requests log
The legacy `GET /queries` route redirects permanently to `GET /requests`.
The full API reference, including payloads and response models, lives in [`docs/api.md`](docs/api.md).
## Configuration

46
docs/api.md
View File

@@ -25,7 +25,7 @@ Example response:
## `GET /api/overview`
Returns summary counters plus recent IP and recent event samples.
Returns summary counters, chart data, and dashboard leaderboards.
Query parameters:
@@ -37,15 +37,6 @@ Query parameters:
- optional
- default: `24`
- used for the top activity leaderboards returned in the same payload
- `show_known_bots`
- optional
- default: `true`
- when `false`, the leaderboards exclude IPs currently identified as known bots
- `show_allowed`
- optional
- default: `true`
- when `false`, the leaderboards exclude IPs whose current state is `allowed`
Main response fields:
- `total_events`
@@ -55,8 +46,13 @@ Main response fields:
- `allowed_ips`
- `observed_ips`
- `activity_since`
- `top_ips_by_events`
- `top_ips_by_traffic`
- `activity_buckets`
- `methods`
- `bots`
- `top_bot_ips_by_events`
- `top_non_bot_ips_by_events`
- `top_bot_ips_by_traffic`
- `top_non_bot_ips_by_traffic`
- `top_sources`
- `top_urls`
- `recent_ips`
@@ -64,7 +60,7 @@ Main response fields:
## `GET /api/events`
Returns recent raw events.
Returns a paginated slice of recent raw events.
Query parameters:
@@ -72,6 +68,30 @@ Query parameters:
- optional
- default: `100`
- maximum: `1000`
- `page`
- optional
- default: `1`
- one-based page number
- `show_known_bots`
- optional
- default: `true`
- when `false`, verified bots are hidden from the log
- `show_allowed`
- optional
- default: `true`
- when `false`, rows whose current IP state is `allowed` are hidden
- `review_only`
- optional
- default: `false`
- when `true`, only requests whose current IP state is `review` are returned
Main response fields:
- `items`
- `page`
- `limit`
- `has_prev`
- `has_next`
Each event includes:

13
internal/model/types.go
View File

@@ -234,6 +234,7 @@ type EventListOptions struct {
ShowKnownBots bool
ShowAllowed bool
ReviewOnly bool
Offset int
}
type SourceOffset struct {
@@ -266,9 +267,21 @@ type Overview struct {
Methods []MethodBreakdownRow `json:"methods"`
Bots []BotBreakdownRow `json:"bots"`
TopIPsByEvents []TopIPRow `json:"top_ips_by_events"`
TopBotIPsByEvents []TopIPRow `json:"top_bot_ips_by_events"`
TopNonBotIPsByEvents []TopIPRow `json:"top_non_bot_ips_by_events"`
TopIPsByTraffic []TopIPRow `json:"top_ips_by_traffic"`
TopBotIPsByTraffic []TopIPRow `json:"top_bot_ips_by_traffic"`
TopNonBotIPsByTraffic []TopIPRow `json:"top_non_bot_ips_by_traffic"`
TopSources []TopSourceRow `json:"top_sources"`
TopURLs []TopURLRow `json:"top_urls"`
RecentIPs []IPState `json:"recent_ips"`
RecentEvents []Event `json:"recent_events"`
}
type EventPage struct {
Items []Event `json:"items"`
Page int `json:"page"`
Limit int `json:"limit"`
HasPrev bool `json:"has_prev"`
HasNext bool `json:"has_next"`
}

37
internal/service/service.go
View File

@@ -725,28 +725,35 @@ func (s *Service) decorateOverviewTopIPs(ctx context.Context, overview *model.Ov
if overview == nil {
return nil
}
ips := append(topIPRowIPs(overview.TopIPsByEvents), topIPRowIPs(overview.TopIPsByTraffic)...)
ips := append([]string{}, topIPRowIPs(overview.TopIPsByEvents)...)
ips = append(ips, topIPRowIPs(overview.TopBotIPsByEvents)...)
ips = append(ips, topIPRowIPs(overview.TopNonBotIPsByEvents)...)
ips = append(ips, topIPRowIPs(overview.TopIPsByTraffic)...)
ips = append(ips, topIPRowIPs(overview.TopBotIPsByTraffic)...)
ips = append(ips, topIPRowIPs(overview.TopNonBotIPsByTraffic)...)
investigations, err := s.store.GetInvestigationsForIPs(ctx, ips)
if err != nil {
return err
}
for index := range overview.TopIPsByEvents {
if investigation, ok := investigations[overview.TopIPsByEvents[index].IP]; ok {
overview.TopIPsByEvents[index].Bot = investigation.Bot
} else {
s.enqueueInvestigation(overview.TopIPsByEvents[index].IP)
}
}
for index := range overview.TopIPsByTraffic {
if investigation, ok := investigations[overview.TopIPsByTraffic[index].IP]; ok {
overview.TopIPsByTraffic[index].Bot = investigation.Bot
} else {
s.enqueueInvestigation(overview.TopIPsByTraffic[index].IP)
}
}
decorateTopIPRows(overview.TopIPsByEvents, investigations, s.enqueueInvestigation)
decorateTopIPRows(overview.TopBotIPsByEvents, investigations, s.enqueueInvestigation)
decorateTopIPRows(overview.TopNonBotIPsByEvents, investigations, s.enqueueInvestigation)
decorateTopIPRows(overview.TopIPsByTraffic, investigations, s.enqueueInvestigation)
decorateTopIPRows(overview.TopBotIPsByTraffic, investigations, s.enqueueInvestigation)
decorateTopIPRows(overview.TopNonBotIPsByTraffic, investigations, s.enqueueInvestigation)
return nil
}
func decorateTopIPRows(items []model.TopIPRow, investigations map[string]model.IPInvestigation, enqueue func(string)) {
for index := range items {
if investigation, ok := investigations[items[index].IP]; ok {
items[index].Bot = investigation.Bot
} else {
enqueue(items[index].IP)
}
}
}
func topIPRowIPs(items []model.TopIPRow) []string {
result := make([]string, 0, len(items))
for _, item := range items {

49
internal/store/store.go
View File

@@ -236,6 +236,16 @@ func knownBotExistsClause(ipExpression string) string {
)`
}
func anyBotExistsClause(ipExpression string) string {
return `EXISTS (
SELECT 1
FROM ip_investigations i
WHERE i.ip = ` + ipExpression + `
AND json_valid(i.payload_json)
AND json_type(i.payload_json, '$.bot') IS NOT NULL
)`
}
func overviewFilterQueryParts(options model.OverviewOptions) (joins []string, clauses []string) {
if !options.ShowAllowed {
joins = append(joins, `LEFT JOIN ip_state s ON s.ip = e.client_ip`)
@@ -443,11 +453,27 @@ func (s *Store) GetOverview(ctx context.Context, since time.Time, limit int, opt
if err != nil {
return model.Overview{}, err
}
topIPsByEvents, err := s.listTopIPRows(ctx, since, limit, "events", options)
topIPsByEvents, err := s.listTopIPRows(ctx, since, limit, "events", options, "all")
if err != nil {
return model.Overview{}, err
}
topIPsByTraffic, err := s.listTopIPRows(ctx, since, limit, "traffic", options)
topBotIPsByEvents, err := s.listTopIPRows(ctx, since, limit, "events", options, "bots")
if err != nil {
return model.Overview{}, err
}
topNonBotIPsByEvents, err := s.listTopIPRows(ctx, since, limit, "events", options, "non-bots")
if err != nil {
return model.Overview{}, err
}
topIPsByTraffic, err := s.listTopIPRows(ctx, since, limit, "traffic", options, "all")
if err != nil {
return model.Overview{}, err
}
topBotIPsByTraffic, err := s.listTopIPRows(ctx, since, limit, "traffic", options, "bots")
if err != nil {
return model.Overview{}, err
}
topNonBotIPsByTraffic, err := s.listTopIPRows(ctx, since, limit, "traffic", options, "non-bots")
if err != nil {
return model.Overview{}, err
}
@@ -477,17 +503,27 @@ func (s *Store) GetOverview(ctx context.Context, since time.Time, limit int, opt
overview.Methods = methods
overview.Bots = bots
overview.TopIPsByEvents = topIPsByEvents
overview.TopBotIPsByEvents = topBotIPsByEvents
overview.TopNonBotIPsByEvents = topNonBotIPsByEvents
overview.TopIPsByTraffic = topIPsByTraffic
overview.TopBotIPsByTraffic = topBotIPsByTraffic
overview.TopNonBotIPsByTraffic = topNonBotIPsByTraffic
overview.TopSources = topSources
overview.TopURLs = topURLs
return overview, nil
}
func (s *Store) listTopIPRows(ctx context.Context, since time.Time, limit int, orderBy string, options model.OverviewOptions) ([]model.TopIPRow, error) {
func (s *Store) listTopIPRows(ctx context.Context, since time.Time, limit int, orderBy string, options model.OverviewOptions, botScope string) ([]model.TopIPRow, error) {
if limit <= 0 {
limit = 10
}
joins, clauses := overviewFilterQueryParts(options)
switch botScope {
case "bots":
clauses = append(clauses, anyBotExistsClause(`e.client_ip`))
case "non-bots":
clauses = append(clauses, `NOT `+anyBotExistsClause(`e.client_ip`))
}
query := fmt.Sprintf(`
SELECT e.client_ip,
COUNT(*) AS event_count,
@@ -797,6 +833,9 @@ func (s *Store) ListEvents(ctx context.Context, since time.Time, limit int, opti
if limit <= 0 {
limit = 100
}
if options.Offset < 0 {
options.Offset = 0
}
joins, clauses := eventFilterQueryParts(options)
query := `
SELECT e.id, e.source_name, e.profile_name, e.occurred_at, e.remote_ip, e.client_ip, e.host,
@@ -815,8 +854,8 @@ func (s *Store) ListEvents(ctx context.Context, since time.Time, limit int, opti
if len(clauses) > 0 {
query += ` WHERE ` + strings.Join(clauses, ` AND `)
}
query += ` ORDER BY e.occurred_at DESC, e.id DESC LIMIT ?`
args = append(args, limit)
query += ` ORDER BY e.occurred_at DESC, e.id DESC LIMIT ? OFFSET ?`
args = append(args, limit, options.Offset)
rows, err := s.db.QueryContext(ctx, query, args...)
if err != nil {

37
internal/store/store_test.go
View File

@@ -109,6 +109,12 @@ func TestStoreRecordsEventsAndState(t *testing.T) {
if len(overview.TopIPsByEvents) != 1 || overview.TopIPsByEvents[0].IP != event.ClientIP {
t.Fatalf("unexpected top ips by events: %+v", overview.TopIPsByEvents)
}
if len(overview.TopBotIPsByEvents) != 0 {
t.Fatalf("expected no bot top ips by events before investigation, got %+v", overview.TopBotIPsByEvents)
}
if len(overview.TopNonBotIPsByEvents) != 1 || overview.TopNonBotIPsByEvents[0].IP != event.ClientIP {
t.Fatalf("unexpected top non-bot ips by events: %+v", overview.TopNonBotIPsByEvents)
}
if len(overview.TopSources) != 1 || overview.TopSources[0].SourceName != event.SourceName {
t.Fatalf("unexpected top sources: %+v", overview.TopSources)
}
@@ -252,6 +258,9 @@ func TestStoreOverviewLeaderboardsUseTrafficFromRawJSON(t *testing.T) {
if overview.TopIPsByEvents[0].IP != "203.0.113.10" || overview.TopIPsByEvents[0].Events != 2 || overview.TopIPsByEvents[0].TrafficBytes != 3072 {
t.Fatalf("unexpected top IP by events row: %+v", overview.TopIPsByEvents[0])
}
if len(overview.TopNonBotIPsByEvents) < 2 || overview.TopNonBotIPsByEvents[0].IP != "203.0.113.10" || overview.TopNonBotIPsByEvents[0].Events != 2 {
t.Fatalf("unexpected top non-bot IP by events rows: %+v", overview.TopNonBotIPsByEvents)
}
if len(overview.TopIPsByTraffic) < 2 {
t.Fatalf("expected at least 2 top IP rows by traffic, got %+v", overview.TopIPsByTraffic)
}
@@ -272,6 +281,22 @@ func TestStoreOverviewLeaderboardsUseTrafficFromRawJSON(t *testing.T) {
}); err != nil {
t.Fatalf("save top bot investigation: %v", err)
}
refreshedOverview, err := db.GetOverview(ctx, baseTime.Add(-time.Minute), 10, model.OverviewOptions{ShowKnownBots: true, ShowAllowed: true})
if err != nil {
t.Fatalf("get refreshed overview: %v", err)
}
if len(refreshedOverview.TopBotIPsByEvents) == 0 || refreshedOverview.TopBotIPsByEvents[0].IP != "203.0.113.10" {
t.Fatalf("unexpected top bot IPs by events after investigation: %+v", refreshedOverview.TopBotIPsByEvents)
}
if len(refreshedOverview.TopNonBotIPsByEvents) == 0 || refreshedOverview.TopNonBotIPsByEvents[0].IP != "203.0.113.20" {
t.Fatalf("unexpected top non-bot IPs by events after investigation: %+v", refreshedOverview.TopNonBotIPsByEvents)
}
if len(refreshedOverview.TopBotIPsByTraffic) == 0 || refreshedOverview.TopBotIPsByTraffic[0].IP != "203.0.113.10" {
t.Fatalf("unexpected top bot IPs by traffic after investigation: %+v", refreshedOverview.TopBotIPsByTraffic)
}
if len(refreshedOverview.TopNonBotIPsByTraffic) == 0 || refreshedOverview.TopNonBotIPsByTraffic[0].IP != "203.0.113.20" {
t.Fatalf("unexpected top non-bot IPs by traffic after investigation: %+v", refreshedOverview.TopNonBotIPsByTraffic)
}
if _, err := db.SetManualOverride(ctx, "203.0.113.20", model.ManualOverrideForceAllow, model.IPStateAllowed, "manual allow"); err != nil {
t.Fatalf("set manual override for filter test: %v", err)
}
@@ -283,9 +308,21 @@ func TestStoreOverviewLeaderboardsUseTrafficFromRawJSON(t *testing.T) {
if len(filtered.TopIPsByEvents) != 0 {
t.Fatalf("expected filtered top IPs by events to be empty, got %+v", filtered.TopIPsByEvents)
}
if len(filtered.TopBotIPsByEvents) != 0 {
t.Fatalf("expected filtered top bot IPs by events to be empty, got %+v", filtered.TopBotIPsByEvents)
}
if len(filtered.TopNonBotIPsByEvents) != 0 {
t.Fatalf("expected filtered top non-bot IPs by events to be empty, got %+v", filtered.TopNonBotIPsByEvents)
}
if len(filtered.TopIPsByTraffic) != 0 {
t.Fatalf("expected filtered top IPs by traffic to be empty, got %+v", filtered.TopIPsByTraffic)
}
if len(filtered.TopBotIPsByTraffic) != 0 {
t.Fatalf("expected filtered top bot IPs by traffic to be empty, got %+v", filtered.TopBotIPsByTraffic)
}
if len(filtered.TopNonBotIPsByTraffic) != 0 {
t.Fatalf("expected filtered top non-bot IPs by traffic to be empty, got %+v", filtered.TopNonBotIPsByTraffic)
}
if len(filtered.TopSources) != 0 {
t.Fatalf("expected filtered top sources to be empty, got %+v", filtered.TopSources)
}

256
internal/web/handler.go
View File

@@ -55,6 +55,7 @@ func NewHandler(app App) http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("/", h.handleOverviewPage)
mux.HandleFunc("/requests", h.handleQueryLogPage)
mux.HandleFunc("/queries", h.handleQueryLogPage)
mux.HandleFunc("/healthz", h.handleHealth)
mux.HandleFunc("/ips/", h.handleIPPage)
@@ -79,7 +80,11 @@ func (h *handler) handleOverviewPage(w http.ResponseWriter, r *http.Request) {
}
func (h *handler) handleQueryLogPage(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/queries" {
if r.URL.Path == "/queries" {
http.Redirect(w, r, "/requests", http.StatusMovedPermanently)
return
}
if r.URL.Path != "/requests" {
http.NotFound(w, r)
return
}
@@ -87,7 +92,7 @@ func (h *handler) handleQueryLogPage(w http.ResponseWriter, r *http.Request) {
methodNotAllowed(w)
return
}
renderTemplate(w, h.queryLogPage, pageData{Title: "Query Log"})
renderTemplate(w, h.queryLogPage, pageData{Title: "Requests Log"})
}
func (h *handler) handleIPPage(w http.ResponseWriter, r *http.Request) {
@@ -122,10 +127,7 @@ func (h *handler) handleAPIOverview(w http.ResponseWriter, r *http.Request) {
hours = 24
}
since := time.Now().UTC().Add(-time.Duration(hours) * time.Hour)
options := model.OverviewOptions{
ShowKnownBots: queryBool(r, "show_known_bots", true),
ShowAllowed: queryBool(r, "show_allowed", true),
}
options := model.OverviewOptions{ShowKnownBots: true, ShowAllowed: true}
overview, err := h.app.GetOverview(r.Context(), since, limit, options)
if err != nil {
writeError(w, http.StatusInternalServerError, err)
@@ -144,18 +146,33 @@ func (h *handler) handleAPIEvents(w http.ResponseWriter, r *http.Request) {
if hours <= 0 {
hours = 24
}
page := queryInt(r, "page", 1)
if page <= 0 {
page = 1
}
since := time.Now().UTC().Add(-time.Duration(hours) * time.Hour)
options := model.EventListOptions{
ShowKnownBots: queryBool(r, "show_known_bots", true),
ShowAllowed: queryBool(r, "show_allowed", true),
ReviewOnly: queryBool(r, "review_only", false),
Offset: (page - 1) * limit,
}
events, err := h.app.ListEvents(r.Context(), since, limit, options)
events, err := h.app.ListEvents(r.Context(), since, limit+1, options)
if err != nil {
writeError(w, http.StatusInternalServerError, err)
return
}
writeJSON(w, http.StatusOK, events)
hasNext := len(events) > limit
if hasNext {
events = events[:limit]
}
writeJSON(w, http.StatusOK, model.EventPage{
Items: events,
Page: page,
Limit: limit,
HasPrev: page > 1,
HasNext: hasNext,
})
}
func (h *handler) handleAPIIPs(w http.ResponseWriter, r *http.Request) {
@@ -481,7 +498,7 @@ const overviewHTML = `<!doctype html>
</div>
<nav class="nav">
<a class="nav-link active" href="/">Dashboard</a>
<a class="nav-link" href="/queries">Query Log</a>
<a class="nav-link" href="/requests">Requests Log</a>
</nav>
</div>
</header>
@@ -494,13 +511,6 @@ const overviewHTML = `<!doctype html>
<div class="card allowed"><div class="muted">Allowed</div><div class="stat-value">—</div></div>
<div class="card observed"><div class="muted">Observed</div><div class="stat-value">—</div></div>
</section>
<section class="panel controls">
<div class="controls-group">
<label class="toggle"><input id="show-bots-toggle" type="checkbox" checked onchange="toggleKnownBots()">Show known bots</label>
<label class="toggle"><input id="show-allowed-toggle" type="checkbox" checked onchange="toggleAllowed()">Show allowed</label>
</div>
<div class="muted">These filters affect all dashboard charts and top lists.</div>
</section>
<section class="dashboard-grid">
<section class="panel chart-card chart-wide">
<div class="panel-head">
@@ -533,7 +543,7 @@ const overviewHTML = `<!doctype html>
</section>
<section class="leaders" id="leaderboards">
<section class="leader-card">
<h2>Top IPs by events</h2>
<h2>Top bot IPs by events</h2>
<div class="muted">Last 24 hours</div>
<ol class="leader-list">
<li class="leader-item"><div class="leader-main"><span class="mono leader-placeholder">Loading…</span><span class="leader-value leader-placeholder">—</span></div></li>
@@ -542,7 +552,25 @@ const overviewHTML = `<!doctype html>
</ol>
</section>
<section class="leader-card">
<h2>Top IPs by traffic</h2>
<h2>Top non-bot IPs by events</h2>
<div class="muted">Last 24 hours</div>
<ol class="leader-list">
<li class="leader-item"><div class="leader-main"><span class="mono leader-placeholder">Loading…</span><span class="leader-value leader-placeholder">—</span></div></li>
<li class="leader-item"><div class="leader-main"><span class="mono leader-placeholder">Loading…</span><span class="leader-value leader-placeholder">—</span></div></li>
<li class="leader-item"><div class="leader-main"><span class="mono leader-placeholder">Loading…</span><span class="leader-value leader-placeholder">—</span></div></li>
</ol>
</section>
<section class="leader-card">
<h2>Top bot IPs by traffic</h2>
<div class="muted">Last 24 hours</div>
<ol class="leader-list">
<li class="leader-item"><div class="leader-main"><span class="mono leader-placeholder">Loading…</span><span class="leader-value leader-placeholder">—</span></div></li>
<li class="leader-item"><div class="leader-main"><span class="mono leader-placeholder">Loading…</span><span class="leader-value leader-placeholder">—</span></div></li>
<li class="leader-item"><div class="leader-main"><span class="mono leader-placeholder">Loading…</span><span class="leader-value leader-placeholder">—</span></div></li>
</ol>
</section>
<section class="leader-card">
<h2>Top non-bot IPs by traffic</h2>
<div class="muted">Last 24 hours</div>
<ol class="leader-list">
<li class="leader-item"><div class="leader-main"><span class="mono leader-placeholder">Loading…</span><span class="leader-value leader-placeholder">—</span></div></li>
@@ -573,21 +601,6 @@ const overviewHTML = `<!doctype html>
<script>
const recentHours = 24;
const sourcePalette = ['#38bdf8', '#22c55e', '#f59e0b', '#a78bfa', '#f97316', '#14b8a6', '#ec4899', '#60a5fa', '#84cc16', '#ef4444'];
let showKnownBots = loadBooleanPreference('cob.dashboard.showKnownBots', true);
let showAllowed = loadBooleanPreference('cob.dashboard.showAllowed', true);
function loadBooleanPreference(key, fallback) {
try {
const raw = localStorage.getItem(key);
return raw === null ? fallback : raw === 'true';
} catch (error) {
return fallback;
}
}
function saveBooleanPreference(key, value) {
localStorage.setItem(key, value ? 'true' : 'false');
}
function escapeHtml(value) {
return String(value ?? '')
@@ -727,8 +740,10 @@ const overviewHTML = `<!doctype html>
function renderLeaderboards(data) {
const cards = [
{ title: 'Top IPs by events', subtitle: 'Last 24 hours', body: renderTopIPs(data.top_ips_by_events, 'events') },
{ title: 'Top IPs by traffic', subtitle: 'Last 24 hours', body: renderTopIPs(data.top_ips_by_traffic, 'traffic') },
{ title: 'Top bot IPs by events', subtitle: 'Last 24 hours', body: renderTopIPs(data.top_bot_ips_by_events, 'events') },
{ title: 'Top non-bot IPs by events', subtitle: 'Last 24 hours', body: renderTopIPs(data.top_non_bot_ips_by_events, 'events') },
{ title: 'Top bot IPs by traffic', subtitle: 'Last 24 hours', body: renderTopIPs(data.top_bot_ips_by_traffic, 'traffic') },
{ title: 'Top non-bot IPs by traffic', subtitle: 'Last 24 hours', body: renderTopIPs(data.top_non_bot_ips_by_traffic, 'traffic') },
{ title: 'Top sources by events', subtitle: 'Last 24 hours', body: renderTopSources(data.top_sources) },
{ title: 'Top URLs by events', subtitle: 'Last 24 hours', body: renderTopURLs(data.top_urls) },
];
@@ -768,7 +783,7 @@ const overviewHTML = `<!doctype html>
}
}
const orderedSources = [...totalsBySource.entries()].sort((left, right) => right[1] - left[1] || left[0].localeCompare(right[0]));
legend.innerHTML = orderedSources.map(entry => '<span class="legend-chip"><span class="legend-dot" style="background:' + colorForSource(entry[0]) + '"></span>' + escapeHtml(entry[0]) + '</span>').join('');
legend.innerHTML = orderedSources.map(entry => '<span class="legend-chip"><span class="legend-dot" style="background:' + colorForSource(entry[0]) + '"></span>' + escapeHtml(entry[0]) + '<span class="muted">' + escapeHtml(String(entry[1])) + '</span></span>').join('');
const width = 1100;
const height = 260;
@@ -848,34 +863,8 @@ const overviewHTML = `<!doctype html>
renderDonut('bots-chart', Array.isArray(data.bots) ? data.bots : [], colors, 'requests');
}
function applyToggles() {
const botsToggle = document.getElementById('show-bots-toggle');
if (botsToggle) {
botsToggle.checked = showKnownBots;
}
const allowedToggle = document.getElementById('show-allowed-toggle');
if (allowedToggle) {
allowedToggle.checked = showAllowed;
}
}
function toggleKnownBots() {
const toggle = document.getElementById('show-bots-toggle');
showKnownBots = !toggle || toggle.checked;
saveBooleanPreference('cob.dashboard.showKnownBots', showKnownBots);
refresh();
}
function toggleAllowed() {
const toggle = document.getElementById('show-allowed-toggle');
showAllowed = !toggle || toggle.checked;
saveBooleanPreference('cob.dashboard.showAllowed', showAllowed);
refresh();
}
async function refresh() {
applyToggles();
const response = await fetch('/api/overview?hours=' + recentHours + '&limit=10&show_known_bots=' + (showKnownBots ? 'true' : 'false') + '&show_allowed=' + (showAllowed ? 'true' : 'false'));
const response = await fetch('/api/overview?hours=' + recentHours + '&limit=10');
const payload = await response.json().catch(() => ({}));
if (!response.ok) {
return;
@@ -886,8 +875,6 @@ const overviewHTML = `<!doctype html>
renderBots(payload || {});
renderLeaderboards(payload || {});
}
applyToggles();
refresh();
setInterval(refresh, 5000);
</script>
@@ -914,16 +901,23 @@ const queryLogHTML = `<!doctype html>
a:hover { text-decoration: underline; }
.muted { color: #94a3b8; }
.mono { font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace; }
.panel { background: #111827; border: 1px solid #334155; border-radius: .85rem; padding: 1rem; overflow: auto; }
.panel { background: #111827; border: 1px solid #334155; border-radius: .85rem; padding: 1rem; overflow: hidden; }
.controls { display: flex; justify-content: space-between; align-items: center; gap: 1rem; flex-wrap: wrap; }
.controls-group { display: flex; align-items: center; gap: 1rem; flex-wrap: wrap; }
.toggle { display: inline-flex; align-items: center; gap: .45rem; font-size: .95rem; color: #cbd5e1; }
.toggle input { margin: 0; }
.toolbar { display: flex; justify-content: space-between; align-items: flex-start; gap: 1rem; margin-bottom: .75rem; flex-wrap: wrap; }
table { width: 100%; border-collapse: collapse; min-width: 1024px; }
.toolbar { display: flex; justify-content: space-between; align-items: center; gap: 1rem; margin-bottom: .75rem; flex-wrap: wrap; }
.toolbar-actions { display: flex; align-items: center; gap: .65rem; flex-wrap: wrap; }
.page-status { color: #cbd5e1; font-size: .92rem; }
.table-shell { overflow: hidden; }
table { width: 100%; border-collapse: collapse; table-layout: fixed; }
th, td { padding: .6rem .65rem; border-bottom: 1px solid #1e293b; text-align: left; vertical-align: top; }
thead th { color: #93c5fd; white-space: nowrap; }
thead th { color: #93c5fd; }
tbody tr:nth-child(even) { background: rgba(15, 23, 42, .55); }
th.tight, td.tight { white-space: nowrap; width: 1%; }
th.request-col, td.request-cell { width: auto; }
td.request-cell { overflow: hidden; }
.request-text { display: block; overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
.status { display: inline-block; padding: .15rem .45rem; border-radius: 999px; font-size: .8rem; background: #1e293b; }
.status.blocked { background: #7f1d1d; }
.status.review { background: #78350f; }
@@ -940,6 +934,7 @@ const queryLogHTML = `<!doctype html>
button { background: #2563eb; color: white; border: 0; cursor: pointer; }
button.secondary { background: #475569; }
button.danger { background: #dc2626; }
button[disabled] { opacity: .5; cursor: default; }
.ip-cell { display: flex; align-items: center; gap: .45rem; min-width: 0; }
.bot-chip { display: inline-flex; align-items: center; justify-content: center; width: 1.25rem; height: 1.25rem; border-radius: 999px; border: 1px solid #334155; background: #0f172a; color: #e2e8f0; font-size: .72rem; font-weight: 700; cursor: help; flex: 0 0 auto; }
.bot-chip.verified { border-color: #2563eb; }
@@ -956,12 +951,17 @@ const queryLogHTML = `<!doctype html>
.bot-chip.yandex { background: #dc2626; color: white; }
.bot-chip.baidu { background: #7c3aed; color: white; }
.bot-chip.bytespider { background: #111827; color: white; }
@media (max-width: 960px) {
.toolbar, .controls { align-items: flex-start; }
.toolbar-actions, .controls-group { width: 100%; justify-content: flex-start; }
th, td { font-size: .88rem; }
}
@media (max-width: 720px) {
header { padding: .9rem 1rem; }
main { padding: 1rem; }
.panel { padding: .85rem; }
.controls { align-items: flex-start; }
.controls-group { width: 100%; justify-content: flex-start; }
.table-shell { overflow-x: auto; }
table { min-width: 900px; }
}
</style>
</head>
@@ -974,7 +974,7 @@ const queryLogHTML = `<!doctype html>
</div>
<nav class="nav">
<a class="nav-link" href="/">Dashboard</a>
<a class="nav-link active" href="/queries">Query Log</a>
<a class="nav-link active" href="/requests">Requests Log</a>
</nav>
</div>
</header>
@@ -984,8 +984,9 @@ const queryLogHTML = `<!doctype html>
<label class="toggle"><input id="show-bots-toggle" type="checkbox" checked onchange="toggleKnownBots()">Show known bots</label>
<label class="toggle"><input id="show-allowed-toggle" type="checkbox" checked onchange="toggleAllowed()">Show allowed</label>
<label class="toggle"><input id="show-review-toggle" type="checkbox" onchange="toggleReviewOnly()">Review only</label>
<label class="toggle"><input id="auto-refresh-toggle" type="checkbox" onchange="toggleAutoRefresh()">Auto refresh</label>
</div>
<div class="muted">These filters affect the full Query Log.</div>
<div class="muted">These filters affect the full Requests Log.</div>
</section>
<section class="panel">
<div class="toolbar">
@@ -993,32 +994,44 @@ const queryLogHTML = `<!doctype html>
<h2>Recent requests</h2>
<div class="muted">Click an IP to open its detail page</div>
</div>
<div class="toolbar-actions">
<button class="secondary" type="button" onclick="refreshNow()">Refresh now</button>
<div class="page-status" id="page-status">Page 1</div>
<button class="secondary" type="button" id="prev-page" onclick="goToPreviousPage()">Previous</button>
<button class="secondary" type="button" id="next-page" onclick="goToNextPage()">Next</button>
</div>
</div>
<div class="table-shell">
<table>
<thead>
<tr>
<th>Time</th>
<th>Source</th>
<th>IP</th>
<th>Method</th>
<th>Request</th>
<th>Status</th>
<th>State</th>
<th>Reason</th>
<th>Actions</th>
<th class="tight">Time</th>
<th class="tight">Source</th>
<th class="tight">IP</th>
<th class="tight">Method</th>
<th class="request-col">Request</th>
<th class="tight">Status</th>
<th class="tight">State</th>
<th class="tight">Reason</th>
<th class="tight">Actions</th>
</tr>
</thead>
<tbody id="events-body">
<tr><td colspan="9" class="muted">Loading query log…</td></tr>
<tr><td colspan="9" class="muted">Loading requests log…</td></tr>
</tbody>
</table>
</div>
</section>
</main>
<script>
const recentHours = 24;
const pageSize = 100;
let showKnownBots = loadBooleanPreference('cob.dashboard.showKnownBots', true);
let showAllowed = loadBooleanPreference('cob.dashboard.showAllowed', true);
let showReviewOnly = loadBooleanPreference('cob.queryLog.showReviewOnly', false);
let autoRefresh = loadBooleanPreference('cob.requests.autoRefresh', false);
let currentPage = 1;
let refreshTimer = null;
function loadBooleanPreference(key, fallback) {
try {
@@ -1111,23 +1124,67 @@ const queryLogHTML = `<!doctype html>
document.getElementById('show-bots-toggle').checked = showKnownBots;
document.getElementById('show-allowed-toggle').checked = showAllowed;
document.getElementById('show-review-toggle').checked = showReviewOnly;
document.getElementById('auto-refresh-toggle').checked = autoRefresh;
}
function updatePager(payload) {
const page = Number(payload.page || currentPage || 1);
document.getElementById('page-status').textContent = 'Page ' + page;
document.getElementById('prev-page').disabled = !payload.has_prev;
document.getElementById('next-page').disabled = !payload.has_next;
}
function scheduleRefresh() {
if (refreshTimer) {
window.clearTimeout(refreshTimer);
refreshTimer = null;
}
if (autoRefresh) {
refreshTimer = window.setTimeout(refresh, 5000);
}
}
function toggleKnownBots() {
showKnownBots = document.getElementById('show-bots-toggle').checked;
saveBooleanPreference('cob.dashboard.showKnownBots', showKnownBots);
currentPage = 1;
refresh();
}
function toggleAllowed() {
showAllowed = document.getElementById('show-allowed-toggle').checked;
saveBooleanPreference('cob.dashboard.showAllowed', showAllowed);
currentPage = 1;
refresh();
}
function toggleReviewOnly() {
showReviewOnly = document.getElementById('show-review-toggle').checked;
saveBooleanPreference('cob.queryLog.showReviewOnly', showReviewOnly);
currentPage = 1;
refresh();
}
function toggleAutoRefresh() {
autoRefresh = document.getElementById('auto-refresh-toggle').checked;
saveBooleanPreference('cob.requests.autoRefresh', autoRefresh);
scheduleRefresh();
}
function goToPreviousPage() {
if (currentPage <= 1) {
return;
}
currentPage -= 1;
refresh();
}
function goToNextPage() {
currentPage += 1;
refresh();
}
function refreshNow() {
refresh();
}
@@ -1149,40 +1206,45 @@ const queryLogHTML = `<!doctype html>
refresh();
}
function renderEvents(items) {
const rows = (Array.isArray(items) ? items : []).map(item => {
function renderEvents(payload) {
const items = Array.isArray(payload.items) ? payload.items : [];
const rows = items.map(item => {
const requestLabel = ((item.host || '') ? (item.host + item.uri) : (item.uri || '—'));
return [
'<tr>',
' <td>' + escapeHtml(formatDate(item.occurred_at)) + '</td>',
' <td>' + escapeHtml(item.source_name || '—') + '</td>',
' <td class="mono"><div class="ip-cell">' + renderBotChip(item.bot) + '<a href="/ips/' + encodeURIComponent(item.client_ip) + '">' + escapeHtml(item.client_ip || '—') + '</a></div></td>',
' <td><span class="method ' + escapeHtml(methodClass(item.method)) + '">' + escapeHtml(item.method || 'OTHER') + '</span></td>',
' <td class="mono">' + escapeHtml(requestLabel) + '</td>',
' <td>' + escapeHtml(String(item.status || 0)) + '</td>',
' <td><span class="status ' + escapeHtml(item.current_state || 'observed') + '">' + escapeHtml(item.current_state || 'observed') + '</span></td>',
' <td>' + escapeHtml(item.decision_reason || '—') + '</td>',
' <td>' + renderActions(item) + '</td>',
' <td class="tight">' + escapeHtml(formatDate(item.occurred_at)) + '</td>',
' <td class="tight">' + escapeHtml(item.source_name || '—') + '</td>',
' <td class="tight mono"><div class="ip-cell">' + renderBotChip(item.bot) + '<a href="/ips/' + encodeURIComponent(item.client_ip) + '">' + escapeHtml(item.client_ip || '—') + '</a></div></td>',
' <td class="tight"><span class="method ' + escapeHtml(methodClass(item.method)) + '">' + escapeHtml(item.method || 'OTHER') + '</span></td>',
' <td class="request-cell mono"><span class="request-text" title="' + escapeHtml(requestLabel) + '">' + escapeHtml(requestLabel) + '</span></td>',
' <td class="tight">' + escapeHtml(String(item.status || 0)) + '</td>',
' <td class="tight"><span class="status ' + escapeHtml(item.current_state || 'observed') + '">' + escapeHtml(item.current_state || 'observed') + '</span></td>',
' <td class="tight">' + escapeHtml(item.decision_reason || '—') + '</td>',
' <td class="tight">' + renderActions(item) + '</td>',
'</tr>'
].join('');
});
document.getElementById('events-body').innerHTML = rows.length ? rows.join('') : '<tr><td colspan="9" class="muted">No requests match the current filters in the last 24 hours.</td></tr>';
updatePager(payload || {});
}
async function refresh() {
applyToggles();
const response = await fetch('/api/events?hours=' + recentHours + '&limit=250&show_known_bots=' + (showKnownBots ? 'true' : 'false') + '&show_allowed=' + (showAllowed ? 'true' : 'false') + '&review_only=' + (showReviewOnly ? 'true' : 'false'));
const payload = await response.json().catch(() => []);
const response = await fetch('/api/events?hours=' + recentHours + '&limit=' + pageSize + '&page=' + currentPage + '&show_known_bots=' + (showKnownBots ? 'true' : 'false') + '&show_allowed=' + (showAllowed ? 'true' : 'false') + '&review_only=' + (showReviewOnly ? 'true' : 'false'));
const payload = await response.json().catch(() => ({ items: [] }));
if (!response.ok) {
document.getElementById('events-body').innerHTML = '<tr><td colspan="9" class="muted">' + escapeHtml(payload.error || response.statusText) + '</td></tr>';
updatePager({ page: currentPage, has_prev: currentPage > 1, has_next: false });
scheduleRefresh();
return;
}
currentPage = Number(payload.page || currentPage || 1);
renderEvents(payload);
scheduleRefresh();
}
applyToggles();
refresh();
setInterval(refresh, 5000);
</script>
</body>
</html>`

110
internal/web/handler_test.go
View File

@@ -52,17 +52,24 @@ func TestHandlerServesOverviewAndManualActions(t *testing.T) {
if recorder.Code != http.StatusOK {
t.Fatalf("unexpected filtered overview status: %d", recorder.Code)
}
if app.lastOverviewOptions.ShowKnownBots || app.lastOverviewOptions.ShowAllowed {
t.Fatalf("overview filter options were not forwarded correctly: %+v", app.lastOverviewOptions)
if !app.lastOverviewOptions.ShowKnownBots || !app.lastOverviewOptions.ShowAllowed {
t.Fatalf("overview should always use the unfiltered dashboard data: %+v", app.lastOverviewOptions)
}
recorder = httptest.NewRecorder()
request = httptest.NewRequest(http.MethodGet, "/api/events?hours=24&limit=250&show_known_bots=false&show_allowed=false&review_only=true", nil)
request = httptest.NewRequest(http.MethodGet, "/api/events?hours=24&limit=250&page=2&show_known_bots=false&show_allowed=false&review_only=true", nil)
handler.ServeHTTP(recorder, request)
if recorder.Code != http.StatusOK {
t.Fatalf("unexpected filtered events status: %d", recorder.Code)
}
if app.lastEventOptions.ShowKnownBots || app.lastEventOptions.ShowAllowed || !app.lastEventOptions.ReviewOnly {
var eventPage model.EventPage
if err := json.Unmarshal(recorder.Body.Bytes(), &eventPage); err != nil {
t.Fatalf("decode event page payload: %v", err)
}
if eventPage.Page != 2 || !eventPage.HasPrev {
t.Fatalf("unexpected event page payload: %+v", eventPage)
}
if app.lastEventOptions.ShowKnownBots || app.lastEventOptions.ShowAllowed || !app.lastEventOptions.ReviewOnly || app.lastEventOptions.Offset != 250 {
t.Fatalf("event filter options were not forwarded correctly: %+v", app.lastEventOptions)
}
@@ -94,11 +101,11 @@ func TestHandlerServesOverviewAndManualActions(t *testing.T) {
if !strings.Contains(recorder.Body.String(), "Local-only review and enforcement console") {
t.Fatalf("overview page did not render expected content")
}
if !strings.Contains(recorder.Body.String(), "Show known bots") {
t.Fatalf("overview page should expose the known bots toggle")
if strings.Contains(recorder.Body.String(), "Show known bots") {
t.Fatalf("overview page should no longer expose the known bots toggle")
}
if !strings.Contains(recorder.Body.String(), "Query Log") {
t.Fatalf("overview page should link to the query log")
if !strings.Contains(recorder.Body.String(), "Requests Log") {
t.Fatalf("overview page should link to the requests log")
}
if !strings.Contains(recorder.Body.String(), "Activity") {
t.Fatalf("overview page should expose the activity chart")
@@ -109,11 +116,11 @@ func TestHandlerServesOverviewAndManualActions(t *testing.T) {
if !strings.Contains(recorder.Body.String(), "Bots") {
t.Fatalf("overview page should expose the bots chart")
}
if !strings.Contains(recorder.Body.String(), "Top IPs by events") {
t.Fatalf("overview page should expose the top IPs by events block")
if !strings.Contains(recorder.Body.String(), "Top bot IPs by events") {
t.Fatalf("overview page should expose the top bot IPs by events block")
}
if !strings.Contains(recorder.Body.String(), "Top IPs by traffic") {
t.Fatalf("overview page should expose the top IPs by traffic block")
if !strings.Contains(recorder.Body.String(), "Top non-bot IPs by traffic") {
t.Fatalf("overview page should expose the split top IP traffic block")
}
if !strings.Contains(recorder.Body.String(), "Top sources by events") {
t.Fatalf("overview page should expose the top sources block")
@@ -124,37 +131,47 @@ func TestHandlerServesOverviewAndManualActions(t *testing.T) {
if !strings.Contains(recorder.Body.String(), "Loading…") {
t.Fatalf("overview page should render stable loading placeholders")
}
if !strings.Contains(recorder.Body.String(), "These filters affect all dashboard charts and top lists") {
t.Fatalf("overview page should explain the scope of the shared filters")
}
if strings.Contains(recorder.Body.String(), "position: sticky") {
t.Fatalf("overview page header should no longer be sticky")
}
if !strings.Contains(recorder.Body.String(), "Show allowed") {
t.Fatalf("overview page should expose the allowed toggle")
}
if strings.Contains(recorder.Body.String(), "Review only") {
t.Fatalf("overview page should not expose the review-only toggle anymore")
}
if !strings.Contains(recorder.Body.String(), "localStorage") {
t.Fatalf("overview page should persist preferences in localStorage")
if strings.Contains(recorder.Body.String(), "Auto refresh") {
t.Fatalf("overview page should not expose requests log controls")
}
recorder = httptest.NewRecorder()
request = httptest.NewRequest(http.MethodGet, "/queries", nil)
handler.ServeHTTP(recorder, request)
if recorder.Code != http.StatusMovedPermanently {
t.Fatalf("unexpected legacy query log redirect status: %d", recorder.Code)
}
if location := recorder.Header().Get("Location"); location != "/requests" {
t.Fatalf("unexpected redirect location: %q", location)
}
recorder = httptest.NewRecorder()
request = httptest.NewRequest(http.MethodGet, "/requests", nil)
handler.ServeHTTP(recorder, request)
if recorder.Code != http.StatusOK {
t.Fatalf("unexpected query log page status: %d", recorder.Code)
t.Fatalf("unexpected requests log page status: %d", recorder.Code)
}
queryLogBody := recorder.Body.String()
if !strings.Contains(queryLogBody, "Review only") {
t.Fatalf("query log page should expose the review-only toggle")
t.Fatalf("requests log page should expose the review-only toggle")
}
if !strings.Contains(queryLogBody, "These filters affect the full Query Log") {
t.Fatalf("query log page should explain its filters")
if !strings.Contains(queryLogBody, "These filters affect the full Requests Log") {
t.Fatalf("requests log page should explain its filters")
}
if !strings.Contains(queryLogBody, "Request") {
t.Fatalf("query log page should render the request table")
t.Fatalf("requests log page should render the request table")
}
if !strings.Contains(queryLogBody, "Auto refresh") {
t.Fatalf("requests log page should expose the auto refresh toggle")
}
if !strings.Contains(queryLogBody, "Previous") || !strings.Contains(queryLogBody, "Next") {
t.Fatalf("requests log page should expose pagination controls")
}
recorder = httptest.NewRecorder()
@@ -217,6 +234,19 @@ func (s *stubApp) GetOverview(_ context.Context, _ time.Time, _ int, options mod
LastSeenAt: now,
Bot: &model.BotMatch{Name: "Googlebot", ProviderID: "google_official", Verified: true},
}},
TopBotIPsByEvents: []model.TopIPRow{{
IP: "203.0.113.10",
Events: 3,
TrafficBytes: 4096,
LastSeenAt: now,
Bot: &model.BotMatch{Name: "Googlebot", ProviderID: "google_official", Verified: true},
}},
TopNonBotIPsByEvents: []model.TopIPRow{{
IP: "198.51.100.20",
Events: 2,
TrafficBytes: 2048,
LastSeenAt: now,
}},
TopIPsByTraffic: []model.TopIPRow{{
IP: "203.0.113.10",
Events: 3,
@@ -224,6 +254,19 @@ func (s *stubApp) GetOverview(_ context.Context, _ time.Time, _ int, options mod
LastSeenAt: now,
Bot: &model.BotMatch{Name: "Googlebot", ProviderID: "google_official", Verified: true},
}},
TopBotIPsByTraffic: []model.TopIPRow{{
IP: "203.0.113.10",
Events: 3,
TrafficBytes: 4096,
LastSeenAt: now,
Bot: &model.BotMatch{Name: "Googlebot", ProviderID: "google_official", Verified: true},
}},
TopNonBotIPsByTraffic: []model.TopIPRow{{
IP: "198.51.100.20",
Events: 2,
TrafficBytes: 2048,
LastSeenAt: now,
}},
TopSources: []model.TopSourceRow{{
SourceName: "main",
Events: 3,
@@ -265,7 +308,22 @@ func (s *stubApp) GetOverview(_ context.Context, _ time.Time, _ int, options mod
func (s *stubApp) ListEvents(ctx context.Context, _ time.Time, limit int, options model.EventListOptions) ([]model.Event, error) {
s.lastEventOptions = options
overview, _ := s.GetOverview(ctx, time.Time{}, limit, model.OverviewOptions{ShowKnownBots: true, ShowAllowed: true})
return overview.RecentEvents, nil
items := overview.RecentEvents
if limit > 1 {
items = append(items, model.Event{
ID: 2,
SourceName: "main",
ClientIP: "198.51.100.20",
OccurredAt: time.Now().UTC().Add(-time.Minute),
Method: http.MethodPost,
URI: "/xmlrpc.php",
Host: "example.test",
Status: http.StatusNotFound,
CurrentState: model.IPStateReview,
Actions: model.ActionAvailability{CanBlock: true},
})
}
return items, nil
}
func (s *stubApp) ListIPs(ctx context.Context, limit int, state string) ([]model.IPState, error) {