infrastructure/caddy-opnsense-blocker
2
Fork 0
Code Issues Activity

Improve requests log filters and sorting

Browse Source
This commit is contained in:
Codex, agent ChatGPT
2026-03-12 18:42:12 +01:00
parent 200fc83831
commit fef2237c49
7 changed files with 589 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

130
internal/store/store.go
View File

@@ -9,6 +9,7 @@ import (
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"time"
@@ -257,18 +258,123 @@ func overviewFilterQueryParts(options model.OverviewOptions) (joins []string, cl
return joins, clauses
}
func eventFilterQueryParts(options model.EventListOptions) (joins []string, clauses []string) {
func normalizedEventStateExpression() string {
return `CASE WHEN COALESCE(s.state, '') = '' THEN '` + string(model.IPStateObserved) + `' ELSE s.state END`
}
func parseStatusFilter(filter string) (clauses []string, args []any, err error) {
normalized := strings.ToLower(strings.TrimSpace(filter))
if normalized == "" {
return nil, nil, nil
}
if len(normalized) == 3 && strings.HasSuffix(normalized, "xx") && normalized[0] >= '1' && normalized[0] <= '5' {
hundred := int(normalized[0]-'0') * 100
return []string{`e.status >= ?`, `e.status < ?`}, []any{hundred, hundred + 100}, nil
}
status, convErr := strconv.Atoi(normalized)
if convErr != nil || status < 100 || status > 599 {
return nil, nil, fmt.Errorf("invalid status filter %q", filter)
}
return []string{`e.status = ?`}, []any{status}, nil
}
func eventOrderClause(options model.EventListOptions) string {
stateExpr := normalizedEventStateExpression()
direction := "ASC"
tieDirection := "ASC"
field := strings.ToLower(strings.TrimSpace(options.SortBy))
if field == "" {
return `e.occurred_at DESC, e.id DESC`
}
if options.SortDesc {
direction = "DESC"
tieDirection = "DESC"
}
switch field {
case "time", "occurred_at":
return fmt.Sprintf(`e.occurred_at %s, e.id %s`, direction, tieDirection)
case "source":
return fmt.Sprintf(`LOWER(e.source_name) %s, e.occurred_at %s, e.id %s`, direction, tieDirection, tieDirection)
case "ip":
return fmt.Sprintf(`e.client_ip %s, e.occurred_at %s, e.id %s`, direction, tieDirection, tieDirection)
case "method":
return fmt.Sprintf(`UPPER(e.method) %s, e.occurred_at %s, e.id %s`, direction, tieDirection, tieDirection)
case "request", "uri":
return fmt.Sprintf(`e.uri %s, e.host %s, e.occurred_at %s, e.id %s`, direction, direction, tieDirection, tieDirection)
case "status":
return fmt.Sprintf(`e.status %s, e.occurred_at %s, e.id %s`, direction, tieDirection, tieDirection)
case "state":
return fmt.Sprintf(`%s %s, e.occurred_at %s, e.id %s`, stateExpr, direction, tieDirection, tieDirection)
case "reason":
return fmt.Sprintf(`LOWER(e.decision_reason) %s, e.occurred_at %s, e.id %s`, direction, tieDirection, tieDirection)
default:
return `e.occurred_at DESC, e.id DESC`
}
}
func eventFilterQueryParts(options model.EventListOptions) (joins []string, clauses []string, args []any, err error) {
joins = append(joins, `LEFT JOIN ip_state s ON s.ip = e.client_ip`)
if !options.ShowAllowed {
clauses = append(clauses, `COALESCE(s.state, '') <> '`+string(model.IPStateAllowed)+`'`)
stateExpr := normalizedEventStateExpression()
stateFilter := strings.ToLower(strings.TrimSpace(options.State))
if stateFilter == "all" {
stateFilter = ""
}
if options.ReviewOnly {
clauses = append(clauses, `COALESCE(s.state, '') = '`+string(model.IPStateReview)+`'`)
if stateFilter != "" {
switch stateFilter {
case string(model.IPStateObserved), string(model.IPStateReview), string(model.IPStateBlocked), string(model.IPStateAllowed):
clauses = append(clauses, stateExpr+` = ?`)
args = append(args, stateFilter)
default:
return nil, nil, nil, fmt.Errorf("invalid state filter %q", options.State)
}
} else {
if !options.ShowAllowed {
clauses = append(clauses, stateExpr+` <> ?`)
args = append(args, string(model.IPStateAllowed))
}
if options.ReviewOnly {
clauses = append(clauses, stateExpr+` = ?`)
args = append(args, string(model.IPStateReview))
}
}
if !options.ShowKnownBots {
if source := strings.TrimSpace(options.Source); source != "" {
clauses = append(clauses, `LOWER(e.source_name) = LOWER(?)`)
args = append(args, source)
}
if method := strings.TrimSpace(options.Method); method != "" {
if strings.EqualFold(method, "other") {
clauses = append(clauses, `UPPER(COALESCE(e.method, '')) NOT IN ('GET', 'POST', 'HEAD', 'PUT', 'DELETE', 'PATCH', 'OPTIONS')`)
} else {
clauses = append(clauses, `UPPER(COALESCE(e.method, '')) = ?`)
args = append(args, strings.ToUpper(method))
}
}
statusClauses, statusArgs, statusErr := parseStatusFilter(options.StatusFilter)
if statusErr != nil {
return nil, nil, nil, statusErr
}
clauses = append(clauses, statusClauses...)
args = append(args, statusArgs...)
botFilter := strings.ToLower(strings.TrimSpace(options.BotFilter))
if botFilter == "" && !options.ShowKnownBots {
botFilter = "no-known"
}
switch botFilter {
case "", "all":
case "known":
clauses = append(clauses, knownBotExistsClause(`e.client_ip`))
case "possible":
clauses = append(clauses, anyBotExistsClause(`e.client_ip`), `NOT `+knownBotExistsClause(`e.client_ip`))
case "any":
clauses = append(clauses, anyBotExistsClause(`e.client_ip`))
case "non-bot":
clauses = append(clauses, `NOT `+anyBotExistsClause(`e.client_ip`))
case "no-known":
clauses = append(clauses, `NOT `+knownBotExistsClause(`e.client_ip`))
default:
return nil, nil, nil, fmt.Errorf("invalid bot filter %q", options.BotFilter)
}
return joins, clauses
return joins, clauses, args, nil
}
func (s *Store) AddDecision(ctx context.Context, decision *model.DecisionRecord) error {
@@ -836,7 +942,10 @@ func (s *Store) ListEvents(ctx context.Context, since time.Time, limit int, opti
if options.Offset < 0 {
options.Offset = 0
}
joins, clauses := eventFilterQueryParts(options)
joins, clauses, filterArgs, err := eventFilterQueryParts(options)
if err != nil {
return nil, err
}
query := `
SELECT e.id, e.source_name, e.profile_name, e.occurred_at, e.remote_ip, e.client_ip, e.host,
e.method, e.uri, e.path, e.status, e.user_agent, e.decision, e.decision_reason,
@@ -846,15 +955,16 @@ func (s *Store) ListEvents(ctx context.Context, since time.Time, limit int, opti
if len(joins) > 0 {
query += ` ` + strings.Join(joins, ` `)
}
args := make([]any, 0, 2)
args := make([]any, 0, len(filterArgs)+3)
if !since.IsZero() {
clauses = append([]string{`e.occurred_at >= ?`}, clauses...)
args = append(args, formatTime(since))
}
args = append(args, filterArgs...)
if len(clauses) > 0 {
query += ` WHERE ` + strings.Join(clauses, ` AND `)
}
query += ` ORDER BY e.occurred_at DESC, e.id DESC LIMIT ? OFFSET ?`
query += ` ORDER BY ` + eventOrderClause(options) + ` LIMIT ? OFFSET ?`
args = append(args, limit, options.Offset)
rows, err := s.db.QueryContext(ctx, query, args...)

120
internal/store/store_test.go
View File

@@ -330,3 +330,123 @@ func TestStoreOverviewLeaderboardsUseTrafficFromRawJSON(t *testing.T) {
t.Fatalf("expected filtered top urls to be empty, got %+v", filtered.TopURLs)
}
}
func TestStoreListEventsSupportsFiltersAndSorting(t *testing.T) {
t.Parallel()
dbPath := filepath.Join(t.TempDir(), "blocker.db")
db, err := Open(dbPath)
if err != nil {
t.Fatalf("open store: %v", err)
}
defer db.Close()
ctx := context.Background()
baseTime := time.Date(2025, 3, 12, 18, 0, 0, 0, time.UTC)
events := []*model.Event{
{
SourceName: "main",
ProfileName: "main",
OccurredAt: baseTime,
RemoteIP: "198.51.100.10",
ClientIP: "203.0.113.10",
Host: "example.test",
Method: "GET",
URI: "/wp-login.php",
Path: "/wp-login.php",
Status: 404,
UserAgent: "curl/8.0",
Decision: model.DecisionActionReview,
DecisionReason: "php_path",
DecisionReasons: []string{"php_path"},
RawJSON: `{"status":404}`,
},
{
SourceName: "main",
ProfileName: "main",
OccurredAt: baseTime.Add(10 * time.Second),
RemoteIP: "198.51.100.11",
ClientIP: "203.0.113.11",
Host: "example.test",
Method: "GET",
URI: "/xmlrpc.php",
Path: "/xmlrpc.php",
Status: 401,
UserAgent: "curl/8.0",
Decision: model.DecisionActionReview,
DecisionReason: "php_path",
DecisionReasons: []string{"php_path"},
RawJSON: `{"status":401}`,
},
{
SourceName: "main",
ProfileName: "main",
OccurredAt: baseTime.Add(20 * time.Second),
RemoteIP: "198.51.100.12",
ClientIP: "203.0.113.20",
Host: "example.test",
Method: "POST",
URI: "/xmlrpc.php",
Path: "/xmlrpc.php",
Status: 403,
UserAgent: "curl/8.0",
Decision: model.DecisionActionReview,
DecisionReason: "unexpected_post",
DecisionReasons: []string{"unexpected_post"},
RawJSON: `{"status":403}`,
},
{
SourceName: "gitea",
ProfileName: "gitea",
OccurredAt: baseTime.Add(30 * time.Second),
RemoteIP: "198.51.100.13",
ClientIP: "203.0.113.30",
Host: "git.example.test",
Method: "GET",
URI: "/install.php",
Path: "/install.php",
Status: 404,
UserAgent: "curl/8.0",
Decision: model.DecisionActionReview,
DecisionReason: "suspicious_path_prefix:/install.php",
DecisionReasons: []string{"suspicious_path_prefix:/install.php"},
RawJSON: `{"status":404}`,
},
}
for _, event := range events {
if err := db.RecordEvent(ctx, event); err != nil {
t.Fatalf("record event %+v: %v", event, err)
}
}
for _, ip := range []string{"203.0.113.10", "203.0.113.11"} {
if err := db.SaveInvestigation(ctx, model.IPInvestigation{
IP: ip,
UpdatedAt: baseTime,
Bot: &model.BotMatch{Name: "Googlebot", ProviderID: "google_official", Verified: true},
}); err != nil {
t.Fatalf("save investigation for %s: %v", ip, err)
}
}
items, err := db.ListEvents(ctx, baseTime.Add(-time.Minute), 10, model.EventListOptions{
Source: "main",
Method: "GET",
StatusFilter: "4xx",
State: string(model.IPStateReview),
BotFilter: "known",
SortBy: "status",
SortDesc: false,
})
if err != nil {
t.Fatalf("list events with filters: %v", err)
}
if len(items) != 2 {
t.Fatalf("expected 2 filtered items, got %+v", items)
}
if items[0].ClientIP != "203.0.113.11" || items[0].Status != 401 {
t.Fatalf("unexpected first filtered row: %+v", items[0])
}
if items[1].ClientIP != "203.0.113.10" || items[1].Status != 404 {
t.Fatalf("unexpected second filtered row: %+v", items[1])
}
}