You've already forked caddy-opnsense-blocker
Build a Pi-hole-style dashboard and query log
This commit is contained in:
@@ -54,26 +54,28 @@ func (d Decision) PrimaryReason() string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type Event struct {
|
type Event struct {
|
||||||
ID int64 `json:"id"`
|
ID int64 `json:"id"`
|
||||||
SourceName string `json:"source_name"`
|
SourceName string `json:"source_name"`
|
||||||
ProfileName string `json:"profile_name"`
|
ProfileName string `json:"profile_name"`
|
||||||
OccurredAt time.Time `json:"occurred_at"`
|
OccurredAt time.Time `json:"occurred_at"`
|
||||||
RemoteIP string `json:"remote_ip"`
|
RemoteIP string `json:"remote_ip"`
|
||||||
ClientIP string `json:"client_ip"`
|
ClientIP string `json:"client_ip"`
|
||||||
Host string `json:"host"`
|
Host string `json:"host"`
|
||||||
Method string `json:"method"`
|
Method string `json:"method"`
|
||||||
URI string `json:"uri"`
|
URI string `json:"uri"`
|
||||||
Path string `json:"path"`
|
Path string `json:"path"`
|
||||||
Status int `json:"status"`
|
Status int `json:"status"`
|
||||||
UserAgent string `json:"user_agent"`
|
UserAgent string `json:"user_agent"`
|
||||||
Decision DecisionAction `json:"decision"`
|
Decision DecisionAction `json:"decision"`
|
||||||
DecisionReason string `json:"decision_reason"`
|
DecisionReason string `json:"decision_reason"`
|
||||||
DecisionReasons []string `json:"decision_reasons,omitempty"`
|
DecisionReasons []string `json:"decision_reasons,omitempty"`
|
||||||
Enforced bool `json:"enforced"`
|
Enforced bool `json:"enforced"`
|
||||||
RawJSON string `json:"raw_json"`
|
RawJSON string `json:"raw_json"`
|
||||||
CreatedAt time.Time `json:"created_at"`
|
CreatedAt time.Time `json:"created_at"`
|
||||||
CurrentState IPStateStatus `json:"current_state"`
|
CurrentState IPStateStatus `json:"current_state"`
|
||||||
ManualOverride ManualOverride `json:"manual_override"`
|
ManualOverride ManualOverride `json:"manual_override"`
|
||||||
|
Bot *BotMatch `json:"bot,omitempty"`
|
||||||
|
Actions ActionAvailability `json:"actions"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type IPState struct {
|
type IPState struct {
|
||||||
@@ -179,11 +181,11 @@ type RecentIPRow struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type TopIPRow struct {
|
type TopIPRow struct {
|
||||||
IP string `json:"ip"`
|
IP string `json:"ip"`
|
||||||
Events int64 `json:"events"`
|
Events int64 `json:"events"`
|
||||||
TrafficBytes int64 `json:"traffic_bytes"`
|
TrafficBytes int64 `json:"traffic_bytes"`
|
||||||
LastSeenAt time.Time `json:"last_seen_at"`
|
LastSeenAt time.Time `json:"last_seen_at"`
|
||||||
Bot *BotMatch `json:"bot,omitempty"`
|
Bot *BotMatch `json:"bot,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type TopSourceRow struct {
|
type TopSourceRow struct {
|
||||||
@@ -201,11 +203,39 @@ type TopURLRow struct {
|
|||||||
LastSeenAt time.Time `json:"last_seen_at"`
|
LastSeenAt time.Time `json:"last_seen_at"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type ActivitySourceCount struct {
|
||||||
|
SourceName string `json:"source_name"`
|
||||||
|
Events int64 `json:"events"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type ActivityBucket struct {
|
||||||
|
BucketStart time.Time `json:"bucket_start"`
|
||||||
|
TotalEvents int64 `json:"total_events"`
|
||||||
|
Sources []ActivitySourceCount `json:"sources,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type MethodBreakdownRow struct {
|
||||||
|
Method string `json:"method"`
|
||||||
|
Events int64 `json:"events"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type BotBreakdownRow struct {
|
||||||
|
Key string `json:"key"`
|
||||||
|
Label string `json:"label"`
|
||||||
|
Events int64 `json:"events"`
|
||||||
|
}
|
||||||
|
|
||||||
type OverviewOptions struct {
|
type OverviewOptions struct {
|
||||||
ShowKnownBots bool
|
ShowKnownBots bool
|
||||||
ShowAllowed bool
|
ShowAllowed bool
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type EventListOptions struct {
|
||||||
|
ShowKnownBots bool
|
||||||
|
ShowAllowed bool
|
||||||
|
ReviewOnly bool
|
||||||
|
}
|
||||||
|
|
||||||
type SourceOffset struct {
|
type SourceOffset struct {
|
||||||
SourceName string
|
SourceName string
|
||||||
Path string
|
Path string
|
||||||
@@ -225,17 +255,20 @@ type IPDetails struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type Overview struct {
|
type Overview struct {
|
||||||
TotalEvents int64 `json:"total_events"`
|
TotalEvents int64 `json:"total_events"`
|
||||||
TotalIPs int64 `json:"total_ips"`
|
TotalIPs int64 `json:"total_ips"`
|
||||||
BlockedIPs int64 `json:"blocked_ips"`
|
BlockedIPs int64 `json:"blocked_ips"`
|
||||||
ReviewIPs int64 `json:"review_ips"`
|
ReviewIPs int64 `json:"review_ips"`
|
||||||
AllowedIPs int64 `json:"allowed_ips"`
|
AllowedIPs int64 `json:"allowed_ips"`
|
||||||
ObservedIPs int64 `json:"observed_ips"`
|
ObservedIPs int64 `json:"observed_ips"`
|
||||||
ActivitySince time.Time `json:"activity_since,omitempty"`
|
ActivitySince time.Time `json:"activity_since,omitempty"`
|
||||||
TopIPsByEvents []TopIPRow `json:"top_ips_by_events"`
|
ActivityBuckets []ActivityBucket `json:"activity_buckets"`
|
||||||
TopIPsByTraffic []TopIPRow `json:"top_ips_by_traffic"`
|
Methods []MethodBreakdownRow `json:"methods"`
|
||||||
TopSources []TopSourceRow `json:"top_sources"`
|
Bots []BotBreakdownRow `json:"bots"`
|
||||||
TopURLs []TopURLRow `json:"top_urls"`
|
TopIPsByEvents []TopIPRow `json:"top_ips_by_events"`
|
||||||
RecentIPs []IPState `json:"recent_ips"`
|
TopIPsByTraffic []TopIPRow `json:"top_ips_by_traffic"`
|
||||||
RecentEvents []Event `json:"recent_events"`
|
TopSources []TopSourceRow `json:"top_sources"`
|
||||||
|
TopURLs []TopURLRow `json:"top_urls"`
|
||||||
|
RecentIPs []IPState `json:"recent_ips"`
|
||||||
|
RecentEvents []Event `json:"recent_events"`
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -102,8 +102,15 @@ func (s *Service) GetOverview(ctx context.Context, since time.Time, limit int, o
|
|||||||
return overview, nil
|
return overview, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Service) ListEvents(ctx context.Context, limit int) ([]model.Event, error) {
|
func (s *Service) ListEvents(ctx context.Context, since time.Time, limit int, options model.EventListOptions) ([]model.Event, error) {
|
||||||
return s.store.ListRecentEvents(ctx, limit)
|
items, err := s.store.ListEvents(ctx, since, limit, options)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := s.decorateEvents(ctx, items); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return items, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Service) ListIPs(ctx context.Context, limit int, state string) ([]model.IPState, error) {
|
func (s *Service) ListIPs(ctx context.Context, limit int, state string) ([]model.IPState, error) {
|
||||||
@@ -673,6 +680,47 @@ func recentRowIPs(items []model.RecentIPRow) []string {
|
|||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func eventIPs(items []model.Event) []string {
|
||||||
|
result := make([]string, 0, len(items))
|
||||||
|
for _, item := range items {
|
||||||
|
result = append(result, item.ClientIP)
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Service) decorateEvents(ctx context.Context, items []model.Event) error {
|
||||||
|
if len(items) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
investigations, err := s.store.GetInvestigationsForIPs(ctx, eventIPs(items))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
actionsByIP := make(map[string]model.ActionAvailability, len(items))
|
||||||
|
for index := range items {
|
||||||
|
ip := items[index].ClientIP
|
||||||
|
if investigation, ok := investigations[ip]; ok {
|
||||||
|
items[index].Bot = investigation.Bot
|
||||||
|
} else {
|
||||||
|
s.enqueueInvestigation(ip)
|
||||||
|
}
|
||||||
|
if _, ok := actionsByIP[ip]; !ok {
|
||||||
|
state := model.IPState{
|
||||||
|
IP: ip,
|
||||||
|
State: items[index].CurrentState,
|
||||||
|
ManualOverride: items[index].ManualOverride,
|
||||||
|
}
|
||||||
|
if state.State == "" {
|
||||||
|
state.State = model.IPStateObserved
|
||||||
|
}
|
||||||
|
backend := s.resolveOPNsenseStatus(ctx, state)
|
||||||
|
actionsByIP[ip] = actionAvailability(state, backend)
|
||||||
|
}
|
||||||
|
items[index].Actions = actionsByIP[ip]
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func (s *Service) decorateOverviewTopIPs(ctx context.Context, overview *model.Overview) error {
|
func (s *Service) decorateOverviewTopIPs(ctx context.Context, overview *model.Overview) error {
|
||||||
if overview == nil {
|
if overview == nil {
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -225,19 +225,38 @@ func (s *Store) RecordEvent(ctx context.Context, event *model.Event) error {
|
|||||||
|
|
||||||
const responseBytesExpression = `CASE WHEN json_valid(e.raw_json) THEN CAST(COALESCE(json_extract(e.raw_json, '$.size'), 0) AS INTEGER) ELSE 0 END`
|
const responseBytesExpression = `CASE WHEN json_valid(e.raw_json) THEN CAST(COALESCE(json_extract(e.raw_json, '$.size'), 0) AS INTEGER) ELSE 0 END`
|
||||||
|
|
||||||
|
func knownBotExistsClause(ipExpression string) string {
|
||||||
|
return `EXISTS (
|
||||||
|
SELECT 1
|
||||||
|
FROM ip_investigations i
|
||||||
|
WHERE i.ip = ` + ipExpression + `
|
||||||
|
AND json_valid(i.payload_json)
|
||||||
|
AND json_type(i.payload_json, '$.bot') IS NOT NULL
|
||||||
|
AND COALESCE(json_extract(i.payload_json, '$.bot.verified'), 0) = 1
|
||||||
|
)`
|
||||||
|
}
|
||||||
|
|
||||||
func overviewFilterQueryParts(options model.OverviewOptions) (joins []string, clauses []string) {
|
func overviewFilterQueryParts(options model.OverviewOptions) (joins []string, clauses []string) {
|
||||||
if !options.ShowAllowed {
|
if !options.ShowAllowed {
|
||||||
joins = append(joins, `LEFT JOIN ip_state s ON s.ip = e.client_ip`)
|
joins = append(joins, `LEFT JOIN ip_state s ON s.ip = e.client_ip`)
|
||||||
clauses = append(clauses, `COALESCE(s.state, '') <> '`+string(model.IPStateAllowed)+`'`)
|
clauses = append(clauses, `COALESCE(s.state, '') <> '`+string(model.IPStateAllowed)+`'`)
|
||||||
}
|
}
|
||||||
if !options.ShowKnownBots {
|
if !options.ShowKnownBots {
|
||||||
clauses = append(clauses, `NOT EXISTS (
|
clauses = append(clauses, `NOT `+knownBotExistsClause(`e.client_ip`))
|
||||||
SELECT 1
|
}
|
||||||
FROM ip_investigations i
|
return joins, clauses
|
||||||
WHERE i.ip = e.client_ip
|
}
|
||||||
AND json_valid(i.payload_json)
|
|
||||||
AND json_type(i.payload_json, '$.bot') IS NOT NULL
|
func eventFilterQueryParts(options model.EventListOptions) (joins []string, clauses []string) {
|
||||||
)`)
|
joins = append(joins, `LEFT JOIN ip_state s ON s.ip = e.client_ip`)
|
||||||
|
if !options.ShowAllowed {
|
||||||
|
clauses = append(clauses, `COALESCE(s.state, '') <> '`+string(model.IPStateAllowed)+`'`)
|
||||||
|
}
|
||||||
|
if options.ReviewOnly {
|
||||||
|
clauses = append(clauses, `COALESCE(s.state, '') = '`+string(model.IPStateReview)+`'`)
|
||||||
|
}
|
||||||
|
if !options.ShowKnownBots {
|
||||||
|
clauses = append(clauses, `NOT `+knownBotExistsClause(`e.client_ip`))
|
||||||
}
|
}
|
||||||
return joins, clauses
|
return joins, clauses
|
||||||
}
|
}
|
||||||
@@ -440,8 +459,23 @@ func (s *Store) GetOverview(ctx context.Context, since time.Time, limit int, opt
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return model.Overview{}, err
|
return model.Overview{}, err
|
||||||
}
|
}
|
||||||
|
activityBuckets, err := s.listActivityBuckets(ctx, since, options)
|
||||||
|
if err != nil {
|
||||||
|
return model.Overview{}, err
|
||||||
|
}
|
||||||
|
methods, err := s.listMethodBreakdown(ctx, since, options)
|
||||||
|
if err != nil {
|
||||||
|
return model.Overview{}, err
|
||||||
|
}
|
||||||
|
bots, err := s.listBotBreakdown(ctx, since, options)
|
||||||
|
if err != nil {
|
||||||
|
return model.Overview{}, err
|
||||||
|
}
|
||||||
overview.RecentIPs = recentIPs
|
overview.RecentIPs = recentIPs
|
||||||
overview.RecentEvents = recentEvents
|
overview.RecentEvents = recentEvents
|
||||||
|
overview.ActivityBuckets = activityBuckets
|
||||||
|
overview.Methods = methods
|
||||||
|
overview.Bots = bots
|
||||||
overview.TopIPsByEvents = topIPsByEvents
|
overview.TopIPsByEvents = topIPsByEvents
|
||||||
overview.TopIPsByTraffic = topIPsByTraffic
|
overview.TopIPsByTraffic = topIPsByTraffic
|
||||||
overview.TopSources = topSources
|
overview.TopSources = topSources
|
||||||
@@ -610,6 +644,200 @@ func (s *Store) listTopURLRows(ctx context.Context, since time.Time, limit int,
|
|||||||
return items, nil
|
return items, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *Store) listActivityBuckets(ctx context.Context, since time.Time, options model.OverviewOptions) ([]model.ActivityBucket, error) {
|
||||||
|
if since.IsZero() {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
joins, clauses := overviewFilterQueryParts(options)
|
||||||
|
query := `
|
||||||
|
SELECT (CAST(strftime('%s', e.occurred_at) AS INTEGER) / 600) * 600 AS bucket_unix,
|
||||||
|
e.source_name,
|
||||||
|
COUNT(*) AS event_count
|
||||||
|
FROM events e`
|
||||||
|
if len(joins) > 0 {
|
||||||
|
query += ` ` + strings.Join(joins, ` `)
|
||||||
|
}
|
||||||
|
args := []any{formatTime(since)}
|
||||||
|
clauses = append([]string{`e.occurred_at >= ?`}, clauses...)
|
||||||
|
if len(clauses) > 0 {
|
||||||
|
query += ` WHERE ` + strings.Join(clauses, ` AND `)
|
||||||
|
}
|
||||||
|
query += ` GROUP BY bucket_unix, e.source_name ORDER BY bucket_unix ASC, event_count DESC, e.source_name ASC`
|
||||||
|
|
||||||
|
rows, err := s.db.QueryContext(ctx, query, args...)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("list activity buckets: %w", err)
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
type bucketKey int64
|
||||||
|
bucketMap := map[bucketKey]*model.ActivityBucket{}
|
||||||
|
for rows.Next() {
|
||||||
|
var bucketUnix int64
|
||||||
|
var sourceName string
|
||||||
|
var events int64
|
||||||
|
if err := rows.Scan(&bucketUnix, &sourceName, &events); err != nil {
|
||||||
|
return nil, fmt.Errorf("scan activity bucket: %w", err)
|
||||||
|
}
|
||||||
|
key := bucketKey(bucketUnix)
|
||||||
|
bucket, ok := bucketMap[key]
|
||||||
|
if !ok {
|
||||||
|
bucket = &model.ActivityBucket{BucketStart: time.Unix(bucketUnix, 0).UTC()}
|
||||||
|
bucketMap[key] = bucket
|
||||||
|
}
|
||||||
|
bucket.TotalEvents += events
|
||||||
|
bucket.Sources = append(bucket.Sources, model.ActivitySourceCount{SourceName: sourceName, Events: events})
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return nil, fmt.Errorf("iterate activity buckets: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
start := since.UTC().Truncate(10 * time.Minute)
|
||||||
|
end := time.Now().UTC().Truncate(10 * time.Minute)
|
||||||
|
buckets := make([]model.ActivityBucket, 0, int(end.Sub(start)/(10*time.Minute))+1)
|
||||||
|
for current := start; !current.After(end); current = current.Add(10 * time.Minute) {
|
||||||
|
bucket, ok := bucketMap[bucketKey(current.Unix())]
|
||||||
|
if !ok {
|
||||||
|
buckets = append(buckets, model.ActivityBucket{BucketStart: current})
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
sort.Slice(bucket.Sources, func(left int, right int) bool {
|
||||||
|
if bucket.Sources[left].Events != bucket.Sources[right].Events {
|
||||||
|
return bucket.Sources[left].Events > bucket.Sources[right].Events
|
||||||
|
}
|
||||||
|
return bucket.Sources[left].SourceName < bucket.Sources[right].SourceName
|
||||||
|
})
|
||||||
|
buckets = append(buckets, *bucket)
|
||||||
|
}
|
||||||
|
return buckets, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) listMethodBreakdown(ctx context.Context, since time.Time, options model.OverviewOptions) ([]model.MethodBreakdownRow, error) {
|
||||||
|
joins, clauses := overviewFilterQueryParts(options)
|
||||||
|
query := `
|
||||||
|
SELECT COALESCE(NULLIF(UPPER(TRIM(e.method)), ''), 'OTHER') AS method,
|
||||||
|
COUNT(*) AS event_count
|
||||||
|
FROM events e`
|
||||||
|
if len(joins) > 0 {
|
||||||
|
query += ` ` + strings.Join(joins, ` `)
|
||||||
|
}
|
||||||
|
args := make([]any, 0, 2)
|
||||||
|
if !since.IsZero() {
|
||||||
|
clauses = append([]string{`e.occurred_at >= ?`}, clauses...)
|
||||||
|
args = append(args, formatTime(since))
|
||||||
|
}
|
||||||
|
if len(clauses) > 0 {
|
||||||
|
query += ` WHERE ` + strings.Join(clauses, ` AND `)
|
||||||
|
}
|
||||||
|
query += ` GROUP BY method ORDER BY event_count DESC, method ASC`
|
||||||
|
|
||||||
|
rows, err := s.db.QueryContext(ctx, query, args...)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("list method breakdown: %w", err)
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
items := make([]model.MethodBreakdownRow, 0, 8)
|
||||||
|
for rows.Next() {
|
||||||
|
var item model.MethodBreakdownRow
|
||||||
|
if err := rows.Scan(&item.Method, &item.Events); err != nil {
|
||||||
|
return nil, fmt.Errorf("scan method breakdown row: %w", err)
|
||||||
|
}
|
||||||
|
items = append(items, item)
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return nil, fmt.Errorf("iterate method breakdown rows: %w", err)
|
||||||
|
}
|
||||||
|
return items, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) listBotBreakdown(ctx context.Context, since time.Time, options model.OverviewOptions) ([]model.BotBreakdownRow, error) {
|
||||||
|
joins, clauses := overviewFilterQueryParts(options)
|
||||||
|
joins = append(joins, `LEFT JOIN ip_investigations i ON i.ip = e.client_ip`)
|
||||||
|
query := `
|
||||||
|
SELECT
|
||||||
|
COALESCE(SUM(CASE WHEN json_valid(i.payload_json)
|
||||||
|
AND json_type(i.payload_json, '$.bot') IS NOT NULL
|
||||||
|
AND COALESCE(json_extract(i.payload_json, '$.bot.verified'), 0) = 1
|
||||||
|
THEN 1 ELSE 0 END), 0) AS known_bots,
|
||||||
|
COALESCE(SUM(CASE WHEN json_valid(i.payload_json)
|
||||||
|
AND json_type(i.payload_json, '$.bot') IS NOT NULL
|
||||||
|
AND COALESCE(json_extract(i.payload_json, '$.bot.verified'), 0) <> 1
|
||||||
|
THEN 1 ELSE 0 END), 0) AS possible_bots,
|
||||||
|
COALESCE(SUM(CASE WHEN NOT json_valid(i.payload_json)
|
||||||
|
OR json_type(i.payload_json, '$.bot') IS NULL
|
||||||
|
THEN 1 ELSE 0 END), 0) AS other_traffic
|
||||||
|
FROM events e`
|
||||||
|
if len(joins) > 0 {
|
||||||
|
query += ` ` + strings.Join(joins, ` `)
|
||||||
|
}
|
||||||
|
args := make([]any, 0, 2)
|
||||||
|
if !since.IsZero() {
|
||||||
|
clauses = append([]string{`e.occurred_at >= ?`}, clauses...)
|
||||||
|
args = append(args, formatTime(since))
|
||||||
|
}
|
||||||
|
if len(clauses) > 0 {
|
||||||
|
query += ` WHERE ` + strings.Join(clauses, ` AND `)
|
||||||
|
}
|
||||||
|
|
||||||
|
var knownBots int64
|
||||||
|
var possibleBots int64
|
||||||
|
var otherTraffic int64
|
||||||
|
if err := s.db.QueryRowContext(ctx, query, args...).Scan(&knownBots, &possibleBots, &otherTraffic); err != nil {
|
||||||
|
return nil, fmt.Errorf("list bot breakdown: %w", err)
|
||||||
|
}
|
||||||
|
return []model.BotBreakdownRow{
|
||||||
|
{Key: "known", Label: "Known bots", Events: knownBots},
|
||||||
|
{Key: "possible", Label: "Possible bots", Events: possibleBots},
|
||||||
|
{Key: "other", Label: "Other traffic", Events: otherTraffic},
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Store) ListEvents(ctx context.Context, since time.Time, limit int, options model.EventListOptions) ([]model.Event, error) {
|
||||||
|
if limit <= 0 {
|
||||||
|
limit = 100
|
||||||
|
}
|
||||||
|
joins, clauses := eventFilterQueryParts(options)
|
||||||
|
query := `
|
||||||
|
SELECT e.id, e.source_name, e.profile_name, e.occurred_at, e.remote_ip, e.client_ip, e.host,
|
||||||
|
e.method, e.uri, e.path, e.status, e.user_agent, e.decision, e.decision_reason,
|
||||||
|
e.decision_reasons_json, e.enforced, e.raw_json, e.created_at,
|
||||||
|
COALESCE(s.state, ''), COALESCE(s.manual_override, '')
|
||||||
|
FROM events e`
|
||||||
|
if len(joins) > 0 {
|
||||||
|
query += ` ` + strings.Join(joins, ` `)
|
||||||
|
}
|
||||||
|
args := make([]any, 0, 2)
|
||||||
|
if !since.IsZero() {
|
||||||
|
clauses = append([]string{`e.occurred_at >= ?`}, clauses...)
|
||||||
|
args = append(args, formatTime(since))
|
||||||
|
}
|
||||||
|
if len(clauses) > 0 {
|
||||||
|
query += ` WHERE ` + strings.Join(clauses, ` AND `)
|
||||||
|
}
|
||||||
|
query += ` ORDER BY e.occurred_at DESC, e.id DESC LIMIT ?`
|
||||||
|
args = append(args, limit)
|
||||||
|
|
||||||
|
rows, err := s.db.QueryContext(ctx, query, args...)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("list events: %w", err)
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
|
||||||
|
items := make([]model.Event, 0, limit)
|
||||||
|
for rows.Next() {
|
||||||
|
item, err := scanEvent(rows)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
items = append(items, item)
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return nil, fmt.Errorf("iterate events: %w", err)
|
||||||
|
}
|
||||||
|
return items, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (s *Store) ListRecentEvents(ctx context.Context, limit int) ([]model.Event, error) {
|
func (s *Store) ListRecentEvents(ctx context.Context, limit int) ([]model.Event, error) {
|
||||||
if limit <= 0 {
|
if limit <= 0 {
|
||||||
limit = 50
|
limit = 50
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -56,6 +56,16 @@ func TestHandlerServesOverviewAndManualActions(t *testing.T) {
|
|||||||
t.Fatalf("overview filter options were not forwarded correctly: %+v", app.lastOverviewOptions)
|
t.Fatalf("overview filter options were not forwarded correctly: %+v", app.lastOverviewOptions)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
recorder = httptest.NewRecorder()
|
||||||
|
request = httptest.NewRequest(http.MethodGet, "/api/events?hours=24&limit=250&show_known_bots=false&show_allowed=false&review_only=true", nil)
|
||||||
|
handler.ServeHTTP(recorder, request)
|
||||||
|
if recorder.Code != http.StatusOK {
|
||||||
|
t.Fatalf("unexpected filtered events status: %d", recorder.Code)
|
||||||
|
}
|
||||||
|
if app.lastEventOptions.ShowKnownBots || app.lastEventOptions.ShowAllowed || !app.lastEventOptions.ReviewOnly {
|
||||||
|
t.Fatalf("event filter options were not forwarded correctly: %+v", app.lastEventOptions)
|
||||||
|
}
|
||||||
|
|
||||||
recorder = httptest.NewRecorder()
|
recorder = httptest.NewRecorder()
|
||||||
request = httptest.NewRequest(http.MethodPost, "/api/ips/203.0.113.10/block", strings.NewReader(`{"reason":"test reason","actor":"tester"}`))
|
request = httptest.NewRequest(http.MethodPost, "/api/ips/203.0.113.10/block", strings.NewReader(`{"reason":"test reason","actor":"tester"}`))
|
||||||
request.Header.Set("Content-Type", "application/json")
|
request.Header.Set("Content-Type", "application/json")
|
||||||
@@ -84,12 +94,21 @@ func TestHandlerServesOverviewAndManualActions(t *testing.T) {
|
|||||||
if !strings.Contains(recorder.Body.String(), "Local-only review and enforcement console") {
|
if !strings.Contains(recorder.Body.String(), "Local-only review and enforcement console") {
|
||||||
t.Fatalf("overview page did not render expected content")
|
t.Fatalf("overview page did not render expected content")
|
||||||
}
|
}
|
||||||
if strings.Contains(recorder.Body.String(), "Recent events") {
|
|
||||||
t.Fatalf("overview page should no longer render recent events block")
|
|
||||||
}
|
|
||||||
if !strings.Contains(recorder.Body.String(), "Show known bots") {
|
if !strings.Contains(recorder.Body.String(), "Show known bots") {
|
||||||
t.Fatalf("overview page should expose the known bots toggle")
|
t.Fatalf("overview page should expose the known bots toggle")
|
||||||
}
|
}
|
||||||
|
if !strings.Contains(recorder.Body.String(), "Query Log") {
|
||||||
|
t.Fatalf("overview page should link to the query log")
|
||||||
|
}
|
||||||
|
if !strings.Contains(recorder.Body.String(), "Activity") {
|
||||||
|
t.Fatalf("overview page should expose the activity chart")
|
||||||
|
}
|
||||||
|
if !strings.Contains(recorder.Body.String(), "Methods") {
|
||||||
|
t.Fatalf("overview page should expose the methods chart")
|
||||||
|
}
|
||||||
|
if !strings.Contains(recorder.Body.String(), "Bots") {
|
||||||
|
t.Fatalf("overview page should expose the bots chart")
|
||||||
|
}
|
||||||
if !strings.Contains(recorder.Body.String(), "Top IPs by events") {
|
if !strings.Contains(recorder.Body.String(), "Top IPs by events") {
|
||||||
t.Fatalf("overview page should expose the top IPs by events block")
|
t.Fatalf("overview page should expose the top IPs by events block")
|
||||||
}
|
}
|
||||||
@@ -105,7 +124,7 @@ func TestHandlerServesOverviewAndManualActions(t *testing.T) {
|
|||||||
if !strings.Contains(recorder.Body.String(), "Loading…") {
|
if !strings.Contains(recorder.Body.String(), "Loading…") {
|
||||||
t.Fatalf("overview page should render stable loading placeholders")
|
t.Fatalf("overview page should render stable loading placeholders")
|
||||||
}
|
}
|
||||||
if !strings.Contains(recorder.Body.String(), "These two filters affect both the leaderboards and the Recent IPs list") {
|
if !strings.Contains(recorder.Body.String(), "These filters affect all dashboard charts and top lists") {
|
||||||
t.Fatalf("overview page should explain the scope of the shared filters")
|
t.Fatalf("overview page should explain the scope of the shared filters")
|
||||||
}
|
}
|
||||||
if strings.Contains(recorder.Body.String(), "position: sticky") {
|
if strings.Contains(recorder.Body.String(), "position: sticky") {
|
||||||
@@ -114,13 +133,30 @@ func TestHandlerServesOverviewAndManualActions(t *testing.T) {
|
|||||||
if !strings.Contains(recorder.Body.String(), "Show allowed") {
|
if !strings.Contains(recorder.Body.String(), "Show allowed") {
|
||||||
t.Fatalf("overview page should expose the allowed toggle")
|
t.Fatalf("overview page should expose the allowed toggle")
|
||||||
}
|
}
|
||||||
if !strings.Contains(recorder.Body.String(), "Review only") {
|
if strings.Contains(recorder.Body.String(), "Review only") {
|
||||||
t.Fatalf("overview page should expose the review-only toggle")
|
t.Fatalf("overview page should not expose the review-only toggle anymore")
|
||||||
}
|
}
|
||||||
if !strings.Contains(recorder.Body.String(), "localStorage") {
|
if !strings.Contains(recorder.Body.String(), "localStorage") {
|
||||||
t.Fatalf("overview page should persist preferences in localStorage")
|
t.Fatalf("overview page should persist preferences in localStorage")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
recorder = httptest.NewRecorder()
|
||||||
|
request = httptest.NewRequest(http.MethodGet, "/queries", nil)
|
||||||
|
handler.ServeHTTP(recorder, request)
|
||||||
|
if recorder.Code != http.StatusOK {
|
||||||
|
t.Fatalf("unexpected query log page status: %d", recorder.Code)
|
||||||
|
}
|
||||||
|
queryLogBody := recorder.Body.String()
|
||||||
|
if !strings.Contains(queryLogBody, "Review only") {
|
||||||
|
t.Fatalf("query log page should expose the review-only toggle")
|
||||||
|
}
|
||||||
|
if !strings.Contains(queryLogBody, "These filters affect the full Query Log") {
|
||||||
|
t.Fatalf("query log page should explain its filters")
|
||||||
|
}
|
||||||
|
if !strings.Contains(queryLogBody, "Request") {
|
||||||
|
t.Fatalf("query log page should render the request table")
|
||||||
|
}
|
||||||
|
|
||||||
recorder = httptest.NewRecorder()
|
recorder = httptest.NewRecorder()
|
||||||
request = httptest.NewRequest(http.MethodGet, "/ips/203.0.113.10", nil)
|
request = httptest.NewRequest(http.MethodGet, "/ips/203.0.113.10", nil)
|
||||||
handler.ServeHTTP(recorder, request)
|
handler.ServeHTTP(recorder, request)
|
||||||
@@ -154,6 +190,7 @@ func TestHandlerServesOverviewAndManualActions(t *testing.T) {
|
|||||||
type stubApp struct {
|
type stubApp struct {
|
||||||
lastAction string
|
lastAction string
|
||||||
lastOverviewOptions model.OverviewOptions
|
lastOverviewOptions model.OverviewOptions
|
||||||
|
lastEventOptions model.EventListOptions
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *stubApp) GetOverview(_ context.Context, _ time.Time, _ int, options model.OverviewOptions) (model.Overview, error) {
|
func (s *stubApp) GetOverview(_ context.Context, _ time.Time, _ int, options model.OverviewOptions) (model.Overview, error) {
|
||||||
@@ -163,17 +200,29 @@ func (s *stubApp) GetOverview(_ context.Context, _ time.Time, _ int, options mod
|
|||||||
TotalEvents: 1,
|
TotalEvents: 1,
|
||||||
TotalIPs: 1,
|
TotalIPs: 1,
|
||||||
BlockedIPs: 1,
|
BlockedIPs: 1,
|
||||||
|
ActivityBuckets: []model.ActivityBucket{{
|
||||||
|
BucketStart: now.Add(-10 * time.Minute),
|
||||||
|
TotalEvents: 1,
|
||||||
|
Sources: []model.ActivitySourceCount{{
|
||||||
|
SourceName: "main",
|
||||||
|
Events: 1,
|
||||||
|
}},
|
||||||
|
}},
|
||||||
|
Methods: []model.MethodBreakdownRow{{Method: "GET", Events: 1}},
|
||||||
|
Bots: []model.BotBreakdownRow{{Key: "non_bot", Label: "Non-bot", Events: 1}},
|
||||||
TopIPsByEvents: []model.TopIPRow{{
|
TopIPsByEvents: []model.TopIPRow{{
|
||||||
IP: "203.0.113.10",
|
IP: "203.0.113.10",
|
||||||
Events: 3,
|
Events: 3,
|
||||||
TrafficBytes: 4096,
|
TrafficBytes: 4096,
|
||||||
LastSeenAt: now,
|
LastSeenAt: now,
|
||||||
|
Bot: &model.BotMatch{Name: "Googlebot", ProviderID: "google_official", Verified: true},
|
||||||
}},
|
}},
|
||||||
TopIPsByTraffic: []model.TopIPRow{{
|
TopIPsByTraffic: []model.TopIPRow{{
|
||||||
IP: "203.0.113.10",
|
IP: "203.0.113.10",
|
||||||
Events: 3,
|
Events: 3,
|
||||||
TrafficBytes: 4096,
|
TrafficBytes: 4096,
|
||||||
LastSeenAt: now,
|
LastSeenAt: now,
|
||||||
|
Bot: &model.BotMatch{Name: "Googlebot", ProviderID: "google_official", Verified: true},
|
||||||
}},
|
}},
|
||||||
TopSources: []model.TopSourceRow{{
|
TopSources: []model.TopSourceRow{{
|
||||||
SourceName: "main",
|
SourceName: "main",
|
||||||
@@ -196,17 +245,25 @@ func (s *stubApp) GetOverview(_ context.Context, _ time.Time, _ int, options mod
|
|||||||
LastSeenAt: now,
|
LastSeenAt: now,
|
||||||
}},
|
}},
|
||||||
RecentEvents: []model.Event{{
|
RecentEvents: []model.Event{{
|
||||||
ID: 1,
|
ID: 1,
|
||||||
SourceName: "main",
|
SourceName: "main",
|
||||||
ClientIP: "203.0.113.10",
|
ClientIP: "203.0.113.10",
|
||||||
OccurredAt: now,
|
OccurredAt: now,
|
||||||
Decision: model.DecisionActionBlock,
|
Method: http.MethodGet,
|
||||||
CurrentState: model.IPStateBlocked,
|
URI: "/wp-login.php",
|
||||||
|
Host: "example.test",
|
||||||
|
Status: http.StatusNotFound,
|
||||||
|
Decision: model.DecisionActionBlock,
|
||||||
|
CurrentState: model.IPStateBlocked,
|
||||||
|
DecisionReason: "php_path",
|
||||||
|
Actions: model.ActionAvailability{CanUnblock: true},
|
||||||
|
Bot: &model.BotMatch{Name: "Googlebot", ProviderID: "google_official", Verified: true},
|
||||||
}},
|
}},
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *stubApp) ListEvents(ctx context.Context, limit int) ([]model.Event, error) {
|
func (s *stubApp) ListEvents(ctx context.Context, _ time.Time, limit int, options model.EventListOptions) ([]model.Event, error) {
|
||||||
|
s.lastEventOptions = options
|
||||||
overview, _ := s.GetOverview(ctx, time.Time{}, limit, model.OverviewOptions{ShowKnownBots: true, ShowAllowed: true})
|
overview, _ := s.GetOverview(ctx, time.Time{}, limit, model.OverviewOptions{ShowKnownBots: true, ShowAllowed: true})
|
||||||
return overview.RecentEvents, nil
|
return overview.RecentEvents, nil
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user