Build a Pi-hole-style dashboard and query log

2026-03-12 17:12:19 +01:00
parent b7943e69db
commit 0a14dd1df9
5 changed files with 1012 additions and 450 deletions
--- a/internal/store/store.go
+++ b/internal/store/store.go
@@ -225,19 +225,38 @@ func (s *Store) RecordEvent(ctx context.Context, event *model.Event) error {

 const responseBytesExpression = `CASE WHEN json_valid(e.raw_json) THEN CAST(COALESCE(json_extract(e.raw_json, '$.size'), 0) AS INTEGER) ELSE 0 END`

+func knownBotExistsClause(ipExpression string) string {
+	return `EXISTS (
+		SELECT 1
+		FROM ip_investigations i
+		WHERE i.ip = ` + ipExpression + `
+		  AND json_valid(i.payload_json)
+		  AND json_type(i.payload_json, '$.bot') IS NOT NULL
+		  AND COALESCE(json_extract(i.payload_json, '$.bot.verified'), 0) = 1
+	)`
+}
+
 func overviewFilterQueryParts(options model.OverviewOptions) (joins []string, clauses []string) {
 	if !options.ShowAllowed {
 		joins = append(joins, `LEFT JOIN ip_state s ON s.ip = e.client_ip`)
 		clauses = append(clauses, `COALESCE(s.state, '') <> '`+string(model.IPStateAllowed)+`'`)
 	}
 	if !options.ShowKnownBots {
-		clauses = append(clauses, `NOT EXISTS (
-			SELECT 1
-			FROM ip_investigations i
-			WHERE i.ip = e.client_ip
-			  AND json_valid(i.payload_json)
-			  AND json_type(i.payload_json, '$.bot') IS NOT NULL
-		)`)
+		clauses = append(clauses, `NOT `+knownBotExistsClause(`e.client_ip`))
+	}
+	return joins, clauses
+}
+
+func eventFilterQueryParts(options model.EventListOptions) (joins []string, clauses []string) {
+	joins = append(joins, `LEFT JOIN ip_state s ON s.ip = e.client_ip`)
+	if !options.ShowAllowed {
+		clauses = append(clauses, `COALESCE(s.state, '') <> '`+string(model.IPStateAllowed)+`'`)
+	}
+	if options.ReviewOnly {
+		clauses = append(clauses, `COALESCE(s.state, '') = '`+string(model.IPStateReview)+`'`)
+	}
+	if !options.ShowKnownBots {
+		clauses = append(clauses, `NOT `+knownBotExistsClause(`e.client_ip`))
 	}
 	return joins, clauses
 }
@@ -440,8 +459,23 @@ func (s *Store) GetOverview(ctx context.Context, since time.Time, limit int, opt
 	if err != nil {
 		return model.Overview{}, err
 	}
+	activityBuckets, err := s.listActivityBuckets(ctx, since, options)
+	if err != nil {
+		return model.Overview{}, err
+	}
+	methods, err := s.listMethodBreakdown(ctx, since, options)
+	if err != nil {
+		return model.Overview{}, err
+	}
+	bots, err := s.listBotBreakdown(ctx, since, options)
+	if err != nil {
+		return model.Overview{}, err
+	}
 	overview.RecentIPs = recentIPs
 	overview.RecentEvents = recentEvents
+	overview.ActivityBuckets = activityBuckets
+	overview.Methods = methods
+	overview.Bots = bots
 	overview.TopIPsByEvents = topIPsByEvents
 	overview.TopIPsByTraffic = topIPsByTraffic
 	overview.TopSources = topSources
@@ -610,6 +644,200 @@ func (s *Store) listTopURLRows(ctx context.Context, since time.Time, limit int,
 	return items, nil
 }

+func (s *Store) listActivityBuckets(ctx context.Context, since time.Time, options model.OverviewOptions) ([]model.ActivityBucket, error) {
+	if since.IsZero() {
+		return nil, nil
+	}
+	joins, clauses := overviewFilterQueryParts(options)
+	query := `
+		SELECT (CAST(strftime('%s', e.occurred_at) AS INTEGER) / 600) * 600 AS bucket_unix,
+		       e.source_name,
+		       COUNT(*) AS event_count
+		FROM events e`
+	if len(joins) > 0 {
+		query += ` ` + strings.Join(joins, ` `)
+	}
+	args := []any{formatTime(since)}
+	clauses = append([]string{`e.occurred_at >= ?`}, clauses...)
+	if len(clauses) > 0 {
+		query += ` WHERE ` + strings.Join(clauses, ` AND `)
+	}
+	query += ` GROUP BY bucket_unix, e.source_name ORDER BY bucket_unix ASC, event_count DESC, e.source_name ASC`
+
+	rows, err := s.db.QueryContext(ctx, query, args...)
+	if err != nil {
+		return nil, fmt.Errorf("list activity buckets: %w", err)
+	}
+	defer rows.Close()
+
+	type bucketKey int64
+	bucketMap := map[bucketKey]*model.ActivityBucket{}
+	for rows.Next() {
+		var bucketUnix int64
+		var sourceName string
+		var events int64
+		if err := rows.Scan(&bucketUnix, &sourceName, &events); err != nil {
+			return nil, fmt.Errorf("scan activity bucket: %w", err)
+		}
+		key := bucketKey(bucketUnix)
+		bucket, ok := bucketMap[key]
+		if !ok {
+			bucket = &model.ActivityBucket{BucketStart: time.Unix(bucketUnix, 0).UTC()}
+			bucketMap[key] = bucket
+		}
+		bucket.TotalEvents += events
+		bucket.Sources = append(bucket.Sources, model.ActivitySourceCount{SourceName: sourceName, Events: events})
+	}
+	if err := rows.Err(); err != nil {
+		return nil, fmt.Errorf("iterate activity buckets: %w", err)
+	}
+
+	start := since.UTC().Truncate(10 * time.Minute)
+	end := time.Now().UTC().Truncate(10 * time.Minute)
+	buckets := make([]model.ActivityBucket, 0, int(end.Sub(start)/(10*time.Minute))+1)
+	for current := start; !current.After(end); current = current.Add(10 * time.Minute) {
+		bucket, ok := bucketMap[bucketKey(current.Unix())]
+		if !ok {
+			buckets = append(buckets, model.ActivityBucket{BucketStart: current})
+			continue
+		}
+		sort.Slice(bucket.Sources, func(left int, right int) bool {
+			if bucket.Sources[left].Events != bucket.Sources[right].Events {
+				return bucket.Sources[left].Events > bucket.Sources[right].Events
+			}
+			return bucket.Sources[left].SourceName < bucket.Sources[right].SourceName
+		})
+		buckets = append(buckets, *bucket)
+	}
+	return buckets, nil
+}
+
+func (s *Store) listMethodBreakdown(ctx context.Context, since time.Time, options model.OverviewOptions) ([]model.MethodBreakdownRow, error) {
+	joins, clauses := overviewFilterQueryParts(options)
+	query := `
+		SELECT COALESCE(NULLIF(UPPER(TRIM(e.method)), ''), 'OTHER') AS method,
+		       COUNT(*) AS event_count
+		FROM events e`
+	if len(joins) > 0 {
+		query += ` ` + strings.Join(joins, ` `)
+	}
+	args := make([]any, 0, 2)
+	if !since.IsZero() {
+		clauses = append([]string{`e.occurred_at >= ?`}, clauses...)
+		args = append(args, formatTime(since))
+	}
+	if len(clauses) > 0 {
+		query += ` WHERE ` + strings.Join(clauses, ` AND `)
+	}
+	query += ` GROUP BY method ORDER BY event_count DESC, method ASC`
+
+	rows, err := s.db.QueryContext(ctx, query, args...)
+	if err != nil {
+		return nil, fmt.Errorf("list method breakdown: %w", err)
+	}
+	defer rows.Close()
+
+	items := make([]model.MethodBreakdownRow, 0, 8)
+	for rows.Next() {
+		var item model.MethodBreakdownRow
+		if err := rows.Scan(&item.Method, &item.Events); err != nil {
+			return nil, fmt.Errorf("scan method breakdown row: %w", err)
+		}
+		items = append(items, item)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, fmt.Errorf("iterate method breakdown rows: %w", err)
+	}
+	return items, nil
+}
+
+func (s *Store) listBotBreakdown(ctx context.Context, since time.Time, options model.OverviewOptions) ([]model.BotBreakdownRow, error) {
+	joins, clauses := overviewFilterQueryParts(options)
+	joins = append(joins, `LEFT JOIN ip_investigations i ON i.ip = e.client_ip`)
+	query := `
+		SELECT
+		  COALESCE(SUM(CASE WHEN json_valid(i.payload_json)
+		                       AND json_type(i.payload_json, '$.bot') IS NOT NULL
+		                       AND COALESCE(json_extract(i.payload_json, '$.bot.verified'), 0) = 1
+		                  THEN 1 ELSE 0 END), 0) AS known_bots,
+		  COALESCE(SUM(CASE WHEN json_valid(i.payload_json)
+		                       AND json_type(i.payload_json, '$.bot') IS NOT NULL
+		                       AND COALESCE(json_extract(i.payload_json, '$.bot.verified'), 0) <> 1
+		                  THEN 1 ELSE 0 END), 0) AS possible_bots,
+		  COALESCE(SUM(CASE WHEN NOT json_valid(i.payload_json)
+		                       OR json_type(i.payload_json, '$.bot') IS NULL
+		                  THEN 1 ELSE 0 END), 0) AS other_traffic
+		FROM events e`
+	if len(joins) > 0 {
+		query += ` ` + strings.Join(joins, ` `)
+	}
+	args := make([]any, 0, 2)
+	if !since.IsZero() {
+		clauses = append([]string{`e.occurred_at >= ?`}, clauses...)
+		args = append(args, formatTime(since))
+	}
+	if len(clauses) > 0 {
+		query += ` WHERE ` + strings.Join(clauses, ` AND `)
+	}
+
+	var knownBots int64
+	var possibleBots int64
+	var otherTraffic int64
+	if err := s.db.QueryRowContext(ctx, query, args...).Scan(&knownBots, &possibleBots, &otherTraffic); err != nil {
+		return nil, fmt.Errorf("list bot breakdown: %w", err)
+	}
+	return []model.BotBreakdownRow{
+		{Key: "known", Label: "Known bots", Events: knownBots},
+		{Key: "possible", Label: "Possible bots", Events: possibleBots},
+		{Key: "other", Label: "Other traffic", Events: otherTraffic},
+	}, nil
+}
+
+func (s *Store) ListEvents(ctx context.Context, since time.Time, limit int, options model.EventListOptions) ([]model.Event, error) {
+	if limit <= 0 {
+		limit = 100
+	}
+	joins, clauses := eventFilterQueryParts(options)
+	query := `
+		SELECT e.id, e.source_name, e.profile_name, e.occurred_at, e.remote_ip, e.client_ip, e.host,
+		       e.method, e.uri, e.path, e.status, e.user_agent, e.decision, e.decision_reason,
+		       e.decision_reasons_json, e.enforced, e.raw_json, e.created_at,
+		       COALESCE(s.state, ''), COALESCE(s.manual_override, '')
+		FROM events e`
+	if len(joins) > 0 {
+		query += ` ` + strings.Join(joins, ` `)
+	}
+	args := make([]any, 0, 2)
+	if !since.IsZero() {
+		clauses = append([]string{`e.occurred_at >= ?`}, clauses...)
+		args = append(args, formatTime(since))
+	}
+	if len(clauses) > 0 {
+		query += ` WHERE ` + strings.Join(clauses, ` AND `)
+	}
+	query += ` ORDER BY e.occurred_at DESC, e.id DESC LIMIT ?`
+	args = append(args, limit)
+
+	rows, err := s.db.QueryContext(ctx, query, args...)
+	if err != nil {
+		return nil, fmt.Errorf("list events: %w", err)
+	}
+	defer rows.Close()
+
+	items := make([]model.Event, 0, limit)
+	for rows.Next() {
+		item, err := scanEvent(rows)
+		if err != nil {
+			return nil, err
+		}
+		items = append(items, item)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, fmt.Errorf("iterate events: %w", err)
+	}
+	return items, nil
+}
+
 func (s *Store) ListRecentEvents(ctx context.Context, limit int) ([]model.Event, error) {
 	if limit <= 0 {
 		limit = 50