You've already forked caddy-opnsense-blocker
Build initial caddy-opnsense-blocker daemon
This commit is contained in:
194
internal/caddylog/parser.go
Normal file
194
internal/caddylog/parser.go
Normal file
@@ -0,0 +1,194 @@
|
||||
package caddylog
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.dern.ovh/infrastructure/caddy-opnsense-blocker/internal/model"
|
||||
)
|
||||
|
||||
var ErrEmptyLine = errors.New("empty log line")
|
||||
|
||||
type accessLogEntry struct {
|
||||
Timestamp json.RawMessage `json:"ts"`
|
||||
Status int `json:"status"`
|
||||
Request accessLogRequest `json:"request"`
|
||||
}
|
||||
|
||||
type accessLogRequest struct {
|
||||
RemoteIP string `json:"remote_ip"`
|
||||
ClientIP string `json:"client_ip"`
|
||||
Host string `json:"host"`
|
||||
Method string `json:"method"`
|
||||
URI string `json:"uri"`
|
||||
Headers map[string][]string `json:"headers"`
|
||||
}
|
||||
|
||||
func ParseLine(line string) (model.AccessLogRecord, error) {
|
||||
trimmed := strings.TrimSpace(line)
|
||||
if trimmed == "" {
|
||||
return model.AccessLogRecord{}, ErrEmptyLine
|
||||
}
|
||||
|
||||
var entry accessLogEntry
|
||||
if err := json.Unmarshal([]byte(trimmed), &entry); err != nil {
|
||||
return model.AccessLogRecord{}, fmt.Errorf("decode caddy log line: %w", err)
|
||||
}
|
||||
if entry.Status == 0 {
|
||||
return model.AccessLogRecord{}, errors.New("missing caddy status")
|
||||
}
|
||||
|
||||
remoteIP, err := normalizeIP(entry.Request.RemoteIP)
|
||||
if err != nil && strings.TrimSpace(entry.Request.RemoteIP) != "" {
|
||||
return model.AccessLogRecord{}, fmt.Errorf("normalize remote ip: %w", err)
|
||||
}
|
||||
|
||||
clientCandidate := entry.Request.ClientIP
|
||||
if strings.TrimSpace(clientCandidate) == "" {
|
||||
clientCandidate = entry.Request.RemoteIP
|
||||
}
|
||||
clientIP, err := normalizeIP(clientCandidate)
|
||||
if err != nil {
|
||||
return model.AccessLogRecord{}, fmt.Errorf("normalize client ip: %w", err)
|
||||
}
|
||||
|
||||
occurredAt, err := parseTimestamp(entry.Timestamp)
|
||||
if err != nil {
|
||||
return model.AccessLogRecord{}, fmt.Errorf("parse timestamp: %w", err)
|
||||
}
|
||||
|
||||
uri := entry.Request.URI
|
||||
if strings.TrimSpace(uri) == "" {
|
||||
uri = "/"
|
||||
}
|
||||
|
||||
return model.AccessLogRecord{
|
||||
OccurredAt: occurredAt,
|
||||
RemoteIP: remoteIP,
|
||||
ClientIP: clientIP,
|
||||
Host: strings.TrimSpace(entry.Request.Host),
|
||||
Method: strings.ToUpper(strings.TrimSpace(entry.Request.Method)),
|
||||
URI: uri,
|
||||
Path: pathFromURI(uri),
|
||||
Status: entry.Status,
|
||||
UserAgent: firstUserAgent(entry.Request.Headers),
|
||||
RawJSON: trimmed,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func ParseLines(lines []string) ([]model.AccessLogRecord, error) {
|
||||
records := make([]model.AccessLogRecord, 0, len(lines))
|
||||
for _, line := range lines {
|
||||
record, err := ParseLine(line)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrEmptyLine) {
|
||||
continue
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
records = append(records, record)
|
||||
}
|
||||
return records, nil
|
||||
}
|
||||
|
||||
func firstUserAgent(headers map[string][]string) string {
|
||||
if len(headers) == 0 {
|
||||
return ""
|
||||
}
|
||||
for _, key := range []string{"User-Agent", "user-agent", "USER-AGENT"} {
|
||||
if values, ok := headers[key]; ok && len(values) > 0 {
|
||||
return strings.TrimSpace(values[0])
|
||||
}
|
||||
}
|
||||
for key, values := range headers {
|
||||
if strings.EqualFold(key, "user-agent") && len(values) > 0 {
|
||||
return strings.TrimSpace(values[0])
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func parseTimestamp(raw json.RawMessage) (time.Time, error) {
|
||||
if len(raw) == 0 {
|
||||
return time.Time{}, errors.New("missing timestamp")
|
||||
}
|
||||
|
||||
var numeric float64
|
||||
if err := json.Unmarshal(raw, &numeric); err == nil {
|
||||
seconds := int64(numeric)
|
||||
nanos := int64((numeric - float64(seconds)) * float64(time.Second))
|
||||
return time.Unix(seconds, nanos).UTC(), nil
|
||||
}
|
||||
|
||||
var text string
|
||||
if err := json.Unmarshal(raw, &text); err == nil {
|
||||
text = strings.TrimSpace(text)
|
||||
if text == "" {
|
||||
return time.Time{}, errors.New("empty timestamp")
|
||||
}
|
||||
if numeric, err := strconv.ParseFloat(text, 64); err == nil {
|
||||
seconds := int64(numeric)
|
||||
nanos := int64((numeric - float64(seconds)) * float64(time.Second))
|
||||
return time.Unix(seconds, nanos).UTC(), nil
|
||||
}
|
||||
for _, layout := range []string{time.RFC3339Nano, time.RFC3339} {
|
||||
parsed, err := time.Parse(layout, text)
|
||||
if err == nil {
|
||||
return parsed.UTC(), nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return time.Time{}, fmt.Errorf("unsupported timestamp payload %s", string(raw))
|
||||
}
|
||||
|
||||
func normalizeIP(value string) (string, error) {
|
||||
trimmed := strings.TrimSpace(value)
|
||||
if trimmed == "" {
|
||||
return "", errors.New("missing ip address")
|
||||
}
|
||||
parsed := net.ParseIP(trimmed)
|
||||
if parsed == nil {
|
||||
return "", fmt.Errorf("invalid ip address %q", value)
|
||||
}
|
||||
return parsed.String(), nil
|
||||
}
|
||||
|
||||
func pathFromURI(rawURI string) string {
|
||||
trimmed := strings.TrimSpace(rawURI)
|
||||
if trimmed == "" {
|
||||
return "/"
|
||||
}
|
||||
|
||||
parsed, err := url.ParseRequestURI(trimmed)
|
||||
if err == nil {
|
||||
if parsed.Path == "" {
|
||||
return "/"
|
||||
}
|
||||
return normalizePath(parsed.Path)
|
||||
}
|
||||
|
||||
value := strings.SplitN(trimmed, "?", 2)[0]
|
||||
value = strings.SplitN(value, "#", 2)[0]
|
||||
return normalizePath(value)
|
||||
}
|
||||
|
||||
func normalizePath(value string) string {
|
||||
trimmed := strings.TrimSpace(value)
|
||||
if trimmed == "" {
|
||||
return "/"
|
||||
}
|
||||
if !strings.HasPrefix(trimmed, "/") {
|
||||
trimmed = "/" + trimmed
|
||||
}
|
||||
if trimmed != "/" {
|
||||
trimmed = strings.TrimRight(trimmed, "/")
|
||||
}
|
||||
return strings.ToLower(trimmed)
|
||||
}
|
||||
57
internal/caddylog/parser_test.go
Normal file
57
internal/caddylog/parser_test.go
Normal file
@@ -0,0 +1,57 @@
|
||||
package caddylog
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestParseLineWithNumericTimestamp(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
line := `{"ts":1710000000.5,"status":404,"request":{"remote_ip":"198.51.100.10","client_ip":"203.0.113.5","host":"example.test","method":"GET","uri":"/wp-login.php?foo=bar","headers":{"User-Agent":["UnitTestBot/1.0"]}}}`
|
||||
record, err := ParseLine(line)
|
||||
if err != nil {
|
||||
t.Fatalf("parse line: %v", err)
|
||||
}
|
||||
|
||||
if got, want := record.ClientIP, "203.0.113.5"; got != want {
|
||||
t.Fatalf("unexpected client ip: got %q want %q", got, want)
|
||||
}
|
||||
if got, want := record.RemoteIP, "198.51.100.10"; got != want {
|
||||
t.Fatalf("unexpected remote ip: got %q want %q", got, want)
|
||||
}
|
||||
if got, want := record.Path, "/wp-login.php"; got != want {
|
||||
t.Fatalf("unexpected path: got %q want %q", got, want)
|
||||
}
|
||||
if got, want := record.UserAgent, "UnitTestBot/1.0"; got != want {
|
||||
t.Fatalf("unexpected user agent: got %q want %q", got, want)
|
||||
}
|
||||
if got, want := record.Method, "GET"; got != want {
|
||||
t.Fatalf("unexpected method: got %q want %q", got, want)
|
||||
}
|
||||
expected := time.Unix(1710000000, 500000000).UTC()
|
||||
if !record.OccurredAt.Equal(expected) {
|
||||
t.Fatalf("unexpected timestamp: got %s want %s", record.OccurredAt, expected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseLineWithRFC3339TimestampAndMissingClientIP(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
line := `{"ts":"2025-03-11T12:13:14.123456Z","status":401,"request":{"remote_ip":"2001:db8::1","host":"git.example.test","method":"POST","uri":"user/login","headers":{"user-agent":["curl/8.0"]}}}`
|
||||
record, err := ParseLine(line)
|
||||
if err != nil {
|
||||
t.Fatalf("parse line: %v", err)
|
||||
}
|
||||
|
||||
if got, want := record.ClientIP, "2001:db8::1"; got != want {
|
||||
t.Fatalf("unexpected fallback client ip: got %q want %q", got, want)
|
||||
}
|
||||
if got, want := record.Path, "/user/login"; got != want {
|
||||
t.Fatalf("unexpected path: got %q want %q", got, want)
|
||||
}
|
||||
if !strings.Contains(record.RawJSON, `"status":401`) {
|
||||
t.Fatalf("raw json was not preserved")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user