You've already forked caddy-opnsense-blocker
Add on-demand IP investigation and richer IP details
This commit is contained in:
@@ -36,11 +36,12 @@ func (d Duration) MarshalYAML() (any, error) {
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
Server ServerConfig `yaml:"server"`
|
||||
Storage StorageConfig `yaml:"storage"`
|
||||
OPNsense OPNsenseConfig `yaml:"opnsense"`
|
||||
Profiles map[string]ProfileConfig `yaml:"profiles"`
|
||||
Sources []SourceConfig `yaml:"sources"`
|
||||
Server ServerConfig `yaml:"server"`
|
||||
Storage StorageConfig `yaml:"storage"`
|
||||
Investigation InvestigationConfig `yaml:"investigation"`
|
||||
OPNsense OPNsenseConfig `yaml:"opnsense"`
|
||||
Profiles map[string]ProfileConfig `yaml:"profiles"`
|
||||
Sources []SourceConfig `yaml:"sources"`
|
||||
}
|
||||
|
||||
type ServerConfig struct {
|
||||
@@ -54,6 +55,14 @@ type StorageConfig struct {
|
||||
Path string `yaml:"path"`
|
||||
}
|
||||
|
||||
type InvestigationConfig struct {
|
||||
Enabled bool `yaml:"enabled"`
|
||||
RefreshAfter Duration `yaml:"refresh_after"`
|
||||
Timeout Duration `yaml:"timeout"`
|
||||
UserAgent string `yaml:"user_agent"`
|
||||
SpamhausEnabled bool `yaml:"spamhaus_enabled"`
|
||||
}
|
||||
|
||||
type OPNsenseConfig struct {
|
||||
Enabled bool `yaml:"enabled"`
|
||||
BaseURL string `yaml:"base_url"`
|
||||
@@ -157,6 +166,21 @@ func (c *Config) applyDefaults() error {
|
||||
if c.Storage.Path == "" {
|
||||
c.Storage.Path = "./data/caddy-opnsense-blocker.db"
|
||||
}
|
||||
if !c.Investigation.Enabled {
|
||||
c.Investigation.Enabled = true
|
||||
}
|
||||
if c.Investigation.RefreshAfter.Duration == 0 {
|
||||
c.Investigation.RefreshAfter.Duration = 24 * time.Hour
|
||||
}
|
||||
if c.Investigation.Timeout.Duration == 0 {
|
||||
c.Investigation.Timeout.Duration = 8 * time.Second
|
||||
}
|
||||
if strings.TrimSpace(c.Investigation.UserAgent) == "" {
|
||||
c.Investigation.UserAgent = "caddy-opnsense-blocker/0.2"
|
||||
}
|
||||
if !c.Investigation.SpamhausEnabled {
|
||||
c.Investigation.SpamhausEnabled = true
|
||||
}
|
||||
|
||||
if c.OPNsense.Timeout.Duration == 0 {
|
||||
c.OPNsense.Timeout.Duration = 8 * time.Second
|
||||
|
||||
822
internal/investigation/service.go
Normal file
822
internal/investigation/service.go
Normal file
@@ -0,0 +1,822 @@
|
||||
package investigation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/csv"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.dern.ovh/infrastructure/caddy-opnsense-blocker/internal/config"
|
||||
"git.dern.ovh/infrastructure/caddy-opnsense-blocker/internal/model"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultRDAPBootstrapIPv4 = "https://data.iana.org/rdap/ipv4.json"
|
||||
defaultRDAPBootstrapIPv6 = "https://data.iana.org/rdap/ipv6.json"
|
||||
spamhausLookupZone = "zen.spamhaus.org"
|
||||
)
|
||||
|
||||
type dnsResolver interface {
|
||||
LookupAddr(ctx context.Context, addr string) ([]string, error)
|
||||
LookupIPAddr(ctx context.Context, host string) ([]net.IPAddr, error)
|
||||
LookupHost(ctx context.Context, host string) ([]string, error)
|
||||
}
|
||||
|
||||
type httpClient interface {
|
||||
Do(req *http.Request) (*http.Response, error)
|
||||
}
|
||||
|
||||
type Service struct {
|
||||
cfg config.InvestigationConfig
|
||||
logger *log.Logger
|
||||
client httpClient
|
||||
resolver dnsResolver
|
||||
|
||||
mu sync.Mutex
|
||||
networkCache map[string]networkCacheEntry
|
||||
bootstrapCache map[string]bootstrapCacheEntry
|
||||
providers []botProvider
|
||||
bootstrapURLs map[string]string
|
||||
}
|
||||
|
||||
type networkCacheEntry struct {
|
||||
updatedAt time.Time
|
||||
networks []netip.Prefix
|
||||
}
|
||||
|
||||
type bootstrapCacheEntry struct {
|
||||
updatedAt time.Time
|
||||
services []rdapService
|
||||
}
|
||||
|
||||
type rdapService struct {
|
||||
prefixes []netip.Prefix
|
||||
urls []string
|
||||
}
|
||||
|
||||
type botProvider struct {
|
||||
ID string
|
||||
Name string
|
||||
Icon string
|
||||
SourceFormat string
|
||||
CacheTTL time.Duration
|
||||
IPRangeURLs []string
|
||||
ReverseDNSSuffixes []string
|
||||
UserAgentPrefixes []string
|
||||
}
|
||||
|
||||
func New(cfg config.InvestigationConfig, logger *log.Logger) *Service {
|
||||
return newService(
|
||||
cfg,
|
||||
&http.Client{Timeout: cfg.Timeout.Duration},
|
||||
net.DefaultResolver,
|
||||
logger,
|
||||
defaultBotProviders(),
|
||||
map[string]string{
|
||||
"ipv4": defaultRDAPBootstrapIPv4,
|
||||
"ipv6": defaultRDAPBootstrapIPv6,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
func newService(
|
||||
cfg config.InvestigationConfig,
|
||||
client httpClient,
|
||||
resolver dnsResolver,
|
||||
logger *log.Logger,
|
||||
providers []botProvider,
|
||||
bootstrapURLs map[string]string,
|
||||
) *Service {
|
||||
if logger == nil {
|
||||
logger = log.New(io.Discard, "", 0)
|
||||
}
|
||||
return &Service{
|
||||
cfg: cfg,
|
||||
logger: logger,
|
||||
client: client,
|
||||
resolver: resolver,
|
||||
networkCache: map[string]networkCacheEntry{},
|
||||
bootstrapCache: map[string]bootstrapCacheEntry{},
|
||||
providers: providers,
|
||||
bootstrapURLs: bootstrapURLs,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) Investigate(ctx context.Context, ip string, userAgents []string) (model.IPInvestigation, error) {
|
||||
parsed, err := netip.ParseAddr(strings.TrimSpace(ip))
|
||||
if err != nil {
|
||||
return model.IPInvestigation{}, fmt.Errorf("invalid ip address %q: %w", ip, err)
|
||||
}
|
||||
|
||||
investigation := model.IPInvestigation{
|
||||
IP: parsed.String(),
|
||||
UpdatedAt: time.Now().UTC(),
|
||||
}
|
||||
if !s.cfg.Enabled {
|
||||
return investigation, nil
|
||||
}
|
||||
|
||||
lookupCtx, cancel := context.WithTimeout(ctx, s.cfg.Timeout.Duration)
|
||||
defer cancel()
|
||||
|
||||
normalizedUserAgents := normalizeUserAgents(userAgents)
|
||||
botMatch, reverseDNSInfo := s.identifyBot(lookupCtx, parsed, normalizedUserAgents)
|
||||
if botMatch != nil {
|
||||
investigation.Bot = botMatch
|
||||
investigation.ReverseDNS = reverseDNSInfo
|
||||
return investigation, nil
|
||||
}
|
||||
|
||||
warnings := make([]string, 0, 2)
|
||||
if reverseDNSInfo == nil {
|
||||
reverseDNSInfo, err = s.lookupReverseDNS(lookupCtx, parsed)
|
||||
if err != nil {
|
||||
warnings = append(warnings, err.Error())
|
||||
}
|
||||
}
|
||||
if reverseDNSInfo != nil {
|
||||
investigation.ReverseDNS = reverseDNSInfo
|
||||
}
|
||||
|
||||
registration, err := s.lookupRegistration(lookupCtx, parsed)
|
||||
if err != nil {
|
||||
warnings = append(warnings, err.Error())
|
||||
} else if registration != nil {
|
||||
investigation.Registration = registration
|
||||
}
|
||||
|
||||
if s.cfg.SpamhausEnabled {
|
||||
reputation, err := s.lookupSpamhaus(lookupCtx, parsed)
|
||||
if err != nil {
|
||||
warnings = append(warnings, err.Error())
|
||||
} else if reputation != nil {
|
||||
investigation.Reputation = reputation
|
||||
}
|
||||
}
|
||||
|
||||
if len(warnings) > 0 {
|
||||
investigation.Error = strings.Join(uniqueStrings(warnings), "; ")
|
||||
}
|
||||
return investigation, nil
|
||||
}
|
||||
|
||||
func (s *Service) identifyBot(ctx context.Context, ip netip.Addr, userAgents []string) (*model.BotMatch, *model.ReverseDNSInfo) {
|
||||
var reverseDNSInfo *model.ReverseDNSInfo
|
||||
for _, provider := range s.providers {
|
||||
if len(provider.IPRangeURLs) > 0 {
|
||||
networks, err := s.loadPublishedNetworks(ctx, provider)
|
||||
if err != nil {
|
||||
s.logger.Printf("bot provider %s: %v", provider.ID, err)
|
||||
} else if ipMatchesPrefixes(ip, networks) {
|
||||
if len(provider.UserAgentPrefixes) == 0 || userAgentMatchesPrefixes(userAgents, provider.UserAgentPrefixes) {
|
||||
method := "published_ranges"
|
||||
if len(provider.UserAgentPrefixes) > 0 {
|
||||
method = "user_agent+published_ranges"
|
||||
}
|
||||
return &model.BotMatch{
|
||||
ProviderID: provider.ID,
|
||||
Name: provider.Name,
|
||||
Icon: provider.Icon,
|
||||
Method: method,
|
||||
Verified: true,
|
||||
}, reverseDNSInfo
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(provider.ReverseDNSSuffixes) == 0 {
|
||||
continue
|
||||
}
|
||||
info, err := s.lookupReverseDNS(ctx, ip)
|
||||
if err != nil {
|
||||
s.logger.Printf("bot provider %s reverse DNS: %v", provider.ID, err)
|
||||
continue
|
||||
}
|
||||
if info == nil {
|
||||
continue
|
||||
}
|
||||
reverseDNSInfo = info
|
||||
ptr := strings.ToLower(strings.TrimSuffix(info.PTR, "."))
|
||||
if ptr == "" || !info.ForwardConfirmed {
|
||||
continue
|
||||
}
|
||||
for _, suffix := range provider.ReverseDNSSuffixes {
|
||||
if strings.HasSuffix(ptr, suffix) {
|
||||
return &model.BotMatch{
|
||||
ProviderID: provider.ID,
|
||||
Name: provider.Name,
|
||||
Icon: provider.Icon,
|
||||
Method: "reverse_dns+fcrdns",
|
||||
Verified: true,
|
||||
}, reverseDNSInfo
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil, reverseDNSInfo
|
||||
}
|
||||
|
||||
func (s *Service) loadPublishedNetworks(ctx context.Context, provider botProvider) ([]netip.Prefix, error) {
|
||||
s.mu.Lock()
|
||||
entry, found := s.networkCache[provider.ID]
|
||||
s.mu.Unlock()
|
||||
if found && time.Since(entry.updatedAt) < provider.CacheTTL {
|
||||
return append([]netip.Prefix(nil), entry.networks...), nil
|
||||
}
|
||||
|
||||
networks := make([]netip.Prefix, 0, 64)
|
||||
errMessages := make([]string, 0, len(provider.IPRangeURLs))
|
||||
for _, sourceURL := range provider.IPRangeURLs {
|
||||
payload, err := s.fetchDocument(ctx, sourceURL)
|
||||
if err != nil {
|
||||
errMessages = append(errMessages, err.Error())
|
||||
continue
|
||||
}
|
||||
parsed, err := parsePublishedNetworks(payload, provider.SourceFormat, sourceURL)
|
||||
if err != nil {
|
||||
errMessages = append(errMessages, err.Error())
|
||||
continue
|
||||
}
|
||||
networks = append(networks, parsed...)
|
||||
}
|
||||
if len(networks) == 0 && len(errMessages) > 0 {
|
||||
return nil, fmt.Errorf("load published ranges for %s: %s", provider.ID, strings.Join(uniqueStrings(errMessages), "; "))
|
||||
}
|
||||
networks = uniquePrefixes(networks)
|
||||
s.mu.Lock()
|
||||
s.networkCache[provider.ID] = networkCacheEntry{updatedAt: time.Now().UTC(), networks: append([]netip.Prefix(nil), networks...)}
|
||||
s.mu.Unlock()
|
||||
return networks, nil
|
||||
}
|
||||
|
||||
func parsePublishedNetworks(payload []byte, sourceFormat string, sourceURL string) ([]netip.Prefix, error) {
|
||||
switch sourceFormat {
|
||||
case "json_prefixes":
|
||||
var document struct {
|
||||
Prefixes []struct {
|
||||
IPv4Prefix string `json:"ipv4Prefix"`
|
||||
IPv6Prefix string `json:"ipv6Prefix"`
|
||||
} `json:"prefixes"`
|
||||
}
|
||||
if err := json.Unmarshal(payload, &document); err != nil {
|
||||
return nil, fmt.Errorf("decode published prefix payload from %s: %w", sourceURL, err)
|
||||
}
|
||||
networks := make([]netip.Prefix, 0, len(document.Prefixes))
|
||||
for _, entry := range document.Prefixes {
|
||||
rawPrefix := strings.TrimSpace(entry.IPv4Prefix)
|
||||
if rawPrefix == "" {
|
||||
rawPrefix = strings.TrimSpace(entry.IPv6Prefix)
|
||||
}
|
||||
if rawPrefix == "" {
|
||||
continue
|
||||
}
|
||||
prefix, err := netip.ParsePrefix(rawPrefix)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse published prefix %q from %s: %w", rawPrefix, sourceURL, err)
|
||||
}
|
||||
networks = append(networks, prefix.Masked())
|
||||
}
|
||||
return networks, nil
|
||||
case "geofeed_csv":
|
||||
reader := csv.NewReader(strings.NewReader(string(payload)))
|
||||
rows, err := reader.ReadAll()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("decode geofeed payload from %s: %w", sourceURL, err)
|
||||
}
|
||||
networks := make([]netip.Prefix, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
if len(row) == 0 {
|
||||
continue
|
||||
}
|
||||
candidate := strings.TrimSpace(row[0])
|
||||
if candidate == "" || strings.HasPrefix(candidate, "#") {
|
||||
continue
|
||||
}
|
||||
prefix, err := netip.ParsePrefix(candidate)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse geofeed prefix %q from %s: %w", candidate, sourceURL, err)
|
||||
}
|
||||
networks = append(networks, prefix.Masked())
|
||||
}
|
||||
return networks, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported source format %q for %s", sourceFormat, sourceURL)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Service) lookupReverseDNS(ctx context.Context, ip netip.Addr) (*model.ReverseDNSInfo, error) {
|
||||
names, err := s.resolver.LookupAddr(ctx, ip.String())
|
||||
if err != nil {
|
||||
if isDNSNotFound(err) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, fmt.Errorf("reverse dns lookup for %s: %w", ip, err)
|
||||
}
|
||||
if len(names) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
sort.Strings(names)
|
||||
ptr := strings.TrimSuffix(strings.TrimSpace(names[0]), ".")
|
||||
if ptr == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
resolvedIPs, err := s.resolver.LookupIPAddr(ctx, ptr)
|
||||
if err != nil && !isDNSNotFound(err) {
|
||||
return &model.ReverseDNSInfo{PTR: ptr, ForwardConfirmed: false}, fmt.Errorf("forward-confirm dns lookup for %s: %w", ptr, err)
|
||||
}
|
||||
forwardConfirmed := false
|
||||
for _, resolved := range resolvedIPs {
|
||||
addr, ok := netip.AddrFromSlice(resolved.IP)
|
||||
if ok && addr.Unmap() == ip.Unmap() {
|
||||
forwardConfirmed = true
|
||||
break
|
||||
}
|
||||
}
|
||||
return &model.ReverseDNSInfo{PTR: ptr, ForwardConfirmed: forwardConfirmed}, nil
|
||||
}
|
||||
|
||||
func (s *Service) lookupRegistration(ctx context.Context, ip netip.Addr) (*model.RegistrationInfo, error) {
|
||||
family := "ipv4"
|
||||
if ip.Is6() {
|
||||
family = "ipv6"
|
||||
}
|
||||
services, err := s.loadBootstrap(ctx, family)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
baseURL := lookupRDAPBaseURL(ip, services)
|
||||
if baseURL == "" {
|
||||
return nil, fmt.Errorf("no RDAP service found for %s", ip)
|
||||
}
|
||||
requestURL := strings.TrimRight(baseURL, "/") + "/ip/" + url.PathEscape(ip.String())
|
||||
payload, err := s.fetchJSONDocument(ctx, requestURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("rdap lookup for %s: %w", ip, err)
|
||||
}
|
||||
registration := &model.RegistrationInfo{
|
||||
Source: requestURL,
|
||||
Handle: strings.TrimSpace(asString(payload["handle"])),
|
||||
Name: strings.TrimSpace(asString(payload["name"])),
|
||||
Country: strings.TrimSpace(asString(payload["country"])),
|
||||
Prefix: extractPrefix(payload),
|
||||
Organization: extractOrganization(payload),
|
||||
AbuseEmail: extractAbuseEmail(payload["entities"]),
|
||||
}
|
||||
if registration.Organization == "" {
|
||||
registration.Organization = registration.Name
|
||||
}
|
||||
if registration.Name == "" && registration.Organization == "" && registration.Handle == "" && registration.Prefix == "" && registration.Country == "" && registration.AbuseEmail == "" {
|
||||
return nil, nil
|
||||
}
|
||||
return registration, nil
|
||||
}
|
||||
|
||||
func (s *Service) loadBootstrap(ctx context.Context, family string) ([]rdapService, error) {
|
||||
s.mu.Lock()
|
||||
entry, found := s.bootstrapCache[family]
|
||||
s.mu.Unlock()
|
||||
if found && time.Since(entry.updatedAt) < 24*time.Hour {
|
||||
return append([]rdapService(nil), entry.services...), nil
|
||||
}
|
||||
|
||||
bootstrapURL := s.bootstrapURLs[family]
|
||||
payload, err := s.fetchDocument(ctx, bootstrapURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fetch %s RDAP bootstrap: %w", family, err)
|
||||
}
|
||||
services, err := parseBootstrap(payload, bootstrapURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.mu.Lock()
|
||||
s.bootstrapCache[family] = bootstrapCacheEntry{updatedAt: time.Now().UTC(), services: append([]rdapService(nil), services...)}
|
||||
s.mu.Unlock()
|
||||
return services, nil
|
||||
}
|
||||
|
||||
func parseBootstrap(payload []byte, sourceURL string) ([]rdapService, error) {
|
||||
var document struct {
|
||||
Services [][][]string `json:"services"`
|
||||
}
|
||||
if err := json.Unmarshal(payload, &document); err != nil {
|
||||
return nil, fmt.Errorf("decode RDAP bootstrap from %s: %w", sourceURL, err)
|
||||
}
|
||||
services := make([]rdapService, 0, len(document.Services))
|
||||
for _, rawService := range document.Services {
|
||||
if len(rawService) < 2 {
|
||||
continue
|
||||
}
|
||||
prefixes := make([]netip.Prefix, 0, len(rawService[0]))
|
||||
for _, candidate := range rawService[0] {
|
||||
prefix, err := netip.ParsePrefix(strings.TrimSpace(candidate))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
prefixes = append(prefixes, prefix.Masked())
|
||||
}
|
||||
if len(prefixes) == 0 || len(rawService[1]) == 0 {
|
||||
continue
|
||||
}
|
||||
services = append(services, rdapService{prefixes: prefixes, urls: append([]string(nil), rawService[1]...)})
|
||||
}
|
||||
if len(services) == 0 {
|
||||
return nil, fmt.Errorf("empty RDAP bootstrap payload from %s", sourceURL)
|
||||
}
|
||||
return services, nil
|
||||
}
|
||||
|
||||
func lookupRDAPBaseURL(ip netip.Addr, services []rdapService) string {
|
||||
bestBits := -1
|
||||
bestURL := ""
|
||||
for _, service := range services {
|
||||
for _, prefix := range service.prefixes {
|
||||
if prefix.Contains(ip) && prefix.Bits() > bestBits && len(service.urls) > 0 {
|
||||
bestBits = prefix.Bits()
|
||||
bestURL = strings.TrimSpace(service.urls[0])
|
||||
}
|
||||
}
|
||||
}
|
||||
return bestURL
|
||||
}
|
||||
|
||||
func (s *Service) lookupSpamhaus(ctx context.Context, ip netip.Addr) (*model.ReputationInfo, error) {
|
||||
if !isPublicIP(ip) {
|
||||
return nil, nil
|
||||
}
|
||||
lookupName, err := spamhausLookupName(ip)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
answers, err := s.resolver.LookupHost(ctx, lookupName)
|
||||
if err != nil {
|
||||
if isDNSNotFound(err) {
|
||||
return &model.ReputationInfo{SpamhausLookup: spamhausLookupZone, SpamhausListed: false}, nil
|
||||
}
|
||||
return &model.ReputationInfo{SpamhausLookup: spamhausLookupZone, Error: err.Error()}, nil
|
||||
}
|
||||
return &model.ReputationInfo{
|
||||
SpamhausLookup: spamhausLookupZone,
|
||||
SpamhausListed: len(answers) > 0,
|
||||
SpamhausCodes: uniqueStrings(answers),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *Service) fetchDocument(ctx context.Context, requestURL string) ([]byte, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, requestURL, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("build request for %s: %w", requestURL, err)
|
||||
}
|
||||
req.Header.Set("Accept", "application/json, text/plain, */*")
|
||||
req.Header.Set("User-Agent", s.cfg.UserAgent)
|
||||
|
||||
resp, err := s.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("request %s: %w", requestURL, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
payload, _ := io.ReadAll(io.LimitReader(resp.Body, 8<<10))
|
||||
return nil, fmt.Errorf("request %s returned %s: %s", requestURL, resp.Status, strings.TrimSpace(string(payload)))
|
||||
}
|
||||
payload, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read response %s: %w", requestURL, err)
|
||||
}
|
||||
return payload, nil
|
||||
}
|
||||
|
||||
func (s *Service) fetchJSONDocument(ctx context.Context, requestURL string) (map[string]any, error) {
|
||||
payload, err := s.fetchDocument(ctx, requestURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var decoded map[string]any
|
||||
if err := json.Unmarshal(payload, &decoded); err != nil {
|
||||
return nil, fmt.Errorf("decode json payload from %s: %w", requestURL, err)
|
||||
}
|
||||
return decoded, nil
|
||||
}
|
||||
|
||||
func defaultBotProviders() []botProvider {
|
||||
return []botProvider{
|
||||
{
|
||||
ID: "google_official",
|
||||
Name: "Googlebot",
|
||||
Icon: "🤖",
|
||||
SourceFormat: "json_prefixes",
|
||||
CacheTTL: 24 * time.Hour,
|
||||
IPRangeURLs: []string{
|
||||
"https://developers.google.com/static/crawling/ipranges/common-crawlers.json",
|
||||
"https://developers.google.com/static/crawling/ipranges/special-crawlers.json",
|
||||
"https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers-google.json",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "bing_official",
|
||||
Name: "Bingbot",
|
||||
Icon: "🤖",
|
||||
SourceFormat: "json_prefixes",
|
||||
CacheTTL: 24 * time.Hour,
|
||||
ReverseDNSSuffixes: []string{".search.msn.com"},
|
||||
},
|
||||
{
|
||||
ID: "apple_official",
|
||||
Name: "Applebot",
|
||||
Icon: "🤖",
|
||||
SourceFormat: "json_prefixes",
|
||||
CacheTTL: 24 * time.Hour,
|
||||
IPRangeURLs: []string{"https://search.developer.apple.com/applebot.json"},
|
||||
ReverseDNSSuffixes: []string{".applebot.apple.com"},
|
||||
},
|
||||
{
|
||||
ID: "facebook_official",
|
||||
Name: "Meta crawler",
|
||||
Icon: "🤖",
|
||||
SourceFormat: "geofeed_csv",
|
||||
CacheTTL: 24 * time.Hour,
|
||||
IPRangeURLs: []string{"https://www.facebook.com/peering/geofeed"},
|
||||
UserAgentPrefixes: []string{
|
||||
"facebookexternalhit/",
|
||||
"meta-webindexer/",
|
||||
"meta-externalads/",
|
||||
"meta-externalagent/",
|
||||
"meta-externalfetcher/",
|
||||
},
|
||||
},
|
||||
{
|
||||
ID: "duckduckgo_official",
|
||||
Name: "DuckDuckBot",
|
||||
Icon: "🤖",
|
||||
SourceFormat: "json_prefixes",
|
||||
CacheTTL: 24 * time.Hour,
|
||||
IPRangeURLs: []string{"https://duckduckgo.com/duckduckbot.json"},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func ipMatchesPrefixes(ip netip.Addr, prefixes []netip.Prefix) bool {
|
||||
for _, prefix := range prefixes {
|
||||
if prefix.Contains(ip) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func userAgentMatchesPrefixes(userAgents []string, prefixes []string) bool {
|
||||
for _, agent := range userAgents {
|
||||
for _, prefix := range prefixes {
|
||||
if strings.HasPrefix(agent, prefix) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func normalizeUserAgents(userAgents []string) []string {
|
||||
items := make([]string, 0, len(userAgents))
|
||||
for _, userAgent := range userAgents {
|
||||
normalized := strings.ToLower(strings.TrimSpace(userAgent))
|
||||
if normalized == "" {
|
||||
continue
|
||||
}
|
||||
items = append(items, normalized)
|
||||
}
|
||||
return uniqueStrings(items)
|
||||
}
|
||||
|
||||
func extractPrefix(payload map[string]any) string {
|
||||
items, ok := payload["cidr0_cidrs"].([]any)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
for _, item := range items {
|
||||
entry, ok := item.(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
prefix := strings.TrimSpace(asString(entry["v4prefix"]))
|
||||
if prefix == "" {
|
||||
prefix = strings.TrimSpace(asString(entry["v6prefix"]))
|
||||
}
|
||||
length := asInt(entry["length"])
|
||||
if prefix == "" || length == 0 {
|
||||
continue
|
||||
}
|
||||
return prefix + "/" + strconv.Itoa(length)
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func extractOrganization(payload map[string]any) string {
|
||||
if organization := extractEntityName(payload["entities"]); organization != "" {
|
||||
return organization
|
||||
}
|
||||
return strings.TrimSpace(asString(payload["name"]))
|
||||
}
|
||||
|
||||
func extractEntityName(value any) string {
|
||||
entities, ok := value.([]any)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
for _, rawEntity := range entities {
|
||||
entity, ok := rawEntity.(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if name := strings.TrimSpace(asString(entity["fn"])); name != "" {
|
||||
return name
|
||||
}
|
||||
if name := extractVCardText(entity["vcardArray"], "fn"); name != "" {
|
||||
return name
|
||||
}
|
||||
if nested := extractEntityName(entity["entities"]); nested != "" {
|
||||
return nested
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func extractAbuseEmail(value any) string {
|
||||
entities, ok := value.([]any)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
for _, rawEntity := range entities {
|
||||
entity, ok := rawEntity.(map[string]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
roles := toStrings(entity["roles"])
|
||||
if containsString(roles, "abuse") {
|
||||
if email := extractVCardText(entity["vcardArray"], "email"); email != "" {
|
||||
return email
|
||||
}
|
||||
}
|
||||
if nested := extractAbuseEmail(entity["entities"]); nested != "" {
|
||||
return nested
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func extractVCardText(value any, field string) string {
|
||||
items, ok := value.([]any)
|
||||
if !ok || len(items) < 2 {
|
||||
return ""
|
||||
}
|
||||
rows, ok := items[1].([]any)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
for _, rawRow := range rows {
|
||||
row, ok := rawRow.([]any)
|
||||
if !ok || len(row) < 4 {
|
||||
continue
|
||||
}
|
||||
name, ok := row[0].(string)
|
||||
if !ok || name != field {
|
||||
continue
|
||||
}
|
||||
textValue, ok := row[3].(string)
|
||||
if ok {
|
||||
return strings.TrimSpace(textValue)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func spamhausLookupName(ip netip.Addr) (string, error) {
|
||||
ip = ip.Unmap()
|
||||
if ip.Is4() {
|
||||
bytes := ip.As4()
|
||||
return fmt.Sprintf("%d.%d.%d.%d.%s", bytes[3], bytes[2], bytes[1], bytes[0], spamhausLookupZone), nil
|
||||
}
|
||||
if ip.Is6() {
|
||||
bytes := ip.As16()
|
||||
hexString := hex.EncodeToString(bytes[:])
|
||||
parts := make([]string, 0, len(hexString))
|
||||
for index := len(hexString) - 1; index >= 0; index-- {
|
||||
parts = append(parts, string(hexString[index]))
|
||||
}
|
||||
return strings.Join(parts, ".") + "." + spamhausLookupZone, nil
|
||||
}
|
||||
return "", fmt.Errorf("unsupported ip family for %s", ip)
|
||||
}
|
||||
|
||||
func uniquePrefixes(items []netip.Prefix) []netip.Prefix {
|
||||
if len(items) == 0 {
|
||||
return nil
|
||||
}
|
||||
seen := make(map[string]struct{}, len(items))
|
||||
result := make([]netip.Prefix, 0, len(items))
|
||||
for _, item := range items {
|
||||
key := item.Masked().String()
|
||||
if _, ok := seen[key]; ok {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
result = append(result, item.Masked())
|
||||
}
|
||||
sort.Slice(result, func(left int, right int) bool {
|
||||
if result[left].Bits() == result[right].Bits() {
|
||||
return result[left].String() < result[right].String()
|
||||
}
|
||||
return result[left].Bits() > result[right].Bits()
|
||||
})
|
||||
return result
|
||||
}
|
||||
|
||||
func uniqueStrings(items []string) []string {
|
||||
if len(items) == 0 {
|
||||
return nil
|
||||
}
|
||||
seen := make(map[string]struct{}, len(items))
|
||||
result := make([]string, 0, len(items))
|
||||
for _, item := range items {
|
||||
if _, ok := seen[item]; ok {
|
||||
continue
|
||||
}
|
||||
seen[item] = struct{}{}
|
||||
result = append(result, item)
|
||||
}
|
||||
sort.Strings(result)
|
||||
return result
|
||||
}
|
||||
|
||||
func containsString(items []string, expected string) bool {
|
||||
for _, item := range items {
|
||||
if item == expected {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func toStrings(value any) []string {
|
||||
rawItems, ok := value.([]any)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
items := make([]string, 0, len(rawItems))
|
||||
for _, rawItem := range rawItems {
|
||||
if text, ok := rawItem.(string); ok {
|
||||
items = append(items, strings.TrimSpace(text))
|
||||
}
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
func asString(value any) string {
|
||||
text, _ := value.(string)
|
||||
return text
|
||||
}
|
||||
|
||||
func asInt(value any) int {
|
||||
switch current := value.(type) {
|
||||
case float64:
|
||||
return int(current)
|
||||
case float32:
|
||||
return int(current)
|
||||
case int:
|
||||
return current
|
||||
case int64:
|
||||
return int(current)
|
||||
case json.Number:
|
||||
converted, _ := current.Int64()
|
||||
return int(converted)
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
func isDNSNotFound(err error) bool {
|
||||
var dnsError *net.DNSError
|
||||
if errors.As(err, &dnsError) {
|
||||
return dnsError.IsNotFound
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func isPublicIP(ip netip.Addr) bool {
|
||||
ip = ip.Unmap()
|
||||
if !ip.IsValid() || ip.IsLoopback() || ip.IsMulticast() || ip.IsPrivate() || ip.IsLinkLocalMulticast() || ip.IsLinkLocalUnicast() || ip.IsUnspecified() {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
229
internal/investigation/service_test.go
Normal file
229
internal/investigation/service_test.go
Normal file
@@ -0,0 +1,229 @@
|
||||
package investigation
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/netip"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.dern.ovh/infrastructure/caddy-opnsense-blocker/internal/config"
|
||||
)
|
||||
|
||||
func TestInvestigateRecognizesBotViaPublishedRanges(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/ranges.json" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"prefixes":[{"ipv4Prefix":"203.0.113.0/24"}]}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
svc := newService(
|
||||
config.InvestigationConfig{Enabled: true, Timeout: config.Duration{Duration: time.Second}, UserAgent: "test-agent", SpamhausEnabled: true},
|
||||
server.Client(),
|
||||
&fakeResolver{},
|
||||
log.New(testWriter{t}, "", 0),
|
||||
[]botProvider{{
|
||||
ID: "google_official",
|
||||
Name: "Googlebot",
|
||||
Icon: "🤖",
|
||||
SourceFormat: "json_prefixes",
|
||||
CacheTTL: time.Hour,
|
||||
IPRangeURLs: []string{server.URL + "/ranges.json"},
|
||||
}},
|
||||
map[string]string{"ipv4": server.URL + "/bootstrap-v4.json", "ipv6": server.URL + "/bootstrap-v6.json"},
|
||||
)
|
||||
|
||||
investigation, err := svc.Investigate(context.Background(), "203.0.113.10", []string{"Mozilla/5.0"})
|
||||
if err != nil {
|
||||
t.Fatalf("investigate ip: %v", err)
|
||||
}
|
||||
if investigation.Bot == nil || investigation.Bot.Name != "Googlebot" {
|
||||
t.Fatalf("expected Googlebot match, got %+v", investigation)
|
||||
}
|
||||
if investigation.Registration != nil || investigation.Reputation != nil {
|
||||
t.Fatalf("expected bot investigation to stop before deep lookups, got %+v", investigation)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInvestigateRecognizesBotViaReverseDNS(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
resolver := &fakeResolver{
|
||||
reverse: map[string][]string{"198.51.100.20": {"crawl.search.example.test."}},
|
||||
forward: map[string][]net.IPAddr{"crawl.search.example.test": {{IP: net.ParseIP("198.51.100.20")}}},
|
||||
}
|
||||
|
||||
svc := newService(
|
||||
config.InvestigationConfig{Enabled: true, Timeout: config.Duration{Duration: time.Second}, UserAgent: "test-agent", SpamhausEnabled: true},
|
||||
http.DefaultClient,
|
||||
resolver,
|
||||
log.New(testWriter{t}, "", 0),
|
||||
[]botProvider{{
|
||||
ID: "bing_official",
|
||||
Name: "Bingbot",
|
||||
Icon: "🤖",
|
||||
CacheTTL: time.Hour,
|
||||
ReverseDNSSuffixes: []string{".search.example.test"},
|
||||
}},
|
||||
map[string]string{},
|
||||
)
|
||||
|
||||
investigation, err := svc.Investigate(context.Background(), "198.51.100.20", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("investigate ip: %v", err)
|
||||
}
|
||||
if investigation.Bot == nil || investigation.Bot.Name != "Bingbot" || investigation.ReverseDNS == nil || !investigation.ReverseDNS.ForwardConfirmed {
|
||||
t.Fatalf("expected reverse DNS bot match, got %+v", investigation)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInvestigateLoadsRegistrationAndSpamhausForNonBot(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
serverURL := ""
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/bootstrap-v4.json":
|
||||
_, _ = w.Write([]byte(`{"services":[[["198.51.100.0/24"],["` + serverURL + `/rdap/"]]]}`))
|
||||
case "/rdap/ip/198.51.100.30":
|
||||
_, _ = w.Write([]byte(`{
|
||||
"handle":"NET-198-51-100-0-1",
|
||||
"name":"Example Network",
|
||||
"country":"FR",
|
||||
"cidr0_cidrs":[{"v4prefix":"198.51.100.0","length":24}],
|
||||
"entities":[{"roles":["abuse"],"vcardArray":["vcard",[["email",{},"text","abuse@example.test"],["fn",{},"text","Example ISP"]]]}]
|
||||
}`))
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
serverURL = server.URL
|
||||
defer server.Close()
|
||||
|
||||
resolver := &fakeResolver{
|
||||
hosts: map[string][]string{spamhausQuery(t, "198.51.100.30"): {"127.0.0.2"}},
|
||||
}
|
||||
|
||||
svc := newService(
|
||||
config.InvestigationConfig{Enabled: true, Timeout: config.Duration{Duration: 2 * time.Second}, UserAgent: "test-agent", SpamhausEnabled: true},
|
||||
server.Client(),
|
||||
resolver,
|
||||
log.New(testWriter{t}, "", 0),
|
||||
nil,
|
||||
map[string]string{"ipv4": server.URL + "/bootstrap-v4.json", "ipv6": server.URL + "/bootstrap-v6.json"},
|
||||
)
|
||||
|
||||
investigation, err := svc.Investigate(context.Background(), "198.51.100.30", []string{"curl/8.0"})
|
||||
if err != nil {
|
||||
t.Fatalf("investigate ip: %v", err)
|
||||
}
|
||||
if investigation.Bot != nil {
|
||||
t.Fatalf("expected no bot match, got %+v", investigation.Bot)
|
||||
}
|
||||
if investigation.Registration == nil || investigation.Registration.Organization != "Example ISP" || investigation.Registration.Prefix != "198.51.100.0/24" {
|
||||
t.Fatalf("unexpected registration info: %+v", investigation.Registration)
|
||||
}
|
||||
if investigation.Reputation == nil || !investigation.Reputation.SpamhausListed {
|
||||
t.Fatalf("expected spamhaus listing, got %+v", investigation.Reputation)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublishedNetworksAreCached(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var mu sync.Mutex
|
||||
requestCount := 0
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
mu.Lock()
|
||||
requestCount++
|
||||
mu.Unlock()
|
||||
_, _ = w.Write([]byte(`{"prefixes":[{"ipv4Prefix":"203.0.113.0/24"}]}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
svc := newService(
|
||||
config.InvestigationConfig{Enabled: true, Timeout: config.Duration{Duration: time.Second}, UserAgent: "test-agent", SpamhausEnabled: true},
|
||||
server.Client(),
|
||||
&fakeResolver{},
|
||||
log.New(testWriter{t}, "", 0),
|
||||
[]botProvider{{
|
||||
ID: "provider",
|
||||
Name: "Provider bot",
|
||||
Icon: "🤖",
|
||||
SourceFormat: "json_prefixes",
|
||||
CacheTTL: time.Hour,
|
||||
IPRangeURLs: []string{server.URL},
|
||||
}},
|
||||
map[string]string{},
|
||||
)
|
||||
|
||||
for index := 0; index < 2; index++ {
|
||||
if _, err := svc.Investigate(context.Background(), "203.0.113.10", nil); err != nil {
|
||||
t.Fatalf("investigate ip #%d: %v", index, err)
|
||||
}
|
||||
}
|
||||
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if requestCount != 1 {
|
||||
t.Fatalf("expected exactly one published-range request, got %d", requestCount)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeResolver struct {
|
||||
reverse map[string][]string
|
||||
forward map[string][]net.IPAddr
|
||||
hosts map[string][]string
|
||||
}
|
||||
|
||||
func (r *fakeResolver) LookupAddr(_ context.Context, addr string) ([]string, error) {
|
||||
if values, ok := r.reverse[addr]; ok {
|
||||
return values, nil
|
||||
}
|
||||
return nil, &net.DNSError{IsNotFound: true}
|
||||
}
|
||||
|
||||
func (r *fakeResolver) LookupIPAddr(_ context.Context, host string) ([]net.IPAddr, error) {
|
||||
if values, ok := r.forward[host]; ok {
|
||||
return values, nil
|
||||
}
|
||||
return nil, &net.DNSError{IsNotFound: true}
|
||||
}
|
||||
|
||||
func (r *fakeResolver) LookupHost(_ context.Context, host string) ([]string, error) {
|
||||
if values, ok := r.hosts[host]; ok {
|
||||
return values, nil
|
||||
}
|
||||
return nil, &net.DNSError{IsNotFound: true}
|
||||
}
|
||||
|
||||
type testWriter struct{ t *testing.T }
|
||||
|
||||
func (w testWriter) Write(payload []byte) (int, error) {
|
||||
w.t.Helper()
|
||||
w.t.Log(strings.TrimSpace(string(payload)))
|
||||
return len(payload), nil
|
||||
}
|
||||
|
||||
func spamhausQuery(t *testing.T, ip string) string {
|
||||
t.Helper()
|
||||
addr, err := netip.ParseAddr(ip)
|
||||
if err != nil {
|
||||
t.Fatalf("parse ip: %v", err)
|
||||
}
|
||||
query, err := spamhausLookupName(addr)
|
||||
if err != nil {
|
||||
t.Fatalf("build spamhaus query: %v", err)
|
||||
}
|
||||
return query
|
||||
}
|
||||
@@ -113,6 +113,59 @@ type OPNsenseAction struct {
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
type BotMatch struct {
|
||||
ProviderID string `json:"provider_id"`
|
||||
Name string `json:"name"`
|
||||
Icon string `json:"icon"`
|
||||
Method string `json:"method"`
|
||||
Verified bool `json:"verified"`
|
||||
}
|
||||
|
||||
type ReverseDNSInfo struct {
|
||||
PTR string `json:"ptr"`
|
||||
ForwardConfirmed bool `json:"forward_confirmed"`
|
||||
}
|
||||
|
||||
type RegistrationInfo struct {
|
||||
Source string `json:"source"`
|
||||
Handle string `json:"handle"`
|
||||
Name string `json:"name"`
|
||||
Prefix string `json:"prefix"`
|
||||
Organization string `json:"organization"`
|
||||
Country string `json:"country"`
|
||||
AbuseEmail string `json:"abuse_email"`
|
||||
}
|
||||
|
||||
type ReputationInfo struct {
|
||||
SpamhausLookup string `json:"spamhaus_lookup"`
|
||||
SpamhausListed bool `json:"spamhaus_listed"`
|
||||
SpamhausCodes []string `json:"spamhaus_codes,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type IPInvestigation struct {
|
||||
IP string `json:"ip"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Bot *BotMatch `json:"bot,omitempty"`
|
||||
ReverseDNS *ReverseDNSInfo `json:"reverse_dns,omitempty"`
|
||||
Registration *RegistrationInfo `json:"registration,omitempty"`
|
||||
Reputation *ReputationInfo `json:"reputation,omitempty"`
|
||||
}
|
||||
|
||||
type OPNsenseStatus struct {
|
||||
Configured bool `json:"configured"`
|
||||
Present bool `json:"present"`
|
||||
CheckedAt time.Time `json:"checked_at,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type ActionAvailability struct {
|
||||
CanBlock bool `json:"can_block"`
|
||||
CanUnblock bool `json:"can_unblock"`
|
||||
CanClearOverride bool `json:"can_clear_override"`
|
||||
}
|
||||
|
||||
type SourceOffset struct {
|
||||
SourceName string
|
||||
Path string
|
||||
@@ -122,10 +175,13 @@ type SourceOffset struct {
|
||||
}
|
||||
|
||||
type IPDetails struct {
|
||||
State IPState `json:"state"`
|
||||
RecentEvents []Event `json:"recent_events"`
|
||||
Decisions []DecisionRecord `json:"decisions"`
|
||||
BackendActions []OPNsenseAction `json:"backend_actions"`
|
||||
State IPState `json:"state"`
|
||||
RecentEvents []Event `json:"recent_events"`
|
||||
Decisions []DecisionRecord `json:"decisions"`
|
||||
BackendActions []OPNsenseAction `json:"backend_actions"`
|
||||
Investigation *IPInvestigation `json:"investigation,omitempty"`
|
||||
OPNsense OPNsenseStatus `json:"opnsense"`
|
||||
Actions ActionAvailability `json:"actions"`
|
||||
}
|
||||
|
||||
type Overview struct {
|
||||
|
||||
@@ -23,23 +23,29 @@ import (
|
||||
)
|
||||
|
||||
type Service struct {
|
||||
cfg *config.Config
|
||||
store *store.Store
|
||||
evaluator *engine.Evaluator
|
||||
blocker opnsense.AliasClient
|
||||
logger *log.Logger
|
||||
cfg *config.Config
|
||||
store *store.Store
|
||||
evaluator *engine.Evaluator
|
||||
blocker opnsense.AliasClient
|
||||
investigator Investigator
|
||||
logger *log.Logger
|
||||
}
|
||||
|
||||
func New(cfg *config.Config, db *store.Store, blocker opnsense.AliasClient, logger *log.Logger) *Service {
|
||||
type Investigator interface {
|
||||
Investigate(ctx context.Context, ip string, userAgents []string) (model.IPInvestigation, error)
|
||||
}
|
||||
|
||||
func New(cfg *config.Config, db *store.Store, blocker opnsense.AliasClient, investigator Investigator, logger *log.Logger) *Service {
|
||||
if logger == nil {
|
||||
logger = log.New(io.Discard, "", 0)
|
||||
}
|
||||
return &Service{
|
||||
cfg: cfg,
|
||||
store: db,
|
||||
evaluator: engine.NewEvaluator(),
|
||||
blocker: blocker,
|
||||
logger: logger,
|
||||
cfg: cfg,
|
||||
store: db,
|
||||
evaluator: engine.NewEvaluator(),
|
||||
blocker: blocker,
|
||||
investigator: investigator,
|
||||
logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -75,7 +81,40 @@ func (s *Service) GetIPDetails(ctx context.Context, ip string) (model.IPDetails,
|
||||
if err != nil {
|
||||
return model.IPDetails{}, err
|
||||
}
|
||||
return s.store.GetIPDetails(ctx, normalized, 100, 100, 100)
|
||||
details, err := s.store.GetIPDetails(ctx, normalized, 0, 100, 100)
|
||||
if err != nil {
|
||||
return model.IPDetails{}, err
|
||||
}
|
||||
return s.decorateDetails(ctx, details)
|
||||
}
|
||||
|
||||
func (s *Service) InvestigateIP(ctx context.Context, ip string) (model.IPDetails, error) {
|
||||
normalized, err := normalizeIP(ip)
|
||||
if err != nil {
|
||||
return model.IPDetails{}, err
|
||||
}
|
||||
details, err := s.store.GetIPDetails(ctx, normalized, 0, 100, 100)
|
||||
if err != nil {
|
||||
return model.IPDetails{}, err
|
||||
}
|
||||
if s.investigator != nil {
|
||||
investigation, found, err := s.store.GetInvestigation(ctx, normalized)
|
||||
if err != nil {
|
||||
return model.IPDetails{}, err
|
||||
}
|
||||
shouldRefresh := !found || time.Since(investigation.UpdatedAt) >= s.cfg.Investigation.RefreshAfter.Duration
|
||||
if shouldRefresh {
|
||||
fresh, err := s.investigator.Investigate(ctx, normalized, collectUserAgents(details.RecentEvents))
|
||||
if err != nil {
|
||||
return model.IPDetails{}, err
|
||||
}
|
||||
if err := s.store.SaveInvestigation(ctx, fresh); err != nil {
|
||||
return model.IPDetails{}, err
|
||||
}
|
||||
details.Investigation = &fresh
|
||||
}
|
||||
}
|
||||
return s.decorateDetails(ctx, details)
|
||||
}
|
||||
|
||||
func (s *Service) ForceBlock(ctx context.Context, ip string, actor string, reason string) error {
|
||||
@@ -410,3 +449,58 @@ func defaultReason(reason string, fallback string) string {
|
||||
}
|
||||
return strings.TrimSpace(reason)
|
||||
}
|
||||
|
||||
func (s *Service) decorateDetails(ctx context.Context, details model.IPDetails) (model.IPDetails, error) {
|
||||
if details.State.IP != "" && details.Investigation == nil {
|
||||
investigation, found, err := s.store.GetInvestigation(ctx, details.State.IP)
|
||||
if err != nil {
|
||||
return model.IPDetails{}, err
|
||||
}
|
||||
if found {
|
||||
details.Investigation = &investigation
|
||||
}
|
||||
}
|
||||
details.OPNsense = s.resolveOPNsenseStatus(ctx, details.State)
|
||||
details.Actions = actionAvailability(details.State, details.OPNsense)
|
||||
return details, nil
|
||||
}
|
||||
|
||||
func (s *Service) resolveOPNsenseStatus(ctx context.Context, state model.IPState) model.OPNsenseStatus {
|
||||
status := model.OPNsenseStatus{Configured: s.blocker != nil}
|
||||
if s.blocker == nil || state.IP == "" {
|
||||
return status
|
||||
}
|
||||
status.CheckedAt = time.Now().UTC()
|
||||
present, err := s.blocker.IsIPPresent(ctx, state.IP)
|
||||
if err != nil {
|
||||
status.Error = err.Error()
|
||||
return status
|
||||
}
|
||||
status.Present = present
|
||||
return status
|
||||
}
|
||||
|
||||
func actionAvailability(state model.IPState, backend model.OPNsenseStatus) model.ActionAvailability {
|
||||
present := false
|
||||
if backend.Configured && backend.Error == "" {
|
||||
present = backend.Present
|
||||
} else {
|
||||
present = state.State == model.IPStateBlocked || state.ManualOverride == model.ManualOverrideForceBlock
|
||||
}
|
||||
return model.ActionAvailability{
|
||||
CanBlock: !present,
|
||||
CanUnblock: present,
|
||||
CanClearOverride: state.ManualOverride != model.ManualOverrideNone,
|
||||
}
|
||||
}
|
||||
|
||||
func collectUserAgents(events []model.Event) []string {
|
||||
items := make([]string, 0, len(events))
|
||||
for _, event := range events {
|
||||
if strings.TrimSpace(event.UserAgent) == "" {
|
||||
continue
|
||||
}
|
||||
items = append(items, event.UserAgent)
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
@@ -87,7 +87,7 @@ sources:
|
||||
t.Fatalf("open store: %v", err)
|
||||
}
|
||||
defer database.Close()
|
||||
svc := New(cfg, database, opnsense.NewClient(cfg.OPNsense), log.New(os.Stderr, "", 0))
|
||||
svc := New(cfg, database, opnsense.NewClient(cfg.OPNsense), nil, log.New(os.Stderr, "", 0))
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
@@ -94,6 +94,14 @@ CREATE TABLE IF NOT EXISTS source_offsets (
|
||||
offset INTEGER NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS ip_investigations (
|
||||
ip TEXT PRIMARY KEY,
|
||||
payload_json TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_ip_investigations_updated_at ON ip_investigations(updated_at DESC, ip ASC);
|
||||
`
|
||||
|
||||
type Store struct {
|
||||
@@ -492,6 +500,53 @@ func (s *Store) GetIPDetails(ctx context.Context, ip string, eventLimit, decisio
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *Store) GetInvestigation(ctx context.Context, ip string) (model.IPInvestigation, bool, error) {
|
||||
row := s.db.QueryRowContext(ctx, `SELECT payload_json, updated_at FROM ip_investigations WHERE ip = ?`, ip)
|
||||
var payload string
|
||||
var updatedAt string
|
||||
if err := row.Scan(&payload, &updatedAt); err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return model.IPInvestigation{}, false, nil
|
||||
}
|
||||
return model.IPInvestigation{}, false, fmt.Errorf("query ip investigation %q: %w", ip, err)
|
||||
}
|
||||
var item model.IPInvestigation
|
||||
if err := json.Unmarshal([]byte(payload), &item); err != nil {
|
||||
return model.IPInvestigation{}, false, fmt.Errorf("decode ip investigation %q: %w", ip, err)
|
||||
}
|
||||
parsed, err := parseTime(updatedAt)
|
||||
if err != nil {
|
||||
return model.IPInvestigation{}, false, fmt.Errorf("parse ip investigation updated_at: %w", err)
|
||||
}
|
||||
item.UpdatedAt = parsed
|
||||
return item, true, nil
|
||||
}
|
||||
|
||||
func (s *Store) SaveInvestigation(ctx context.Context, item model.IPInvestigation) error {
|
||||
if item.UpdatedAt.IsZero() {
|
||||
item.UpdatedAt = time.Now().UTC()
|
||||
}
|
||||
payload, err := json.Marshal(item)
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode ip investigation: %w", err)
|
||||
}
|
||||
_, err = s.db.ExecContext(
|
||||
ctx,
|
||||
`INSERT INTO ip_investigations (ip, payload_json, updated_at)
|
||||
VALUES (?, ?, ?)
|
||||
ON CONFLICT(ip) DO UPDATE SET
|
||||
payload_json = excluded.payload_json,
|
||||
updated_at = excluded.updated_at`,
|
||||
item.IP,
|
||||
string(payload),
|
||||
formatTime(item.UpdatedAt),
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("upsert ip investigation %q: %w", item.IP, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) GetSourceOffset(ctx context.Context, sourceName string) (model.SourceOffset, bool, error) {
|
||||
row := s.db.QueryRowContext(ctx, `SELECT source_name, path, inode, offset, updated_at FROM source_offsets WHERE source_name = ?`, sourceName)
|
||||
var offset model.SourceOffset
|
||||
@@ -536,10 +591,7 @@ func (s *Store) SaveSourceOffset(ctx context.Context, offset model.SourceOffset)
|
||||
}
|
||||
|
||||
func (s *Store) listEventsForIP(ctx context.Context, ip string, limit int) ([]model.Event, error) {
|
||||
if limit <= 0 {
|
||||
limit = 50
|
||||
}
|
||||
rows, err := s.db.QueryContext(ctx, `
|
||||
query := `
|
||||
SELECT e.id, e.source_name, e.profile_name, e.occurred_at, e.remote_ip, e.client_ip, e.host,
|
||||
e.method, e.uri, e.path, e.status, e.user_agent, e.decision, e.decision_reason,
|
||||
e.decision_reasons_json, e.enforced, e.raw_json, e.created_at,
|
||||
@@ -547,17 +599,19 @@ func (s *Store) listEventsForIP(ctx context.Context, ip string, limit int) ([]mo
|
||||
FROM events e
|
||||
LEFT JOIN ip_state s ON s.ip = e.client_ip
|
||||
WHERE e.client_ip = ?
|
||||
ORDER BY e.occurred_at DESC, e.id DESC
|
||||
LIMIT ?`,
|
||||
ip,
|
||||
limit,
|
||||
)
|
||||
ORDER BY e.occurred_at DESC, e.id DESC`
|
||||
args := []any{ip}
|
||||
if limit > 0 {
|
||||
query += ` LIMIT ?`
|
||||
args = append(args, limit)
|
||||
}
|
||||
rows, err := s.db.QueryContext(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list events for ip %q: %w", ip, err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
items := make([]model.Event, 0, limit)
|
||||
items := make([]model.Event, 0, max(limit, 0))
|
||||
for rows.Next() {
|
||||
item, err := scanEvent(rows)
|
||||
if err != nil {
|
||||
@@ -651,6 +705,13 @@ func (s *Store) listBackendActionsForIP(ctx context.Context, ip string, limit in
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func max(left int, right int) int {
|
||||
if left > right {
|
||||
return left
|
||||
}
|
||||
return right
|
||||
}
|
||||
|
||||
func getIPStateDB(ctx context.Context, db queryer, ip string) (model.IPState, bool, error) {
|
||||
row := db.QueryRowContext(ctx, `
|
||||
SELECT ip, first_seen_at, last_seen_at, last_source_name, last_user_agent, latest_status,
|
||||
|
||||
@@ -113,4 +113,20 @@ func TestStoreRecordsEventsAndState(t *testing.T) {
|
||||
if len(details.RecentEvents) != 1 || len(details.Decisions) != 1 || len(details.BackendActions) != 1 {
|
||||
t.Fatalf("unexpected ip details: %+v", details)
|
||||
}
|
||||
|
||||
investigation := model.IPInvestigation{
|
||||
IP: event.ClientIP,
|
||||
UpdatedAt: occurredAt,
|
||||
Bot: &model.BotMatch{Name: "Googlebot", ProviderID: "google_official", Icon: "🤖", Method: "published_ranges", Verified: true},
|
||||
}
|
||||
if err := db.SaveInvestigation(ctx, investigation); err != nil {
|
||||
t.Fatalf("save investigation: %v", err)
|
||||
}
|
||||
loadedInvestigation, found, err := db.GetInvestigation(ctx, event.ClientIP)
|
||||
if err != nil {
|
||||
t.Fatalf("get investigation: %v", err)
|
||||
}
|
||||
if !found || loadedInvestigation.Bot == nil || loadedInvestigation.Bot.Name != "Googlebot" {
|
||||
t.Fatalf("unexpected investigation payload: found=%v investigation=%+v", found, loadedInvestigation)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ type App interface {
|
||||
ListEvents(ctx context.Context, limit int) ([]model.Event, error)
|
||||
ListIPs(ctx context.Context, limit int, state string) ([]model.IPState, error)
|
||||
GetIPDetails(ctx context.Context, ip string) (model.IPDetails, error)
|
||||
InvestigateIP(ctx context.Context, ip string) (model.IPDetails, error)
|
||||
ForceBlock(ctx context.Context, ip string, actor string, reason string) error
|
||||
ForceAllow(ctx context.Context, ip string, actor string, reason string) error
|
||||
ClearOverride(ctx context.Context, ip string, actor string, reason string) error
|
||||
@@ -165,6 +166,16 @@ func (h *handler) handleAPIIP(w http.ResponseWriter, r *http.Request) {
|
||||
methodNotAllowed(w)
|
||||
return
|
||||
}
|
||||
if action == "investigate" {
|
||||
details, err := h.app.InvestigateIP(r.Context(), ip)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, details)
|
||||
return
|
||||
}
|
||||
|
||||
payload, err := decodeActionPayload(r)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err)
|
||||
@@ -175,7 +186,7 @@ func (h *handler) handleAPIIP(w http.ResponseWriter, r *http.Request) {
|
||||
err = h.app.ForceBlock(r.Context(), ip, payload.Actor, payload.Reason)
|
||||
case "unblock":
|
||||
err = h.app.ForceAllow(r.Context(), ip, payload.Actor, payload.Reason)
|
||||
case "reset":
|
||||
case "clear-override", "reset":
|
||||
err = h.app.ClearOverride(r.Context(), ip, payload.Actor, payload.Reason)
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
@@ -199,26 +210,50 @@ func decodeActionPayload(r *http.Request) (actionPayload, error) {
|
||||
if r.ContentLength == 0 {
|
||||
return payload, nil
|
||||
}
|
||||
decoder := json.NewDecoder(io.LimitReader(r.Body, 1<<20))
|
||||
if err := decoder.Decode(&payload); err != nil {
|
||||
if errors.Is(err, io.EOF) {
|
||||
return payload, nil
|
||||
}
|
||||
return actionPayload{}, fmt.Errorf("decode request body: %w", err)
|
||||
limited := io.LimitReader(r.Body, 1<<20)
|
||||
if err := json.NewDecoder(limited).Decode(&payload); err != nil {
|
||||
return actionPayload{}, fmt.Errorf("decode action payload: %w", err)
|
||||
}
|
||||
return payload, nil
|
||||
}
|
||||
|
||||
func queryLimit(r *http.Request, fallback int) int {
|
||||
if fallback <= 0 {
|
||||
fallback = 50
|
||||
}
|
||||
value := strings.TrimSpace(r.URL.Query().Get("limit"))
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
parsed, err := strconv.Atoi(value)
|
||||
if err != nil || parsed <= 0 {
|
||||
return fallback
|
||||
}
|
||||
if parsed > 1000 {
|
||||
return 1000
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
func writeJSON(w http.ResponseWriter, status int, payload any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(payload)
|
||||
}
|
||||
|
||||
func writeError(w http.ResponseWriter, status int, err error) {
|
||||
writeJSON(w, status, map[string]any{"error": err.Error()})
|
||||
}
|
||||
|
||||
func extractPathValue(path string, prefix string) (string, bool) {
|
||||
if !strings.HasPrefix(path, prefix) {
|
||||
return "", false
|
||||
}
|
||||
rest := strings.TrimPrefix(path, prefix)
|
||||
rest = strings.Trim(rest, "/")
|
||||
if rest == "" {
|
||||
value := strings.Trim(strings.TrimPrefix(path, prefix), "/")
|
||||
if value == "" {
|
||||
return "", false
|
||||
}
|
||||
decoded, err := url.PathUnescape(rest)
|
||||
decoded, err := url.PathUnescape(value)
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
@@ -229,48 +264,19 @@ func extractAPIPath(path string) (ip string, action string, ok bool) {
|
||||
if !strings.HasPrefix(path, "/api/ips/") {
|
||||
return "", "", false
|
||||
}
|
||||
rest := strings.TrimPrefix(path, "/api/ips/")
|
||||
rest = strings.Trim(rest, "/")
|
||||
rest := strings.Trim(strings.TrimPrefix(path, "/api/ips/"), "/")
|
||||
if rest == "" {
|
||||
return "", "", false
|
||||
}
|
||||
parts := strings.Split(rest, "/")
|
||||
decoded, err := url.PathUnescape(parts[0])
|
||||
decodedIP, err := url.PathUnescape(parts[0])
|
||||
if err != nil {
|
||||
return "", "", false
|
||||
}
|
||||
if len(parts) == 1 {
|
||||
return decoded, "", true
|
||||
return decodedIP, "", true
|
||||
}
|
||||
if len(parts) == 2 {
|
||||
return decoded, parts[1], true
|
||||
}
|
||||
return "", "", false
|
||||
}
|
||||
|
||||
func queryLimit(r *http.Request, fallback int) int {
|
||||
value := strings.TrimSpace(r.URL.Query().Get("limit"))
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
parsed, err := strconv.Atoi(value)
|
||||
if err != nil || parsed <= 0 {
|
||||
return fallback
|
||||
}
|
||||
if parsed > 500 {
|
||||
return 500
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
func writeJSON(w http.ResponseWriter, status int, payload any) {
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(payload)
|
||||
}
|
||||
|
||||
func writeError(w http.ResponseWriter, status int, err error) {
|
||||
writeJSON(w, status, map[string]any{"error": err.Error()})
|
||||
return decodedIP, parts[1], true
|
||||
}
|
||||
|
||||
func methodNotAllowed(w http.ResponseWriter) {
|
||||
@@ -309,10 +315,6 @@ const overviewHTML = `<!doctype html>
|
||||
.status.review { background: #78350f; }
|
||||
.status.allowed { background: #14532d; }
|
||||
.status.observed { background: #1e293b; }
|
||||
.actions { display: flex; gap: .35rem; flex-wrap: wrap; }
|
||||
button { background: #2563eb; color: white; border: 0; border-radius: .45rem; padding: .35rem .6rem; cursor: pointer; }
|
||||
button.secondary { background: #475569; }
|
||||
button.danger { background: #dc2626; }
|
||||
.muted { color: #94a3b8; }
|
||||
.mono { font-family: ui-monospace, monospace; }
|
||||
.panel { background: #111827; border: 1px solid #334155; border-radius: .75rem; padding: 1rem; overflow: auto; }
|
||||
@@ -329,13 +331,13 @@ const overviewHTML = `<!doctype html>
|
||||
<h2>Recent IPs</h2>
|
||||
<table>
|
||||
<thead>
|
||||
<tr><th>IP</th><th>State</th><th>Override</th><th>Events</th><th>Last seen</th><th>Reason</th><th>Actions</th></tr>
|
||||
<tr><th>IP</th><th>State</th><th>Override</th><th>Events</th><th>Last seen</th><th>Reason</th></tr>
|
||||
</thead>
|
||||
<tbody id="ips-body"></tbody>
|
||||
</table>
|
||||
</section>
|
||||
<section class="panel">
|
||||
<h2>Recent Events</h2>
|
||||
<h2>Recent events</h2>
|
||||
<table>
|
||||
<thead>
|
||||
<tr><th>Time</th><th>Source</th><th>IP</th><th>Host</th><th>Method</th><th>Path</th><th>Status</th><th>Decision</th></tr>
|
||||
@@ -349,18 +351,11 @@ const overviewHTML = `<!doctype html>
|
||||
return String(value ?? '').replace(/[&<>"']/g, character => ({'&':'&','<':'<','>':'>','"':'"',"'":'''}[character]));
|
||||
}
|
||||
|
||||
async function sendAction(ip, action) {
|
||||
const reason = window.prompt('Optional reason', '');
|
||||
const response = await fetch('/api/ips/' + encodeURIComponent(ip) + '/' + action, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ reason, actor: 'web-ui' }),
|
||||
});
|
||||
if (!response.ok) {
|
||||
const payload = await response.json().catch(() => ({ error: response.statusText }));
|
||||
window.alert(payload.error || 'Request failed');
|
||||
function formatDate(value) {
|
||||
if (!value) {
|
||||
return '—';
|
||||
}
|
||||
await refresh();
|
||||
return new Date(value).toLocaleString();
|
||||
}
|
||||
|
||||
function renderStats(data) {
|
||||
@@ -387,15 +382,8 @@ const overviewHTML = `<!doctype html>
|
||||
' <td><span class="status ' + escapeHtml(item.state) + '">' + escapeHtml(item.state) + '</span></td>',
|
||||
' <td>' + escapeHtml(item.manual_override) + '</td>',
|
||||
' <td>' + escapeHtml(item.total_events) + '</td>',
|
||||
' <td>' + escapeHtml(new Date(item.last_seen_at).toLocaleString()) + '</td>',
|
||||
' <td>' + escapeHtml(formatDate(item.last_seen_at)) + '</td>',
|
||||
' <td>' + escapeHtml(item.state_reason) + '</td>',
|
||||
' <td>',
|
||||
' <div class="actions">',
|
||||
' <button class="danger" onclick="sendAction("' + escapeHtml(item.ip) + '", "block")">Block</button>',
|
||||
' <button onclick="sendAction("' + escapeHtml(item.ip) + '", "unblock")">Unblock</button>',
|
||||
' <button class="secondary" onclick="sendAction("' + escapeHtml(item.ip) + '", "reset")">Reset</button>',
|
||||
' </div>',
|
||||
' </td>',
|
||||
'</tr>'
|
||||
].join('')).join('');
|
||||
}
|
||||
@@ -403,7 +391,7 @@ const overviewHTML = `<!doctype html>
|
||||
function renderEvents(items) {
|
||||
document.getElementById('events-body').innerHTML = items.map(item => [
|
||||
'<tr>',
|
||||
' <td>' + escapeHtml(new Date(item.occurred_at).toLocaleString()) + '</td>',
|
||||
' <td>' + escapeHtml(formatDate(item.occurred_at)) + '</td>',
|
||||
' <td>' + escapeHtml(item.source_name) + '</td>',
|
||||
' <td class="mono"><a href="/ips/' + encodeURIComponent(item.client_ip) + '">' + escapeHtml(item.client_ip) + '</a></td>',
|
||||
' <td>' + escapeHtml(item.host) + '</td>',
|
||||
@@ -437,24 +425,29 @@ const ipDetailsHTML = `<!doctype html>
|
||||
<title>{{ .Title }}</title>
|
||||
<style>
|
||||
:root { color-scheme: dark; }
|
||||
body { font-family: system-ui, sans-serif; margin: 0; background: #0f172a; color: #e2e8f0; }
|
||||
header { padding: 1rem 1.5rem; border-bottom: 1px solid #334155; }
|
||||
body { font-family: system-ui, sans-serif; margin: 0; background: #020617; color: #e2e8f0; }
|
||||
header { padding: 1rem 1.5rem; border-bottom: 1px solid #334155; position: sticky; top: 0; background: rgba(2,6,23,.97); }
|
||||
main { padding: 1.5rem; display: grid; gap: 1.5rem; }
|
||||
.panel { background: #111827; border: 1px solid #334155; border-radius: .75rem; padding: 1rem; overflow: auto; }
|
||||
table { width: 100%; border-collapse: collapse; }
|
||||
th, td { padding: .55rem .65rem; border-bottom: 1px solid #1e293b; text-align: left; vertical-align: top; }
|
||||
th { color: #93c5fd; }
|
||||
h1, h2 { margin: 0 0 .75rem 0; }
|
||||
.status { display: inline-block; padding: .15rem .45rem; border-radius: 999px; font-size: .8rem; background: #1e293b; }
|
||||
.status.blocked { background: #7f1d1d; }
|
||||
.status.review { background: #78350f; }
|
||||
.status.allowed { background: #14532d; }
|
||||
.status.observed { background: #1e293b; }
|
||||
.actions { display: flex; gap: .35rem; flex-wrap: wrap; }
|
||||
.muted { color: #94a3b8; }
|
||||
.badge { display: inline-flex; align-items: center; gap: .35rem; padding: .2rem .55rem; border-radius: 999px; background: #1d4ed8; color: white; font-size: .8rem; }
|
||||
.kv { display: grid; gap: .45rem; }
|
||||
.actions { display: flex; gap: .35rem; flex-wrap: wrap; margin-top: .9rem; }
|
||||
button { background: #2563eb; color: white; border: 0; border-radius: .45rem; padding: .35rem .6rem; cursor: pointer; }
|
||||
button.secondary { background: #475569; }
|
||||
button.danger { background: #dc2626; }
|
||||
table { width: 100%; border-collapse: collapse; }
|
||||
th, td { padding: .55rem .65rem; border-bottom: 1px solid #1e293b; text-align: left; vertical-align: top; }
|
||||
th { color: #93c5fd; }
|
||||
.mono { font-family: ui-monospace, monospace; }
|
||||
a { color: #93c5fd; text-decoration: none; }
|
||||
.hint { font-size: .9rem; color: #94a3b8; margin-top: .75rem; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
@@ -465,18 +458,19 @@ const ipDetailsHTML = `<!doctype html>
|
||||
<main>
|
||||
<section class="panel">
|
||||
<h2>State</h2>
|
||||
<div id="state"></div>
|
||||
<div class="actions">
|
||||
<button class="danger" onclick="sendAction('block')">Block</button>
|
||||
<button onclick="sendAction('unblock')">Unblock</button>
|
||||
<button class="secondary" onclick="sendAction('reset')">Reset</button>
|
||||
</div>
|
||||
<div id="state" class="kv"></div>
|
||||
<div id="actions" class="actions"></div>
|
||||
<div class="hint">Clear override removes the local manual override only. It does not change the current OPNsense alias entry.</div>
|
||||
</section>
|
||||
<section class="panel">
|
||||
<h2>Recent events</h2>
|
||||
<h2>Investigation</h2>
|
||||
<div id="investigation" class="kv"></div>
|
||||
</section>
|
||||
<section class="panel">
|
||||
<h2>Requests from this IP</h2>
|
||||
<table>
|
||||
<thead>
|
||||
<tr><th>Time</th><th>Source</th><th>Method</th><th>Path</th><th>Status</th><th>Decision</th></tr>
|
||||
<tr><th>Time</th><th>Source</th><th>Host</th><th>Method</th><th>URI</th><th>Status</th><th>Decision</th><th>User agent</th></tr>
|
||||
</thead>
|
||||
<tbody id="events-body"></tbody>
|
||||
</table>
|
||||
@@ -507,8 +501,18 @@ const ipDetailsHTML = `<!doctype html>
|
||||
return String(value ?? '').replace(/[&<>"']/g, character => ({'&':'&','<':'<','>':'>','"':'"',"'":'''}[character]));
|
||||
}
|
||||
|
||||
async function sendAction(action) {
|
||||
const reason = window.prompt('Optional reason', '');
|
||||
function formatDate(value) {
|
||||
if (!value) {
|
||||
return '—';
|
||||
}
|
||||
return new Date(value).toLocaleString();
|
||||
}
|
||||
|
||||
async function sendAction(action, promptLabel) {
|
||||
const reason = window.prompt(promptLabel, '');
|
||||
if (reason === null) {
|
||||
return;
|
||||
}
|
||||
const response = await fetch('/api/ips/' + encodeURIComponent(ip) + '/' + action, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
@@ -517,67 +521,148 @@ const ipDetailsHTML = `<!doctype html>
|
||||
if (!response.ok) {
|
||||
const payload = await response.json().catch(() => ({ error: response.statusText }));
|
||||
window.alert(payload.error || 'Request failed');
|
||||
return;
|
||||
}
|
||||
await refresh();
|
||||
const data = await response.json();
|
||||
renderAll(data);
|
||||
}
|
||||
|
||||
function renderState(state) {
|
||||
async function investigate() {
|
||||
document.getElementById('investigation').innerHTML = '<div class="muted">Refreshing investigation…</div>';
|
||||
const response = await fetch('/api/ips/' + encodeURIComponent(ip) + '/investigate', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ actor: 'web-ui' }),
|
||||
});
|
||||
if (!response.ok) {
|
||||
const payload = await response.json().catch(() => ({ error: response.statusText }));
|
||||
document.getElementById('investigation').innerHTML = '<div class="muted">' + escapeHtml(payload.error || 'Investigation failed') + '</div>';
|
||||
return;
|
||||
}
|
||||
const data = await response.json();
|
||||
renderAll(data);
|
||||
}
|
||||
|
||||
function renderState(data) {
|
||||
const state = data.state || {};
|
||||
const opnsense = data.opnsense || {};
|
||||
const opnsenseLabel = opnsense.configured ? (opnsense.error ? ('unknown (' + opnsense.error + ')') : (opnsense.present ? 'blocked' : 'not blocked')) : 'disabled';
|
||||
document.getElementById('state').innerHTML = [
|
||||
'<div><strong>State</strong>: <span class="status ' + escapeHtml(state.state) + '">' + escapeHtml(state.state) + '</span></div>',
|
||||
'<div><strong>Override</strong>: ' + escapeHtml(state.manual_override) + '</div>',
|
||||
'<div><strong>Total events</strong>: ' + escapeHtml(state.total_events) + '</div>',
|
||||
'<div><strong>Last seen</strong>: ' + escapeHtml(new Date(state.last_seen_at).toLocaleString()) + '</div>',
|
||||
'<div><strong>Reason</strong>: ' + escapeHtml(state.state_reason) + '</div>'
|
||||
'<div><strong>Last seen</strong>: ' + escapeHtml(formatDate(state.last_seen_at)) + '</div>',
|
||||
'<div><strong>Reason</strong>: ' + escapeHtml(state.state_reason) + '</div>',
|
||||
'<div><strong>OPNsense alias</strong>: ' + escapeHtml(opnsenseLabel) + '</div>'
|
||||
].join('');
|
||||
}
|
||||
|
||||
function renderActions(data) {
|
||||
const actions = data.actions || {};
|
||||
const buttons = [];
|
||||
if (actions.can_unblock) {
|
||||
buttons.push('<button onclick="sendAction("unblock", "Reason for manual unblock")">Unblock</button>');
|
||||
} else if (actions.can_block) {
|
||||
buttons.push('<button class="danger" onclick="sendAction("block", "Reason for manual block")">Block</button>');
|
||||
}
|
||||
if (actions.can_clear_override) {
|
||||
buttons.push('<button class="secondary" onclick="sendAction("clear-override", "Reason for clearing the manual override")">Clear override</button>');
|
||||
}
|
||||
buttons.push('<button class="secondary" onclick="investigate()">Refresh investigation</button>');
|
||||
document.getElementById('actions').innerHTML = buttons.join('');
|
||||
}
|
||||
|
||||
function renderInvestigation(investigation) {
|
||||
if (!investigation) {
|
||||
document.getElementById('investigation').innerHTML = '<div class="muted">No cached investigation yet.</div>';
|
||||
return;
|
||||
}
|
||||
const rows = [];
|
||||
if (investigation.bot) {
|
||||
rows.push('<div><strong>Bot</strong>: <span class="badge">' + escapeHtml(investigation.bot.icon || '🤖') + ' ' + escapeHtml(investigation.bot.name) + '</span> via ' + escapeHtml(investigation.bot.method) + '</div>');
|
||||
} else {
|
||||
rows.push('<div><strong>Bot</strong>: no verified bot match</div>');
|
||||
}
|
||||
if (investigation.reverse_dns) {
|
||||
rows.push('<div><strong>Reverse DNS</strong>: <span class="mono">' + escapeHtml(investigation.reverse_dns.ptr || '—') + '</span>' + (investigation.reverse_dns.forward_confirmed ? ' · forward-confirmed' : '') + '</div>');
|
||||
}
|
||||
if (investigation.registration) {
|
||||
rows.push('<div><strong>Registration</strong>: ' + escapeHtml(investigation.registration.organization || investigation.registration.name || '—') + '</div>');
|
||||
rows.push('<div><strong>Prefix</strong>: <span class="mono">' + escapeHtml(investigation.registration.prefix || '—') + '</span></div>');
|
||||
rows.push('<div><strong>Country</strong>: ' + escapeHtml(investigation.registration.country || '—') + '</div>');
|
||||
rows.push('<div><strong>Abuse contact</strong>: ' + escapeHtml(investigation.registration.abuse_email || '—') + '</div>');
|
||||
}
|
||||
if (investigation.reputation) {
|
||||
const label = investigation.reputation.spamhaus_listed ? ('listed (' + (investigation.reputation.spamhaus_codes || []).join(', ') + ')') : 'not listed';
|
||||
rows.push('<div><strong>Spamhaus</strong>: ' + escapeHtml(label) + '</div>');
|
||||
if (investigation.reputation.error) {
|
||||
rows.push('<div><strong>Spamhaus error</strong>: ' + escapeHtml(investigation.reputation.error) + '</div>');
|
||||
}
|
||||
}
|
||||
rows.push('<div><strong>Updated</strong>: ' + escapeHtml(formatDate(investigation.updated_at)) + '</div>');
|
||||
if (investigation.error) {
|
||||
rows.push('<div><strong>Lookup warning</strong>: ' + escapeHtml(investigation.error) + '</div>');
|
||||
}
|
||||
document.getElementById('investigation').innerHTML = rows.join('');
|
||||
}
|
||||
|
||||
function renderEvents(items) {
|
||||
document.getElementById('events-body').innerHTML = items.map(item => [
|
||||
const rows = items.map(item => [
|
||||
'<tr>',
|
||||
' <td>' + escapeHtml(new Date(item.occurred_at).toLocaleString()) + '</td>',
|
||||
' <td>' + escapeHtml(formatDate(item.occurred_at)) + '</td>',
|
||||
' <td>' + escapeHtml(item.source_name) + '</td>',
|
||||
' <td>' + escapeHtml(item.host) + '</td>',
|
||||
' <td>' + escapeHtml(item.method) + '</td>',
|
||||
' <td class="mono">' + escapeHtml(item.path) + '</td>',
|
||||
' <td class="mono">' + escapeHtml(item.uri || item.path) + '</td>',
|
||||
' <td>' + escapeHtml(item.status) + '</td>',
|
||||
' <td>' + escapeHtml(item.decision) + (item.enforced ? ' · enforced' : '') + '</td>',
|
||||
' <td>' + escapeHtml(item.user_agent || '—') + '</td>',
|
||||
'</tr>'
|
||||
].join('')).join('');
|
||||
].join(''));
|
||||
document.getElementById('events-body').innerHTML = rows.length > 0 ? rows.join('') : '<tr><td colspan="8" class="muted">No requests stored for this IP yet.</td></tr>';
|
||||
}
|
||||
|
||||
function renderDecisions(items) {
|
||||
document.getElementById('decisions-body').innerHTML = items.map(item => [
|
||||
const rows = items.map(item => [
|
||||
'<tr>',
|
||||
' <td>' + escapeHtml(new Date(item.created_at).toLocaleString()) + '</td>',
|
||||
' <td>' + escapeHtml(formatDate(item.created_at)) + '</td>',
|
||||
' <td>' + escapeHtml(item.kind) + '</td>',
|
||||
' <td>' + escapeHtml(item.action) + '</td>',
|
||||
' <td>' + escapeHtml(item.reason) + '</td>',
|
||||
' <td>' + escapeHtml(item.actor) + '</td>',
|
||||
'</tr>'
|
||||
].join('')).join('');
|
||||
].join(''));
|
||||
document.getElementById('decisions-body').innerHTML = rows.length > 0 ? rows.join('') : '<tr><td colspan="5" class="muted">No decisions recorded for this IP yet.</td></tr>';
|
||||
}
|
||||
|
||||
function renderBackend(items) {
|
||||
document.getElementById('backend-body').innerHTML = items.map(item => [
|
||||
const rows = items.map(item => [
|
||||
'<tr>',
|
||||
' <td>' + escapeHtml(new Date(item.created_at).toLocaleString()) + '</td>',
|
||||
' <td>' + escapeHtml(formatDate(item.created_at)) + '</td>',
|
||||
' <td>' + escapeHtml(item.action) + '</td>',
|
||||
' <td>' + escapeHtml(item.result) + '</td>',
|
||||
' <td>' + escapeHtml(item.message) + '</td>',
|
||||
'</tr>'
|
||||
].join('')).join('');
|
||||
].join(''));
|
||||
document.getElementById('backend-body').innerHTML = rows.length > 0 ? rows.join('') : '<tr><td colspan="4" class="muted">No backend actions recorded for this IP yet.</td></tr>';
|
||||
}
|
||||
|
||||
function renderAll(data) {
|
||||
renderState(data || {});
|
||||
renderActions(data || {});
|
||||
renderInvestigation((data || {}).investigation || null);
|
||||
renderEvents((data || {}).recent_events || []);
|
||||
renderDecisions((data || {}).decisions || []);
|
||||
renderBackend((data || {}).backend_actions || []);
|
||||
}
|
||||
|
||||
async function refresh() {
|
||||
const response = await fetch('/api/ips/' + encodeURIComponent(ip));
|
||||
const data = await response.json();
|
||||
renderState(data.state || {});
|
||||
renderEvents(data.recent_events || []);
|
||||
renderDecisions(data.decisions || []);
|
||||
renderBackend(data.backend_actions || []);
|
||||
renderAll(data);
|
||||
}
|
||||
|
||||
refresh();
|
||||
setInterval(refresh, 2000);
|
||||
refresh().then(() => investigate());
|
||||
</script>
|
||||
</body>
|
||||
</html>`
|
||||
|
||||
@@ -43,6 +43,14 @@ func TestHandlerServesOverviewAndManualActions(t *testing.T) {
|
||||
t.Fatalf("unexpected recorded action: %q", app.lastAction)
|
||||
}
|
||||
|
||||
recorder = httptest.NewRecorder()
|
||||
request = httptest.NewRequest(http.MethodPost, "/api/ips/203.0.113.10/investigate", strings.NewReader(`{"actor":"tester"}`))
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
handler.ServeHTTP(recorder, request)
|
||||
if recorder.Code != http.StatusOK {
|
||||
t.Fatalf("unexpected investigate status: %d body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
|
||||
recorder = httptest.NewRecorder()
|
||||
request = httptest.NewRequest(http.MethodGet, "/", nil)
|
||||
handler.ServeHTTP(recorder, request)
|
||||
@@ -105,9 +113,16 @@ func (s *stubApp) GetIPDetails(context.Context, string) (model.IPDetails, error)
|
||||
RecentEvents: []model.Event{{ID: 1, ClientIP: "203.0.113.10", OccurredAt: now, Decision: model.DecisionActionBlock}},
|
||||
Decisions: []model.DecisionRecord{{ID: 1, IP: "203.0.113.10", Action: model.DecisionActionBlock, CreatedAt: now}},
|
||||
BackendActions: []model.OPNsenseAction{{ID: 1, IP: "203.0.113.10", Action: "block", Result: "added", CreatedAt: now}},
|
||||
OPNsense: model.OPNsenseStatus{Configured: true, Present: true, CheckedAt: now},
|
||||
Actions: model.ActionAvailability{CanUnblock: true},
|
||||
Investigation: &model.IPInvestigation{IP: "203.0.113.10", UpdatedAt: now},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *stubApp) InvestigateIP(context.Context, string) (model.IPDetails, error) {
|
||||
return s.GetIPDetails(context.Background(), "203.0.113.10")
|
||||
}
|
||||
|
||||
func (s *stubApp) ForceBlock(_ context.Context, ip string, actor string, reason string) error {
|
||||
s.lastAction = "block:" + ip + ":" + actor + ":" + reason
|
||||
return nil
|
||||
|
||||
Reference in New Issue
Block a user