Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions docs/blocky-grafana.json
Original file line number Diff line number Diff line change
Expand Up @@ -1777,7 +1777,7 @@
"legendFormat": "{{reason}}"
}
],
"description": "Detailed answer reason, including the upstream resolver or the denylist group that matched."
"description": "Answer reason, including the upstream resolver or the denylist group that matched. For blocked queries the `reason` label is group-level (to keep its cardinality bounded); the specific matched rule/domain is available in the query-log dashboard."
},
{
"id": 22,
Expand Down Expand Up @@ -2599,7 +2599,7 @@
"legendFormat": "{{reason}}"
}
],
"description": "Which denylist groups blocked the most queries in the dashboard time range."
"description": "Which denylist groups blocked the most queries in the dashboard time range. The `reason` metric label is intentionally group-level to keep its cardinality bounded; for the specific matched rule/domain, use the query-log dashboard."
},
{
"id": 30,
Expand Down
7 changes: 7 additions & 0 deletions docs/prometheus_grafana.md
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,13 @@ Following metrics will be exported:
| blocky_rate_limit_cap_exhausted_total | Counter of queries dropped because the rate limiter bucket store was full |
| blocky_rate_limit_active_buckets | Gauge of token buckets (≈ distinct clients) currently tracked by the rate limiter |

!!! note "`reason` label for blocked responses"

To keep the `reason` label of `blocky_response_total` bounded, blocked responses use the matched
group names only (e.g. `BLOCKED (ads)`), **not** the matched rule. The full reason including the
matched rule (e.g. `BLOCKED (ads: *.docler.com)`) is still available in the [query log](configuration.md#query-log).
This avoids unbounded metric cardinality when large deny lists are used.

### Grafana dashboard

Example [Grafana](https://grafana.com/) dashboard
Expand Down
9 changes: 9 additions & 0 deletions helpertest/helper.go
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,15 @@ func HaveReason(reason string) types.GomegaMatcher {
)
}

func HaveReasonLabel(reasonLabel string) types.GomegaMatcher {
return gcustom.MakeMatcher(func(m *model.Response) (bool, error) {
return m.ReasonLabel == reasonLabel, nil
}).WithTemplate(
"Expected:\n{{.Actual}}\n{{.To}} have reason label:\n{{format .Data 1}}",
reasonLabel,
)
}

func HaveResponseType(c model.ResponseType) types.GomegaMatcher {
return gcustom.MakeMatcher(func(m *model.Response) (bool, error) {
return m.RType == c, nil
Expand Down
7 changes: 6 additions & 1 deletion model/models.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,12 @@ func (t ResponseType) ToExtendedErrorCode() uint16 {
type Response struct {
Res *dns.Msg
Reason string
RType ResponseType
// ReasonLabel is a low-cardinality variant of Reason, used as a Prometheus
// metric label. When empty, metrics fall back to Reason. Blocked responses
// set this to the matched group names only (without the matched rule), to
// keep the `reason` label bounded even with large deny lists.
ReasonLabel string
RType ResponseType
}

// RequestProtocol represents the server protocol ENUM(
Expand Down
29 changes: 24 additions & 5 deletions resolver/blocking_resolver.go
Original file line number Diff line number Diff line change
Expand Up @@ -426,9 +426,10 @@ func determineAllowlistOnlyGroups(cfg *config.Blocking) (result map[string]bool)

// sets answer and/or return code for DNS response, if request should be blocked
func (r *BlockingResolver) handleBlocked(logger *logrus.Entry,
request *model.Request, question dns.Question, reason string,
request *model.Request, question dns.Question, reason, reasonLabel string,
) (*model.Response, error) {
modelResp := model.NewResponseWithReason(request, model.ResponseTypeBLOCKED, reason)
modelResp.ReasonLabel = reasonLabel
r.blockHandler.handleBlock(question, modelResp.Res)

logger.Debugf("blocking request '%s'", reason)
Expand Down Expand Up @@ -479,7 +480,7 @@ func (r *BlockingResolver) handleDenylist(ctx context.Context, groupsToCheck []s
}

if allowlistOnlyAllowed {
resp, err := r.handleBlocked(logger, request, question, "BLOCKED (ALLOWLIST ONLY)")
resp, err := r.handleBlocked(logger, request, question, "BLOCKED (ALLOWLIST ONLY)", "")
if err != nil {
err = fmt.Errorf("failed to handle allowlist-only block for %s: %w", domain, err)
}
Expand All @@ -488,7 +489,8 @@ func (r *BlockingResolver) handleDenylist(ctx context.Context, groupsToCheck []s
}

if matches := r.matches(groupsToCheck, r.denylistMatcher, domain); len(matches) > 0 {
resp, err := r.handleBlocked(logger, request, question, formatBlockReason(matches, ""))
resp, err := r.handleBlocked(logger, request, question,
formatBlockReason(matches, ""), formatBlockReasonLabel(matches, ""))

return true, resp, err
}
Expand Down Expand Up @@ -520,7 +522,8 @@ func (r *BlockingResolver) Resolve(ctx context.Context, request *model.Request)
if matches := r.matches(groupsToCheck, r.allowlistMatcher, entryToCheck); len(matches) > 0 {
logger.WithField("matches", matches).Debugf("%s is allowlisted", tName)
} else if matches := r.matches(groupsToCheck, r.denylistMatcher, entryToCheck); len(matches) > 0 {
return r.handleBlocked(logger, request, request.Req.Question[0], formatBlockReason(matches, tName))
return r.handleBlocked(logger, request, request.Req.Question[0],
formatBlockReason(matches, tName), formatBlockReasonLabel(matches, tName))
}
}
}
Expand Down Expand Up @@ -656,6 +659,18 @@ func (r *BlockingResolver) matches(groupsToCheck []string, m lists.Matcher,
// "BLOCKED[ TYPE] (group: rule[, group: rule...])". Matched groups are sorted
// so the rendered reason is deterministic when more than one group matches.
func formatBlockReason(matches map[string]string, typeName string) string {
return renderBlockReason(matches, typeName, true)
}

// formatBlockReasonLabel renders a low-cardinality block reason of the form
// "BLOCKED[ TYPE] (group[, group...])" — group names only, without the matched
// rule. It is used as the `reason` Prometheus metric label so the label stays
// bounded by the number of configured groups, regardless of deny-list size.
func formatBlockReasonLabel(matches map[string]string, typeName string) string {
return renderBlockReason(matches, typeName, false)
}

func renderBlockReason(matches map[string]string, typeName string, includeRules bool) string {
reason := "BLOCKED"
if typeName != "" {
reason += " " + typeName
Expand All @@ -667,7 +682,11 @@ func formatBlockReason(matches map[string]string, typeName string) string {

entries := make([]string, 0, len(matches))
for _, group := range slices.Sorted(maps.Keys(matches)) {
entries = append(entries, fmt.Sprintf("%s: %s", group, matches[group]))
if includeRules {
entries = append(entries, fmt.Sprintf("%s: %s", group, matches[group]))
} else {
entries = append(entries, group)
}
}

return fmt.Sprintf("%s (%s)", reason, strings.Join(entries, ", "))
Expand Down
41 changes: 41 additions & 0 deletions resolver/blocking_resolver_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -285,6 +285,15 @@ var _ = Describe("BlockingResolver", Label("blockingResolver"), func() {
HaveReturnCode(dns.RcodeSuccess),
))
})
It("should set a group-only ReasonLabel while keeping the detailed Reason", func() {
Expect(sut.Resolve(ctx, newRequestWithClient("domain1.com.", A, "1.2.1.2", "client1"))).
Should(
SatisfyAll(
HaveResponseType(ResponseTypeBLOCKED),
HaveReason("BLOCKED (gr1: domain1.com)"),
HaveReasonLabel("BLOCKED (gr1)"),
))
})
It("should block the A query if domain is on the denylist (multipart 1)", func() {
Expect(sut.Resolve(ctx, newRequestWithClient("domain1.com.", A, "1.2.1.2", "client2"))).
Should(
Expand Down Expand Up @@ -1542,3 +1551,35 @@ var _ = Describe("formatBlockReason", func() {
}
})
})

var _ = Describe("formatBlockReasonLabel", func() {
It("renders the matched group name only, without the rule", func() {
Expect(formatBlockReasonLabel(map[string]string{"gr1": "domain1.com"}, "")).
Should(Equal("BLOCKED (gr1)"))
})

It("includes the entry type when given (e.g. CNAME/IP)", func() {
Expect(formatBlockReasonLabel(map[string]string{"ads": "*.docler.com"}, "CNAME")).
Should(Equal("BLOCKED CNAME (ads)"))
})

It("keeps the label bounded: different rules in the same group yield the same label", func() {
// This is the whole point of the label: the rule (regex/domain) is
// unbounded, but the group set is not. Both must map to one metric series.
a := formatBlockReasonLabel(map[string]string{"ads": "/snapchat/"}, "")
b := formatBlockReasonLabel(map[string]string{"ads": "tracker.example.com"}, "")

Expect(a).Should(Equal("BLOCKED (ads)"))
Expect(b).Should(Equal(a))
})

It("renders multiple matched groups sorted by group for deterministic output", func() {
matches := map[string]string{"zeta": "z.com", "alpha": "a.com", "mid": "m.com"}

// repeat to guard against Go's randomized map iteration order
for range 20 {
Expect(formatBlockReasonLabel(matches, "")).
Should(Equal("BLOCKED (alpha, mid, zeta)"))
}
})
})
9 changes: 7 additions & 2 deletions resolver/caching_resolver.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,12 @@ import (
"github.com/sirupsen/logrus"
)

const defaultCachingCleanUpInterval = 5 * time.Second
const (
defaultCachingCleanUpInterval = 5 * time.Second

// cachedReason is the response reason used for cache hits.
cachedReason = "CACHED"
)

// CacheDecorator optionally wraps the result cache (e.g., with Redis sync).
type CacheDecorator func(cache.ExpiringCache[[]byte]) (cache.ExpiringCache[[]byte], error)
Expand Down Expand Up @@ -217,7 +222,7 @@ func (r *CachingResolver) Resolve(ctx context.Context, request *model.Request) (
setTTLInCachedResponse(val, ttl)

if val.Rcode == dns.RcodeSuccess {
return &model.Response{Res: val, RType: model.ResponseTypeCACHED, Reason: "CACHED"}, nil
return &model.Response{Res: val, RType: model.ResponseTypeCACHED, Reason: cachedReason}, nil
}

return &model.Response{Res: val, RType: model.ResponseTypeCACHED, Reason: "CACHED NEGATIVE"}, nil
Expand Down
49 changes: 30 additions & 19 deletions resolver/metrics_resolver.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,30 +41,41 @@ type MetricsResolver struct {
func (r *MetricsResolver) Resolve(ctx context.Context, request *model.Request) (*model.Response, error) {
response, err := r.next.Resolve(ctx, request)

if r.cfg.Enable {
r.totalQueries.With(prometheus.Labels{
labelClient: strings.Join(request.ClientNames, ","),
labelType: dns.TypeToString[request.Req.Question[0].Qtype],
}).Inc()
if !r.cfg.Enable {
return response, err
}

reqDuration := time.Since(request.RequestTS)
responseType := "err"
r.totalQueries.With(prometheus.Labels{
labelClient: strings.Join(request.ClientNames, ","),
labelType: dns.TypeToString[request.Req.Question[0].Qtype],
}).Inc()

if response != nil {
responseType = response.RType.String()
}
reqDuration := time.Since(request.RequestTS)
responseType := "err"

r.durationHistogram.WithLabelValues(responseType).Observe(reqDuration.Seconds())
if response != nil {
responseType = response.RType.String()
}

if err != nil {
r.totalErrors.Inc()
} else {
r.totalResponse.With(prometheus.Labels{
labelReason: response.Reason,
labelResponseCode: dns.RcodeToString[response.Res.Rcode],
labelResponseType: response.RType.String(),
}).Inc()
r.durationHistogram.WithLabelValues(responseType).Observe(reqDuration.Seconds())

if err != nil {
r.totalErrors.Inc()
} else {
// Prefer the low-cardinality ReasonLabel for the metric label; fall
// back to Reason when it is not set. Blocked responses embed the
// matched rule in Reason (e.g. a regex or domain), which is unbounded
// for large deny lists, so they set ReasonLabel to the group names only.
reasonLabel := response.ReasonLabel
if reasonLabel == "" {
reasonLabel = response.Reason
}

r.totalResponse.With(prometheus.Labels{
labelReason: reasonLabel,
labelResponseCode: dns.RcodeToString[response.Res.Rcode],
labelResponseType: response.RType.String(),
}).Inc()
}

return response, err
Expand Down
47 changes: 47 additions & 0 deletions resolver/metrics_resolver_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,53 @@ var _ = Describe("MetricResolver", func() {
m.AssertExpectations(GinkgoT())
})
})
When("Response has a low-cardinality ReasonLabel", func() {
BeforeEach(func() {
m = &mockResolver{}
m.On("Resolve", mock.Anything).Return(&Response{
Res: new(dns.Msg),
RType: ResponseTypeBLOCKED,
Reason: "BLOCKED (ads: /snapchat/)",
ReasonLabel: "BLOCKED (ads)",
}, nil)
sut.Next(m)
})
It("uses ReasonLabel for the reason metric label, not the detailed Reason", func() {
_, err := sut.Resolve(ctx, newRequestWithClient("example.com.", A, "", "client"))
Expect(err).Should(Succeed())

cnt, err := sut.totalResponse.GetMetricWith(prometheus.Labels{
"reason": "BLOCKED (ads)",
"response_code": "NOERROR",
"response_type": "BLOCKED",
})
Expect(err).Should(Succeed())
Expect(testutil.ToFloat64(cnt)).Should(BeNumerically("==", 1))
})
})
When("Response has no ReasonLabel", func() {
BeforeEach(func() {
m = &mockResolver{}
m.On("Resolve", mock.Anything).Return(&Response{
Res: new(dns.Msg),
RType: ResponseTypeRESOLVED,
Reason: "CACHED",
}, nil)
sut.Next(m)
})
It("falls back to Reason for the reason metric label", func() {
_, err := sut.Resolve(ctx, newRequestWithClient("example.com.", A, "", "client"))
Expect(err).Should(Succeed())

cnt, err := sut.totalResponse.GetMetricWith(prometheus.Labels{
"reason": "CACHED",
"response_code": "NOERROR",
"response_type": "RESOLVED",
})
Expect(err).Should(Succeed())
Expect(testutil.ToFloat64(cnt)).Should(BeNumerically("==", 1))
})
})
When("Error occurs while request processing", func() {
BeforeEach(func() {
m = &mockResolver{}
Expand Down