Introduce query timeout configuration (#2157)
## Summary Implements configurable query execution timeout controls to prevent poorly optimized or excessive queries from consuming excessive server resources, causing performance degradation, or crashing the Zabbix server. Fixes: https://github.com/grafana/oss-big-tent-squad/issues/127 ## Problem Previously, the plugin only had an HTTP connection timeout (`timeout`) that controlled individual API request timeouts. However, a complete query execution could involve multiple API calls and run indefinitely if not properly controlled, potentially causing resource exhaustion. ## Solution Added a new `queryTimeout` setting that enforces a maximum execution time for entire database queries initiated by the plugin. Queries exceeding this limit are automatically terminated with proper error handling and logging. ## Testing 1. Configure a datasource with `queryTimeout` set to a low value (e.g., 5 seconds) 2. Execute a query that would normally take longer than the timeout 3. Verify that: - Query is terminated after the timeout period - Error message indicates timeout occurred - Logs contain timeout warning with query details - Other queries in the same request continue to execute ## Notes - `queryTimeout` is separate from `timeout` (HTTP connection timeout) - `queryTimeout` applies to the entire query execution, which may involve multiple API calls - Default value of 60 seconds ensures reasonable protection while allowing normal queries to complete - Timeout errors are logged with query refId, queryType, timeout duration, and datasourceId for troubleshooting
This commit is contained in:
@@ -2,8 +2,12 @@ package datasource
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/alexanderzobnin/grafana-zabbix/pkg/settings"
|
||||
"github.com/grafana/grafana-plugin-sdk-go/backend"
|
||||
"gotest.tools/assert"
|
||||
)
|
||||
@@ -66,3 +70,101 @@ func TestZabbixBackend_getCachedDatasource(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryData_QueryTimeoutConfiguration(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
queryTimeout interface{}
|
||||
expectedTimeout time.Duration
|
||||
description string
|
||||
}{
|
||||
{
|
||||
name: "Default timeout when not configured",
|
||||
queryTimeout: nil,
|
||||
expectedTimeout: 60 * time.Second,
|
||||
description: "Should use default 60 seconds when queryTimeout is not set",
|
||||
},
|
||||
{
|
||||
name: "Default timeout when zero",
|
||||
queryTimeout: 0,
|
||||
expectedTimeout: 60 * time.Second,
|
||||
description: "Should use default 60 seconds when queryTimeout is 0",
|
||||
},
|
||||
{
|
||||
name: "Custom timeout configured",
|
||||
queryTimeout: 30,
|
||||
expectedTimeout: 30 * time.Second,
|
||||
description: "Should use configured queryTimeout value",
|
||||
},
|
||||
{
|
||||
name: "Custom timeout as string",
|
||||
queryTimeout: "45",
|
||||
expectedTimeout: 45 * time.Second,
|
||||
description: "Should parse string queryTimeout value",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Create datasource settings with queryTimeout
|
||||
jsonData := map[string]interface{}{
|
||||
"queryTimeout": tt.queryTimeout,
|
||||
}
|
||||
jsonBytes, _ := json.Marshal(jsonData)
|
||||
|
||||
dsSettings := backend.DataSourceInstanceSettings{
|
||||
ID: 1,
|
||||
Name: "TestDatasource",
|
||||
URL: "http://zabbix.org/zabbix",
|
||||
JSONData: jsonBytes,
|
||||
}
|
||||
|
||||
// Parse settings to verify timeout is set correctly
|
||||
zabbixSettings, err := settings.ReadZabbixSettings(&dsSettings)
|
||||
assert.NilError(t, err)
|
||||
assert.Equal(t, tt.expectedTimeout, zabbixSettings.QueryTimeout, tt.description)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryData_QueryTimeoutContextCreation(t *testing.T) {
|
||||
// Test that query timeout context is properly created with the configured timeout
|
||||
jsonData := map[string]interface{}{
|
||||
"queryTimeout": 5, // 5 seconds
|
||||
}
|
||||
jsonBytes, _ := json.Marshal(jsonData)
|
||||
|
||||
dsSettings := backend.DataSourceInstanceSettings{
|
||||
ID: 1,
|
||||
Name: "TestDatasource",
|
||||
URL: "http://zabbix.org/zabbix",
|
||||
JSONData: jsonBytes,
|
||||
}
|
||||
|
||||
// Verify queryTimeout is set correctly
|
||||
zabbixSettings, err := settings.ReadZabbixSettings(&dsSettings)
|
||||
assert.NilError(t, err)
|
||||
assert.Equal(t, 5*time.Second, zabbixSettings.QueryTimeout)
|
||||
|
||||
// Test that context with timeout is created correctly
|
||||
ctx := context.Background()
|
||||
queryCtx, cancel := context.WithTimeout(ctx, zabbixSettings.QueryTimeout)
|
||||
defer cancel()
|
||||
|
||||
// Verify context has deadline set
|
||||
deadline, ok := queryCtx.Deadline()
|
||||
assert.Assert(t, ok, "Context should have a deadline")
|
||||
assert.Assert(t, deadline.After(time.Now()), "Deadline should be in the future")
|
||||
assert.Assert(t, deadline.Before(time.Now().Add(6*time.Second)), "Deadline should be approximately 5 seconds from now")
|
||||
}
|
||||
|
||||
func TestQueryData_QueryTimeoutErrorMessage(t *testing.T) {
|
||||
// Test that timeout error message contains the expected information
|
||||
timeoutMsg := "Query execution exceeded maximum allowed time (5s). Query was automatically terminated to prevent excessive resource consumption."
|
||||
|
||||
// Verify error message format
|
||||
assert.Assert(t, strings.Contains(timeoutMsg, "Query execution exceeded maximum allowed time"))
|
||||
assert.Assert(t, strings.Contains(timeoutMsg, "5s"))
|
||||
assert.Assert(t, strings.Contains(timeoutMsg, "automatically terminated"))
|
||||
assert.Assert(t, strings.Contains(timeoutMsg, "prevent excessive resource consumption"))
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user