From a3ab102a06b6f13ffb4ad032ad26aa4afa174ee2 Mon Sep 17 00:00:00 2001 From: Alexander Zobnin Date: Sun, 28 Oct 2018 21:03:56 +0300 Subject: [PATCH 01/21] influxdb connector WIP --- src/datasource-zabbix/config.controller.js | 2 +- .../connectors/influxdb/influxdbConnector.js | 152 ++++++++++++++++++ src/datasource-zabbix/zabbix/zabbix.js | 24 ++- 3 files changed, 170 insertions(+), 8 deletions(-) create mode 100644 src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js diff --git a/src/datasource-zabbix/config.controller.js b/src/datasource-zabbix/config.controller.js index 02de9d2..fcd77e9 100644 --- a/src/datasource-zabbix/config.controller.js +++ b/src/datasource-zabbix/config.controller.js @@ -1,7 +1,7 @@ import _ from 'lodash'; import { migrateDSConfig } from './migrations'; -const SUPPORTED_SQL_DS = ['mysql', 'postgres']; +const SUPPORTED_SQL_DS = ['mysql', 'postgres', 'influxdb']; const zabbixVersions = [ { name: '2.x', value: 2 }, { name: '3.x', value: 3 }, diff --git a/src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js b/src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js new file mode 100644 index 0000000..9ae38a5 --- /dev/null +++ b/src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js @@ -0,0 +1,152 @@ +import _ from 'lodash'; +import DBConnector from '../dbConnector'; + +const DEFAULT_QUERY_LIMIT = 10000; +const HISTORY_TO_TABLE_MAP = { + '0': 'history', + '1': 'history_str', + '2': 'history_log', + '3': 'history_uint', + '4': 'history_text' +}; + +const TREND_TO_TABLE_MAP = { + '0': 'trends', + '3': 'trends_uint' +}; + +const consolidateByFunc = { + 'avg': 'AVG', + 'min': 'MIN', + 'max': 'MAX', + 'sum': 'SUM', + 'count': 'COUNT' +}; + +const consolidateByTrendColumns = { + 'avg': 'value_avg', + 'min': 'value_min', + 'max': 'value_max', + 'sum': 'num*value_avg' // sum of sums inside the one-hour trend period +}; + +export class InfluxDBConnector extends DBConnector { + constructor(options, backendSrv, datasourceSrv) { + super(options, backendSrv, datasourceSrv); + this.limit = options.limit || DEFAULT_QUERY_LIMIT; + super.loadDBDataSource().then(ds => { + console.log(ds); + this.ds = ds; + return ds; + }); + } + + /** + * Try to invoke test query for one of Zabbix database tables. + */ + testDataSource() { + return this.ds.testDatasource(); + } + + getHistory(items, timeFrom, timeTill, options) { + let {intervalMs, consolidateBy} = options; + const intervalSec = Math.ceil(intervalMs / 1000); + + consolidateBy = consolidateBy || 'avg'; + const aggFunction = consolidateByFunc[consolidateBy]; + + // Group items by value type and perform request for each value type + const grouped_items = _.groupBy(items, 'value_type'); + const promises = _.map(grouped_items, (items, value_type) => { + const itemids = _.map(items, 'itemid'); + const table = HISTORY_TO_TABLE_MAP[value_type]; + const query = this.buildHistoryQuery(itemids, table, timeFrom, timeTill, intervalSec, aggFunction); + console.log(query); + return this.invokeInfluxDBQuery(query); + }); + + return Promise.all(promises).then(results => { + return _.flatten(results); + }); + } + + getTrends(items, timeFrom, timeTill, options) { + let { intervalMs, consolidateBy } = options; + const intervalSec = Math.ceil(intervalMs / 1000); + + consolidateBy = consolidateBy || 'avg'; + const aggFunction = consolidateByFunc[consolidateBy]; + + // Group items by value type and perform request for each value type + const grouped_items = _.groupBy(items, 'value_type'); + const promises = _.map(grouped_items, (items, value_type) => { + const itemids = _.map(items, 'itemid'); + const table = TREND_TO_TABLE_MAP[value_type]; + let valueColumn = _.includes(['avg', 'min', 'max', 'sum'], consolidateBy) ? consolidateBy : 'avg'; + valueColumn = consolidateByTrendColumns[valueColumn]; + const query = this.buildTrendsQuery(itemids, table, timeFrom, timeTill, intervalSec, aggFunction, valueColumn); + console.log(query); + return this.invokeInfluxDBQuery(query); + }); + + return Promise.all(promises).then(results => { + return _.flatten(results); + }); + } + + buildHistoryQuery(itemids, table, timeFrom, timeTill, intervalSec, aggFunction) { + const AGG = aggFunction === 'AVG' ? 'MEAN' : aggFunction; + const where_clause = itemids.map(itemid => `"itemid" = ${itemid}`).join(' AND '); + const query = `SELECT "itemid", "time", ${AGG}("value") FROM "${table}" + WHERE ${where_clause} AND "time" >= ${timeFrom} AND "time" <= ${timeTill} + GROUP BY time(${intervalSec}s)`; + return compactQuery(query); + } + + buildTrendsQuery(itemids, table, timeFrom, timeTill, intervalSec, aggFunction, valueColumn) { + const AGG = aggFunction === 'AVG' ? 'MEAN' : aggFunction; + const where_clause = itemids.map(itemid => `"itemid" = ${itemid}`).join(' AND '); + const query = `SELECT "itemid", "time", ${AGG}("${valueColumn}") FROM "${table}" + WHERE ${where_clause} AND "time" >= ${timeFrom} AND "time" <= ${timeTill} + GROUP BY time(${intervalSec}s)`; + return compactQuery(query); + } + + handleGrafanaTSResponse(history, items, addHostName = true) { + return convertGrafanaTSResponse(history, items, addHostName); + } + + invokeInfluxDBQuery(query) { + return this.ds._seriesQuery(query); + } +} + +/////////////////////////////////////////////////////////////////////////////// + +function convertGrafanaTSResponse(time_series, items, addHostName) { + //uniqBy is needed to deduplicate + var hosts = _.uniqBy(_.flatten(_.map(items, 'hosts')), 'hostid'); + let grafanaSeries = _.map(_.compact(time_series), series => { + let itemid = series.name; + var item = _.find(items, {'itemid': itemid}); + var alias = item.name; + //only when actual multi hosts selected + if (_.keys(hosts).length > 1 && addHostName) { + var host = _.find(hosts, {'hostid': item.hostid}); + alias = host.name + ": " + alias; + } + // CachingProxy deduplicates requests and returns one time series for equal queries. + // Clone is needed to prevent changing of series object shared between all targets. + let datapoints = _.cloneDeep(series.points); + return { + target: alias, + datapoints: datapoints + }; + }); + + return _.sortBy(grafanaSeries, 'target'); +} + +function compactQuery(query) { + return query.replace(/\s+/g, ' '); +} diff --git a/src/datasource-zabbix/zabbix/zabbix.js b/src/datasource-zabbix/zabbix/zabbix.js index b6cb66b..e720580 100644 --- a/src/datasource-zabbix/zabbix/zabbix.js +++ b/src/datasource-zabbix/zabbix/zabbix.js @@ -1,8 +1,10 @@ import _ from 'lodash'; import * as utils from '../utils'; import responseHandler from '../responseHandler'; +import DBConnector from './connectors/dbConnector'; import { ZabbixAPIConnector } from './connectors/zabbix_api/zabbixAPIConnector'; import { SQLConnector } from './connectors/sql/sqlConnector'; +import { InfluxDBConnector } from './connectors/influxdb/influxdbConnector'; import { CachingProxy } from './proxy/cachingProxy'; import { ZabbixNotImplemented } from './connectors/dbConnector'; @@ -48,19 +50,27 @@ export class Zabbix { this.zabbixAPI = new ZabbixAPIConnector(url, username, password, zabbixVersion, basicAuth, withCredentials, backendSrv); + this.proxyfyRequests(); + this.cacheRequests(); + this.bindRequests(); + if (enableDirectDBConnection) { let dbConnectorOptions = { datasourceId: dbConnectionDatasourceId, datasourceName: dbConnectionDatasourceName }; - this.dbConnector = new SQLConnector(dbConnectorOptions, backendSrv, datasourceSrv); - this.getHistoryDB = this.cachingProxy.proxyfyWithCache(this.dbConnector.getHistory, 'getHistory', this.dbConnector); - this.getTrendsDB = this.cachingProxy.proxyfyWithCache(this.dbConnector.getTrends, 'getTrends', this.dbConnector); + this.dbConnector = new DBConnector(dbConnectorOptions, backendSrv, datasourceSrv); + this.dbConnector.loadDBDataSource().then(ds => { + if (ds.type === 'influxdb') { + this.dbConnector = new InfluxDBConnector(dbConnectorOptions, backendSrv, datasourceSrv); + } else { + this.dbConnector = new SQLConnector(dbConnectorOptions, backendSrv, datasourceSrv); + } + }).then(() => { + this.getHistoryDB = this.cachingProxy.proxyfyWithCache(this.dbConnector.getHistory, 'getHistory', this.dbConnector); + this.getTrendsDB = this.cachingProxy.proxyfyWithCache(this.dbConnector.getTrends, 'getTrends', this.dbConnector); + }); } - - this.proxyfyRequests(); - this.cacheRequests(); - this.bindRequests(); } proxyfyRequests() { From ec436cb3d565500fcdf10e4b5603c76a2be1961d Mon Sep 17 00:00:00 2001 From: Alexander Zobnin Date: Tue, 30 Oct 2018 11:31:44 +0300 Subject: [PATCH 02/21] initial influxdb connector --- .../connectors/influxdb/influxdbConnector.js | 69 +++++++++++++++---- 1 file changed, 57 insertions(+), 12 deletions(-) diff --git a/src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js b/src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js index 9ae38a5..4f7bbfa 100644 --- a/src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js +++ b/src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js @@ -1,5 +1,6 @@ import _ from 'lodash'; import DBConnector from '../dbConnector'; +// import InfluxSeries from 'grafana/app/plugins/datasource/influxdb/influx_series'; const DEFAULT_QUERY_LIMIT = 10000; const HISTORY_TO_TABLE_MAP = { @@ -65,8 +66,10 @@ export class InfluxDBConnector extends DBConnector { return this.invokeInfluxDBQuery(query); }); - return Promise.all(promises).then(results => { - return _.flatten(results); + return Promise.all(promises) + .then(_.flatten) + .then(results => { + return handleInfluxHistoryResponse(results); }); } @@ -89,40 +92,82 @@ export class InfluxDBConnector extends DBConnector { return this.invokeInfluxDBQuery(query); }); - return Promise.all(promises).then(results => { - return _.flatten(results); + return Promise.all(promises) + .then(_.flatten) + .then(results => { + return handleInfluxHistoryResponse(results); }); } buildHistoryQuery(itemids, table, timeFrom, timeTill, intervalSec, aggFunction) { const AGG = aggFunction === 'AVG' ? 'MEAN' : aggFunction; - const where_clause = itemids.map(itemid => `"itemid" = ${itemid}`).join(' AND '); - const query = `SELECT "itemid", "time", ${AGG}("value") FROM "${table}" - WHERE ${where_clause} AND "time" >= ${timeFrom} AND "time" <= ${timeTill} - GROUP BY time(${intervalSec}s)`; + const where_clause = this.buildWhereClause(itemids); + const query = `SELECT ${AGG}("value") FROM "${table}" + WHERE ${where_clause} AND "time" >= ${timeFrom}s AND "time" <= ${timeTill}s + GROUP BY time(${intervalSec}s), "itemid" fill(linear)`; return compactQuery(query); } buildTrendsQuery(itemids, table, timeFrom, timeTill, intervalSec, aggFunction, valueColumn) { const AGG = aggFunction === 'AVG' ? 'MEAN' : aggFunction; - const where_clause = itemids.map(itemid => `"itemid" = ${itemid}`).join(' AND '); - const query = `SELECT "itemid", "time", ${AGG}("${valueColumn}") FROM "${table}" - WHERE ${where_clause} AND "time" >= ${timeFrom} AND "time" <= ${timeTill} + const where_clause = this.buildWhereClause(itemids); + const query = `SELECT ${AGG}("${valueColumn}") FROM "${table}" + WHERE ${where_clause} AND "time" >= ${timeFrom}s AND "time" <= ${timeTill}s GROUP BY time(${intervalSec}s)`; return compactQuery(query); } + buildWhereClause(itemids) { + const itemidsWhere = itemids.map(itemid => `"itemid" = '${itemid}'`).join(' OR '); + return `(${itemidsWhere})`; + } + handleGrafanaTSResponse(history, items, addHostName = true) { return convertGrafanaTSResponse(history, items, addHostName); } invokeInfluxDBQuery(query) { - return this.ds._seriesQuery(query); + return this.ds._seriesQuery(query).then(data => { + return data && data.results ? data.results : []; + }); } } /////////////////////////////////////////////////////////////////////////////// +function handleInfluxHistoryResponse(results) { + if (!results) { + return []; + } + + const seriesList = []; + for (let i = 0; i < results.length; i++) { + const result = results[i]; + if (!result || !result.series) { + continue; + } + + const influxSeriesList = results[i].series; + + for (let y = 0; y < influxSeriesList.length; y++) { + const influxSeries = influxSeriesList[y]; + const datapoints = []; + if (influxSeries.values) { + for (i = 0; i < influxSeries.values.length; i++) { + datapoints[i] = [influxSeries.values[i][1], influxSeries.values[i][0]]; + } + } + const timeSeries = { + name: influxSeries.tags.itemid, + points: datapoints + }; + seriesList.push(timeSeries); + } + } + + return seriesList; +} + function convertGrafanaTSResponse(time_series, items, addHostName) { //uniqBy is needed to deduplicate var hosts = _.uniqBy(_.flatten(_.map(items, 'hosts')), 'hostid'); From 6da2479e9d74efa5c7245eeb541660822735ca34 Mon Sep 17 00:00:00 2001 From: Alexander Zobnin Date: Tue, 30 Oct 2018 19:35:23 +0300 Subject: [PATCH 03/21] influx connector: refactor --- .../zabbix/connectors/dbConnector.js | 38 +++++++++++++++++ .../connectors/influxdb/influxdbConnector.js | 41 ++----------------- .../zabbix/connectors/sql/sqlConnector.js | 29 ------------- 3 files changed, 42 insertions(+), 66 deletions(-) diff --git a/src/datasource-zabbix/zabbix/connectors/dbConnector.js b/src/datasource-zabbix/zabbix/connectors/dbConnector.js index 28fa264..8b23429 100644 --- a/src/datasource-zabbix/zabbix/connectors/dbConnector.js +++ b/src/datasource-zabbix/zabbix/connectors/dbConnector.js @@ -54,6 +54,10 @@ export default class DBConnector { getTrends() { throw new ZabbixNotImplemented('getTrends()'); } + + handleGrafanaTSResponse(history, items, addHostName = true) { + return convertGrafanaTSResponse(history, items, addHostName); + } } // Define Zabbix DB Connector exception type for non-implemented methods @@ -68,3 +72,37 @@ export class ZabbixNotImplemented { return this.message; } } + +/** + * Converts time series returned by the data source into format that Grafana expects + * time_series is Array of series: + * ``` + * [{ + * name: string, + * points: Array<[value: number, timestamp: number]> + * }] + * ``` + */ +function convertGrafanaTSResponse(time_series, items, addHostName) { + //uniqBy is needed to deduplicate + var hosts = _.uniqBy(_.flatten(_.map(items, 'hosts')), 'hostid'); + let grafanaSeries = _.map(_.compact(time_series), series => { + let itemid = series.name; + var item = _.find(items, {'itemid': itemid}); + var alias = item.name; + //only when actual multi hosts selected + if (_.keys(hosts).length > 1 && addHostName) { + var host = _.find(hosts, {'hostid': item.hostid}); + alias = host.name + ": " + alias; + } + // CachingProxy deduplicates requests and returns one time series for equal queries. + // Clone is needed to prevent changing of series object shared between all targets. + let datapoints = _.cloneDeep(series.points); + return { + target: alias, + datapoints: datapoints + }; + }); + + return _.sortBy(grafanaSeries, 'target'); +} diff --git a/src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js b/src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js index 4f7bbfa..97e6a14 100644 --- a/src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js +++ b/src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js @@ -1,6 +1,5 @@ import _ from 'lodash'; import DBConnector from '../dbConnector'; -// import InfluxSeries from 'grafana/app/plugins/datasource/influxdb/influx_series'; const DEFAULT_QUERY_LIMIT = 10000; const HISTORY_TO_TABLE_MAP = { @@ -36,8 +35,7 @@ export class InfluxDBConnector extends DBConnector { super(options, backendSrv, datasourceSrv); this.limit = options.limit || DEFAULT_QUERY_LIMIT; super.loadDBDataSource().then(ds => { - console.log(ds); - this.ds = ds; + this.influxDS = ds; return ds; }); } @@ -46,7 +44,7 @@ export class InfluxDBConnector extends DBConnector { * Try to invoke test query for one of Zabbix database tables. */ testDataSource() { - return this.ds.testDatasource(); + return this.influxDS.testDatasource(); } getHistory(items, timeFrom, timeTill, options) { @@ -62,7 +60,6 @@ export class InfluxDBConnector extends DBConnector { const itemids = _.map(items, 'itemid'); const table = HISTORY_TO_TABLE_MAP[value_type]; const query = this.buildHistoryQuery(itemids, table, timeFrom, timeTill, intervalSec, aggFunction); - console.log(query); return this.invokeInfluxDBQuery(query); }); @@ -88,7 +85,6 @@ export class InfluxDBConnector extends DBConnector { let valueColumn = _.includes(['avg', 'min', 'max', 'sum'], consolidateBy) ? consolidateBy : 'avg'; valueColumn = consolidateByTrendColumns[valueColumn]; const query = this.buildTrendsQuery(itemids, table, timeFrom, timeTill, intervalSec, aggFunction, valueColumn); - console.log(query); return this.invokeInfluxDBQuery(query); }); @@ -122,14 +118,9 @@ export class InfluxDBConnector extends DBConnector { return `(${itemidsWhere})`; } - handleGrafanaTSResponse(history, items, addHostName = true) { - return convertGrafanaTSResponse(history, items, addHostName); - } - invokeInfluxDBQuery(query) { - return this.ds._seriesQuery(query).then(data => { - return data && data.results ? data.results : []; - }); + return this.influxDS._seriesQuery(query) + .then(data => data && data.results ? data.results : []); } } @@ -168,30 +159,6 @@ function handleInfluxHistoryResponse(results) { return seriesList; } -function convertGrafanaTSResponse(time_series, items, addHostName) { - //uniqBy is needed to deduplicate - var hosts = _.uniqBy(_.flatten(_.map(items, 'hosts')), 'hostid'); - let grafanaSeries = _.map(_.compact(time_series), series => { - let itemid = series.name; - var item = _.find(items, {'itemid': itemid}); - var alias = item.name; - //only when actual multi hosts selected - if (_.keys(hosts).length > 1 && addHostName) { - var host = _.find(hosts, {'hostid': item.hostid}); - alias = host.name + ": " + alias; - } - // CachingProxy deduplicates requests and returns one time series for equal queries. - // Clone is needed to prevent changing of series object shared between all targets. - let datapoints = _.cloneDeep(series.points); - return { - target: alias, - datapoints: datapoints - }; - }); - - return _.sortBy(grafanaSeries, 'target'); -} - function compactQuery(query) { return query.replace(/\s+/g, ' '); } diff --git a/src/datasource-zabbix/zabbix/connectors/sql/sqlConnector.js b/src/datasource-zabbix/zabbix/connectors/sql/sqlConnector.js index f536c47..127789f 100644 --- a/src/datasource-zabbix/zabbix/connectors/sql/sqlConnector.js +++ b/src/datasource-zabbix/zabbix/connectors/sql/sqlConnector.js @@ -112,10 +112,6 @@ export class SQLConnector extends DBConnector { }); } - handleGrafanaTSResponse(history, items, addHostName = true) { - return convertGrafanaTSResponse(history, items, addHostName); - } - invokeSQLQuery(query) { let queryDef = { refId: 'A', @@ -143,31 +139,6 @@ export class SQLConnector extends DBConnector { } } -/////////////////////////////////////////////////////////////////////////////// - -function convertGrafanaTSResponse(time_series, items, addHostName) { - //uniqBy is needed to deduplicate - var hosts = _.uniqBy(_.flatten(_.map(items, 'hosts')), 'hostid'); - let grafanaSeries = _.map(_.compact(time_series), series => { - let itemid = series.name; - var item = _.find(items, {'itemid': itemid}); - var alias = item.name; - //only when actual multi hosts selected - if (_.keys(hosts).length > 1 && addHostName) { - var host = _.find(hosts, {'hostid': item.hostid}); - alias = host.name + ": " + alias; - } - // CachingProxy deduplicates requests and returns one time series for equal queries. - // Clone is needed to prevent changing of series object shared between all targets. - let datapoints = _.cloneDeep(series.points); - return { - target: alias, - datapoints: datapoints - }; - }); - - return _.sortBy(grafanaSeries, 'target'); -} function compactSQLQuery(query) { return query.replace(/\s+/g, ' '); From 6eb52619b94daec2daab920c984bf0fa0f1acb75 Mon Sep 17 00:00:00 2001 From: Alexander Zobnin Date: Tue, 30 Oct 2018 19:58:48 +0300 Subject: [PATCH 04/21] db connector: refactor --- src/datasource-zabbix/datasource.js | 2 +- .../specs/dbConnector.test.js | 11 ++- .../zabbix/connectors/dbConnector.js | 45 ++++++++++++- .../connectors/influxdb/influxdbConnector.js | 67 ++----------------- .../zabbix/connectors/sql/sqlConnector.js | 42 ++---------- src/datasource-zabbix/zabbix/zabbix.js | 10 +-- 6 files changed, 64 insertions(+), 113 deletions(-) diff --git a/src/datasource-zabbix/datasource.js b/src/datasource-zabbix/datasource.js index 27387f6..31b1eee 100644 --- a/src/datasource-zabbix/datasource.js +++ b/src/datasource-zabbix/datasource.js @@ -69,7 +69,7 @@ export class ZabbixDatasource { dbConnectionDatasourceName: this.dbConnectionDatasourceName }; - this.zabbix = new Zabbix(zabbixOptions, backendSrv, datasourceSrv); + this.zabbix = new Zabbix(zabbixOptions, datasourceSrv, backendSrv); } //////////////////////// diff --git a/src/datasource-zabbix/specs/dbConnector.test.js b/src/datasource-zabbix/specs/dbConnector.test.js index 0f8a302..cca08f1 100644 --- a/src/datasource-zabbix/specs/dbConnector.test.js +++ b/src/datasource-zabbix/specs/dbConnector.test.js @@ -1,9 +1,8 @@ import mocks from '../../test-setup/mocks'; -import DBConnector from '../zabbix/connectors/dbConnector'; +import { DBConnector } from '../zabbix/connectors/dbConnector'; describe('DBConnector', () => { let ctx = {}; - const backendSrv = mocks.backendSrvMock; const datasourceSrv = mocks.datasourceSrvMock; datasourceSrv.loadDatasource.mockResolvedValue({ id: 42, name: 'foo', meta: {} }); datasourceSrv.getAll.mockReturnValue([{ id: 42, name: 'foo' }]); @@ -20,14 +19,14 @@ describe('DBConnector', () => { ctx.options = { datasourceName: 'bar' }; - const dbConnector = new DBConnector(ctx.options, backendSrv, datasourceSrv); + const dbConnector = new DBConnector(ctx.options, datasourceSrv); dbConnector.loadDBDataSource(); expect(datasourceSrv.getAll).not.toHaveBeenCalled(); expect(datasourceSrv.loadDatasource).toHaveBeenCalledWith('bar'); }); it('should load datasource by id if name not present', () => { - const dbConnector = new DBConnector(ctx.options, backendSrv, datasourceSrv); + const dbConnector = new DBConnector(ctx.options, datasourceSrv); dbConnector.loadDBDataSource(); expect(datasourceSrv.getAll).toHaveBeenCalled(); expect(datasourceSrv.loadDatasource).toHaveBeenCalledWith('foo'); @@ -35,13 +34,13 @@ describe('DBConnector', () => { it('should throw error if no name and id specified', () => { ctx.options = {}; - const dbConnector = new DBConnector(ctx.options, backendSrv, datasourceSrv); + const dbConnector = new DBConnector(ctx.options, datasourceSrv); return expect(dbConnector.loadDBDataSource()).rejects.toBe('SQL Data Source name should be specified'); }); it('should throw error if datasource with given id is not found', () => { ctx.options.datasourceId = 45; - const dbConnector = new DBConnector(ctx.options, backendSrv, datasourceSrv); + const dbConnector = new DBConnector(ctx.options, datasourceSrv); return expect(dbConnector.loadDBDataSource()).rejects.toBe('SQL Data Source with ID 45 not found'); }); }); diff --git a/src/datasource-zabbix/zabbix/connectors/dbConnector.js b/src/datasource-zabbix/zabbix/connectors/dbConnector.js index 8b23429..5006ce0 100644 --- a/src/datasource-zabbix/zabbix/connectors/dbConnector.js +++ b/src/datasource-zabbix/zabbix/connectors/dbConnector.js @@ -1,12 +1,40 @@ import _ from 'lodash'; +export const DEFAULT_QUERY_LIMIT = 10000; +export const HISTORY_TO_TABLE_MAP = { + '0': 'history', + '1': 'history_str', + '2': 'history_log', + '3': 'history_uint', + '4': 'history_text' +}; + +export const TREND_TO_TABLE_MAP = { + '0': 'trends', + '3': 'trends_uint' +}; + +export const consolidateByFunc = { + 'avg': 'AVG', + 'min': 'MIN', + 'max': 'MAX', + 'sum': 'SUM', + 'count': 'COUNT' +}; + +export const consolidateByTrendColumns = { + 'avg': 'value_avg', + 'min': 'value_min', + 'max': 'value_max', + 'sum': 'num*value_avg' // sum of sums inside the one-hour trend period +}; + /** * Base class for external history database connectors. Subclasses should implement `getHistory()`, `getTrends()` and * `testDataSource()` methods, which describe how to fetch data from source other than Zabbix API. */ -export default class DBConnector { - constructor(options, backendSrv, datasourceSrv) { - this.backendSrv = backendSrv; +export class DBConnector { + constructor(options, datasourceSrv) { this.datasourceSrv = datasourceSrv; this.datasourceId = options.datasourceId; this.datasourceName = options.datasourceName; @@ -106,3 +134,14 @@ function convertGrafanaTSResponse(time_series, items, addHostName) { return _.sortBy(grafanaSeries, 'target'); } + +const defaults = { + DBConnector, + DEFAULT_QUERY_LIMIT, + HISTORY_TO_TABLE_MAP, + TREND_TO_TABLE_MAP, + consolidateByFunc, + consolidateByTrendColumns +}; + +export default defaults; diff --git a/src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js b/src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js index 97e6a14..d8aea1e 100644 --- a/src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js +++ b/src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js @@ -1,38 +1,9 @@ import _ from 'lodash'; -import DBConnector from '../dbConnector'; - -const DEFAULT_QUERY_LIMIT = 10000; -const HISTORY_TO_TABLE_MAP = { - '0': 'history', - '1': 'history_str', - '2': 'history_log', - '3': 'history_uint', - '4': 'history_text' -}; - -const TREND_TO_TABLE_MAP = { - '0': 'trends', - '3': 'trends_uint' -}; - -const consolidateByFunc = { - 'avg': 'AVG', - 'min': 'MIN', - 'max': 'MAX', - 'sum': 'SUM', - 'count': 'COUNT' -}; - -const consolidateByTrendColumns = { - 'avg': 'value_avg', - 'min': 'value_min', - 'max': 'value_max', - 'sum': 'num*value_avg' // sum of sums inside the one-hour trend period -}; +import { DBConnector, DEFAULT_QUERY_LIMIT, HISTORY_TO_TABLE_MAP, consolidateByFunc } from '../dbConnector'; export class InfluxDBConnector extends DBConnector { - constructor(options, backendSrv, datasourceSrv) { - super(options, backendSrv, datasourceSrv); + constructor(options, datasourceSrv) { + super(options, datasourceSrv); this.limit = options.limit || DEFAULT_QUERY_LIMIT; super.loadDBDataSource().then(ds => { this.influxDS = ds; @@ -71,28 +42,7 @@ export class InfluxDBConnector extends DBConnector { } getTrends(items, timeFrom, timeTill, options) { - let { intervalMs, consolidateBy } = options; - const intervalSec = Math.ceil(intervalMs / 1000); - - consolidateBy = consolidateBy || 'avg'; - const aggFunction = consolidateByFunc[consolidateBy]; - - // Group items by value type and perform request for each value type - const grouped_items = _.groupBy(items, 'value_type'); - const promises = _.map(grouped_items, (items, value_type) => { - const itemids = _.map(items, 'itemid'); - const table = TREND_TO_TABLE_MAP[value_type]; - let valueColumn = _.includes(['avg', 'min', 'max', 'sum'], consolidateBy) ? consolidateBy : 'avg'; - valueColumn = consolidateByTrendColumns[valueColumn]; - const query = this.buildTrendsQuery(itemids, table, timeFrom, timeTill, intervalSec, aggFunction, valueColumn); - return this.invokeInfluxDBQuery(query); - }); - - return Promise.all(promises) - .then(_.flatten) - .then(results => { - return handleInfluxHistoryResponse(results); - }); + return this.getHistory(items, timeFrom, timeTill, options); } buildHistoryQuery(itemids, table, timeFrom, timeTill, intervalSec, aggFunction) { @@ -104,15 +54,6 @@ export class InfluxDBConnector extends DBConnector { return compactQuery(query); } - buildTrendsQuery(itemids, table, timeFrom, timeTill, intervalSec, aggFunction, valueColumn) { - const AGG = aggFunction === 'AVG' ? 'MEAN' : aggFunction; - const where_clause = this.buildWhereClause(itemids); - const query = `SELECT ${AGG}("${valueColumn}") FROM "${table}" - WHERE ${where_clause} AND "time" >= ${timeFrom}s AND "time" <= ${timeTill}s - GROUP BY time(${intervalSec}s)`; - return compactQuery(query); - } - buildWhereClause(itemids) { const itemidsWhere = itemids.map(itemid => `"itemid" = '${itemid}'`).join(' OR '); return `(${itemidsWhere})`; diff --git a/src/datasource-zabbix/zabbix/connectors/sql/sqlConnector.js b/src/datasource-zabbix/zabbix/connectors/sql/sqlConnector.js index 127789f..72af163 100644 --- a/src/datasource-zabbix/zabbix/connectors/sql/sqlConnector.js +++ b/src/datasource-zabbix/zabbix/connectors/sql/sqlConnector.js @@ -1,45 +1,17 @@ import _ from 'lodash'; import mysql from './mysql'; import postgres from './postgres'; -import DBConnector from '../dbConnector'; +import dbConnector, { DBConnector, DEFAULT_QUERY_LIMIT, HISTORY_TO_TABLE_MAP, TREND_TO_TABLE_MAP } from '../dbConnector'; const supportedDatabases = { mysql: 'mysql', postgres: 'postgres' }; -const DEFAULT_QUERY_LIMIT = 10000; -const HISTORY_TO_TABLE_MAP = { - '0': 'history', - '1': 'history_str', - '2': 'history_log', - '3': 'history_uint', - '4': 'history_text' -}; - -const TREND_TO_TABLE_MAP = { - '0': 'trends', - '3': 'trends_uint' -}; - -const consolidateByFunc = { - 'avg': 'AVG', - 'min': 'MIN', - 'max': 'MAX', - 'sum': 'SUM', - 'count': 'COUNT' -}; - -const consolidateByTrendColumns = { - 'avg': 'value_avg', - 'min': 'value_min', - 'max': 'value_max', - 'sum': 'num*value_avg' // sum of sums inside the one-hour trend period -}; - export class SQLConnector extends DBConnector { - constructor(options, backendSrv, datasourceSrv) { - super(options, backendSrv, datasourceSrv); + constructor(options, datasourceSrv, backendSrv) { + super(options, datasourceSrv); + this.backendSrv = backendSrv; this.limit = options.limit || DEFAULT_QUERY_LIMIT; this.sqlDialect = null; @@ -69,7 +41,7 @@ export class SQLConnector extends DBConnector { let intervalSec = Math.ceil(intervalMs / 1000); consolidateBy = consolidateBy || 'avg'; - let aggFunction = consolidateByFunc[consolidateBy]; + let aggFunction = dbConnector.consolidateByFunc[consolidateBy]; // Group items by value type and perform request for each value type let grouped_items = _.groupBy(items, 'value_type'); @@ -92,7 +64,7 @@ export class SQLConnector extends DBConnector { let intervalSec = Math.ceil(intervalMs / 1000); consolidateBy = consolidateBy || 'avg'; - let aggFunction = consolidateByFunc[consolidateBy]; + let aggFunction = dbConnector.consolidateByFunc[consolidateBy]; // Group items by value type and perform request for each value type let grouped_items = _.groupBy(items, 'value_type'); @@ -100,7 +72,7 @@ export class SQLConnector extends DBConnector { let itemids = _.map(items, 'itemid').join(', '); let table = TREND_TO_TABLE_MAP[value_type]; let valueColumn = _.includes(['avg', 'min', 'max', 'sum'], consolidateBy) ? consolidateBy : 'avg'; - valueColumn = consolidateByTrendColumns[valueColumn]; + valueColumn = dbConnector.consolidateByTrendColumns[valueColumn]; let query = this.sqlDialect.trendsQuery(itemids, table, timeFrom, timeTill, intervalSec, aggFunction, valueColumn); query = compactSQLQuery(query); diff --git a/src/datasource-zabbix/zabbix/zabbix.js b/src/datasource-zabbix/zabbix/zabbix.js index e720580..7d986a9 100644 --- a/src/datasource-zabbix/zabbix/zabbix.js +++ b/src/datasource-zabbix/zabbix/zabbix.js @@ -1,7 +1,7 @@ import _ from 'lodash'; import * as utils from '../utils'; import responseHandler from '../responseHandler'; -import DBConnector from './connectors/dbConnector'; +import { DBConnector } from './connectors/dbConnector'; import { ZabbixAPIConnector } from './connectors/zabbix_api/zabbixAPIConnector'; import { SQLConnector } from './connectors/sql/sqlConnector'; import { InfluxDBConnector } from './connectors/influxdb/influxdbConnector'; @@ -25,7 +25,7 @@ const REQUESTS_TO_BIND = [ ]; export class Zabbix { - constructor(options, backendSrv, datasourceSrv) { + constructor(options, datasourceSrv, backendSrv) { let { url, username, @@ -59,12 +59,12 @@ export class Zabbix { datasourceId: dbConnectionDatasourceId, datasourceName: dbConnectionDatasourceName }; - this.dbConnector = new DBConnector(dbConnectorOptions, backendSrv, datasourceSrv); + this.dbConnector = new DBConnector(dbConnectorOptions, datasourceSrv); this.dbConnector.loadDBDataSource().then(ds => { if (ds.type === 'influxdb') { - this.dbConnector = new InfluxDBConnector(dbConnectorOptions, backendSrv, datasourceSrv); + this.dbConnector = new InfluxDBConnector(dbConnectorOptions, datasourceSrv); } else { - this.dbConnector = new SQLConnector(dbConnectorOptions, backendSrv, datasourceSrv); + this.dbConnector = new SQLConnector(dbConnectorOptions, datasourceSrv, backendSrv); } }).then(() => { this.getHistoryDB = this.cachingProxy.proxyfyWithCache(this.dbConnector.getHistory, 'getHistory', this.dbConnector); From 34ba8bba1c4fe31f2522db8f6c6a6586c51d8dda Mon Sep 17 00:00:00 2001 From: Alexander Zobnin Date: Tue, 30 Oct 2018 21:05:48 +0300 Subject: [PATCH 05/21] db connector: ds loading refactor --- .../zabbix/connectors/dbConnector.js | 33 +++++++++++-------- .../zabbix/connectors/sql/sqlConnector.js | 8 +++-- src/datasource-zabbix/zabbix/zabbix.js | 31 +++++++++-------- 3 files changed, 42 insertions(+), 30 deletions(-) diff --git a/src/datasource-zabbix/zabbix/connectors/dbConnector.js b/src/datasource-zabbix/zabbix/connectors/dbConnector.js index 5006ce0..8f3ee5a 100644 --- a/src/datasource-zabbix/zabbix/connectors/dbConnector.js +++ b/src/datasource-zabbix/zabbix/connectors/dbConnector.js @@ -42,26 +42,33 @@ export class DBConnector { this.datasourceTypeName = null; } - loadDBDataSource() { - if (!this.datasourceName && this.datasourceId !== undefined) { - let ds = _.find(this.datasourceSrv.getAll(), {'id': this.datasourceId}); + static loadDatasource(dsId, dsName, datasourceSrv) { + if (!dsName && dsId !== undefined) { + let ds = _.find(datasourceSrv.getAll(), {'id': dsId}); if (!ds) { - return Promise.reject(`SQL Data Source with ID ${this.datasourceId} not found`); + return Promise.reject(`Data Source with ID ${dsId} not found`); } - this.datasourceName = ds.name; + dsName = ds.name; } - if (this.datasourceName) { - return this.datasourceSrv.loadDatasource(this.datasourceName) - .then(ds => { - this.datasourceTypeId = ds.meta.id; - this.datasourceTypeName = ds.meta.name; - return ds; - }); + if (dsName) { + return datasourceSrv.loadDatasource(dsName); } else { - return Promise.reject(`SQL Data Source name should be specified`); + return Promise.reject(`Data Source name should be specified`); } } + loadDBDataSource() { + return DBConnector.loadDatasource(this.datasourceId, this.datasourceName, this.datasourceSrv) + .then(ds => { + this.datasourceTypeId = ds.meta.id; + this.datasourceTypeName = ds.meta.name; + if (!this.datasourceName) { + this.datasourceName = ds.name; + } + return ds; + }); + } + /** * Send test request to datasource in order to ensure it's working. */ diff --git a/src/datasource-zabbix/zabbix/connectors/sql/sqlConnector.js b/src/datasource-zabbix/zabbix/connectors/sql/sqlConnector.js index 72af163..575f72d 100644 --- a/src/datasource-zabbix/zabbix/connectors/sql/sqlConnector.js +++ b/src/datasource-zabbix/zabbix/connectors/sql/sqlConnector.js @@ -9,15 +9,17 @@ const supportedDatabases = { }; export class SQLConnector extends DBConnector { - constructor(options, datasourceSrv, backendSrv) { + constructor(options, datasourceSrv) { super(options, datasourceSrv); - this.backendSrv = backendSrv; this.limit = options.limit || DEFAULT_QUERY_LIMIT; this.sqlDialect = null; super.loadDBDataSource() - .then(() => this.loadSQLDialect()); + .then(ds => { + this.backendSrv = ds.backendSrv; + this.loadSQLDialect(); + }); } loadSQLDialect() { diff --git a/src/datasource-zabbix/zabbix/zabbix.js b/src/datasource-zabbix/zabbix/zabbix.js index 7d986a9..c68e8a0 100644 --- a/src/datasource-zabbix/zabbix/zabbix.js +++ b/src/datasource-zabbix/zabbix/zabbix.js @@ -1,12 +1,12 @@ import _ from 'lodash'; import * as utils from '../utils'; import responseHandler from '../responseHandler'; +import { CachingProxy } from './proxy/cachingProxy'; +import { ZabbixNotImplemented } from './connectors/dbConnector'; import { DBConnector } from './connectors/dbConnector'; import { ZabbixAPIConnector } from './connectors/zabbix_api/zabbixAPIConnector'; import { SQLConnector } from './connectors/sql/sqlConnector'; import { InfluxDBConnector } from './connectors/influxdb/influxdbConnector'; -import { CachingProxy } from './proxy/cachingProxy'; -import { ZabbixNotImplemented } from './connectors/dbConnector'; const REQUESTS_TO_PROXYFY = [ 'getHistory', 'getTrend', 'getGroups', 'getHosts', 'getApps', 'getItems', 'getMacros', 'getItemsByIDs', @@ -55,24 +55,27 @@ export class Zabbix { this.bindRequests(); if (enableDirectDBConnection) { - let dbConnectorOptions = { - datasourceId: dbConnectionDatasourceId, - datasourceName: dbConnectionDatasourceName - }; - this.dbConnector = new DBConnector(dbConnectorOptions, datasourceSrv); - this.dbConnector.loadDBDataSource().then(ds => { - if (ds.type === 'influxdb') { - this.dbConnector = new InfluxDBConnector(dbConnectorOptions, datasourceSrv); - } else { - this.dbConnector = new SQLConnector(dbConnectorOptions, datasourceSrv, backendSrv); - } - }).then(() => { + this.initDBConnector(dbConnectionDatasourceId, dbConnectionDatasourceName, datasourceSrv) + .then(() => { this.getHistoryDB = this.cachingProxy.proxyfyWithCache(this.dbConnector.getHistory, 'getHistory', this.dbConnector); this.getTrendsDB = this.cachingProxy.proxyfyWithCache(this.dbConnector.getTrends, 'getTrends', this.dbConnector); }); } } + initDBConnector(datasourceId, datasourceName, datasourceSrv) { + return DBConnector.loadDatasource(datasourceId, datasourceName, datasourceSrv) + .then(ds => { + const options = { datasourceId, datasourceName }; + if (ds.type === 'influxdb') { + this.dbConnector = new InfluxDBConnector(options, datasourceSrv); + } else { + this.dbConnector = new SQLConnector(options, datasourceSrv); + } + return this.dbConnector; + }); + } + proxyfyRequests() { for (let request of REQUESTS_TO_PROXYFY) { this.zabbixAPI[request] = this.cachingProxy.proxyfy(this.zabbixAPI[request], request, this.zabbixAPI); From d55757b3f158e41df0a232c4cbefd07c38df34cf Mon Sep 17 00:00:00 2001 From: Alexander Zobnin Date: Tue, 30 Oct 2018 21:26:14 +0300 Subject: [PATCH 06/21] influx: use fill(none) by default --- .../zabbix/connectors/influxdb/influxdbConnector.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js b/src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js index d8aea1e..921b228 100644 --- a/src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js +++ b/src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js @@ -50,7 +50,7 @@ export class InfluxDBConnector extends DBConnector { const where_clause = this.buildWhereClause(itemids); const query = `SELECT ${AGG}("value") FROM "${table}" WHERE ${where_clause} AND "time" >= ${timeFrom}s AND "time" <= ${timeTill}s - GROUP BY time(${intervalSec}s), "itemid" fill(linear)`; + GROUP BY time(${intervalSec}s), "itemid" fill(none)`; return compactQuery(query); } From 369565ab7f9b59b47dbd2d257f6d2437f048bba9 Mon Sep 17 00:00:00 2001 From: Alexander Zobnin Date: Tue, 30 Oct 2018 21:31:49 +0300 Subject: [PATCH 07/21] influx: able to use arbitrary aggregation --- .../zabbix/connectors/influxdb/influxdbConnector.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js b/src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js index 921b228..472d6e1 100644 --- a/src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js +++ b/src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js @@ -23,7 +23,7 @@ export class InfluxDBConnector extends DBConnector { const intervalSec = Math.ceil(intervalMs / 1000); consolidateBy = consolidateBy || 'avg'; - const aggFunction = consolidateByFunc[consolidateBy]; + const aggFunction = consolidateByFunc[consolidateBy] || consolidateBy; // Group items by value type and perform request for each value type const grouped_items = _.groupBy(items, 'value_type'); From 6af18e493ecd6e3df5b118e468ce5634726c2cf1 Mon Sep 17 00:00:00 2001 From: Alexander Zobnin Date: Tue, 30 Oct 2018 21:43:14 +0300 Subject: [PATCH 08/21] influx: remove unused limit option --- .../zabbix/connectors/influxdb/influxdbConnector.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js b/src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js index 472d6e1..614eb83 100644 --- a/src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js +++ b/src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js @@ -1,10 +1,9 @@ import _ from 'lodash'; -import { DBConnector, DEFAULT_QUERY_LIMIT, HISTORY_TO_TABLE_MAP, consolidateByFunc } from '../dbConnector'; +import { DBConnector, HISTORY_TO_TABLE_MAP, consolidateByFunc } from '../dbConnector'; export class InfluxDBConnector extends DBConnector { constructor(options, datasourceSrv) { super(options, datasourceSrv); - this.limit = options.limit || DEFAULT_QUERY_LIMIT; super.loadDBDataSource().then(ds => { this.influxDS = ds; return ds; From 42839648c4e9f9897e9b3cc35a866eaad06f5862 Mon Sep 17 00:00:00 2001 From: Alexander Zobnin Date: Tue, 30 Oct 2018 21:48:39 +0300 Subject: [PATCH 09/21] db connector: fix tests --- src/datasource-zabbix/specs/dbConnector.test.js | 6 +++--- src/datasource-zabbix/zabbix/connectors/sql/sqlConnector.js | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/datasource-zabbix/specs/dbConnector.test.js b/src/datasource-zabbix/specs/dbConnector.test.js index cca08f1..b41276c 100644 --- a/src/datasource-zabbix/specs/dbConnector.test.js +++ b/src/datasource-zabbix/specs/dbConnector.test.js @@ -15,7 +15,7 @@ describe('DBConnector', () => { }; }); - it('should load datasource by name by default', () => { + it('should try to load datasource by name first', () => { ctx.options = { datasourceName: 'bar' }; @@ -35,13 +35,13 @@ describe('DBConnector', () => { it('should throw error if no name and id specified', () => { ctx.options = {}; const dbConnector = new DBConnector(ctx.options, datasourceSrv); - return expect(dbConnector.loadDBDataSource()).rejects.toBe('SQL Data Source name should be specified'); + return expect(dbConnector.loadDBDataSource()).rejects.toBe('Data Source name should be specified'); }); it('should throw error if datasource with given id is not found', () => { ctx.options.datasourceId = 45; const dbConnector = new DBConnector(ctx.options, datasourceSrv); - return expect(dbConnector.loadDBDataSource()).rejects.toBe('SQL Data Source with ID 45 not found'); + return expect(dbConnector.loadDBDataSource()).rejects.toBe('Data Source with ID 45 not found'); }); }); }); diff --git a/src/datasource-zabbix/zabbix/connectors/sql/sqlConnector.js b/src/datasource-zabbix/zabbix/connectors/sql/sqlConnector.js index 575f72d..7c03e4e 100644 --- a/src/datasource-zabbix/zabbix/connectors/sql/sqlConnector.js +++ b/src/datasource-zabbix/zabbix/connectors/sql/sqlConnector.js @@ -113,7 +113,6 @@ export class SQLConnector extends DBConnector { } } - function compactSQLQuery(query) { return query.replace(/\s+/g, ' '); } From c5b690188f7ac514e06cc532eb2b41ce18d1ce57 Mon Sep 17 00:00:00 2001 From: Alexander Zobnin Date: Tue, 30 Oct 2018 22:12:04 +0300 Subject: [PATCH 10/21] influx: handle influxdb errors --- .../zabbix/connectors/influxdb/influxdbConnector.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js b/src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js index 614eb83..682268d 100644 --- a/src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js +++ b/src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js @@ -74,6 +74,12 @@ function handleInfluxHistoryResponse(results) { const seriesList = []; for (let i = 0; i < results.length; i++) { const result = results[i]; + + if (result.error) { + const error = `InfluxDB error: ${result.error}`; + return Promise.reject(new Error(error)); + } + if (!result || !result.series) { continue; } From 4bb8df19f68a8b8664201bac0482e26a8f8a66b2 Mon Sep 17 00:00:00 2001 From: Alexander Zobnin Date: Tue, 30 Oct 2018 22:44:09 +0300 Subject: [PATCH 11/21] datasource: query performance logging --- src/datasource-zabbix/datasource.js | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/datasource-zabbix/datasource.js b/src/datasource-zabbix/datasource.js index 31b1eee..8133f18 100644 --- a/src/datasource-zabbix/datasource.js +++ b/src/datasource-zabbix/datasource.js @@ -1,4 +1,5 @@ import _ from 'lodash'; +import config from 'grafana/app/core/config'; import * as dateMath from 'grafana/app/core/utils/datemath'; import * as utils from './utils'; import * as migrations from './migrations'; @@ -18,6 +19,8 @@ export class ZabbixDatasource { this.templateSrv = templateSrv; this.zabbixAlertingSrv = zabbixAlertingSrv; + this.enableDebugLog = config.buildInfo.env === 'development'; + // Use custom format for template variables this.replaceTemplateVars = _.partial(replaceTemplateVars, this.templateSrv); @@ -165,11 +168,21 @@ export class ZabbixDatasource { * Query target data for Metrics mode */ queryNumericData(target, timeRange, useTrends, options) { + let queryStart, queryEnd; let getItemOptions = { itemtype: 'num' }; return this.zabbix.getItemsFromTarget(target, getItemOptions) - .then(items => this.queryNumericDataForItems(items, target, timeRange, useTrends, options)); + .then(items => { + queryStart = new Date().getTime(); + return this.queryNumericDataForItems(items, target, timeRange, useTrends, options); + }).then(result => { + queryEnd = new Date().getTime(); + if (this.enableDebugLog) { + console.log(`Datasource::Performance Query Time (${this.name}): ${queryEnd - queryStart}`); + } + return result; + }); } /** From 9708ec459c82636a9c65032246501bdcefccabf6 Mon Sep 17 00:00:00 2001 From: Alexander Zobnin Date: Tue, 30 Oct 2018 23:01:41 +0300 Subject: [PATCH 12/21] fix tests (mock for app/core/config) --- src/test-setup/jest-setup.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/test-setup/jest-setup.js b/src/test-setup/jest-setup.js index dca0135..23a32b5 100644 --- a/src/test-setup/jest-setup.js +++ b/src/test-setup/jest-setup.js @@ -54,6 +54,12 @@ jest.mock('grafana/app/core/table_model', () => { }; }, {virtual: true}); +jest.mock('grafana/app/core/config', () => { + return { + buildInfo: { env: 'development' } + }; +}, {virtual: true}); + jest.mock('jquery', () => 'module not found', {virtual: true}); // Required for loading angularjs From eeaef5c2ee4d15548daf85e86ba47d9074605f26 Mon Sep 17 00:00:00 2001 From: Alexander Zobnin Date: Wed, 31 Oct 2018 09:11:24 +0300 Subject: [PATCH 13/21] codecov: adjust threshold --- codecov.yml => .codecov.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) rename codecov.yml => .codecov.yml (70%) diff --git a/codecov.yml b/.codecov.yml similarity index 70% rename from codecov.yml rename to .codecov.yml index f5db33d..a641bd5 100644 --- a/codecov.yml +++ b/.codecov.yml @@ -6,10 +6,9 @@ coverage: status: project: default: - target: 30% - threshold: 5% - patch: no - changes: no + threshold: 5 + patch: off + changes: off comment: false From 054e5518c5cf2dae77b6553dc7ddbefa68e1a63a Mon Sep 17 00:00:00 2001 From: Alexander Zobnin Date: Wed, 31 Oct 2018 10:17:23 +0300 Subject: [PATCH 14/21] tests for influxdb connector --- .../specs/influxdbConnector.test.js | 95 +++++++++++++++++++ src/datasource-zabbix/utils.js | 7 ++ .../connectors/influxdb/influxdbConnector.js | 5 +- .../zabbix/connectors/sql/sqlConnector.js | 9 +- 4 files changed, 106 insertions(+), 10 deletions(-) create mode 100644 src/datasource-zabbix/specs/influxdbConnector.test.js diff --git a/src/datasource-zabbix/specs/influxdbConnector.test.js b/src/datasource-zabbix/specs/influxdbConnector.test.js new file mode 100644 index 0000000..96084ab --- /dev/null +++ b/src/datasource-zabbix/specs/influxdbConnector.test.js @@ -0,0 +1,95 @@ +import { InfluxDBConnector } from '../zabbix/connectors/influxdb/influxdbConnector'; +import { compactQuery } from '../utils'; + +describe('InfluxDBConnector', () => { + let ctx = {}; + + beforeEach(() => { + ctx.options = { datasourceName: 'InfluxDB DS' }; + ctx.datasourceSrvMock = { + loadDatasource: jest.fn().mockResolvedValue( + { id: 42, name: 'InfluxDB DS', meta: {} } + ), + }; + ctx.influxDBConnector = new InfluxDBConnector(ctx.options, ctx.datasourceSrvMock); + ctx.influxDBConnector.invokeInfluxDBQuery = jest.fn().mockResolvedValue([]); + ctx.defaultQueryParams = { + itemids: ['123', '234'], + timeFrom: 15000, timeTill: 15100, intervalSec: 5, + table: 'history', aggFunction: 'MAX' + }; + }); + + describe('When building InfluxDB query', () => { + it('should build proper query', () => { + const { itemids, timeFrom, timeTill, intervalSec, table, aggFunction } = ctx.defaultQueryParams; + const query = ctx.influxDBConnector.buildHistoryQuery(itemids, table, timeFrom, timeTill, intervalSec, aggFunction); + const expected = compactQuery(`SELECT MAX("value") + FROM "history" WHERE ("itemid" = '123' OR "itemid" = '234') AND "time" >= 15000s AND "time" <= 15100s + GROUP BY time(5s), "itemid" fill(none) + `); + expect(query).toBe(expected); + }); + + it('should use MEAN instead of AVG', () => { + const { itemids, timeFrom, timeTill, intervalSec, table } = ctx.defaultQueryParams; + const aggFunction = 'AVG'; + const query = ctx.influxDBConnector.buildHistoryQuery(itemids, table, timeFrom, timeTill, intervalSec, aggFunction); + const expected = compactQuery(`SELECT MEAN("value") + FROM "history" WHERE ("itemid" = '123' OR "itemid" = '234') AND "time" >= 15000s AND "time" <= 15100s + GROUP BY time(5s), "itemid" fill(none) + `); + expect(query).toBe(expected); + }); + }); + + describe('When invoking InfluxDB query', () => { + it('should query proper table depending on item type', () => { + const { timeFrom, timeTill} = ctx.defaultQueryParams; + const options = { intervalMs: 5000 }; + const items = [ + { itemid: '123', value_type: 3 } + ]; + const expectedQuery = compactQuery(`SELECT MEAN("value") + FROM "history_uint" WHERE ("itemid" = '123') AND "time" >= 15000s AND "time" <= 15100s + GROUP BY time(5s), "itemid" fill(none) + `); + ctx.influxDBConnector.getHistory(items, timeFrom, timeTill, options); + expect(ctx.influxDBConnector.invokeInfluxDBQuery).toHaveBeenCalledWith(expectedQuery); + }); + + it('should split query if different item types are used', () => { + const { timeFrom, timeTill} = ctx.defaultQueryParams; + const options = { intervalMs: 5000 }; + const items = [ + { itemid: '123', value_type: 0 }, + { itemid: '234', value_type: 3 }, + ]; + const sharedQueryPart = `AND "time" >= 15000s AND "time" <= 15100s GROUP BY time(5s), "itemid" fill(none)`; + const expectedQueryFirst = compactQuery(`SELECT MEAN("value") + FROM "history" WHERE ("itemid" = '123') ${sharedQueryPart} + `); + const expectedQuerySecond = compactQuery(`SELECT MEAN("value") + FROM "history_uint" WHERE ("itemid" = '234') ${sharedQueryPart} + `); + ctx.influxDBConnector.getHistory(items, timeFrom, timeTill, options); + expect(ctx.influxDBConnector.invokeInfluxDBQuery).toHaveBeenCalledTimes(2); + expect(ctx.influxDBConnector.invokeInfluxDBQuery).toHaveBeenNthCalledWith(1, expectedQueryFirst); + expect(ctx.influxDBConnector.invokeInfluxDBQuery).toHaveBeenNthCalledWith(2, expectedQuerySecond); + }); + + it('should use the same table for trends query', () => { + const { timeFrom, timeTill} = ctx.defaultQueryParams; + const options = { intervalMs: 5000 }; + const items = [ + { itemid: '123', value_type: 3 } + ]; + const expectedQuery = compactQuery(`SELECT MEAN("value") + FROM "history_uint" WHERE ("itemid" = '123') AND "time" >= 15000s AND "time" <= 15100s + GROUP BY time(5s), "itemid" fill(none) + `); + ctx.influxDBConnector.getTrends(items, timeFrom, timeTill, options); + expect(ctx.influxDBConnector.invokeInfluxDBQuery).toHaveBeenCalledWith(expectedQuery); + }); + }); +}); diff --git a/src/datasource-zabbix/utils.js b/src/datasource-zabbix/utils.js index 53f0bea..ab6639e 100644 --- a/src/datasource-zabbix/utils.js +++ b/src/datasource-zabbix/utils.js @@ -258,6 +258,13 @@ export function parseVersion(version) { return { major, minor, patch, meta }; } +/** + * Replaces any space-like symbols (tabs, new lines, spaces) by single whitespace. + */ +export function compactQuery(query) { + return query.replace(/\s+/g, ' ').trim(); +} + // Fix for backward compatibility with lodash 2.4 if (!_.includes) { _.includes = _.contains; diff --git a/src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js b/src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js index 682268d..2e957db 100644 --- a/src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js +++ b/src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js @@ -1,4 +1,5 @@ import _ from 'lodash'; +import { compactQuery } from '../../../utils'; import { DBConnector, HISTORY_TO_TABLE_MAP, consolidateByFunc } from '../dbConnector'; export class InfluxDBConnector extends DBConnector { @@ -104,7 +105,3 @@ function handleInfluxHistoryResponse(results) { return seriesList; } - -function compactQuery(query) { - return query.replace(/\s+/g, ' '); -} diff --git a/src/datasource-zabbix/zabbix/connectors/sql/sqlConnector.js b/src/datasource-zabbix/zabbix/connectors/sql/sqlConnector.js index 7c03e4e..0ec5dc6 100644 --- a/src/datasource-zabbix/zabbix/connectors/sql/sqlConnector.js +++ b/src/datasource-zabbix/zabbix/connectors/sql/sqlConnector.js @@ -1,4 +1,5 @@ import _ from 'lodash'; +import { compactQuery } from '../../../utils'; import mysql from './mysql'; import postgres from './postgres'; import dbConnector, { DBConnector, DEFAULT_QUERY_LIMIT, HISTORY_TO_TABLE_MAP, TREND_TO_TABLE_MAP } from '../dbConnector'; @@ -52,7 +53,7 @@ export class SQLConnector extends DBConnector { let table = HISTORY_TO_TABLE_MAP[value_type]; let query = this.sqlDialect.historyQuery(itemids, table, timeFrom, timeTill, intervalSec, aggFunction); - query = compactSQLQuery(query); + query = compactQuery(query); return this.invokeSQLQuery(query); }); @@ -77,7 +78,7 @@ export class SQLConnector extends DBConnector { valueColumn = dbConnector.consolidateByTrendColumns[valueColumn]; let query = this.sqlDialect.trendsQuery(itemids, table, timeFrom, timeTill, intervalSec, aggFunction, valueColumn); - query = compactSQLQuery(query); + query = compactQuery(query); return this.invokeSQLQuery(query); }); @@ -112,7 +113,3 @@ export class SQLConnector extends DBConnector { }); } } - -function compactSQLQuery(query) { - return query.replace(/\s+/g, ' '); -} From d061a3ddce3a1053c99822212780aced2527ea4b Mon Sep 17 00:00:00 2001 From: Alexander Zobnin Date: Wed, 31 Oct 2018 10:35:56 +0300 Subject: [PATCH 15/21] config page: rename SQL data sources to DB data sources --- src/datasource-zabbix/config.controller.js | 5 +++-- src/datasource-zabbix/partials/config.html | 12 ++++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/datasource-zabbix/config.controller.js b/src/datasource-zabbix/config.controller.js index fcd77e9..fe9815f 100644 --- a/src/datasource-zabbix/config.controller.js +++ b/src/datasource-zabbix/config.controller.js @@ -2,6 +2,7 @@ import _ from 'lodash'; import { migrateDSConfig } from './migrations'; const SUPPORTED_SQL_DS = ['mysql', 'postgres', 'influxdb']; + const zabbixVersions = [ { name: '2.x', value: 2 }, { name: '3.x', value: 3 }, @@ -27,12 +28,12 @@ export class ZabbixDSConfigController { this.current.jsonData = migrateDSConfig(this.current.jsonData); _.defaults(this.current.jsonData, defaultConfig); - this.sqlDataSources = this.getSupportedSQLDataSources(); + this.dbDataSources = this.getSupportedDBDataSources(); this.zabbixVersions = _.cloneDeep(zabbixVersions); this.autoDetectZabbixVersion(); } - getSupportedSQLDataSources() { + getSupportedDBDataSources() { let datasources = this.datasourceSrv.getAll(); return _.filter(datasources, ds => { return _.includes(SUPPORTED_SQL_DS, ds.type); diff --git a/src/datasource-zabbix/partials/config.html b/src/datasource-zabbix/partials/config.html index 1530d2f..043c7c7 100644 --- a/src/datasource-zabbix/partials/config.html +++ b/src/datasource-zabbix/partials/config.html @@ -91,20 +91,20 @@ checked="ctrl.current.jsonData.dbConnectionEnable">
-
+
- SQL Data Source + Data Source - Select SQL Data Source for Zabbix database. - In order to use this feature you should create and - configure it first. Zabbix plugin uses this data source for querying history data directly from database. + Select Data Source for Zabbix history database. + In order to use this feature it should be created and + configured first. Zabbix plugin uses this data source for querying history data directly from the database. This way usually faster than pulling data from Zabbix API, especially on the wide time ranges, and reduces amount of data transfered.
From d5a224d4fcf42b48d3d53fbfe909c3387fb9abfe Mon Sep 17 00:00:00 2001 From: Alexander Zobnin Date: Wed, 31 Oct 2018 12:25:24 +0300 Subject: [PATCH 16/21] docs for influxdb connector --- docs/sources/configuration/index.md | 4 ++-- docs/sources/reference/direct-db-connection.md | 16 +++++++++++++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/docs/sources/configuration/index.md b/docs/sources/configuration/index.md index 602b7c3..c96c3ec 100644 --- a/docs/sources/configuration/index.md +++ b/docs/sources/configuration/index.md @@ -61,11 +61,11 @@ amount of data transfered. Read [how to configure](./sql_datasource) SQL data source in Grafana. - **Enable**: enable Direct DB Connection. -- **SQL Data Source**: Select SQL Data Source for Zabbix database. +- **Data Source**: Select Data Source for Zabbix history database. #### Supported databases -**MySQL** and **PostgreSQL** are supported by Grafana. +**MySQL**, **PostgreSQL**, **InfluxDB** are supported as sources of historical data for the plugin. ### Alerting diff --git a/docs/sources/reference/direct-db-connection.md b/docs/sources/reference/direct-db-connection.md index 16112a6..ec91ed5 100644 --- a/docs/sources/reference/direct-db-connection.md +++ b/docs/sources/reference/direct-db-connection.md @@ -1,6 +1,6 @@ # Direct DB Connection -Since version 4.3 Grafana can use MySQL as a native data source. The Grafana-Zabbix plugin can use this data source for querying data directly from a Zabbix database. +Since version 4.3 Grafana can use MySQL as a native data source. The idea of Direct DB Connection is that Grafana-Zabbix plugin can use this data source for querying data directly from a Zabbix database. One of the most resource intensive queries for Zabbix API is the history query. For long time intervals `history.get` returns a huge amount of data. In order to display it, the plugin should adjust time series resolution @@ -10,6 +10,8 @@ time series, but that data should be loaded and processed on the client side fir Also, many users see better performance from direct database queries versus API calls. This could be the result of several reasons, such as the additional PHP layer and additional SQL queries (user permissions checks). +Direct DB Connection feature allows using database transparently for querying historical data. Now Grafana-Zabbix plugin supports few databases for history queries: MySQL, PostgreSQL and InfluxDB. Regardless of the database type, idea and data flow remain the same. + ## Data Flow This chart illustrates how the plugin uses both Zabbix API and the MySQL data source for querying different types @@ -76,6 +78,18 @@ ORDER BY time ASC As you can see, the Grafana-Zabbix plugin uses aggregation by a given time interval. This interval is provided by Grafana and depends on the panel width in pixels. Thus, Grafana displays the data in the proper resolution. +## InfluxDB +Zabbix supports loadable modules which makes possible to write history data into an external database. There's a [module](https://github.com/i-ky/effluence) for InfluxDB written by [@i-ky](https://github.com/i-ky) which can export history into InfluxDB in real-time. + +InfluxDB Query Example: +```sql +SELECT MEAN("value") +FROM "history" +WHERE ("itemid" = '10073' OR "itemid" = '10074') + AND "time" >= 1540000000000s AND "time" <= 1540000000060s +GROUP BY time(10s), "itemid" fill(none) +``` + ## Functions usage with Direct DB Connection There's only one function that directly affects the backend data. This function is `consolidateBy`. Other functions work on the client side and transform data that comes from the backend. So you should clearly understand that this is pre-aggregated data (by AVG, MAX, MIN, etc). From 089700d227cadf6e4c496fe1f5f913cf053eab57 Mon Sep 17 00:00:00 2001 From: Alexander Zobnin Date: Wed, 31 Oct 2018 20:56:36 +0300 Subject: [PATCH 17/21] influx: support retention policy for long-term stored data --- src/datasource-zabbix/config.controller.js | 7 +++ src/datasource-zabbix/datasource.js | 4 +- src/datasource-zabbix/partials/config.html | 44 +++++++++++++------ .../specs/influxdbConnector.test.js | 36 ++++++++++----- .../connectors/influxdb/influxdbConnector.js | 13 ++++-- src/datasource-zabbix/zabbix/zabbix.js | 13 +++--- 6 files changed, 83 insertions(+), 34 deletions(-) diff --git a/src/datasource-zabbix/config.controller.js b/src/datasource-zabbix/config.controller.js index fe9815f..9f7cfcc 100644 --- a/src/datasource-zabbix/config.controller.js +++ b/src/datasource-zabbix/config.controller.js @@ -31,6 +31,7 @@ export class ZabbixDSConfigController { this.dbDataSources = this.getSupportedDBDataSources(); this.zabbixVersions = _.cloneDeep(zabbixVersions); this.autoDetectZabbixVersion(); + console.log(this.dbDataSources); } getSupportedDBDataSources() { @@ -40,6 +41,12 @@ export class ZabbixDSConfigController { }); } + getCurrentDatasourceType() { + const dsId = this.current.jsonData.dbConnectionDatasourceId; + const currentDs = _.find(this.dbDataSources, { 'id': dsId }); + return currentDs ? currentDs.type : null; + } + autoDetectZabbixVersion() { if (!this.current.id) { return; diff --git a/src/datasource-zabbix/datasource.js b/src/datasource-zabbix/datasource.js index 8133f18..cd46410 100644 --- a/src/datasource-zabbix/datasource.js +++ b/src/datasource-zabbix/datasource.js @@ -58,6 +58,7 @@ export class ZabbixDatasource { this.enableDirectDBConnection = jsonData.dbConnectionEnable || false; this.dbConnectionDatasourceId = jsonData.dbConnectionDatasourceId; this.dbConnectionDatasourceName = jsonData.dbConnectionDatasourceName; + this.dbConnectionRetentionPolicy = jsonData.dbConnectionRetentionPolicy; let zabbixOptions = { url: this.url, @@ -69,7 +70,8 @@ export class ZabbixDatasource { cacheTTL: this.cacheTTL, enableDirectDBConnection: this.enableDirectDBConnection, dbConnectionDatasourceId: this.dbConnectionDatasourceId, - dbConnectionDatasourceName: this.dbConnectionDatasourceName + dbConnectionDatasourceName: this.dbConnectionDatasourceName, + dbConnectionRetentionPolicy: this.dbConnectionRetentionPolicy, }; this.zabbix = new Zabbix(zabbixOptions, datasourceSrv, backendSrv); diff --git a/src/datasource-zabbix/partials/config.html b/src/datasource-zabbix/partials/config.html index 043c7c7..f9920f1 100644 --- a/src/datasource-zabbix/partials/config.html +++ b/src/datasource-zabbix/partials/config.html @@ -92,22 +92,38 @@
- - Data Source - - Select Data Source for Zabbix history database. - In order to use this feature it should be created and - configured first. Zabbix plugin uses this data source for querying history data directly from the database. - This way usually faster than pulling data from Zabbix API, especially on the wide time ranges, and reduces - amount of data transfered. - - -
- + + Data Source + + Select Data Source for Zabbix history database. + In order to use this feature it should be created and + configured first. Zabbix plugin uses this data source for querying history data directly from the database. + This way usually faster than pulling data from Zabbix API, especially on the wide time ranges, and reduces + amount of data transfered. + + +
+ +
+
+
+ + Retention Policy + + Specify retention policy name for fetching long-term stored data (optional). + Leave it blank if only default retention policy is using. + + + + +
diff --git a/src/datasource-zabbix/specs/influxdbConnector.test.js b/src/datasource-zabbix/specs/influxdbConnector.test.js index 96084ab..089770e 100644 --- a/src/datasource-zabbix/specs/influxdbConnector.test.js +++ b/src/datasource-zabbix/specs/influxdbConnector.test.js @@ -5,7 +5,7 @@ describe('InfluxDBConnector', () => { let ctx = {}; beforeEach(() => { - ctx.options = { datasourceName: 'InfluxDB DS' }; + ctx.options = { datasourceName: 'InfluxDB DS', retentionPolicy: 'longterm' }; ctx.datasourceSrvMock = { loadDatasource: jest.fn().mockResolvedValue( { id: 42, name: 'InfluxDB DS', meta: {} } @@ -15,15 +15,16 @@ describe('InfluxDBConnector', () => { ctx.influxDBConnector.invokeInfluxDBQuery = jest.fn().mockResolvedValue([]); ctx.defaultQueryParams = { itemids: ['123', '234'], - timeFrom: 15000, timeTill: 15100, intervalSec: 5, + range: { timeFrom: 15000, timeTill: 15100 }, + intervalSec: 5, table: 'history', aggFunction: 'MAX' }; }); describe('When building InfluxDB query', () => { it('should build proper query', () => { - const { itemids, timeFrom, timeTill, intervalSec, table, aggFunction } = ctx.defaultQueryParams; - const query = ctx.influxDBConnector.buildHistoryQuery(itemids, table, timeFrom, timeTill, intervalSec, aggFunction); + const { itemids, range, intervalSec, table, aggFunction } = ctx.defaultQueryParams; + const query = ctx.influxDBConnector.buildHistoryQuery(itemids, table, range, intervalSec, aggFunction); const expected = compactQuery(`SELECT MAX("value") FROM "history" WHERE ("itemid" = '123' OR "itemid" = '234') AND "time" >= 15000s AND "time" <= 15100s GROUP BY time(5s), "itemid" fill(none) @@ -32,9 +33,9 @@ describe('InfluxDBConnector', () => { }); it('should use MEAN instead of AVG', () => { - const { itemids, timeFrom, timeTill, intervalSec, table } = ctx.defaultQueryParams; + const { itemids, range, intervalSec, table } = ctx.defaultQueryParams; const aggFunction = 'AVG'; - const query = ctx.influxDBConnector.buildHistoryQuery(itemids, table, timeFrom, timeTill, intervalSec, aggFunction); + const query = ctx.influxDBConnector.buildHistoryQuery(itemids, table, range, intervalSec, aggFunction); const expected = compactQuery(`SELECT MEAN("value") FROM "history" WHERE ("itemid" = '123' OR "itemid" = '234') AND "time" >= 15000s AND "time" <= 15100s GROUP BY time(5s), "itemid" fill(none) @@ -45,7 +46,7 @@ describe('InfluxDBConnector', () => { describe('When invoking InfluxDB query', () => { it('should query proper table depending on item type', () => { - const { timeFrom, timeTill} = ctx.defaultQueryParams; + const { timeFrom, timeTill } = ctx.defaultQueryParams.range; const options = { intervalMs: 5000 }; const items = [ { itemid: '123', value_type: 3 } @@ -59,7 +60,7 @@ describe('InfluxDBConnector', () => { }); it('should split query if different item types are used', () => { - const { timeFrom, timeTill} = ctx.defaultQueryParams; + const { timeFrom, timeTill } = ctx.defaultQueryParams.range; const options = { intervalMs: 5000 }; const items = [ { itemid: '123', value_type: 0 }, @@ -78,8 +79,9 @@ describe('InfluxDBConnector', () => { expect(ctx.influxDBConnector.invokeInfluxDBQuery).toHaveBeenNthCalledWith(2, expectedQuerySecond); }); - it('should use the same table for trends query', () => { - const { timeFrom, timeTill} = ctx.defaultQueryParams; + it('should use the same table for trends query if no retention policy set', () => { + ctx.influxDBConnector.retentionPolicy = ''; + const { timeFrom, timeTill } = ctx.defaultQueryParams.range; const options = { intervalMs: 5000 }; const items = [ { itemid: '123', value_type: 3 } @@ -91,5 +93,19 @@ describe('InfluxDBConnector', () => { ctx.influxDBConnector.getTrends(items, timeFrom, timeTill, options); expect(ctx.influxDBConnector.invokeInfluxDBQuery).toHaveBeenCalledWith(expectedQuery); }); + + it('should use retention policy name for trends query if it was set', () => { + const { timeFrom, timeTill } = ctx.defaultQueryParams.range; + const options = { intervalMs: 5000 }; + const items = [ + { itemid: '123', value_type: 3 } + ]; + const expectedQuery = compactQuery(`SELECT MEAN("value") + FROM "longterm"."history_uint" WHERE ("itemid" = '123') AND "time" >= 15000s AND "time" <= 15100s + GROUP BY time(5s), "itemid" fill(none) + `); + ctx.influxDBConnector.getTrends(items, timeFrom, timeTill, options); + expect(ctx.influxDBConnector.invokeInfluxDBQuery).toHaveBeenCalledWith(expectedQuery); + }); }); }); diff --git a/src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js b/src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js index 2e957db..e80fdf6 100644 --- a/src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js +++ b/src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js @@ -5,6 +5,7 @@ import { DBConnector, HISTORY_TO_TABLE_MAP, consolidateByFunc } from '../dbConne export class InfluxDBConnector extends DBConnector { constructor(options, datasourceSrv) { super(options, datasourceSrv); + this.retentionPolicy = options.retentionPolicy; super.loadDBDataSource().then(ds => { this.influxDS = ds; return ds; @@ -19,9 +20,10 @@ export class InfluxDBConnector extends DBConnector { } getHistory(items, timeFrom, timeTill, options) { - let {intervalMs, consolidateBy} = options; + let { intervalMs, consolidateBy, retentionPolicy } = options; const intervalSec = Math.ceil(intervalMs / 1000); + const range = { timeFrom, timeTill }; consolidateBy = consolidateBy || 'avg'; const aggFunction = consolidateByFunc[consolidateBy] || consolidateBy; @@ -30,7 +32,7 @@ export class InfluxDBConnector extends DBConnector { const promises = _.map(grouped_items, (items, value_type) => { const itemids = _.map(items, 'itemid'); const table = HISTORY_TO_TABLE_MAP[value_type]; - const query = this.buildHistoryQuery(itemids, table, timeFrom, timeTill, intervalSec, aggFunction); + const query = this.buildHistoryQuery(itemids, table, range, intervalSec, aggFunction, retentionPolicy); return this.invokeInfluxDBQuery(query); }); @@ -42,13 +44,16 @@ export class InfluxDBConnector extends DBConnector { } getTrends(items, timeFrom, timeTill, options) { + options.retentionPolicy = this.retentionPolicy; return this.getHistory(items, timeFrom, timeTill, options); } - buildHistoryQuery(itemids, table, timeFrom, timeTill, intervalSec, aggFunction) { + buildHistoryQuery(itemids, table, range, intervalSec, aggFunction, retentionPolicy) { + const { timeFrom, timeTill } = range; + const measurement = retentionPolicy ? `"${retentionPolicy}"."${table}"` : `"${table}"`; const AGG = aggFunction === 'AVG' ? 'MEAN' : aggFunction; const where_clause = this.buildWhereClause(itemids); - const query = `SELECT ${AGG}("value") FROM "${table}" + const query = `SELECT ${AGG}("value") FROM ${measurement} WHERE ${where_clause} AND "time" >= ${timeFrom}s AND "time" <= ${timeTill}s GROUP BY time(${intervalSec}s), "itemid" fill(none)`; return compactQuery(query); diff --git a/src/datasource-zabbix/zabbix/zabbix.js b/src/datasource-zabbix/zabbix/zabbix.js index c68e8a0..35bb96a 100644 --- a/src/datasource-zabbix/zabbix/zabbix.js +++ b/src/datasource-zabbix/zabbix/zabbix.js @@ -37,6 +37,7 @@ export class Zabbix { enableDirectDBConnection, dbConnectionDatasourceId, dbConnectionDatasourceName, + dbConnectionRetentionPolicy, } = options; this.enableDirectDBConnection = enableDirectDBConnection; @@ -55,7 +56,8 @@ export class Zabbix { this.bindRequests(); if (enableDirectDBConnection) { - this.initDBConnector(dbConnectionDatasourceId, dbConnectionDatasourceName, datasourceSrv) + const connectorOptions = { dbConnectionRetentionPolicy }; + this.initDBConnector(dbConnectionDatasourceId, dbConnectionDatasourceName, datasourceSrv, connectorOptions) .then(() => { this.getHistoryDB = this.cachingProxy.proxyfyWithCache(this.dbConnector.getHistory, 'getHistory', this.dbConnector); this.getTrendsDB = this.cachingProxy.proxyfyWithCache(this.dbConnector.getTrends, 'getTrends', this.dbConnector); @@ -63,14 +65,15 @@ export class Zabbix { } } - initDBConnector(datasourceId, datasourceName, datasourceSrv) { + initDBConnector(datasourceId, datasourceName, datasourceSrv, options) { return DBConnector.loadDatasource(datasourceId, datasourceName, datasourceSrv) .then(ds => { - const options = { datasourceId, datasourceName }; + let connectorOptions = { datasourceId, datasourceName }; if (ds.type === 'influxdb') { - this.dbConnector = new InfluxDBConnector(options, datasourceSrv); + connectorOptions.retentionPolicy = options.dbConnectionRetentionPolicy; + this.dbConnector = new InfluxDBConnector(connectorOptions, datasourceSrv); } else { - this.dbConnector = new SQLConnector(options, datasourceSrv); + this.dbConnector = new SQLConnector(connectorOptions, datasourceSrv); } return this.dbConnector; }); From a41df652b26e9fca38f6c5612b4085570acf3113 Mon Sep 17 00:00:00 2001 From: Alexander Zobnin Date: Wed, 31 Oct 2018 21:08:22 +0300 Subject: [PATCH 18/21] influx: docs for retention policy option --- docs/sources/configuration/index.md | 1 + docs/sources/configuration/provisioning.md | 5 ++++- src/datasource-zabbix/partials/config.html | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/sources/configuration/index.md b/docs/sources/configuration/index.md index c96c3ec..a75be97 100644 --- a/docs/sources/configuration/index.md +++ b/docs/sources/configuration/index.md @@ -62,6 +62,7 @@ Read [how to configure](./sql_datasource) SQL data source in Grafana. - **Enable**: enable Direct DB Connection. - **Data Source**: Select Data Source for Zabbix history database. +- **Retention Policy** (InfluxDB only): Specify retention policy name for fetching long-term stored data. Grafana will fetch data from this retention policy if query time range suitable for trends query. Leave it blank if only default retention policy used. #### Supported databases diff --git a/docs/sources/configuration/provisioning.md b/docs/sources/configuration/provisioning.md index 7bfa1a2..582bf35 100644 --- a/docs/sources/configuration/provisioning.md +++ b/docs/sources/configuration/provisioning.md @@ -34,8 +34,11 @@ datasources: disableReadOnlyUsersAck: true # Direct DB Connection options dbConnectionEnable: true - # Name of existing SQL datasource + # Name of existing datasource for Direct DB Connection dbConnectionDatasourceName: MySQL Zabbix + # Retention policy name (InfluxDB only) for fetching long-term stored data. + # Leave it blank if only default retention policy used. + dbConnectionRetentionPolicy: one_year version: 1 editable: false diff --git a/src/datasource-zabbix/partials/config.html b/src/datasource-zabbix/partials/config.html index f9920f1..1e9f682 100644 --- a/src/datasource-zabbix/partials/config.html +++ b/src/datasource-zabbix/partials/config.html @@ -115,7 +115,7 @@ Retention Policy Specify retention policy name for fetching long-term stored data (optional). - Leave it blank if only default retention policy is using. + Leave it blank if only default retention policy used. Date: Wed, 31 Oct 2018 21:19:42 +0300 Subject: [PATCH 19/21] use trends only if interval strictly greater than specified --- src/datasource-zabbix/datasource.js | 4 ++-- src/datasource-zabbix/specs/datasource.spec.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/datasource-zabbix/datasource.js b/src/datasource-zabbix/datasource.js index cd46410..d2bf766 100644 --- a/src/datasource-zabbix/datasource.js +++ b/src/datasource-zabbix/datasource.js @@ -618,8 +618,8 @@ export class ZabbixDatasource { let useTrendsFrom = Math.ceil(dateMath.parse('now-' + this.trendsFrom) / 1000); let useTrendsRange = Math.ceil(utils.parseInterval(this.trendsRange) / 1000); let useTrends = this.trends && ( - (timeFrom <= useTrendsFrom) || - (timeTo - timeFrom >= useTrendsRange) + (timeFrom < useTrendsFrom) || + (timeTo - timeFrom > useTrendsRange) ); return useTrends; } diff --git a/src/datasource-zabbix/specs/datasource.spec.js b/src/datasource-zabbix/specs/datasource.spec.js index f03da65..7c95335 100644 --- a/src/datasource-zabbix/specs/datasource.spec.js +++ b/src/datasource-zabbix/specs/datasource.spec.js @@ -56,7 +56,7 @@ describe('ZabbixDatasource', () => { }); it('should use trends if it enabled and time more than trendsFrom', (done) => { - let ranges = ['now-7d', 'now-168h', 'now-1M', 'now-1y']; + let ranges = ['now-8d', 'now-169h', 'now-1M', 'now-1y']; _.forEach(ranges, range => { ctx.options.range.from = range; @@ -73,7 +73,7 @@ describe('ZabbixDatasource', () => { }); it('shouldnt use trends if it enabled and time less than trendsFrom', (done) => { - let ranges = ['now-6d', 'now-167h', 'now-1h', 'now-30m', 'now-30s']; + let ranges = ['now-7d', 'now-168h', 'now-1h', 'now-30m', 'now-30s']; _.forEach(ranges, range => { ctx.options.range.from = range; From b6a5aea71350d707cf4971d5e449f0d18640c9ab Mon Sep 17 00:00:00 2001 From: Alexander Zobnin Date: Thu, 14 Feb 2019 11:00:30 +0300 Subject: [PATCH 20/21] codecov: disable PR comments --- .codecov.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.codecov.yml b/.codecov.yml index a641bd5..b39b33d 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -10,7 +10,7 @@ coverage: patch: off changes: off -comment: false +comment: off ignore: - "dist/test/test-setup/.*" From 125cbe876c120d7b5a2931c859b6bb732a2389bd Mon Sep 17 00:00:00 2001 From: Alexander Zobnin Date: Thu, 14 Feb 2019 14:05:03 +0300 Subject: [PATCH 21/21] docs: add retention policy notes --- docs/mkdocs.yml | 2 +- .../{sql_datasource.md => direct_db_datasource.md} | 7 +++++++ docs/sources/img/configuration-influxdb_ds_config.png | 3 +++ docs/sources/reference/direct-db-connection.md | 8 +++++++- 4 files changed, 18 insertions(+), 2 deletions(-) rename docs/sources/configuration/{sql_datasource.md => direct_db_datasource.md} (80%) create mode 100644 docs/sources/img/configuration-influxdb_ds_config.png diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 7ae5a07..e47ccd5 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -26,7 +26,7 @@ nav: - 'Upgrade': 'installation/upgrade.md' - Configuration: - 'Configuration': 'configuration/index.md' - - 'SQL Data Source Configuration': 'configuration/sql_datasource.md' + - 'Direct DB Connection Configuration': 'configuration/direct_db_datasource.md' - 'Provisioning': 'configuration/provisioning.md' - 'Troubleshooting': 'configuration/troubleshooting.md' - User Guides: diff --git a/docs/sources/configuration/sql_datasource.md b/docs/sources/configuration/direct_db_datasource.md similarity index 80% rename from docs/sources/configuration/sql_datasource.md rename to docs/sources/configuration/direct_db_datasource.md index 68eab39..e78ff77 100644 --- a/docs/sources/configuration/sql_datasource.md +++ b/docs/sources/configuration/direct_db_datasource.md @@ -32,3 +32,10 @@ database name (usually, `zabbix`) and specify credentials. ### Security notes Make sure you use read-only user for Zabbix database. + +## InfluxDB + +Select _InfluxDB_ data source type and provide your InfluxDB instance host address and port (8086 is default). Fill +database name you configured in the [effluence](https://github.com/i-ky/effluence) module config (usually, `zabbix`) and specify credentials. + +![Configure InfluxDB data source](../img/configuration-influxdb_ds_config.png) diff --git a/docs/sources/img/configuration-influxdb_ds_config.png b/docs/sources/img/configuration-influxdb_ds_config.png new file mode 100644 index 0000000..b1578c0 --- /dev/null +++ b/docs/sources/img/configuration-influxdb_ds_config.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:216a5d1a495fbd2093446c67e09a5dc669c65f541e65fcc13802facd411b5434 +size 111917 diff --git a/docs/sources/reference/direct-db-connection.md b/docs/sources/reference/direct-db-connection.md index ec91ed5..12d62f7 100644 --- a/docs/sources/reference/direct-db-connection.md +++ b/docs/sources/reference/direct-db-connection.md @@ -81,7 +81,13 @@ As you can see, the Grafana-Zabbix plugin uses aggregation by a given time inter ## InfluxDB Zabbix supports loadable modules which makes possible to write history data into an external database. There's a [module](https://github.com/i-ky/effluence) for InfluxDB written by [@i-ky](https://github.com/i-ky) which can export history into InfluxDB in real-time. -InfluxDB Query Example: +#### InfluxDB retention policy +In order to keep database size under control, you should use InfluxDB retention policy mechanism. It's possible to create retention policy for long-term data and write aggregated data in the same manner as Zabbix does (trends). Then this retention policy can be used in plugin for getting data after a certain period ([Retention Policy](../../configuration/#direct-db-connection) option in data source config). Read more about how to configure retention policy for using with plugin in effluence module [docs](https://github.com/i-ky/effluence#database-sizing). + +#### InfluxDB Query + +Eventually, plugin generates InfluxDB query similar to this: + ```sql SELECT MEAN("value") FROM "history"