From 089700d227cadf6e4c496fe1f5f913cf053eab57 Mon Sep 17 00:00:00 2001 From: Alexander Zobnin Date: Wed, 31 Oct 2018 20:56:36 +0300 Subject: [PATCH] influx: support retention policy for long-term stored data --- src/datasource-zabbix/config.controller.js | 7 +++ src/datasource-zabbix/datasource.js | 4 +- src/datasource-zabbix/partials/config.html | 44 +++++++++++++------ .../specs/influxdbConnector.test.js | 36 ++++++++++----- .../connectors/influxdb/influxdbConnector.js | 13 ++++-- src/datasource-zabbix/zabbix/zabbix.js | 13 +++--- 6 files changed, 83 insertions(+), 34 deletions(-) diff --git a/src/datasource-zabbix/config.controller.js b/src/datasource-zabbix/config.controller.js index fe9815f..9f7cfcc 100644 --- a/src/datasource-zabbix/config.controller.js +++ b/src/datasource-zabbix/config.controller.js @@ -31,6 +31,7 @@ export class ZabbixDSConfigController { this.dbDataSources = this.getSupportedDBDataSources(); this.zabbixVersions = _.cloneDeep(zabbixVersions); this.autoDetectZabbixVersion(); + console.log(this.dbDataSources); } getSupportedDBDataSources() { @@ -40,6 +41,12 @@ export class ZabbixDSConfigController { }); } + getCurrentDatasourceType() { + const dsId = this.current.jsonData.dbConnectionDatasourceId; + const currentDs = _.find(this.dbDataSources, { 'id': dsId }); + return currentDs ? currentDs.type : null; + } + autoDetectZabbixVersion() { if (!this.current.id) { return; diff --git a/src/datasource-zabbix/datasource.js b/src/datasource-zabbix/datasource.js index 8133f18..cd46410 100644 --- a/src/datasource-zabbix/datasource.js +++ b/src/datasource-zabbix/datasource.js @@ -58,6 +58,7 @@ export class ZabbixDatasource { this.enableDirectDBConnection = jsonData.dbConnectionEnable || false; this.dbConnectionDatasourceId = jsonData.dbConnectionDatasourceId; this.dbConnectionDatasourceName = jsonData.dbConnectionDatasourceName; + this.dbConnectionRetentionPolicy = jsonData.dbConnectionRetentionPolicy; let zabbixOptions = { url: this.url, @@ -69,7 +70,8 @@ export class ZabbixDatasource { cacheTTL: this.cacheTTL, enableDirectDBConnection: this.enableDirectDBConnection, dbConnectionDatasourceId: this.dbConnectionDatasourceId, - dbConnectionDatasourceName: this.dbConnectionDatasourceName + dbConnectionDatasourceName: this.dbConnectionDatasourceName, + dbConnectionRetentionPolicy: this.dbConnectionRetentionPolicy, }; this.zabbix = new Zabbix(zabbixOptions, datasourceSrv, backendSrv); diff --git a/src/datasource-zabbix/partials/config.html b/src/datasource-zabbix/partials/config.html index 043c7c7..f9920f1 100644 --- a/src/datasource-zabbix/partials/config.html +++ b/src/datasource-zabbix/partials/config.html @@ -92,22 +92,38 @@
- - Data Source - - Select Data Source for Zabbix history database. - In order to use this feature it should be created and - configured first. Zabbix plugin uses this data source for querying history data directly from the database. - This way usually faster than pulling data from Zabbix API, especially on the wide time ranges, and reduces - amount of data transfered. - - -
- + + Data Source + + Select Data Source for Zabbix history database. + In order to use this feature it should be created and + configured first. Zabbix plugin uses this data source for querying history data directly from the database. + This way usually faster than pulling data from Zabbix API, especially on the wide time ranges, and reduces + amount of data transfered. + + +
+ +
+
+
+ + Retention Policy + + Specify retention policy name for fetching long-term stored data (optional). + Leave it blank if only default retention policy is using. + + + + +
diff --git a/src/datasource-zabbix/specs/influxdbConnector.test.js b/src/datasource-zabbix/specs/influxdbConnector.test.js index 96084ab..089770e 100644 --- a/src/datasource-zabbix/specs/influxdbConnector.test.js +++ b/src/datasource-zabbix/specs/influxdbConnector.test.js @@ -5,7 +5,7 @@ describe('InfluxDBConnector', () => { let ctx = {}; beforeEach(() => { - ctx.options = { datasourceName: 'InfluxDB DS' }; + ctx.options = { datasourceName: 'InfluxDB DS', retentionPolicy: 'longterm' }; ctx.datasourceSrvMock = { loadDatasource: jest.fn().mockResolvedValue( { id: 42, name: 'InfluxDB DS', meta: {} } @@ -15,15 +15,16 @@ describe('InfluxDBConnector', () => { ctx.influxDBConnector.invokeInfluxDBQuery = jest.fn().mockResolvedValue([]); ctx.defaultQueryParams = { itemids: ['123', '234'], - timeFrom: 15000, timeTill: 15100, intervalSec: 5, + range: { timeFrom: 15000, timeTill: 15100 }, + intervalSec: 5, table: 'history', aggFunction: 'MAX' }; }); describe('When building InfluxDB query', () => { it('should build proper query', () => { - const { itemids, timeFrom, timeTill, intervalSec, table, aggFunction } = ctx.defaultQueryParams; - const query = ctx.influxDBConnector.buildHistoryQuery(itemids, table, timeFrom, timeTill, intervalSec, aggFunction); + const { itemids, range, intervalSec, table, aggFunction } = ctx.defaultQueryParams; + const query = ctx.influxDBConnector.buildHistoryQuery(itemids, table, range, intervalSec, aggFunction); const expected = compactQuery(`SELECT MAX("value") FROM "history" WHERE ("itemid" = '123' OR "itemid" = '234') AND "time" >= 15000s AND "time" <= 15100s GROUP BY time(5s), "itemid" fill(none) @@ -32,9 +33,9 @@ describe('InfluxDBConnector', () => { }); it('should use MEAN instead of AVG', () => { - const { itemids, timeFrom, timeTill, intervalSec, table } = ctx.defaultQueryParams; + const { itemids, range, intervalSec, table } = ctx.defaultQueryParams; const aggFunction = 'AVG'; - const query = ctx.influxDBConnector.buildHistoryQuery(itemids, table, timeFrom, timeTill, intervalSec, aggFunction); + const query = ctx.influxDBConnector.buildHistoryQuery(itemids, table, range, intervalSec, aggFunction); const expected = compactQuery(`SELECT MEAN("value") FROM "history" WHERE ("itemid" = '123' OR "itemid" = '234') AND "time" >= 15000s AND "time" <= 15100s GROUP BY time(5s), "itemid" fill(none) @@ -45,7 +46,7 @@ describe('InfluxDBConnector', () => { describe('When invoking InfluxDB query', () => { it('should query proper table depending on item type', () => { - const { timeFrom, timeTill} = ctx.defaultQueryParams; + const { timeFrom, timeTill } = ctx.defaultQueryParams.range; const options = { intervalMs: 5000 }; const items = [ { itemid: '123', value_type: 3 } @@ -59,7 +60,7 @@ describe('InfluxDBConnector', () => { }); it('should split query if different item types are used', () => { - const { timeFrom, timeTill} = ctx.defaultQueryParams; + const { timeFrom, timeTill } = ctx.defaultQueryParams.range; const options = { intervalMs: 5000 }; const items = [ { itemid: '123', value_type: 0 }, @@ -78,8 +79,9 @@ describe('InfluxDBConnector', () => { expect(ctx.influxDBConnector.invokeInfluxDBQuery).toHaveBeenNthCalledWith(2, expectedQuerySecond); }); - it('should use the same table for trends query', () => { - const { timeFrom, timeTill} = ctx.defaultQueryParams; + it('should use the same table for trends query if no retention policy set', () => { + ctx.influxDBConnector.retentionPolicy = ''; + const { timeFrom, timeTill } = ctx.defaultQueryParams.range; const options = { intervalMs: 5000 }; const items = [ { itemid: '123', value_type: 3 } @@ -91,5 +93,19 @@ describe('InfluxDBConnector', () => { ctx.influxDBConnector.getTrends(items, timeFrom, timeTill, options); expect(ctx.influxDBConnector.invokeInfluxDBQuery).toHaveBeenCalledWith(expectedQuery); }); + + it('should use retention policy name for trends query if it was set', () => { + const { timeFrom, timeTill } = ctx.defaultQueryParams.range; + const options = { intervalMs: 5000 }; + const items = [ + { itemid: '123', value_type: 3 } + ]; + const expectedQuery = compactQuery(`SELECT MEAN("value") + FROM "longterm"."history_uint" WHERE ("itemid" = '123') AND "time" >= 15000s AND "time" <= 15100s + GROUP BY time(5s), "itemid" fill(none) + `); + ctx.influxDBConnector.getTrends(items, timeFrom, timeTill, options); + expect(ctx.influxDBConnector.invokeInfluxDBQuery).toHaveBeenCalledWith(expectedQuery); + }); }); }); diff --git a/src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js b/src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js index 2e957db..e80fdf6 100644 --- a/src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js +++ b/src/datasource-zabbix/zabbix/connectors/influxdb/influxdbConnector.js @@ -5,6 +5,7 @@ import { DBConnector, HISTORY_TO_TABLE_MAP, consolidateByFunc } from '../dbConne export class InfluxDBConnector extends DBConnector { constructor(options, datasourceSrv) { super(options, datasourceSrv); + this.retentionPolicy = options.retentionPolicy; super.loadDBDataSource().then(ds => { this.influxDS = ds; return ds; @@ -19,9 +20,10 @@ export class InfluxDBConnector extends DBConnector { } getHistory(items, timeFrom, timeTill, options) { - let {intervalMs, consolidateBy} = options; + let { intervalMs, consolidateBy, retentionPolicy } = options; const intervalSec = Math.ceil(intervalMs / 1000); + const range = { timeFrom, timeTill }; consolidateBy = consolidateBy || 'avg'; const aggFunction = consolidateByFunc[consolidateBy] || consolidateBy; @@ -30,7 +32,7 @@ export class InfluxDBConnector extends DBConnector { const promises = _.map(grouped_items, (items, value_type) => { const itemids = _.map(items, 'itemid'); const table = HISTORY_TO_TABLE_MAP[value_type]; - const query = this.buildHistoryQuery(itemids, table, timeFrom, timeTill, intervalSec, aggFunction); + const query = this.buildHistoryQuery(itemids, table, range, intervalSec, aggFunction, retentionPolicy); return this.invokeInfluxDBQuery(query); }); @@ -42,13 +44,16 @@ export class InfluxDBConnector extends DBConnector { } getTrends(items, timeFrom, timeTill, options) { + options.retentionPolicy = this.retentionPolicy; return this.getHistory(items, timeFrom, timeTill, options); } - buildHistoryQuery(itemids, table, timeFrom, timeTill, intervalSec, aggFunction) { + buildHistoryQuery(itemids, table, range, intervalSec, aggFunction, retentionPolicy) { + const { timeFrom, timeTill } = range; + const measurement = retentionPolicy ? `"${retentionPolicy}"."${table}"` : `"${table}"`; const AGG = aggFunction === 'AVG' ? 'MEAN' : aggFunction; const where_clause = this.buildWhereClause(itemids); - const query = `SELECT ${AGG}("value") FROM "${table}" + const query = `SELECT ${AGG}("value") FROM ${measurement} WHERE ${where_clause} AND "time" >= ${timeFrom}s AND "time" <= ${timeTill}s GROUP BY time(${intervalSec}s), "itemid" fill(none)`; return compactQuery(query); diff --git a/src/datasource-zabbix/zabbix/zabbix.js b/src/datasource-zabbix/zabbix/zabbix.js index c68e8a0..35bb96a 100644 --- a/src/datasource-zabbix/zabbix/zabbix.js +++ b/src/datasource-zabbix/zabbix/zabbix.js @@ -37,6 +37,7 @@ export class Zabbix { enableDirectDBConnection, dbConnectionDatasourceId, dbConnectionDatasourceName, + dbConnectionRetentionPolicy, } = options; this.enableDirectDBConnection = enableDirectDBConnection; @@ -55,7 +56,8 @@ export class Zabbix { this.bindRequests(); if (enableDirectDBConnection) { - this.initDBConnector(dbConnectionDatasourceId, dbConnectionDatasourceName, datasourceSrv) + const connectorOptions = { dbConnectionRetentionPolicy }; + this.initDBConnector(dbConnectionDatasourceId, dbConnectionDatasourceName, datasourceSrv, connectorOptions) .then(() => { this.getHistoryDB = this.cachingProxy.proxyfyWithCache(this.dbConnector.getHistory, 'getHistory', this.dbConnector); this.getTrendsDB = this.cachingProxy.proxyfyWithCache(this.dbConnector.getTrends, 'getTrends', this.dbConnector); @@ -63,14 +65,15 @@ export class Zabbix { } } - initDBConnector(datasourceId, datasourceName, datasourceSrv) { + initDBConnector(datasourceId, datasourceName, datasourceSrv, options) { return DBConnector.loadDatasource(datasourceId, datasourceName, datasourceSrv) .then(ds => { - const options = { datasourceId, datasourceName }; + let connectorOptions = { datasourceId, datasourceName }; if (ds.type === 'influxdb') { - this.dbConnector = new InfluxDBConnector(options, datasourceSrv); + connectorOptions.retentionPolicy = options.dbConnectionRetentionPolicy; + this.dbConnector = new InfluxDBConnector(connectorOptions, datasourceSrv); } else { - this.dbConnector = new SQLConnector(options, datasourceSrv); + this.dbConnector = new SQLConnector(connectorOptions, datasourceSrv); } return this.dbConnector; });