Misthios/grafana-zabbix
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

refactor: proxy and cache WIP

Browse Source
This commit is contained in:
Alexander Zobnin
2018-03-17 22:10:59 +03:00
parent 24e10e885d
commit d47f729e1c
20 changed files with 445 additions and 613 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/datasource-zabbix/datasource.js
View File

@@ -54,6 +54,7 @@ class ZabbixAPIDatasource {
this.sqlDatasourceId = dbConnectionOptions.datasourceId;
let zabbixOptions = {
url: this.url,
username: this.username,
password: this.password,
basicAuth: this.basicAuth,
@@ -63,7 +64,7 @@ class ZabbixAPIDatasource {
sqlDatasourceId: this.sqlDatasourceId
};
this.zabbix = new Zabbix(this.url, zabbixOptions, backendSrv, datasourceSrv);
this.zabbix = new Zabbix(zabbixOptions, backendSrv, datasourceSrv);
}
////////////////////////

2
src/datasource-zabbix/zabbix/connectors/sql/zabbixDBConnector.js
View File

@@ -164,7 +164,7 @@ function convertGrafanaTSResponse(time_series, items, addHostName) {
var host = _.find(hosts, {'hostid': item.hostid});
alias = host.name + ": " + alias;
}
// zabbixCachingProxy deduplicates requests and returns one time series for equal queries.
// CachingProxy deduplicates requests and returns one time series for equal queries.
// Clone is needed to prevent changing of series object shared between all targets.
let datapoints = _.cloneDeep(series.points);
return {

4
src/datasource-zabbix/zabbix/connectors/zabbix_api/zabbixAPIConnector.js
View File

@@ -1,6 +1,6 @@
import _ from 'lodash';
import * as utils from '../../../utils';
import { ZabbixAPICoreService } from './zabbixAPICore';
import { ZabbixAPICore } from './zabbixAPICore';
/**
* Zabbix API Wrapper.
@@ -25,7 +25,7 @@ export class ZabbixAPIConnector {
this.loginErrorCount = 0;
this.maxLoginAttempts = 3;
this.zabbixAPICore = new ZabbixAPICoreService(backendSrv);
this.zabbixAPICore = new ZabbixAPICore(backendSrv);
this.getTrend = this.getTrend_ZBXNEXT1193;
//getTrend = getTrend_30;

2
src/datasource-zabbix/zabbix/connectors/zabbix_api/zabbixAPICore.js
View File

@@ -2,7 +2,7 @@
* General Zabbix API methods
*/
export class ZabbixAPICoreService {
export class ZabbixAPICore {
/** @ngInject */
constructor(backendSrv) {

108
src/datasource-zabbix/zabbix/proxy/cachingProxy.js Normal file
View File

@@ -0,0 +1,108 @@
/**
* This module allows to deduplicate function calls with the same params and
* cache result of function call.
*/
export class CachingProxy {
constructor(cacheOptions) {
this.cacheEnabled = cacheOptions.enabled;
this.ttl = cacheOptions.ttl || 600000; // 10 minutes by default
// Internal objects for data storing
this.cache = {};
this.promises = {};
}
/**
* Check that result is present in the cache and is up to date or send request otherwise.
*/
cacheRequest(func, funcName, funcScope) {
return cacheRequest(func, funcName, funcScope, this);
}
/**
* Wrap request to prevent multiple calls with same params when request is waiting for response.
*/
proxyfy(func, funcName, funcScope) {
if (!this.promises[funcName]) {
this.promises[funcName] = {};
}
const promiseKeeper = this.promises[funcName];
return callOnce(func, promiseKeeper, funcScope);
}
proxyfyWithCache(func, funcName, funcScope) {
let proxyfied = this.proxyfy(func, funcName, funcScope);
return this.cacheRequest(proxyfied, funcName, funcScope);
}
_isExpired(cacheObject) {
if (cacheObject) {
let object_age = Date.now() - cacheObject.timestamp;
return !(cacheObject.timestamp && object_age < this.ttl);
} else {
return true;
}
}
}
/**
* Wrap request to prevent multiple calls
* with same params when waiting for result.
*/
function callOnce(func, promiseKeeper, funcScope) {
return function() {
var hash = getRequestHash(arguments);
if (!promiseKeeper[hash]) {
promiseKeeper[hash] = Promise.resolve(
func.apply(funcScope, arguments)
.then(result => {
promiseKeeper[hash] = null;
return result;
})
);
}
return promiseKeeper[hash];
};
}
function cacheRequest(func, funcName, funcScope, self) {
return function() {
if (!self.cache[funcName]) {
self.cache[funcName] = {};
}
let cacheObject = self.cache[funcName];
let hash = getRequestHash(arguments);
if (self.cacheEnabled && !self._isExpired(cacheObject[hash])) {
return Promise.resolve(cacheObject[hash].value);
} else {
return func.apply(funcScope, arguments)
.then(result => {
cacheObject[hash] = {
value: result,
timestamp: Date.now()
};
return result;
});
}
};
}
function getRequestHash(args) {
const argsJson = JSON.stringify(args);
return argsJson.getHash();
}
String.prototype.getHash = function() {
var hash = 0, i, chr, len;
if (this.length !== 0) {
for (i = 0, len = this.length; i < len; i++) {
chr = this.charCodeAt(i);
hash = ((hash << 5) - hash) + chr;
hash |= 0; // Convert to 32bit integer
}
}
return hash;
};

231
src/datasource-zabbix/zabbix/proxy/zabbixCachingProxy.js
View File

@@ -1,231 +0,0 @@
import _ from 'lodash';
export class ZabbixCachingProxy {
constructor(zabbixAPI, zabbixDBConnector, cacheOptions) {
this.zabbixAPI = zabbixAPI;
this.dbConnector = zabbixDBConnector;
this.cacheEnabled = cacheOptions.enabled;
this.ttl = cacheOptions.ttl || 600000; // 10 minutes by default
// Internal objects for data storing
this.cache = {
groups: {},
hosts: {},
applications: {},
items: {},
history: {},
trends: {},
macros: {},
globalMacros: {},
itServices: {}
};
this.historyPromises = {};
// Don't run duplicated history requests
this.getHistory = callAPIRequestOnce(_.bind(this.zabbixAPI.getHistory, this.zabbixAPI),
this.historyPromises, getHistoryRequestHash);
if (this.dbConnector) {
this.getHistoryDB = callAPIRequestOnce(_.bind(this.dbConnector.getHistory, this.dbConnector),
this.historyPromises, getDBQueryHash);
this.getTrendsDB = callAPIRequestOnce(_.bind(this.dbConnector.getTrends, this.dbConnector),
this.historyPromises, getDBQueryHash);
}
// Don't run duplicated requests
this.groupPromises = {};
this.getGroupsOnce = callAPIRequestOnce(_.bind(this.zabbixAPI.getGroups, this.zabbixAPI),
this.groupPromises, getRequestHash);
this.hostPromises = {};
this.getHostsOnce = callAPIRequestOnce(_.bind(this.zabbixAPI.getHosts, this.zabbixAPI),
this.hostPromises, getRequestHash);
this.appPromises = {};
this.getAppsOnce = callAPIRequestOnce(_.bind(this.zabbixAPI.getApps, this.zabbixAPI),
this.appPromises, getRequestHash);
this.itemPromises = {};
this.getItemsOnce = callAPIRequestOnce(_.bind(this.zabbixAPI.getItems, this.zabbixAPI),
this.itemPromises, getRequestHash);
this.itemByIdPromises = {};
this.getItemsByIdOnce = callAPIRequestOnce(_.bind(this.zabbixAPI.getItemsByIDs, this.zabbixAPI),
this.itemPromises, getRequestHash);
this.itServicesPromises = {};
this.getITServicesOnce = callAPIRequestOnce(_.bind(this.zabbixAPI.getITService, this.zabbixAPI),
this.itServicesPromises, getRequestHash);
this.macroPromises = {};
this.getMacrosOnce = callAPIRequestOnce(_.bind(this.zabbixAPI.getMacros, this.zabbixAPI),
this.macroPromises, getRequestHash);
this.globalMacroPromises = {};
this.getGlobalMacrosOnce = callAPIRequestOnce(_.bind(this.zabbixAPI.getGlobalMacros, this.zabbixAPI),
this.globalMacroPromises, getRequestHash);
}
isExpired(cacheObject) {
if (cacheObject) {
let object_age = Date.now() - cacheObject.timestamp;
return !(cacheObject.timestamp && object_age < this.ttl);
} else {
return true;
}
}
/**
* Check that result is present in cache and up to date
* or send request to API.
*/
proxyRequest(request, params, cacheObject) {
let hash = getRequestHash(params);
if (this.cacheEnabled && !this.isExpired(cacheObject[hash])) {
return Promise.resolve(cacheObject[hash].value);
} else {
return request(...params)
.then(result => {
cacheObject[hash] = {
value: result,
timestamp: Date.now()
};
return result;
});
}
}
getGroups() {
return this.proxyRequest(this.getGroupsOnce, [], this.cache.groups);
}
getHosts(groupids) {
return this.proxyRequest(this.getHostsOnce, [groupids], this.cache.hosts);
}
getApps(hostids) {
return this.proxyRequest(this.getAppsOnce, [hostids], this.cache.applications);
}
getItems(hostids, appids, itemtype) {
let params = [hostids, appids, itemtype];
return this.proxyRequest(this.getItemsOnce, params, this.cache.items);
}
getItemsByIDs(itemids) {
let params = [itemids];
return this.proxyRequest(this.getItemsByIdOnce, params, this.cache.items);
}
getITServices() {
return this.proxyRequest(this.getITServicesOnce, [], this.cache.itServices);
}
getMacros(hostids) {
// Merge global macros and host macros
let promises = [
this.proxyRequest(this.getMacrosOnce, [hostids], this.cache.macros),
this.proxyRequest(this.getGlobalMacrosOnce, [], this.cache.globalMacros)
];
return Promise.all(promises).then(_.flatten);
}
getHistoryFromCache(items, time_from, time_till) {
var historyStorage = this.cache.history;
var full_history;
var expired = _.filter(_.keyBy(items, 'itemid'), (item, itemid) => {
return !historyStorage[itemid];
});
if (expired.length) {
return this.zabbixAPI.getHistory(expired, time_from, time_till).then(function(history) {
var grouped_history = _.groupBy(history, 'itemid');
_.forEach(expired, item => {
var itemid = item.itemid;
historyStorage[itemid] = item;
historyStorage[itemid].time_from = time_from;
historyStorage[itemid].time_till = time_till;
historyStorage[itemid].history = grouped_history[itemid];
});
full_history = _.map(items, item => {
return historyStorage[item.itemid].history;
});
return _.flatten(full_history, true);
});
} else {
full_history = _.map(items, function(item) {
return historyStorage[item.itemid].history;
});
return Promise.resolve(_.flatten(full_history, true));
}
}
getHistoryFromAPI(items, time_from, time_till) {
return this.zabbixAPI.getHistory(items, time_from, time_till);
}
}
/**
* Wrap zabbix API request to prevent multiple calls
* with same params when waiting for result.
*/
function callAPIRequestOnce(func, promiseKeeper, argsHashFunc) {
return function() {
var hash = argsHashFunc(arguments);
if (!promiseKeeper[hash]) {
promiseKeeper[hash] = Promise.resolve(
func.apply(this, arguments)
.then(result => {
promiseKeeper[hash] = null;
return result;
})
);
}
return promiseKeeper[hash];
};
}
function getRequestHash(args) {
var requestStamp = _.map(args, arg => {
if (arg === undefined) {
return 'undefined';
} else {
if (_.isArray(arg)) {
return arg.sort().toString();
} else {
return arg.toString();
}
}
}).join();
return requestStamp.getHash();
}
function getHistoryRequestHash(args) {
let itemids = _.map(args[0], 'itemid');
let stamp = itemids.join() + args[1] + args[2];
return stamp.getHash();
}
function getDBQueryHash(args) {
let itemids = _.map(args[0], 'itemid');
let consolidateBy = args[3].consolidateBy;
let intervalMs = args[3].intervalMs;
let stamp = itemids.join() + args[1] + args[2] + consolidateBy + intervalMs;
return stamp.getHash();
}
String.prototype.getHash = function() {
var hash = 0, i, chr, len;
if (this.length !== 0) {
for (i = 0, len = this.length; i < len; i++) {
chr = this.charCodeAt(i);
hash = ((hash << 5) - hash) + chr;
hash |= 0; // Convert to 32bit integer
}
}
return hash;
};
// Fix for backward compatibility with lodash 2.4
if (!_.keyBy) {_.keyBy = _.indexBy;}

96
src/datasource-zabbix/zabbix/zabbix.js
View File

@@ -1,52 +1,74 @@
// import angular from 'angular';
import _ from 'lodash';
import * as utils from '../utils';
import { ZabbixAPIConnector } from './connectors/zabbix_api/zabbixAPIConnector';
import { ZabbixDBConnector } from './connectors/sql/zabbixDBConnector';
import { ZabbixCachingProxy } from './proxy/zabbixCachingProxy';
import { CachingProxy } from './proxy/cachingProxy';
const REQUESTS_TO_PROXYFY = [
'getHistory', 'getTrend', 'getGroups', 'getHosts', 'getApps', 'getItems', 'getMacros', 'getItemsByIDs',
'getEvents', 'getAlerts', 'getHostAlerts', 'getAcknowledges', 'getITService', 'getSLA', 'getVersion'
];
const REQUESTS_TO_CACHE = [
'getGroups', 'getHosts', 'getApps', 'getItems', 'getMacros', 'getItemsByIDs', 'getITService'
];
const REQUESTS_TO_BIND = [
'getHistory', 'getTrend', 'getMacros', 'getItemsByIDs', 'getEvents', 'getAlerts', 'getHostAlerts',
'getAcknowledges', 'getITService', 'getSLA', 'getVersion', 'login'
];
export class Zabbix {
/** @ngInject */
constructor(url, options, backendSrv, datasourceSrv) {
constructor(options, backendSrv, datasourceSrv) {
let {
username, password, basicAuth, withCredentials, cacheTTL,
enableDirectDBConnection, sqlDatasourceId
url,
username,
password,
basicAuth,
withCredentials,
cacheTTL,
enableDirectDBConnection,
sqlDatasourceId
} = options;
// Initialize Zabbix API
this.zabbixAPI = new ZabbixAPIConnector(url, username, password, basicAuth, withCredentials, backendSrv);
if (enableDirectDBConnection) {
this.dbConnector = new ZabbixDBConnector(sqlDatasourceId, {}, backendSrv, datasourceSrv);
}
// Initialize caching proxy for requests
let cacheOptions = {
enabled: true,
ttl: cacheTTL
};
this.cachingProxy = new ZabbixCachingProxy(this.zabbixAPI, this.dbConnector, cacheOptions);
this.cachingProxy = new CachingProxy(cacheOptions);
// Proxy methods
this.getHistory = this.cachingProxy.getHistory.bind(this.cachingProxy);
this.getMacros = this.cachingProxy.getMacros.bind(this.cachingProxy);
this.getItemsByIDs = this.cachingProxy.getItemsByIDs.bind(this.cachingProxy);
this.zabbixAPI = new ZabbixAPIConnector(url, username, password, basicAuth, withCredentials, backendSrv);
if (enableDirectDBConnection) {
this.getHistoryDB = this.cachingProxy.getHistoryDB.bind(this.cachingProxy);
this.getTrendsDB = this.cachingProxy.getTrendsDB.bind(this.cachingProxy);
this.dbConnector = new ZabbixDBConnector(sqlDatasourceId, {}, backendSrv, datasourceSrv);
this.getHistoryDB = this.cachingProxy.proxyfyWithCache(this.dbConnector.getHistory, 'getHistory', this.dbConnector);
this.getTrendsDB = this.cachingProxy.proxyfyWithCache(this.dbConnector.getTrends, 'getTrends', this.dbConnector);
}
this.getTrend = this.zabbixAPI.getTrend.bind(this.zabbixAPI);
this.getEvents = this.zabbixAPI.getEvents.bind(this.zabbixAPI);
this.getAlerts = this.zabbixAPI.getAlerts.bind(this.zabbixAPI);
this.getHostAlerts = this.zabbixAPI.getHostAlerts.bind(this.zabbixAPI);
this.getAcknowledges = this.zabbixAPI.getAcknowledges.bind(this.zabbixAPI);
this.getITService = this.zabbixAPI.getITService.bind(this.zabbixAPI);
this.getSLA = this.zabbixAPI.getSLA.bind(this.zabbixAPI);
this.getVersion = this.zabbixAPI.getVersion.bind(this.zabbixAPI);
this.login = this.zabbixAPI.login.bind(this.zabbixAPI);
this.proxyfyRequests();
this.cacheRequests();
this.bindRequests();
}
proxyfyRequests() {
for (let request of REQUESTS_TO_PROXYFY) {
this.zabbixAPI[request] = this.cachingProxy.proxyfy(this.zabbixAPI[request], request, this.zabbixAPI);
}
}
cacheRequests() {
for (let request of REQUESTS_TO_CACHE) {
this.zabbixAPI[request] = this.cachingProxy.cacheRequest(this.zabbixAPI[request], request, this.zabbixAPI);
}
}
bindRequests() {
for (let request of REQUESTS_TO_BIND) {
this[request] = this.zabbixAPI[request].bind(this.zabbixAPI);
}
}
getItemsFromTarget(target, options) {
@@ -71,7 +93,7 @@ export class Zabbix {
}
getAllGroups() {
return this.cachingProxy.getGroups();
return this.zabbixAPI.getGroups();
}
getGroups(groupFilter) {
@@ -86,7 +108,7 @@ export class Zabbix {
return this.getGroups(groupFilter)
.then(groups => {
let groupids = _.map(groups, 'groupid');
return this.cachingProxy.getHosts(groupids);
return this.zabbixAPI.getHosts(groupids);
});
}
@@ -102,7 +124,7 @@ export class Zabbix {
return this.getHosts(groupFilter, hostFilter)
.then(hosts => {
let hostids = _.map(hosts, 'hostid');
return this.cachingProxy.getApps(hostids);
return this.zabbixAPI.getApps(hostids);
});
}
@@ -111,7 +133,7 @@ export class Zabbix {
.then(hosts => {
let hostids = _.map(hosts, 'hostid');
if (appFilter) {
return this.cachingProxy.getApps(hostids)
return this.zabbixAPI.getApps(hostids)
.then(apps => filterByQuery(apps, appFilter));
} else {
return {
@@ -126,10 +148,10 @@ export class Zabbix {
return this.getApps(groupFilter, hostFilter, appFilter)
.then(apps => {
if (apps.appFilterEmpty) {
return this.cachingProxy.getItems(apps.hostids, undefined, options.itemtype);
return this.zabbixAPI.getItems(apps.hostids, undefined, options.itemtype);
} else {
let appids = _.map(apps, 'applicationid');
return this.cachingProxy.getItems(undefined, appids, options.itemtype);
return this.zabbixAPI.getItems(undefined, appids, options.itemtype);
}
})
.then(items => {
@@ -161,7 +183,7 @@ export class Zabbix {
}
getITServices(itServiceFilter) {
return this.cachingProxy.getITServices()
return this.zabbixAPI.getITService()
.then(itServices => findByFilter(itServices, itServiceFilter));
}
@@ -199,10 +221,6 @@ export class Zabbix {
}
}
// angular
// .module('grafana.services')
// .factory('Zabbix', ZabbixFactory);
///////////////////////////////////////////////////////////////////////////////
/**