From fbe7480c073101a5a0fd7bf2fda75a8913de1a35 Mon Sep 17 00:00:00 2001 From: Alexander Zobnin Date: Thu, 17 Mar 2016 22:36:22 +0300 Subject: [PATCH] Improved performance of metric filtering and building history query. --- Gruntfile.js | 4 +- .../queryProcessor.service.js | 127 ++++++++++++------ src/datasource-zabbix/zabbixAPI.service.js | 13 +- src/datasource-zabbix/zabbixCache.service.js | 71 +++++++--- 4 files changed, 149 insertions(+), 66 deletions(-) diff --git a/Gruntfile.js b/Gruntfile.js index 9b20b76..672b82d 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -21,6 +21,7 @@ module.exports = function(grunt) { '!**/utils.js', '!**/zabbixAPICore.service.js', '!**/zabbixAPI.service.js', + //'!**/dataProcessing.service.js', '!**/metricFunctions.js', '!**/*.scss' ], @@ -43,7 +44,7 @@ module.exports = function(grunt) { babel: { options: { - sourceMap: false, + sourceMap: true, presets: ["es2015"], plugins: ['transform-es2015-modules-systemjs', "transform-es2015-for-of"], }, @@ -58,6 +59,7 @@ module.exports = function(grunt) { '**/**/utils.js', '**/**/zabbixAPICore.service.js', '**/**/zabbixAPI.service.js', + //'**/**/dataProcessing.service.js', '**/**/metricFunctions.js' ], dest: 'dist/' diff --git a/src/datasource-zabbix/queryProcessor.service.js b/src/datasource-zabbix/queryProcessor.service.js index ecedaee..05f5cd1 100644 --- a/src/datasource-zabbix/queryProcessor.service.js +++ b/src/datasource-zabbix/queryProcessor.service.js @@ -70,7 +70,7 @@ function (angular, _, utils) { var hostids = _.flatten(_.map(groups, 'hosts')); if (hostids.length) { - return self.cache.getHostsExtend().then(function(hosts) { + return self.cache.getIndexedHosts().then(function(hosts) { return _.map(hostids, function(hostid) { return hosts[hostid]; }); @@ -121,65 +121,91 @@ function (angular, _, utils) { }); }; + /** + * Find group, host, app or item by given name. + * @param list list of groups, apps or other + * @param name visible name + * @return array with finded element or undefined + */ + function findByName(list, name) { + var finded = _.find(list, {'name': name}); + if (finded) { + return [finded]; + } else { + return undefined; + } + } + + function findByRegex(list, regex) { + var filterPattern = utils.buildRegex(regex); + return _.filter(list, function (zbx_obj) { + return filterPattern.test(zbx_obj.name); + }); + } + + function findByFilter(list, filter) { + if (utils.isRegex(filter)) { + return findByRegex(list, filter); + } else { + return findByName(list, filter); + } + } + + function getFromIndex(index, objids) { + return _.map(objids, function(id) { + return index[id]; + }); + } + this.filterItems = function (groupFilter, hostFilter, appFilter, itemType, showDisabledItems) { - var hosts = []; - var apps = []; - var items = []; + var hosts; + var apps; + var items; var promises = [ this.filterHosts(groupFilter), - this.filterApplications(groupFilter, hostFilter) + this.filterApplications(groupFilter, hostFilter), + this.cache.getIndexedHosts(), + this.cache.getIndexedApplications() ]; return $q.all(promises).then(function(results) { var hostList = results[0]; var applicationList = results[1]; + var idx_hosts = results[2]; + var idx_apps = results[3]; - // Filter hosts by regex - if (utils.isRegex(hostFilter)) { - var hostFilterPattern = utils.buildRegex(hostFilter); - hosts = _.filter(hostList, function (hostObj) { - return hostFilterPattern.test(hostObj.name); - }); - } else { - var findedHosts = _.find(hostList, {'name': hostFilter}); - if (findedHosts) { - hosts.push(findedHosts); - } else { - hosts = undefined; - } - } + // Filter hosts + hosts = findByFilter(hostList, hostFilter); + idx_hosts = getFromIndex(idx_hosts, _.map(hosts, 'hostid')); - // Filter applications by regex - if (utils.isRegex(appFilter)) { - var filterPattern = utils.buildRegex(appFilter); - apps = _.filter(applicationList, function (appObj) { - return filterPattern.test(appObj.name); - }); - } - // Find items in selected application - else if (appFilter) { - var finded = _.find(applicationList, {'name': appFilter}); - if (finded) { - apps.push(finded); - } else { - apps = undefined; - } - } else { + // Filter applications + if (appFilter === "") { + // Get all items apps = undefined; if (hosts) { - items = _.flatten(_.map(hosts, 'items'), true); + // Get all items in given hosts + items = _.flatten(_.map(idx_hosts, function(host) { + return _.values(host.idx_items); + }), true); } + } else { + apps = findByFilter(applicationList, appFilter); } if (apps) { - /*var appids = _.flatten(_.map(apps, 'applicationids')); - items = _.filter(cachedItems, function (itemObj) { - return _.intersection(appids, itemObj.applications).length; - }); - items = _.filter(items, function (itemObj) { - return _.find(hosts, {'hostid': itemObj.hostid }); - });*/ + // Get ids for finded applications + var appids = _.flatten(_.map(apps, 'applicationids')); + appids = _.flatten(_.map(_.map(hosts, 'applications'), function(apps) { + return _.intersection(apps, appids); + })); + + // For each finded host get list of items in finded applications + items = _.flatten(_.map(idx_hosts, function(host) { + var host_apps = _.intersection(appids, host.applications); + var host_itemids = _.flatten(_.map(getFromIndex(idx_apps, host_apps), 'itemids')); + return _.values(getFromIndex(host.idx_items, host_itemids)); + }), true); } if (!showDisabledItems) { @@ -194,6 +220,21 @@ function (angular, _, utils) { * Build query - convert target filters to array of Zabbix items */ this.buildFromCache = function (groupFilter, hostFilter, appFilter, itemFilter) { + return this.filterItems(groupFilter, hostFilter, appFilter).then(function(items) { + if (items.length) { + if (utils.isRegex(itemFilter)) { + return findByFilter(items, itemFilter); + } else { + return _.filter(items, {'name': itemFilter}); + } + } else { + return []; + } + }); + }; + + // DEPRECATED + this._buildFromCache = function (groupFilter, hostFilter, appFilter, itemFilter) { // Find items by item names and perform queries var groups = []; diff --git a/src/datasource-zabbix/zabbixAPI.service.js b/src/datasource-zabbix/zabbixAPI.service.js index ca74d11..3047a7a 100644 --- a/src/datasource-zabbix/zabbixAPI.service.js +++ b/src/datasource-zabbix/zabbixAPI.service.js @@ -127,7 +127,8 @@ function ZabbixAPIService($q, alertSrv, zabbixAPICoreService) { var params = { output: ['name', 'host'], sortfield: 'name', - selectGroups: [] + selectGroups: [], + selectApplications: ['applicationid'] }; return this.request('host.get', params); @@ -140,7 +141,8 @@ function ZabbixAPIService($q, alertSrv, zabbixAPICoreService) { // Hack for supporting different apis (2.2 vs 2.4 vs 3.0) selectHost: [], - selectHosts: [] + selectHosts: [], + selectItems: ['itemid'] }; return this.request('application.get', params); @@ -170,9 +172,12 @@ function ZabbixAPIService($q, alertSrv, zabbixAPICoreService) { var params = { output: ['name', 'host'], sortfield: 'name', - selectGroups: [], + selectGroups: ['groupid'], + selectApplications: ['applicationid'], selectItems: [ - 'name', 'key_', + 'itemid', + 'name', + 'key_', 'value_type', 'hostid', 'status', diff --git a/src/datasource-zabbix/zabbixCache.service.js b/src/datasource-zabbix/zabbixCache.service.js index 6427d07..4369d47 100644 --- a/src/datasource-zabbix/zabbixCache.service.js +++ b/src/datasource-zabbix/zabbixCache.service.js @@ -62,7 +62,8 @@ function (angular, _, utils) { self._hosts = convertHosts(results[1]); self._applications = convertApplications(results[2]); self._items = convertItems(results[3]); - self._hostsExtend = convertHostsExtend(results[4]); + self._idx_apps = indexApps(results[2]); + self._idx_hosts = indexHosts(results[4]); } self._initialized = true; }); @@ -90,13 +91,24 @@ function (angular, _, utils) { } }; - p.getHostsExtend = function() { + p.getIndexedHosts = function() { var self = this; - if (this._hostsExtend) { - return $q.when(self._hostsExtend); + if (this._idx_hosts) { + return $q.when(self._idx_hosts); } else { return this.refresh().then(function() { - return self._hostsExtend; + return self._idx_hosts; + }); + } + }; + + p.getIndexedApplications = function() { + var self = this; + if (this._idx_apps) { + return $q.when(self._idx_apps); + } else { + return this.refresh().then(function() { + return self._idx_apps; }); } }; @@ -202,25 +214,13 @@ function (angular, _, utils) { }); } - function convertHostsExtend(hosts) { - return _.indexBy(_.map(hosts, function(host) { - host.items = _.forEach(host.items, function(item) { - item.applications = _.map(item.applications, 'applicationid'); - item.item = item.name; - item.name = utils.expandItemName(item.item, item.key_); - return item; - }); - return host; - }), 'hostid'); - } - /** * Group Zabbix applications by name * host.hosts - array of host ids */ function convertApplications(applications) { return _.map(_.groupBy(applications, 'name'), function(value, key) { - + //console.log(value); // Hack for supporting different apis (2.2 vs 2.4 vs 3.0) var hostField = 'host'; if (value[0] && value[0]['hosts']) { @@ -231,11 +231,46 @@ function (angular, _, utils) { return { name: key, applicationids: _.map(value, 'applicationid'), + itemids: _.uniq(_.map(_.flatten(value, 'items'), 'itemid')), hosts: _.uniq(_.map(_.flatten(value, hostField), 'hostid')) }; }); } + function indexHosts(hosts) { + return _.indexBy(_.map(hosts, function(host) { + + // Expand item names + host.items = _.forEach(host.items, function(item) { + item.item = item.name; + item.name = utils.expandItemName(item.item, item.key_); + return item; + }); + + host.applications = _.map(host.applications, 'applicationid'); + host.idx_items = indexItems(host.items); + host.items = _.map(host.items, 'itemid'); + return host; + }), 'hostid'); + } + + function indexApps(applications) { + return _.indexBy(_.map(applications, function(app) { + return { + name: app.name, + applicationid: app.applicationid, + host: _.first(_.map(app.hosts, 'hostid')), + itemids: _.map(app.items, 'itemid') + }; + }), 'applicationid'); + } + + function indexItems(items) { + return _.indexBy(_.map(items, function(item) { + return item; + }), 'itemid'); + } + /** * Convert item.get response to cache format * item.applications - array of application ids