Improved performance of metric filtering and building history query.

Gruntfile.js

@@ -21,6 +21,7 @@ module.exports = function(grunt) {
           '!**/utils.js',
           '!**/zabbixAPICore.service.js',
           '!**/zabbixAPI.service.js',
+          //'!**/dataProcessing.service.js',
           '!**/metricFunctions.js',
           '!**/*.scss'
         ],
@@ -43,7 +44,7 @@ module.exports = function(grunt) {
 
     babel: {
       options: {
-        sourceMap: false,
+        sourceMap: true,
         presets:  ["es2015"],
         plugins: ['transform-es2015-modules-systemjs', "transform-es2015-for-of"],
       },
@@ -58,6 +59,7 @@ module.exports = function(grunt) {
             '**/**/utils.js',
             '**/**/zabbixAPICore.service.js',
             '**/**/zabbixAPI.service.js',
+            //'**/**/dataProcessing.service.js',
             '**/**/metricFunctions.js'
           ],
           dest: 'dist/'

src/datasource-zabbix/queryProcessor.service.js

@@ -70,7 +70,7 @@ function (angular, _, utils) {
 
           var hostids = _.flatten(_.map(groups, 'hosts'));
           if (hostids.length) {
-            return self.cache.getHostsExtend().then(function(hosts) {
+            return self.cache.getIndexedHosts().then(function(hosts) {
               return _.map(hostids, function(hostid) {
                 return hosts[hostid];
               });
@@ -121,65 +121,91 @@ function (angular, _, utils) {
         });
       };
 
+      /**
+       * Find group, host, app or item by given name.
+       * @param  list list of groups, apps or other
+       * @param  name visible name
+       * @return      array with finded element or undefined
+       */
+      function findByName(list, name) {
+        var finded = _.find(list, {'name': name});
+        if (finded) {
+          return [finded];
+        } else {
+          return undefined;
+        }
+      }
+
+      function findByRegex(list, regex) {
+        var filterPattern = utils.buildRegex(regex);
+        return _.filter(list, function (zbx_obj) {
+          return filterPattern.test(zbx_obj.name);
+        });
+      }
+
+      function findByFilter(list, filter) {
+        if (utils.isRegex(filter)) {
+          return findByRegex(list, filter);
+        } else {
+          return findByName(list, filter);
+        }
+      }
+
+      function getFromIndex(index, objids) {
+        return _.map(objids, function(id) {
+          return index[id];
+        });
+      }
+
       this.filterItems = function (groupFilter, hostFilter, appFilter, itemType, showDisabledItems) {
-        var hosts = [];
+        var hosts;
-        var apps = [];
+        var apps;
-        var items = [];
+        var items;
 
         var promises = [
           this.filterHosts(groupFilter),
-          this.filterApplications(groupFilter, hostFilter)
+          this.filterApplications(groupFilter, hostFilter),
+          this.cache.getIndexedHosts(),
+          this.cache.getIndexedApplications()
         ];
 
         return $q.all(promises).then(function(results) {
           var hostList = results[0];
           var applicationList = results[1];
+          var idx_hosts = results[2];
+          var idx_apps = results[3];
 
-          // Filter hosts by regex
+          // Filter hosts
-          if (utils.isRegex(hostFilter)) {
+          hosts = findByFilter(hostList, hostFilter);
-            var hostFilterPattern = utils.buildRegex(hostFilter);
+          idx_hosts = getFromIndex(idx_hosts, _.map(hosts, 'hostid'));
-            hosts = _.filter(hostList, function (hostObj) {
-              return hostFilterPattern.test(hostObj.name);
-            });
-          } else {
-            var findedHosts = _.find(hostList, {'name': hostFilter});
-            if (findedHosts) {
-              hosts.push(findedHosts);
-            } else {
-              hosts = undefined;
-            }
-          }
 
-          // Filter applications by regex
+          // Filter applications
-          if (utils.isRegex(appFilter)) {
+          if (appFilter === "") {
-            var filterPattern = utils.buildRegex(appFilter);
+            // Get all items
-            apps = _.filter(applicationList, function (appObj) {
-              return filterPattern.test(appObj.name);
-            });
-          }
-          // Find items in selected application
-          else if (appFilter) {
-            var finded = _.find(applicationList, {'name': appFilter});
-            if (finded) {
-              apps.push(finded);
-            } else {
-              apps = undefined;
-            }
-          } else {
             apps = undefined;
             if (hosts) {
-              items = _.flatten(_.map(hosts, 'items'), true);
+              // Get all items in given hosts
+              items = _.flatten(_.map(idx_hosts, function(host) {
+                return _.values(host.idx_items);
+              }), true);
             }
+          } else {
+            apps = findByFilter(applicationList, appFilter);
           }
 
           if (apps) {
-            /*var appids = _.flatten(_.map(apps, 'applicationids'));
+            // Get ids for finded applications
-            items = _.filter(cachedItems, function (itemObj) {
+            var appids = _.flatten(_.map(apps, 'applicationids'));
-              return _.intersection(appids, itemObj.applications).length;
+            appids = _.flatten(_.map(_.map(hosts, 'applications'), function(apps) {
-            });
+              return _.intersection(apps, appids);
-            items = _.filter(items, function (itemObj) {
+            }));
-              return _.find(hosts, {'hostid': itemObj.hostid });
+
-            });*/
+            // For each finded host get list of items in finded applications
+            items = _.flatten(_.map(idx_hosts, function(host) {
+              var host_apps = _.intersection(appids, host.applications);
+              var host_itemids = _.flatten(_.map(getFromIndex(idx_apps, host_apps), 'itemids'));
+              return _.values(getFromIndex(host.idx_items, host_itemids));
+            }), true);
           }
 
           if (!showDisabledItems) {
@@ -194,6 +220,21 @@ function (angular, _, utils) {
        * Build query - convert target filters to array of Zabbix items
        */
       this.buildFromCache = function (groupFilter, hostFilter, appFilter, itemFilter) {
+        return this.filterItems(groupFilter, hostFilter, appFilter).then(function(items) {
+          if (items.length) {
+            if (utils.isRegex(itemFilter)) {
+              return findByFilter(items, itemFilter);
+            } else {
+              return _.filter(items, {'name': itemFilter});
+            }
+          } else {
+            return [];
+          }
+        });
+      };
+
+      // DEPRECATED
+      this._buildFromCache = function (groupFilter, hostFilter, appFilter, itemFilter) {
 
         // Find items by item names and perform queries
         var groups = [];

src/datasource-zabbix/zabbixAPI.service.js

@@ -127,7 +127,8 @@ function ZabbixAPIService($q, alertSrv, zabbixAPICoreService) {
       var params = {
         output: ['name', 'host'],
         sortfield: 'name',
-        selectGroups: []
+        selectGroups: [],
+        selectApplications: ['applicationid']
       };
 
       return this.request('host.get', params);
@@ -140,7 +141,8 @@ function ZabbixAPIService($q, alertSrv, zabbixAPICoreService) {
 
         // Hack for supporting different apis (2.2 vs 2.4 vs 3.0)
         selectHost: [],
-        selectHosts: []
+        selectHosts: [],
+        selectItems: ['itemid']
       };
 
       return this.request('application.get', params);
@@ -170,9 +172,12 @@ function ZabbixAPIService($q, alertSrv, zabbixAPICoreService) {
       var params = {
         output: ['name', 'host'],
         sortfield: 'name',
-        selectGroups: [],
+        selectGroups: ['groupid'],
+        selectApplications: ['applicationid'],
         selectItems: [
-          'name', 'key_',
+          'itemid',
+          'name',
+          'key_',
           'value_type',
           'hostid',
           'status',

src/datasource-zabbix/zabbixCache.service.js

@@ -62,7 +62,8 @@ function (angular, _, utils) {
           self._hosts         = convertHosts(results[1]);
           self._applications  = convertApplications(results[2]);
           self._items         = convertItems(results[3]);
-          self._hostsExtend   = convertHostsExtend(results[4]);
+          self._idx_apps      = indexApps(results[2]);
+          self._idx_hosts     = indexHosts(results[4]);
         }
         self._initialized = true;
       });
@@ -90,13 +91,24 @@ function (angular, _, utils) {
       }
     };
 
-    p.getHostsExtend = function() {
+    p.getIndexedHosts = function() {
       var self = this;
-      if (this._hostsExtend) {
+      if (this._idx_hosts) {
-        return $q.when(self._hostsExtend);
+        return $q.when(self._idx_hosts);
       } else {
         return this.refresh().then(function() {
-          return self._hostsExtend;
+          return self._idx_hosts;
+        });
+      }
+    };
+
+    p.getIndexedApplications = function() {
+      var self = this;
+      if (this._idx_apps) {
+        return $q.when(self._idx_apps);
+      } else {
+        return this.refresh().then(function() {
+          return self._idx_apps;
         });
       }
     };
@@ -202,25 +214,13 @@ function (angular, _, utils) {
       });
     }
 
-    function convertHostsExtend(hosts) {
-      return _.indexBy(_.map(hosts, function(host) {
-        host.items = _.forEach(host.items, function(item) {
-          item.applications = _.map(item.applications, 'applicationid');
-          item.item = item.name;
-          item.name = utils.expandItemName(item.item, item.key_);
-          return item;
-        });
-        return host;
-      }), 'hostid');
-    }
-
     /**
      * Group Zabbix applications by name
      * host.hosts - array of host ids
      */
     function convertApplications(applications) {
       return _.map(_.groupBy(applications, 'name'), function(value, key) {
-
+        //console.log(value);
         // Hack for supporting different apis (2.2 vs 2.4 vs 3.0)
         var hostField = 'host';
         if (value[0] && value[0]['hosts']) {
@@ -231,11 +231,46 @@ function (angular, _, utils) {
         return {
           name: key,
           applicationids: _.map(value, 'applicationid'),
+          itemids: _.uniq(_.map(_.flatten(value, 'items'), 'itemid')),
           hosts: _.uniq(_.map(_.flatten(value, hostField), 'hostid'))
         };
       });
     }
 
+    function indexHosts(hosts) {
+      return _.indexBy(_.map(hosts, function(host) {
+
+        // Expand item names
+        host.items = _.forEach(host.items, function(item) {
+          item.item = item.name;
+          item.name = utils.expandItemName(item.item, item.key_);
+          return item;
+        });
+
+        host.applications = _.map(host.applications, 'applicationid');
+        host.idx_items = indexItems(host.items);
+        host.items = _.map(host.items, 'itemid');
+        return host;
+      }), 'hostid');
+    }
+
+    function indexApps(applications) {
+      return _.indexBy(_.map(applications, function(app) {
+        return {
+          name: app.name,
+          applicationid: app.applicationid,
+          host: _.first(_.map(app.hosts, 'hostid')),
+          itemids: _.map(app.items, 'itemid')
+        };
+      }), 'applicationid');
+    }
+
+    function indexItems(items) {
+      return _.indexBy(_.map(items, function(item) {
+        return item;
+      }), 'itemid');
+    }
+
     /**
      * Convert item.get response to cache format
      * item.applications - array of application ids