Improve tags performance (#1526)

Co-authored-by: Erik De Neve <erik.de.neve@telenet.be>

pkg/datasource/zabbix.go

@@ -7,9 +7,10 @@ import (
 	"github.com/alexanderzobnin/grafana-zabbix/pkg/timeseries"
 
 	"github.com/alexanderzobnin/grafana-zabbix/pkg/zabbix"
+	"golang.org/x/net/context"
+
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
 	"github.com/grafana/grafana-plugin-sdk-go/data"
-	"golang.org/x/net/context"
 )
 
 // ZabbixAPIQuery handles query requests to Zabbix API
@@ -52,7 +53,15 @@ func (ds *ZabbixDatasourceInstance) queryNumericItems(ctx context.Context, query
 	itemFilter := query.Item.Filter
 	showDisabled := query.Options.ShowDisabledItems
 
-	items, err := ds.zabbix.GetItems(ctx, groupFilter, hostFilter, appFilter, itemTagFilter, itemFilter, "num", showDisabled)
+	var items []*zabbix.Item
+	var err error
+	zabbixVersion, err := ds.zabbix.GetVersion(ctx)
+	if zabbixVersion >= 54 {
+		items, err = ds.zabbix.GetItems(ctx, groupFilter, hostFilter, itemTagFilter, itemFilter, "num", showDisabled)
+	} else {
+		items, err = ds.zabbix.GetItemsBefore54(ctx, groupFilter, hostFilter, appFilter, itemFilter, "num", showDisabled)
+	}
+
 	if err != nil {
 		return nil, err
 	}

pkg/zabbix/methods.go

@@ -2,6 +2,7 @@ package zabbix
 
 import (
 	"context"
+	"regexp"
 	"strconv"
 	"strings"
 
@@ -84,11 +85,49 @@ func (ds *Zabbix) GetItems(
 	ctx context.Context,
 	groupFilter string,
 	hostFilter string,
-	appFilter string,
 	itemTagFilter string,
 	itemFilter string,
 	itemType string,
 	showDisabled bool,
+) ([]*Item, error) {
+	var allItems []*Item
+	hosts, err := ds.GetHosts(ctx, groupFilter, hostFilter)
+	if err != nil {
+		return nil, err
+	}
+	if len(hosts) == 0 {
+		return allItems, nil
+	}
+
+	hostids := make([]string, 0)
+	for _, host := range hosts {
+		hostids = append(hostids, host.ID)
+	}
+
+	if isRegex(itemTagFilter) {
+		tags, err := ds.GetItemTags(ctx, groupFilter, hostFilter, itemTagFilter)
+		if err != nil {
+			return nil, err
+		}
+		var tagStrs []string
+		for _, t := range tags {
+			tagStrs = append(tagStrs, itemTagToString(t))
+		}
+		itemTagFilter = strings.Join(tagStrs, ",")
+	}
+	allItems, err = ds.GetAllItems(ctx, hostids, nil, itemType, showDisabled, itemTagFilter)
+
+	return filterItemsByQuery(allItems, itemFilter)
+}
+
+func (ds *Zabbix) GetItemsBefore54(
+	ctx context.Context,
+	groupFilter string,
+	hostFilter string,
+	appFilter string,
+	itemFilter string,
+	itemType string,
+	showDisabled bool,
 ) ([]*Item, error) {
 	hosts, err := ds.GetHosts(ctx, groupFilter, hostFilter)
 	if err != nil {
@@ -100,13 +139,10 @@ func (ds *Zabbix) GetItems(
 	}
 
 	apps, err := ds.GetApps(ctx, groupFilter, hostFilter, appFilter)
-	// Apps not supported in Zabbix 5.4 and higher
+	if err != nil {
-	isZabbix54orHigher := isAppMethodNotFoundError(err)
-	if isZabbix54orHigher {
-		apps = []Application{}
-	} else if err != nil {
 		return nil, err
 	}
+
 	appids := make([]string, 0)
 	for _, app := range apps {
 		appids = append(appids, app.ID)
@@ -114,59 +150,14 @@ func (ds *Zabbix) GetItems(
 
 	var allItems []*Item
 	if len(appids) > 0 {
-		allItems, err = ds.GetAllItems(ctx, nil, appids, itemType, showDisabled)
+		allItems, err = ds.GetAllItems(ctx, nil, appids, itemType, showDisabled, "")
 	} else if len(hostids) > 0 {
-		allItems, err = ds.GetAllItems(ctx, hostids, nil, itemType, showDisabled)
+		allItems, err = ds.GetAllItems(ctx, hostids, nil, itemType, showDisabled, "")
-	}
-
-	if isZabbix54orHigher && itemTagFilter != "" {
-		allItems, err = filterItemsByTag(allItems, itemTagFilter)
-		if err != nil {
-			return nil, err
-		}
 	}
 
 	return filterItemsByQuery(allItems, itemFilter)
 }
 
-func filterItemsByTag(items []*Item, filter string) ([]*Item, error) {
-	re, err := parseFilter(filter)
-	if err != nil {
-		return nil, err
-	}
-
-	filteredItems := make([]*Item, 0)
-	for _, i := range items {
-		if len(i.Tags) == 0 && filter == "/.*/" {
-			filteredItems = append(filteredItems, i)
-		}
-
-		if len(i.Tags) > 0 {
-			tags := make([]string, 0)
-			for _, t := range i.Tags {
-				tags = append(tags, itemTagToString(t))
-			}
-			for _, t := range tags {
-				if re != nil {
-					match, err := re.MatchString(t)
-					if err != nil {
-						return nil, err
-					}
-					if match {
-						filteredItems = append(filteredItems, i)
-						break
-					}
-				} else if t == filter {
-					filteredItems = append(filteredItems, i)
-					break
-				}
-			}
-		}
-	}
-
-	return filteredItems, nil
-}
-
 func filterItemsByQuery(items []*Item, filter string) ([]*Item, error) {
 	re, err := parseFilter(filter)
 	if err != nil {
@@ -236,6 +227,55 @@ func filterAppsByQuery(items []Application, filter string) ([]Application, error
 	return filteredItems, nil
 }
 
+func (ds *Zabbix) GetItemTags(ctx context.Context, groupFilter string, hostFilter string, tagFilter string) ([]ItemTag, error) {
+	hosts, err := ds.GetHosts(ctx, groupFilter, hostFilter)
+	if err != nil {
+		return nil, err
+	}
+	hostids := make([]string, 0)
+	for _, host := range hosts {
+		hostids = append(hostids, host.ID)
+	}
+
+	var allItems []*Item
+	itemType := "num"
+	showDisabled := false
+	allItems, err = ds.GetAllItems(ctx, hostids, nil, itemType, showDisabled, "")
+
+	var allTags []ItemTag
+	for _, item := range allItems {
+		allTags = append(allTags, item.Tags...)
+	}
+
+	return filterTags(allTags, tagFilter)
+}
+
+func filterTags(items []ItemTag, filter string) ([]ItemTag, error) {
+	re, err := parseFilter(filter)
+	if err != nil {
+		return nil, err
+	}
+
+	filteredItems := make([]ItemTag, 0)
+	for _, i := range items {
+		tagStr := itemTagToString(i)
+		if re != nil {
+			match, err := re.MatchString(tagStr)
+			if err != nil {
+				return nil, err
+			}
+			if match {
+				filteredItems = append(filteredItems, i)
+			}
+		} else if tagStr == filter {
+			filteredItems = append(filteredItems, i)
+		}
+
+	}
+
+	return filteredItems, nil
+}
+
 func (ds *Zabbix) GetHosts(ctx context.Context, groupFilter string, hostFilter string) ([]Host, error) {
 	groups, err := ds.GetGroups(ctx, groupFilter)
 	if err != nil {
@@ -314,7 +354,7 @@ func filterGroupsByQuery(items []Group, filter string) ([]Group, error) {
 	return filteredItems, nil
 }
 
-func (ds *Zabbix) GetAllItems(ctx context.Context, hostids []string, appids []string, itemtype string, showDisabled bool) ([]*Item, error) {
+func (ds *Zabbix) GetAllItems(ctx context.Context, hostids []string, appids []string, itemtype string, showDisabled bool, itemTagFilter string) ([]*Item, error) {
 	params := ZabbixAPIParams{
 		"output":         []string{"itemid", "name", "key_", "value_type", "hostid", "status", "state", "units", "valuemapid", "delay"},
 		"sortfield":      "name",
@@ -334,6 +374,24 @@ func (ds *Zabbix) GetAllItems(ctx context.Context, hostids []string, appids []st
 
 	if ds.version >= 54 {
 		params["selectTags"] = "extend"
+		if len(itemTagFilter) > 0 {
+			allTags := strings.Split(itemTagFilter, ",")
+			re := regexp.MustCompile(`(?m).*?([a-zA-Z0-9\s\-_]*):\s*([a-zA-Z0-9\-_\/:]*)`)
+			var tagsParams []map[string]string
+			for i := 0; i < len(allTags); i++ {
+				res := re.FindAllStringSubmatch(allTags[i], -1)
+				for i := range res {
+					tagParam := map[string]string{
+						"tag":      res[i][1],
+						"value":    res[i][2],
+						"operator": "1",
+					}
+					tagsParams = append(tagsParams, tagParam)
+				}
+			}
+			params["tags"] = tagsParams
+			params["evaltype"] = 2
+		}
 	}
 
 	if showDisabled == false {

pkg/zabbix/utils.go

@@ -88,6 +88,11 @@ func parseFilter(filter string) (*regexp2.Regexp, error) {
 	return regexp2.Compile(pattern, regexp2.RE2)
 }
 
+func isRegex(filter string) bool {
+	regex := regexp.MustCompile(`^/(.+)/([imncsxrde]*)$`)
+	return regex.MatchString(filter)
+}
+
 func itemTagToString(tag ItemTag) string {
 	if tag.Value != "" {
 		return fmt.Sprintf("%s: %s", tag.Tag, tag.Value)

src/datasource/components/QueryEditor/MetricsQueryEditor.tsx

@@ -70,7 +70,8 @@ export const MetricsQueryEditor = ({ query, datasource, onChange }: Props) => {
   }, [query.group.filter, query.host.filter]);
 
   const loadTagOptions = async (group: string, host: string) => {
-    if (!datasource.zabbix.isZabbix54OrHigher()) {
+    const tagsAvailable = await datasource.zabbix.isZabbix54OrHigher();
+    if (!tagsAvailable) {
       return [];
     }
 
@@ -78,6 +79,7 @@ export const MetricsQueryEditor = ({ query, datasource, onChange }: Props) => {
     const hostFilter = datasource.replaceTemplateVars(host);
     const items = await datasource.zabbix.getAllItems(groupFilter, hostFilter, null, null, {});
     const tags: ZBXItemTag[] = _.flatten(items.map((item: ZBXItem) => item.tags || []));
+    // const tags: ZBXItemTag[] = await datasource.zabbix.getItemTags(groupFilter, hostFilter, null);
 
     const tagList = _.uniqBy(tags, (t) => t.tag + t.value || '').map((t) => itemTagToString(t));
     let options: Array<SelectableValue<string>> = tagList?.map((tag) => ({

src/datasource/components/QueryEditor/TriggersQueryEditor.tsx

@@ -85,7 +85,8 @@ export const TriggersQueryEditor = ({ query, datasource, onChange }: Props) => {
   }, [query.group.filter, query.host.filter]);
 
   const loadTagOptions = async (group: string, host: string) => {
-    if (!datasource.zabbix.isZabbix54OrHigher()) {
+    const tagsAvailable = await datasource.zabbix.isZabbix54OrHigher();
+    if (!tagsAvailable) {
       return [];
     }
 

src/datasource/components/VariableQueryEditor.tsx

@@ -87,10 +87,10 @@ export class ZabbixVariableQueryEditor extends PureComponent<VariableQueryProps,
     this.props.onChange(queryModel, `Zabbix - ${queryType}`);
   };
 
-  render() {
+  async render() {
     const { selectedQueryType, legacyQuery, group, host, application, itemTag, item } = this.state;
     const { datasource } = this.props;
-    const supportsItemTags = datasource?.zabbix?.isZabbix54OrHigher() || false;
+    const supportsItemTags = (await datasource?.zabbix?.isZabbix54OrHigher()) || false;
 
     return (
       <>

src/datasource/utils.ts

@@ -41,7 +41,7 @@ export function expandItemName(name: string, key: string): string {
   return name;
 }
 
-export function expandItems(items) {
+export function expandItems(items: any[]) {
   _.forEach(items, (item) => {
     item.item = item.name;
     item.name = expandItemName(item.item, item.key_);

src/datasource/zabbix/connectors/zabbix_api/zabbixAPIConnector.ts

@@ -88,6 +88,10 @@ export class ZabbixAPIConnector {
   }
 
   initVersion(): Promise<string> {
+    if (this.version) {
+      return Promise.resolve(this.version);
+    }
+
     if (!this.getVersionPromise) {
       this.getVersionPromise = Promise.resolve(
         this.getVersion().then((version) => {
@@ -142,7 +146,7 @@ export class ZabbixAPIConnector {
     return this.request('hostgroup.get', params);
   }
 
-  getHosts(groupids) {
+  getHosts(groupids): Promise<any[]> {
     const params: any = {
       output: ['hostid', 'name', 'host'],
       sortfield: 'name',
@@ -174,7 +178,7 @@ export class ZabbixAPIConnector {
    * @param  {String} itemtype 'num' or 'text'
    * @return {[type]}          array of items
    */
-  getItems(hostids, appids, itemtype) {
+  getItems(hostids, appids, itemtype, itemTagFilter?: string): Promise<any[]> {
     const params: any = {
       output: ['itemid', 'name', 'key_', 'value_type', 'hostid', 'status', 'state', 'units', 'valuemapid', 'delay'],
       sortfield: 'name',
@@ -199,6 +203,18 @@ export class ZabbixAPIConnector {
 
     if (this.isZabbix54OrHigher()) {
       params.selectTags = 'extend';
+      if (itemTagFilter) {
+        const allTags = itemTagFilter.split(',');
+        let tagsParam = [];
+        const regex = /.*?([a-zA-Z0-9\s\-_]*):\s*([a-zA-Z0-9\-_\/:]*)/;
+        for (let i = 0; i < allTags.length; i++) {
+          const matches = allTags[i].match(regex);
+          tagsParam.push({ tag: matches[1].replace('/', ''), value: matches[2].trim(), operator: '1' });
+        }
+        params.tags = tagsParam;
+        // Use OR eval type
+        params.evaltype = 2;
+      }
     }
 
     return this.request('item.get', params).then(utils.expandItems);

src/datasource/zabbix/zabbix.ts

@@ -252,8 +252,8 @@ export class Zabbix implements ZabbixConnector {
     return version ? semver.gte(version, '6.0.0') : true;
   }
 
-  isZabbix54OrHigher() {
+  async isZabbix54OrHigher() {
-    const version = this.version || this.zabbixAPI.version;
+    const version = await this.zabbixAPI.initVersion();
     return version ? semver.gte(version, '5.4.0') : false;
   }
 
@@ -301,28 +301,28 @@ export class Zabbix implements ZabbixConnector {
     return this.zabbixAPI.getGroups();
   }
 
-  getGroups(groupFilter) {
+  getGroups(groupFilter): Promise<any[]> {
     return this.getAllGroups().then((groups) => findByFilter(groups, groupFilter));
   }
 
   /**
    * Get list of host belonging to given groups.
    */
-  getAllHosts(groupFilter) {
+  getAllHosts(groupFilter): Promise<any[]> {
     return this.getGroups(groupFilter).then((groups) => {
       const groupids = _.map(groups, 'groupid');
       return this.zabbixAPI.getHosts(groupids);
     });
   }
 
-  getHosts(groupFilter?, hostFilter?) {
+  getHosts(groupFilter?, hostFilter?): Promise<any[]> {
     return this.getAllHosts(groupFilter).then((hosts) => findByFilter(hosts, hostFilter));
   }
 
   /**
    * Get list of applications belonging to given groups and hosts.
    */
-  async getAllApps(groupFilter, hostFilter) {
+  async getAllApps(groupFilter, hostFilter): Promise<any[]> {
     await this.getVersion();
     if (!this.supportsApplications()) {
       return [];
@@ -378,22 +378,44 @@ export class Zabbix implements ZabbixConnector {
     return findByFilter(tagsStr, itemTagFilter);
   }
 
-  async getAllItems(groupFilter, hostFilter, appFilter, itemTagFilter, options: any = {}) {
+  async getAllItems(
+    groupFilter: string,
+    hostFilter: string,
+    appFilter: string,
+    itemTagFilter: string,
+    options: any = {}
+  ): Promise<any[]> {
+    if (!this.isZabbix54OrHigher()) {
+      return this.getAllItemsBefore54(groupFilter, hostFilter, appFilter, itemTagFilter, options);
+    }
+
+    const hosts = await this.getHosts(groupFilter, hostFilter);
+    const hostids = _.map(hosts, 'hostid');
+
+    // Support regexp in tags
+    if (utils.isRegex(itemTagFilter)) {
+      const tags = await this.getItemTags(groupFilter, hostFilter, itemTagFilter);
+      itemTagFilter = tags.map((t) => t.name).join(',');
+    }
+
+    let items = await this.zabbixAPI.getItems(hostids, undefined, options.itemtype, itemTagFilter);
+
+    if (!options.showDisabledItems) {
+      items = _.filter(items, { status: '0' });
+    }
+
+    return await this.expandUserMacro(items, false);
+  }
+
+  async getAllItemsBefore54(groupFilter, hostFilter, appFilter, itemTagFilter, options: any = {}) {
     const apps = await this.getApps(groupFilter, hostFilter, appFilter);
     let items: any[];
 
-    if (this.isZabbix54OrHigher()) {
+    if (apps.appFilterEmpty) {
-      items = await this.zabbixAPI.getItems(apps.hostids, undefined, options.itemtype);
+      items = await this.zabbixAPI.getItems(apps.hostids, undefined, options.itemtype, undefined);
-      if (itemTagFilter) {
-        items = filterItemsByTag(items, itemTagFilter);
-      }
     } else {
-      if (apps.appFilterEmpty) {
+      const appids = _.map(apps, 'applicationid');
-        items = await this.zabbixAPI.getItems(apps.hostids, undefined, options.itemtype);
+      items = await this.zabbixAPI.getItems(undefined, appids, options.itemtype, undefined);
-      } else {
-        const appids = _.map(apps, 'applicationid');
-        items = await this.zabbixAPI.getItems(undefined, appids, options.itemtype);
-      }
     }
 
     if (!options.showDisabledItems) {
@@ -746,28 +768,3 @@ function getHostIds(items) {
   });
   return _.uniq(_.flatten(hostIds));
 }
-
-function filterItemsByTag(items: any[], itemTagFilter: string) {
-  if (utils.isRegex(itemTagFilter)) {
-    const filterPattern = utils.buildRegex(itemTagFilter);
-    return items.filter((item) => {
-      if (item.tags) {
-        const tags: string[] = item.tags.map((t) => utils.itemTagToString(t));
-        return tags.some((tag) => {
-          return filterPattern.test(tag);
-        });
-      } else {
-        return false;
-      }
-    });
-  } else {
-    return items.filter((item) => {
-      if (item.tags) {
-        const tags: string[] = item.tags.map((t) => utils.itemTagToString(t));
-        return tags.includes(itemTagFilter);
-      } else {
-        return false;
-      }
-    });
-  }
-}