How to exclude the buckets having doc count equal to 0

ESCoder 0
Viewed 14

I want to exclude those buckets from the date histogram aggregation response, whose doc count is equal to 0. And then, get the count of the filtered buckets.

The query is :

GET metricbeat-*/_search
{
  "size": 0,
  "query": {
    "bool": {
      "filter": [
        {
          "range": {
            "host.cpu.usage": {
              "gte": 0.8
            }
          }
        },
        {
          "range": {
            "@timestamp": {
              "gte": "2022-09-22T10:16:00.000Z",
              "lte": "2022-09-22T10:18:00.000Z"
            }
          }
        }
      ]
    }
  },
  "aggs": {
    "hostName": {
      "terms": {
        "field": "host.name"
      },
      "aggs": {
        "docsOverTimeFrame": {
          "date_histogram": {
            "field": "@timestamp",
            "fixed_interval": "10s"
          }
        },
        "min_bucket_selector": {
          "bucket_selector": {
            "buckets_path": {
              "count": "docsOverTimeFrame._bucket_count" 
            },
            "script": {
              "source": "params.count == 12"
            }
          }
        }
      }
    }
  }
}

The response that I get right now is :

{
  "took" : 8,
  "timed_out" : false,
  "_shards" : {
    "total" : 3,
    "successful" : 3,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 38,
      "relation" : "eq"
    },
    "max_score" : null,
    "hits" : [ ]
  },
  "aggregations" : {
    "hostName" : {
      "doc_count_error_upper_bound" : 0,
      "sum_other_doc_count" : 0,
      "buckets" : [
        {
          "key" : "datahot01",
          "doc_count" : 3,
          "docsOverTimeFrame" : {
            "buckets" : [
              {
                "key_as_string" : "2022-09-22T10:16:00.000Z",
                "key" : 1663841760000,
                "doc_count" : 1
              },
              {
                "key_as_string" : "2022-09-22T10:16:10.000Z",
                "key" : 1663841770000,
                "doc_count" : 1
              },
              {
                "key_as_string" : "2022-09-22T10:16:20.000Z",
                "key" : 1663841780000,
                "doc_count" : 0
              },
              {
                "key_as_string" : "2022-09-22T10:16:30.000Z",
                "key" : 1663841790000,
                "doc_count" : 0
              },
              {
                "key_as_string" : "2022-09-22T10:16:40.000Z",
                "key" : 1663841800000,
                "doc_count" : 0
              },
              {
                "key_as_string" : "2022-09-22T10:16:50.000Z",
                "key" : 1663841810000,
                "doc_count" : 0
              },
              {
                "key_as_string" : "2022-09-22T10:17:00.000Z",
                "key" : 1663841820000,
                "doc_count" : 0
              },
              {
                "key_as_string" : "2022-09-22T10:17:10.000Z",
                "key" : 1663841830000,
                "doc_count" : 0
              },
              {
                "key_as_string" : "2022-09-22T10:17:20.000Z",
                "key" : 1663841840000,
                "doc_count" : 0
              },
              {
                "key_as_string" : "2022-09-22T10:17:30.000Z",
                "key" : 1663841850000,
                "doc_count" : 0
              },
              {
                "key_as_string" : "2022-09-22T10:17:40.000Z",
                "key" : 1663841860000,
                "doc_count" : 0
              },
              {
                "key_as_string" : "2022-09-22T10:17:50.000Z",
                "key" : 1663841870000,
                "doc_count" : 0
              }
            ]
          }
        }
      ]
    }
  }
}

So, if I am able to exclude those buckets that have doc count = 0, then on the basis of the number of buckets (that is bucket count), I want to check whether the count of buckets formed is equal to 12 or not (which I am doing using the bucket selector aggregation).

Is there some way to exclude the buckets having doc count = 0, and get the bucket count = 2 instead of 12

elasticsearch-aggregation elasticsearch
1 Answers

I was able to solve the above use case, by using a pipeline aggregation (i.e a bucket_selector aggregation) inside of the date histogram aggregation.

The modified query is :

{
  "query": {
    "bool": {
      "must": [
        {
          "range": {
            "@timestamp": {
              "gte": "2022-09-22T10:16:00.000Z",
              "lte": "2022-09-22T10:22:00.000Z"
            }
          }
        },
        {
          "range": {
            "system.cpu.total.norm.pct": {
              "gte": 0.8
            }
          }
        }
      ]
    }
  },
  "aggs": {
    "hostName": {
      "terms": {
        "field": "host.name"
      },
      "aggs": {
        "docsOverTimeFrame": {
          "date_histogram": {
            "field": "@timestamp",
            "fixed_interval": "10s"
          },
          "aggs": {
            "histogram_doc_count": {
              "bucket_selector": {
                "buckets_path": {
                  "the_doc_count": "_count"
                },
                "script": "params.the_doc_count > 0"
              }
            }
          }
        },
        "min_bucket_selector": {
          "bucket_selector": {
            "buckets_path": {
              "count": "docsOverTimeFrame._bucket_count"
            },
            "script": {
              "source": "params.count == 12"
            }
          }
        }
      }
    }
  }
}
ESCoder0
Related