kush-elastic · Feb 8, 2023 · Jan 20, 2023 · Jan 20, 2023 · Jan 31, 2023 · Feb 2, 2023
@@ -6,21 +6,22 @@
 
 Use the Ceph integration to:
 
-- Collect metrics related to the cluster health, Object Storage Daemons (OSD) performance, Object Storage Daemons (OSD) pool stats and Object Storage Daemons (OSD) tree.
+- Collect metrics related to the cluster health, Object Storage Daemons (OSD) performance, Object Storage Daemons (OSD) pool stats, Object Storage Daemons (OSD) tree and pool disk.
 - Create visualizations to monitor, measure and analyze the usage trend and key data, and derive business insights.
 - Create alerts to reduce the MTTD and also the MTTR by referencing relevant logs when troubleshooting an issue.
 
 ## Data streams
 
 The Ceph integration collects metrics data.
 
-Metrics give you insight into the statistics of the Ceph. The Metric data streams collected by the Ceph integration are `cluster_health`, `osd_performance`, `osd_pool_stats` and `osd_tree`, so that the user can monitor and troubleshoot the performance of the Ceph instance.
+Metrics give you insight into the statistics of the Ceph. The Metric data streams collected by the Ceph integration are `cluster_health`, `osd_performance`, `osd_pool_stats`, `osd_tree` and `pool_disk`, so that the user can monitor and troubleshoot the performance of the Ceph instance.
 
 Data stream:
 - `cluster_health`: Represents information related to the health of the cluster.
 - `osd_performance`: Tracks Object Storage Daemons (OSD) performance.
 - `osd_pool_stats`: Represents information related to client I/O rates.
 - `osd_tree`: Represents information related to structure of the Object Storage Daemons (OSD) tree.
+- `pool_disk`: Tracks memory of each pool.
 
 Note:
 - Users can monitor and see the metrics inside the ingested documents for Ceph in the `logs-*` index pattern from `Discover`.
@@ -116,4 +117,12 @@ This is the `osd_tree` data stream. This data stream collects metrics related to
 
 {{event "osd_tree"}}
 
-{{fields "osd_tree"}}
+{{fields "osd_tree"}}
+
+### Pool Disk
+
+This is the `pool_disk` data stream. This data stream collects metrics related to pool id, pool name, pool objects, used bytes and available bytes of the pool disk.
+
+{{event "pool_disk"}}
+
+{{fields "pool_disk"}}
@@ -26,6 +26,15 @@ rules:
       - status_code: 200
         body: |-
           {"failed":[],"finished":[{"command":"osd pool stats format=json","outb":"[{\"pool_name\":\"device_health_metrics\",\"pool_id\":1,\"recovery\":{},\"recovery_rate\":{},\"client_io_rate\":{\"read_bytes_sec\":6622518,\"write_bytes_sec\":6622518,\"read_op_per_sec\":11,\"write_op_per_sec\":11}},{\"pool_name\":\"elastic\",\"pool_id\":9,\"recovery\":{},\"recovery_rate\":{},\"client_io_rate\":{}}]","outs":""}],"has_failed":false,"id":"140222622573952","is_finished":true,"is_waiting":false,"running":[],"state":"success","waiting":[]}
+  - path: /request
+    query_params:
+      wait: 1
+    methods: ['POST']
+    request_body: '{"format":"json","prefix":"df"}'
+    responses:
+      - status_code: 200
+        body: |-
+          {"failed":[],"finished":[{"command":"df format=json","outb":"{\"stats\":{\"total_bytes\":85882568704,\"total_avail_bytes\":81199562752,\"total_used_bytes\":388038656,\"total_used_raw_bytes\":4683005952,\"total_used_raw_ratio\":0.054528012871742249,\"num_osds\":4,\"num_per_pool_osds\":4,\"num_per_pool_omap_osds\":4},\"stats_by_class\":{\"hdd\":{\"total_bytes\":85882568704,\"total_avail_bytes\":81199562752,\"total_used_bytes\":388038656,\"total_used_raw_bytes\":4683005952,\"total_used_raw_ratio\":0.054528012871742249}},\"pools\":[{\"name\":\"device_health_metrics\",\"id\":1,\"stats\":{\"stored\":2142673,\"objects\":4,\"kb_used\":6336,\"bytes_used\":6488064,\"percent_used\":8.4362458437681198e-05,\"max_avail\":25633505280}},{\"name\":\"elk\",\"id\":4,\"stats\":{\"stored\":1176572,\"objects\":3,\"kb_used\":3648,\"bytes_used\":3735552,\"percent_used\":4.8574063839623705e-05,\"max_avail\":25633505280}},{\"name\":\"elastic\",\"id\":9,\"stats\":{\"stored\":1349210,\"objects\":5,\"kb_used\":4224,\"bytes_used\":4325376,\"percent_used\":5.6243221479235217e-05,\"max_avail\":25633505280}}]}\n","outs":""}],"has_failed":false,"id":"140222991325696","is_finished":true,"is_waiting":false,"running":[],"state":"success","waiting":[]}
   - path: /request
     query_params:
       wait: 1

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.5.0"
+  changes:
+    - description: Ceph integration package with "pool_disk" data stream.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/5070
 - version: "0.4.0"
   changes:
     - description: Ceph integration package with "osd_tree" data stream.

@@ -0,0 +1,3 @@
+fields:
+  tags:
+    - preserve_original_event
@@ -0,0 +1 @@
+{"name":"device_health_metrics","id":1,"stats":{"stored":2142673,"objects":4,"kb_used":6336,"bytes_used":6488064,"percent_used":8.4363608038984239e-05,"max_avail":25633155072}}
@@ -0,0 +1,39 @@
+{
+    "expected": [
+        {
+            "ceph": {
+                "pool_disk": {
+                    "available": {
+                        "bytes": 25633155072
+                    },
+                    "object": {
+                        "count": 4
+                    },
+                    "pool_id": 1,
+                    "pool_name": "device_health_metrics",
+                    "stored": {
+                        "bytes": 2142673
+                    },
+                    "used": {
+                        "bytes": 6488064,
+                        "pct": 8.436360803898424E-5
+                    }
+                }
+            },
+            "ecs": {
+                "version": "8.5.1"
+            },
+            "event": {
+                "kind": "metric",
+                "module": "ceph",
+                "original": "{\"name\":\"device_health_metrics\",\"id\":1,\"stats\":{\"stored\":2142673,\"objects\":4,\"kb_used\":6336,\"bytes_used\":6488064,\"percent_used\":8.4363608038984239e-05,\"max_avail\":25633155072}}",
+                "type": [
+                    "info"
+                ]
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
+        }
+    ]
+}
@@ -0,0 +1,7 @@
+input: httpjson
+service: ceph
+vars:
+  hostname: http://{{Hostname}}:{{Port}}
+data_stream:
+  vars:
+    preserve_original_event: true
@@ -0,0 +1,50 @@
+config_version: 2
+interval: {{period}}
+{{#if api_user}}
+{{#if api_secret}}
+auth.basic.user: {{api_user}}
+auth.basic.password: {{api_secret}}
+{{/if}}
+{{/if}}
+{{#if ssl}}
+request.ssl: {{ssl}}
+{{/if}}
+request.method: POST
+request.url: {{hostname}}/request?wait=1
+request.transforms:
+  - set:
+      target: body.prefix
+      value: "df"
+  - set:
+      target: body.format
+      value: "json"
+response.split:
+  target: body.finished
+  ignore_empty_value: true
+  transforms:
+    - set:
+        target: body.outb
+        value: '[[.body.outb]]'
+        value_type: json
+  split:
+    target: body.outb.pools
+    value_type: json
+    ignore_empty_value: true
+tags:
+{{#if preserve_original_event}}
+  - preserve_original_event
+{{/if}}
+{{#each tags as |tag|}}
+  - {{tag}}
+{{/each}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+processors:
+- add_fields:
+    target: service
+    fields:
+      address: {{hostname}}
+{{#if processors}}
+{{processors}}
+{{/if}}
@@ -0,0 +1,87 @@
+---
+description: Pipeline for processing Ceph Pool Disk metrics.
+processors:
+  - set:
+      field: ecs.version
+      value: 8.5.1
+  - set:
+      field: event.type
+      value: [info]
+  - set:
+      field: event.kind
+      value: metric
+  - set:
+      field: event.module
+      value: ceph
+  - rename:
+      field: message
+      target_field: event.original
+      ignore_missing: true
+  - json:
+      field: event.original
+      target_field: json
+      on_failure:
+        - append:
+            field: error.message
+            value: '{{{_ingest.on_failure_message}}}'
+  - rename: 
+      field: json.id
+      target_field: ceph.pool_disk.pool_id
+      ignore_missing: true
+  - rename: 
+      field: json.name
+      target_field: ceph.pool_disk.pool_name
+      ignore_missing: true
+  - rename: 
+      field: json.stats.bytes_used
+      target_field: ceph.pool_disk.used.bytes
+      ignore_missing: true
+  - rename: 
+      field: json.stats.max_avail
+      target_field: ceph.pool_disk.available.bytes
+      ignore_missing: true
+  - rename: 
+      field: json.stats.objects
+      target_field: ceph.pool_disk.object.count
+      ignore_missing: true
+  - rename: 
+      field: json.stats.stored
+      target_field: ceph.pool_disk.stored.bytes
+      ignore_missing: true
+  - rename: 
+      field: json.stats.percent_used
+      target_field: ceph.pool_disk.used.pct
+      ignore_missing: true
+  - script:
+      description: Drops null/empty values recursively.
+      lang: painless
+      source: |
+        boolean drop(Object o) {
+          if (o == null || o == "") {
+            return true;
+          } else if (o instanceof Map) {
+            ((Map) o).values().removeIf(v -> drop(v));
+            return (((Map) o).size() == 0);
+          } else if (o instanceof List) {
+            ((List) o).removeIf(v -> drop(v));
+            return (((List) o).length == 0);
+          }
+          return false;
+        }
+        drop(ctx);
+  - remove:
+      field: event.original
+      if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))"
+      ignore_missing: true
+  - remove:
+      field:
+        - json
+      ignore_missing: true
+on_failure:
+  - set:
+      field: error.message
+      value: "{{{_ingest.on_failure_message}}}"
+  - append:
+      field: event.kind
+      value: pipeline_error
+      allow_duplicates: false
@@ -0,0 +1,18 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: input.type
+  type: keyword
+  description: Type of Filebeat input.
+- name: tags
+  type: keyword
+  description: List of keywords used to tag each event.
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.
@@ -0,0 +1,22 @@
+- external: ecs
+  name: ecs.version
+- external: ecs
+  name: error.message
+- external: ecs
+  name: event.agent_id_status
+- external: ecs
+  name: event.created
+- external: ecs
+  name: event.dataset
+- external: ecs
+  name: event.ingested
+- external: ecs
+  name: event.kind
+- external: ecs
+  name: event.module
+- external: ecs
+  name: event.original
+- external: ecs
+  name: event.type
+- external: ecs
+  name: service.address
@@ -0,0 +1,39 @@
+- name: ceph
+  type: group
+  fields:
+    - name: pool_disk
+      type: group
+      fields:
+        - name: available.bytes
+          type: long
+          description: Available bytes of the pool.
+          metric_type: gauge
+          unit: byte
+        - name: object.count
+          type: long
+          description: Number of objects of the pool.
+          metric_type: gauge
+        - name: pool_id
+          type: long
+          description: Id of the pool.
+        - name: pool_name
+          type: keyword
+          description: Name of the pool.
+        - name: stored.bytes
+          type: long
+          description: Stored data of the pool.
+          metric_type: gauge
+          unit: byte
+        - name: used
+          type: group
+          fields:
+            - name: bytes
+              type: long
+              description: Used bytes of the pool.
+              metric_type: gauge
+              unit: byte
+            - name: pct
+              type: double
+              description: Used bytes in percentage of the pool.
+              metric_type: gauge
+              unit: percent
@@ -0,0 +1,39 @@
+title: Pool Disk metrics
+type: logs
+streams:
+  - input: httpjson
+    vars:
+      - name: preserve_original_event
+        required: true
+        show_user: true
+        title: Preserve original event
+        description: Preserves a raw copy of the original event, added to the field `event.original`.
+        type: bool
+        multi: false
+        default: false
+      - name: period
+        type: text
+        required: true
+        title: Period
+        default: 10s
+        show_user: true
+        description: Period of fetching metrics, i.e. 1s/1m/1h.
+      - name: tags
+        type: text
+        title: Tags
+        multi: true
+        required: true
+        show_user: false
+        default:
+          - ceph-pool_disk
+          - forwarded
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
+    template_path: httpjson.yml.hbs
+    title: Ceph Pool Disk metrics
+    description: Collect Pool Disk metrics.
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		{"name":"device_health_metrics","id":1,"stats":{"stored":2142673,"objects":4,"kb_used":6336,"bytes_used":6488064,"percent_used":8.4363608038984239e-05,"max_avail":25633155072}}