[add_cloud_metadata] Warnings in logs for "error fetching EC2 Tags"

[andrewkroh]

The add_cloud_metadata is logging warnings about failure to fetch EC2 tags. There is nothing in the add_cloud_metadata docs about a requirement to give Beats ec2:DescribeTags permissions.

Users find these warnings when looking for other problems, and often think they are the cause of their issue. This is very similar to errors the processor emits during cloud detection (#38679). If this is a nice-to-have enrichment, I would suggest moving the log message to the debug level.

The docs need to state what AWS permissions are needed on the node for the processor to operate cleanly without warnings and errors."

error fetching cluster name metadata: error fetching EC2 Tags: operation error EC2: DescribeTags, https response error StatusCode: 403

Full JSON log

{
  "agent": {
    "name": "cloudflare-logpush-b710-6884d849df-644g2",
    "id": "07cc0788-853d-4e25-8085-34146f520bab",
    "type": "filebeat",
    "ephemeral_id": "d427248b-f459-4d01-a2f2-294748bb15da",
    "version": "8.15.1"
  },
  "service.name": "filebeat",
  "log": {
    "file": {
      "inode": "34603797",
      "path": "/usr/share/elastic-agent/state/data/logs/elastic-agent-20240916.ndjson",
      "device_id": "66305"
    },
    "offset": 29346,
    "source": "aws-s3-default"
  },
  "elastic_agent": {
    "id": "07cc0788-853d-4e25-8085-34146f520bab",
    "version": "8.15.1",
    "snapshot": false
  },
  "message": "error fetching cluster name metadata: error fetching EC2 Tags: operation error EC2: DescribeTags, https response error StatusCode: 403, RequestID: e61d17b1-02be-40e1-b653-7d6ebef7706a, api error UnauthorizedOperation: You are not authorized to perform this operation. User: arn:aws:sts::144492464627:assumed-role/ElasticSITK8sAgentCloudflareLogpush-20240916193919497100000002/1726518813847340601 is not authorized to perform: ec2:DescribeTags because no identity-based policy allows the ec2:DescribeTags action.",
  "log.logger": "add_cloud_metadata",
  "cloud": {
    "image": {
      "id": "ami-06868002018b8d7a7"
    },
    "availability_zone": "us-east-2a",
    "instance": {
      "id": "i-xxxxx"
    },
    "provider": "aws",
    "service": {
      "name": "EC2"
    },
    "machine": {
      "type": "t3.medium"
    },
    "region": "us-east-2",
    "account": {
      "id": "xxxxx"
    }
  },
  "input": {
    "type": "filestream"
  },
  "log.origin": {
    "file.line": 108,
    "function": "github.com/elastic/beats/v7/libbeat/processors/add_cloud_metadata.fetchRawProviderMetadata",
    "file.name": "add_cloud_metadata/provider_aws_ec2.go"
  },
  "component": {
    "binary": "filebeat",
    "id": "aws-s3-default",
    "type": "aws-s3",
    "dataset": "elastic_agent.filebeat"
  },
  "@timestamp": "2024-09-16T20:33:34.076Z",
  "ecs": {
    "version": "8.0.0"
  },
  "data_stream": {
    "namespace": "default",
    "type": "logs",
    "dataset": "elastic_agent.filebeat"
  },
  "host": {
    "hostname": "cloudflare-logpush-b710-6884d849df-644g2",
    "os": {
      "kernel": "5.10.213-201.855.amzn2.x86_64",
      "codename": "focal",
      "name": "Ubuntu",
      "family": "debian",
      "type": "linux",
      "version": "20.04.6 LTS (Focal Fossa)",
      "platform": "ubuntu"
    },
    "containerized": true,
    "ip": [
      "10.90.0.145",
      "fe80::7c63:8aff:fe0a:7694"
    ],
    "name": "cloudflare-logpush-b710-6884d849df-644g2",
    "id": "ec20de1d65b343847dc5e1f56410bb5f",
    "mac": [
      "7E-63-8A-0A-76-94"
    ],
    "architecture": "x86_64"
  },
  "log.level": "warn",
  "event": {
    "agent_id_status": "verified",
    "ingested": "2024-09-16T20:33:46Z",
    "dataset": "elastic_agent.filebeat"
  }
}