forked from vectordotdev/vector
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathaws_s3.toml
More file actions
204 lines (176 loc) · 8.12 KB
/
Copy pathaws_s3.toml
File metadata and controls
204 lines (176 loc) · 8.12 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
[sinks.aws_s3]
title = "AWS S3"
beta = true
common = true
delivery_guarantee = "at_least_once"
egress_method = "batching"
function_category = "transmit"
healthcheck = true
input_types = ["log"]
requirements = {}
service_limits_short_link = "aws_s3_service_limits"
service_providers = ["AWS"]
write_to_description = "[Amazon Web Service's S3 service][urls.aws_s3] via the [`PutObject` API endpoint](https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html)"
<%= render("_partials/_aws_env_vars.toml", namespace: "sinks.aws_s3.env_vars") %>
<%= render("_partials/_aws_options.toml", namespace: "sinks.aws_s3.options") %>
<%= render("_partials/_component_options.toml", type: "sink", name: "aws_s3") %>
<%= render("_partials/_batch_options.toml", namespace: "sinks.aws_s3.options", common: true, max_events: nil, max_size: 10490000, timeout_secs: 300) %>
<%= render("_partials/_buffer_options.toml", namespace: "sinks.aws_s3.options") %>
<%= render(
"_partials/_request_options.toml",
namespace: "sinks.aws_s3.options",
common: false,
in_flight_limit: 5,
rate_limit_duration_secs: 1,
rate_limit_num: 5,
retry_attempts: -1,
retry_initial_backoff_secs: 1,
retry_max_duration_secs: 10,
timeout_secs: 30
) %>
[sinks.aws_s3.options.bucket]
type = "string"
common = true
examples = ["my-bucket"]
required = true
description = "The S3 bucket name. Do not include a leading `s3://` or a trailing `/`."
[sinks.aws_s3.options.encoding]
type = "string"
category = "requests"
common = true
required = true
description = """\
The encoding format used to serialize the events before outputting.\
"""
[sinks.aws_s3.options.encoding.enum]
ndjson = "Each event is encoded into JSON and the payload is new line delimited."
text = "Each event is encoded into text via the `message` key and the payload is new line delimited."
[sinks.aws_s3.options.filename_append_uuid]
type = "bool"
category = "Object Names"
default = true
required = true
description = "Whether or not to append a UUID v4 token to the end of the file. This ensures there are no name collisions high volume use cases."
[sinks.aws_s3.options.filename_extension]
type = "string"
category = "Object Names"
default = "log"
required = true
description = "The filename extension to use in the object name."
[sinks.aws_s3.options.compression]
type = "string"
common = true
required = true
description = "The compression mechanism to use."
[sinks.aws_s3.options.compression.enum]
gzip = "GZIP compression"
none = "No compression"
[sinks.aws_s3.options.filename_time_format]
type = "string"
category = "Object Names"
default = "%s"
required = true
description = "The format of the resulting object file name. [`strftime` specifiers][urls.strptime_specifiers] are supported."
[sinks.aws_s3.options.key_prefix]
type = "string"
category = "Object Names"
common = true
default = "date=%F/"
examples = [
"date=%F/",
"date=%F/hour=%H/",
"year=%Y/month=%m/day=%d/",
"application_id={{ application_id }}/date=%F/",
]
partition_key = true
templateable = true
description = "A prefix to apply to all object key names. This should be used to partition your objects, and it's important to end this value with a `/` if you want this to be the root S3 \"folder\"."
[sinks.aws_s3.options.acl]
type = "string"
category = "Object Attributes"
common = false
required = false
description = "Canned ACL to apply to the created objects. For more information, see [Canned ACL][urls.aws_s3_canned_acl]."
[sinks.aws_s3.options.acl.enum]
private = "Owner gets FULL_CONTROL. No one else has access rights (default)."
public-read = "Owner gets FULL_CONTROL. The AllUsers group gets READ access."
public-read-write = "Owner gets FULL_CONTROL. The AllUsers group gets READ and WRITE access. Granting this on a bucket is generally not recommended."
aws-exec-read = "Owner gets FULL_CONTROL. Amazon EC2 gets READ access to GET an Amazon Machine Image (AMI) bundle from Amazon S3."
authenticated-read = "Owner gets FULL_CONTROL. The AuthenticatedUsers group gets READ access."
log-delivery-write = "The LogDelivery group gets WRITE and READ_ACP permissions on the bucket. For more information about logs, see [Amazon S3 Server Access Logging](https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerLogs.html)."
[sinks.aws_s3.options.grant_full_control]
type = "string"
category = "Object Attributes"
common = false
examples = ["79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", "person@email.com", "http://acs.amazonaws.com/groups/global/AllUsers"]
required = false
description = "Gives the named [grantee][urls.aws_s3_grantee] READ, READ_ACP, and WRITE_ACP permissions on the created objects."
[sinks.aws_s3.options.grant_read]
type = "string"
category = "Object Attributes"
common = false
examples = ["79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", "person@email.com", "http://acs.amazonaws.com/groups/global/AllUsers"]
required = false
description = "Allows the named [grantee][urls.aws_s3_grantee] to read the created objects and their metadata."
[sinks.aws_s3.options.grant_read_acp]
type = "string"
category = "Object Attributes"
common = false
examples = ["79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", "person@email.com", "http://acs.amazonaws.com/groups/global/AllUsers"]
required = false
description = "Allows the named [grantee][urls.aws_s3_grantee] to read the created objects' ACL."
[sinks.aws_s3.options.grant_write_acp]
type = "string"
category = "Object Attributes"
common = false
examples = ["79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", "person@email.com", "http://acs.amazonaws.com/groups/global/AllUsers"]
required = false
description = "Allows the named [grantee][urls.aws_s3_grantee] to write the created objects' ACL."
[sinks.aws_s3.options.server_side_encryption]
type = "string"
category = "Object Attributes"
common = false
required = false
examples = [ "AES256", "aws:kms" ]
description = "The server-side encryption algorithm used when storing these objects."
[sinks.aws_s3.options.server_side_encryption.enum]
AES256 = "256-bit Advanced Encryption Standard"
"aws:kms" = "AWS managed key encryption"
[sinks.aws_s3.options.ssekms_key_id]
type = "string"
category = "Object Attributes"
common = false
examples = ["abcd1234"]
required = false
description = """\
If `server_side_encryption` has the value `"aws.kms"`, this specifies \
the ID of the AWS Key Management Service (AWS KMS) symmetrical customer \
managed customer master key (CMK) that will used for the created \
objects. If not specified, Amazon S3 uses the AWS managed CMK in AWS to \
protect the data.\
"""
[sinks.aws_s3.options.storage_class]
type = "string"
category = "Object Attributes"
common = false
required = false
description = "The storage class for the created objects. See [the S3 Storage Classes](https://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html) for more details."
[sinks.aws_s3.options.storage_class.enum]
STANDARD = "The default storage class. If you don't specify the storage class when you upload an object, Amazon S3 assigns the STANDARD storage class."
REDUCED_REDUNDANCY = "Designed for noncritical, reproducible data that can be stored with less redundancy than the STANDARD storage class. AWS recommends that you not use this storage class. The STANDARD storage class is more cost effective. "
INTELLIGENT_TIERING = "Stores objects in two access tiers: one tier that is optimized for frequent access and another lower-cost tier that is optimized for infrequently accessed data."
STANDARD_IA = "Amazon S3 stores the object data redundantly across multiple geographically separated Availability Zones (similar to the STANDARD storage class)."
ONEZONE_IA = "Amazon S3 stores the object data in only one Availability Zone."
GLACIER = "Use for archives where portions of the data might need to be retrieved in minutes."
DEEP_ARCHIVE = "Use for archiving data that rarely needs to be accessed."
[sinks.aws_s3.options.tags]
type = "table"
category = "Object Attributes"
common = false
required = false
description = "The tag-set for the object."
[sinks.aws_s3.options.tags.children."`[tag-name]`"]
type = "string"
required = false
examples = [ {"Tag1" = "Value1"} ]
description = "A custom tag to be added to the created objects."