-
Notifications
You must be signed in to change notification settings - Fork 20
Expand file tree
/
Copy pathgateway.yml
More file actions
409 lines (387 loc) 路 11.8 KB
/
Copy pathgateway.yml
File metadata and controls
409 lines (387 loc) 路 11.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
allow_multi_instance: true
#for more config examples, please visit: https://github.com/infinilabs/testing
#the env section used for setup default settings, it can be overwritten by system environments.
#eg: PROD_ES_ENDPOINT=http://192.168.3.185:9200 LOGGING_ES_ENDPOINT=http://192.168.3.185:9201 ./bin/gateway
env: #use $[[env.LOGGING_ES_ENDPOINT]] in config instead
LOGGING_ES_ENDPOINT: http://localhost:9200/
LOGGING_ES_USER: admin
LOGGING_ES_PASS: admin
PROD_ES_ENDPOINT: http://localhost:9200/
PROD_ES_USER: admin
PROD_ES_PASS: admin
GW_BINDING: "0.0.0.0:8000"
API_BINDING: "0.0.0.0:2900"
path.data: data
path.logs: log
path.configs: config # directory of additional gateway configurations
configs.auto_reload: false # set true to auto reload gateway configurations
## modules can be disabled by setting enabled to false
#modules:
# - name: pipeline
# enabled: false
# - name: elasticsearch
# enabled: false
#plugins:
# - name: gateway
# enabled: false
# - name: metrics
# enabled: false
gateway:
# By default, gateway will always set entry.network.reuse_port as true.
# If the host doesn't support SO_REUSEPORT, set `true` to disable this behavior
disable_reuse_port_by_default: false
# Gateway internal stats collecting
stats:
enabled: true
# save stats under path.data
persist: true
# disable stats operations cache
no_buffer: true
# stats operations cache size
buffer_size: 1000
# stats operations cache flush interval
flush_interval_ms: 1000
# Statsd integration
statsd:
enabled: false
host: localhost
port: 8125
namespace: "gateway."
protocol: "udp"
# flush interval
interval_in_seconds: 1
##json logging layout
#log.format: '{"timestamp":"%UTCDateT%UTCTime","level":"%Level","message":"%EscapedMsg","file":"%File:%Line","func":"%FuncShort"}%n'
#system api
api:
enabled: true
network:
binding: $[[env.API_BINDING]]
# tls:
# enabled: true
# skip_insecure_verify: true
security: #basic auth for system api
enabled: false
username: admin
password: $[[keystore.API_PASS]] #./bin/gateway keystore add API_PASS
##elasticsearch servers
elasticsearch:
- name: prod
enabled: true
endpoints:
- $[[env.PROD_ES_ENDPOINT]]
discovery:
enabled: false
basic_auth:
username: $[[env.PROD_ES_USER]]
password: $[[env.PROD_ES_PASS]]
traffic_control.max_bytes_per_node: 1010485760
metadata_cache_enabled: false # Whether to cache the cluster info in memory cache
- name: logging-server
enabled: true
endpoints:
- $[[env.LOGGING_ES_ENDPOINT]]
basic_auth:
username: $[[env.LOGGING_ES_USER]]
password: $[[env.LOGGING_ES_PASS]]
discovery:
enabled: false
entry:
- name: my_es_entry
enabled: true
router: my_router
max_concurrency: 10000
network:
binding: $[[env.GW_BINDING]]
# See `gateway.disable_reuse_port_by_default` for more information.
reuse_port: true
# tls:
# enabled: true #auto generate certs, cert_file and key_file are optional
# cert_file: /data/gateway/cert/elasticsearch.pem
# key_file: /data/gateway/cert/elasticsearch.key
# skip_insecure_verify: false
router:
- name: my_router
default_flow: default_flow
tracing_flow: logging_flow
rules:
- method:
- "*"
pattern:
- "/_bulk"
- "/{any_index}/_bulk"
flow:
- async_bulk_flow
flow:
- name: default_flow
filter:
- elasticsearch:
elasticsearch: prod
max_connection_per_node: 1000
# - http: ##another general option to proxy requests
# schema: "http" #https or http
# max_idle_conn_duration: "900s"
# skip_failure_host: false
# hosts:
# - "localhost:9200"
- name: logging_flow
filter:
- logging:
queue_name: request_logging
max_request_body_size: 4096
max_response_body_size: 4096
# when: #>1s or none-200 requests will be logged
# or:
# - not:
# or:
# - equals:
# _ctx.request.path: "/favicon.ico"
# - in:
# _ctx.response.status: [404,200,201]
# - suffix:
# _ctx.request.path: ".js"
# - suffix:
# _ctx.request.path: ".css"
# - range:
# _ctx.elapsed.gte: 1000
- name: async_bulk_flow
filter:
- bulk_reshuffle:
when:
contains:
_ctx.request.path: /_bulk
elasticsearch: prod
# Options: cluster,node,index,shard
# NOTE: node/shard level requires elasticsearch cluster info
level: node
partition_size: 10 #extra partition within level
#shards: [1,3,5,7,9,11,13] #filter shards to ingest for node or shard level
continue_metadata_missing: true # If true, will continue to execute following processors if cluster info missing (level: node, shard)
fix_null_id: true
- elasticsearch:
elasticsearch: prod
max_connection_per_node: 1000
# - http: #fallback for non-bulk requests
# schema: "http" #https or http
# hosts:
# - localhost:9200
##background jobs
pipeline:
- name: pipeline_logging_merge
auto_start: true
keep_running: true
processor:
- indexing_merge:
input_queue: "logging"
idle_timeout_in_seconds: 1
elasticsearch: "logging-server"
index_name: ".infini_logs"
output_queue:
name: "gateway-pipeline-logs"
label:
tag: "pipeline_logging"
worker_size: 1
bulk_size_in_kb: 1
- name: ingest_pipeline_logging
auto_start: true
keep_running: true
processor:
- bulk_indexing:
bulk:
compress: true
batch_size_in_mb: 1
batch_size_in_docs: 1
consumer:
fetch_max_messages: 100
queues:
type: indexing_merge
tag: "pipeline_logging"
## system logging and metrics
- name: async_messages_merge
auto_start: true
keep_running: true
processor:
- indexing_merge:
input_queue: "bulk_result_messages"
elasticsearch: "logging-server"
index_name: ".infini_async_bulk_results"
output_queue:
name: "bulk_requests"
label:
tag: "bulk_logging"
worker_size: 1
bulk_size_in_mb: 10
- name: metrics_merge
auto_start: true
keep_running: true
processor:
- indexing_merge:
input_queue: "metrics"
elasticsearch: "logging-server"
index_name: ".infini_metrics"
output_queue:
name: "bulk_requests"
label:
tag: "metrics"
worker_size: 1
bulk_size_in_mb: 10
- name: request_logging_merge
auto_start: true
keep_running: true
processor:
- indexing_merge:
input_queue: "request_logging"
elasticsearch: "logging-server"
index_name: ".infini_requests_logging"
output_queue:
name: "bulk_requests"
label:
tag: "request_logging"
worker_size: 1
bulk_size_in_mb: 10
- name: ingest_merged_requests
auto_start: true
keep_running: true
processor:
- bulk_indexing:
num_of_slices: 1 #runtime slicing
bulk:
compress: false
batch_size_in_mb: 10
batch_size_in_docs: 500
#remove_duplicated_newlines: true
invalid_queue: "invalid_request"
response_handle:
bulk_result_message_queue: "system_failure_messages"
max_request_body_size: 10240
max_response_body_size: 10240
save_success_results: false
max_error_details_count: 5
consumer:
fetch_max_messages: 100
queues:
type: indexing_merge
when:
cluster_available: ["logging-server"]
##async way to ingest bulk requests handled by async_bulk_flow
- name: async_ingest_bulk_requests
auto_start: true
keep_running: true
retry_delay_in_ms: 1000
processor:
- bulk_indexing:
max_connection_per_node: 1000
num_of_slices: 1 #runtime slice
max_worker_size: 200
idle_timeout_in_seconds: 10
bulk:
compress: false
batch_size_in_mb: 20
batch_size_in_docs: 5000
invalid_queue: "bulk_invalid_requests"
dead_letter_queue: "bulk_dead_requests"
response_handle:
bulk_result_message_queue: "bulk_result_messages"
max_request_body_size: 1024
max_response_body_size: 1024
save_success_results: true
max_error_details_count: 5
retry_rules:
default: true
retry_429: true
retry_4xx: false
denied:
status: []
keyword:
- illegal_state_exception
consumer:
fetch_max_messages: 100
eof_retry_delay_in_ms: 500
queue_selector:
labels:
type: bulk_reshuffle
##metrics
metrics:
enabled: true
queue: metrics
logging_queue: logging
instance:
enabled: true
network:
enabled: true
summary: true
sockets: true
##diskqueue
disk_queue:
prepare_files_to_read: true
#max_bytes_per_file: 20971520
eof_retry_delay_in_ms: 500
cleanup_files_on_init: false
retention:
max_num_of_local_files: 20 # automatically cleanup consumed files
compress:
segment:
enabled: true
delete_after_compress: true # trigger cleanup after compression.
idle_threshold: 20 # max number of uncompressed consumed files to preserve.
# upload_to_s3: true
# s3:
# server: my_blob_store #name defined in s3 servers
# location: your_location
# bucket: your_bucket_name
##s3 servers
#s3:
# my_blob_store: #name of s3 server
# endpoint: "localhost:9021"
# access_key: "your_access_key"
# access_secret: "your_access_secret"
# token: "your_token"
# skip_insecure_verify: true
# ssl: true
## badger kv storage configuration
badger:
enabled: true
single_bucket_mode: true
path: '' # defaults to {path.data}/gateway/node/{nodeID}/badger/
memory_mode: false # don't persist data to disk
sync_writes: false # flush to disk on every write
mem_table_size: 10485760
num_mem_tables: 1
# lsm tuning options
value_log_max_entries: 1000000
value_log_file_size: 536870912
value_threshold: 1048576
num_level0_tables: 1
num_level0_tables_stall: 2
## floating ip configuration
floating_ip:
enabled: false # enable floating ip, requires root privilege
netmask: 255.255.255.0
ip: '' # if empty, will automatically bind to x.x.x.234
interface: '' # if empty, will automatically find an interface to bind
local_ip: '' # if empty, will automatically bind to interface's ip address
priority: 0 # if <= 0, will random set current node's priority. master will switch to standby if other nodes has higher priority
echo: # gateway will automatically ping master's echo.port to check whether it's still alive
port: 61111
dial_timeout_in_ms: 1000
timeout_in_ms: 5000
broadcast:
binding: 224.3.2.2:7654 # broadcast address for floating ip status broadcast
## elasticsearch module global configurations
elastic:
# elasticsearch for gateway's system info storage
elasticsearch: prod
enabled: true
remote_configs: false
health_check:
enabled: true
interval: 30s
availability_check:
enabled: true
interval: 30s
metadata_refresh:
enabled: true
interval: 60s
cluster_settings_check:
enabled: false
interval: 60s