小菜鸟

java菜鸟号正在起航

0%

配置

发表于 更新于 分类于 阅读次数： Valine：
本文字数： 15k 阅读时长 ≈ 13 分钟

配置

elasticsearch配置有两种配置方式，一种是静态配置，只能在配置文件中进行配置；一种是动态配置，可以通过_cluster/settings进行设置

网络配置

本地网关配置

本地网关是当所有集群重新启动时存储集群状态和分片数据的模块，属于静态配置

1
2
3
4
5
6
7
8
9
"gateway": {
            "recover_after_master_nodes": "0",// 集群中存在多少个主节点才启动恢复过程
            "expected_nodes": "-1", // 集群中的预计节点数量
            "recover_after_data_nodes": "-1", // 集群中存在多少个数据节点才启动恢复过程
            "expected_data_nodes": "-1",// 集群中数据节点预计数量
            "recover_after_time": "0ms", // 节点数满足后，等待多长时间开始恢复过程
            "expected_master_nodes": "-1",// 集群中主节点预计数量
            "recover_after_nodes": "-1" // 集群中存在多少个节点才启动恢复过程
        }

HTTP配置

HTTP配置只能进行静态配置

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
"http": {
            "cors": { // 跨域的配置
                "max-age": "1728000",// 缓存时间
                "allow-origin": "",// 允许的请求源
                "allow-headers": "X-Requested-With,Content-Type,Content-Length",// 允许的请求头
                "allow-credentials": "false",// 是否Acess-Control-Allow-Credentials请求头应该被返回，设为false，则只返回这个请求头
                "allow-methods": "OPTIONS,HEAD,GET,POST,PUT,DELETE",// 允许的请求方法
                "enabled": "false" // 启用或禁用跨源资源共享
            },
            "max_chunk_size": "8kb",
            "compression_level": "3",// 压缩级别
            "max_initial_line_length": "4kb",
            "type": "security4",
            "pipelining": "true",//启用或禁用HTTP管道
            "enabled": "true",// 禁用http模块，只能进行内部通信而不是HTTP接口，可以在数据节点禁用HTTP
            "type.default": "netty4",
            "content_type": {
                "required": "true"
            },
            "host": [], // 用于设置bind_host和publish_host
            "publish_port": "-1",// 与此节点通信时，HTTP客户端使用的端口，默认是http.port
            "read_timeout": "0ms",
            "max_content_length": "100mb",// 请求的最大内容
            "netty": {
                "receive_predictor_size": "64kb",
                "max_composite_buffer_components": "69905",
                "receive_predictor_max": "64kb",
                "worker_count": "64",
                "receive_predictor_min": "64kb"
            },
            "tcp": {
                "reuse_address": "true",
                "keep_alive": "true",
                "receive_buffer_size": "-1b",
                "no_delay": "true",
                "send_buffer_size": "-1b"
            },
            "bind_host": [],//绑定主机地址,默认是http.host或者network.bind_host
            "reset_cookies": "false",
            "max_warning_header_count": "-1",
            "max_warning_header_size": "-1b",
            "detailed_errors": { // 启用或禁用响应输出中的详细错误信息和堆栈跟踪输出
                "enabled": "true"
            },
            "port": "9200-9300", //绑定端口
            "max_header_size": "8kb",// 请求头的最大长度
            "pipelining.max_events": "10000",// 管道中排队事件的最大数量
            "tcp_no_delay": "true",
            "compression": "true", // 是否支持压缩
            "publish_host": []// 给客户端用来连接的主机地址，默认是http.host或者network.publish_host
        }

网络配置

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
"network": {
            "host": [
                "0.0.0.0"
            ], // 节点会绑定这个主机名或ip并且发布到集群中的其他节点
            "tcp": {
                "reuse_address": "true",// 地址是否被重用
                "keep_alive": "true",//TCP长连接设置
                "connect_timeout": "30s",
                "receive_buffer_size": "-1b",//接收缓存的大小
                "no_delay": "true",// TCP延迟设置
                "send_buffer_size": "-1b"// 发送缓存的大小
            },
            "bind_host": [
                "0.0.0.0"
            ],// 指定节点应该绑定的网络接口来监听请求
            "server": "true",
            "breaker": {
                "inflight_requests": {
                    "limit": "100%",
                    "overhead": "1.0"
                }
            },
            "publish_host": [ // 发布主机是节点发布到集群中其他节点的单独接口，其他节点可以单独连接到本节点
                "0.0.0.0"
            ]
        }

节点间的发现机制

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
"discovery": {
            "type": "zen",
            "zen": {
                "commit_timeout": "30s",
                "no_master_block": "write",
                "join_retry_delay": "100ms",
                "join_retry_attempts": "3",
                "ping": {
                    "unicast": { // 单播，使用单播的话需要关闭组播
                        "concurrent_connects": "10", // 单播发现使用的最大并发连接数，默认10个，如果初始连接阶段有大量的节点需要连接，需要调高该值
                        "hosts": [ // 集群初始节点列表，为了加入一个集群，节点需要知道集群中其他的节点的主机名或ip。节点列表不必包含集群中全部的节点，因为一旦节点连接上了列表中的任意节点，就会被告知集群中所有节点的信息
                            "0.0.0.0"
                        ],
                        "hosts.resolve_timeout": "5s"
                    },
                      "multicast":{ // 组播
                          "address":   //用来通信的网络接口，默认是所有可用的网络接口
                          "port":  // 用来通信的端口，默认是54328
                          "group":  // 组播消息需要发送到的地址，默认224.2.2.4
                          "buffer_size":  // 组播消息使用的缓冲区大小，默认是2028
                          "ttl":  // 组播消息的生存时间默认值是3，消息包每次通过一个路由器，TTS就减一
                          "enabled":  // 是否开启组播，默认值是true
                    }
                },
                "master_election": {
                    "ignore_non_master_pings": "false",
                    "wait_for_joins_timeout": "30000ms"
                },
                "send_leave_request": "true",
                "ping_timeout": "3s",
                "join_timeout": "60000ms",
                "publish_diff": {
                    "enable": "true"
                },
                "publish": {
                    "max_pending_cluster_states": "25"
                },
                "minimum_master_nodes": "-1",
                "hosts_provider": [],
                "publish_timeout": "30s",
                "fd": {
                    "connect_on_network_disconnect": "false",
                    "ping_interval": "1s",
                    "ping_retries": "3",
                    "register_connection_listener": "true",
                    "ping_timeout": "30s"
                },
                "max_pings_from_another_master": "3"
            },
            "initial_state_timeout": "30s"
        }

节点间的传输通信，用于内部通信，每个请求使用传输模块从一个节点发送到另一个节点

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
"transport": {
            "tcp": {
                "reuse_address": "true",
                "connect_timeout": "30s",// 端口连接超时设置
                "compress": "false",// 是否启用压缩
                "port": "9300-9400",// 绑定节点间通信的端口，如果是范围，会绑定第一个可用的端口
                "no_delay": "true",
                "keep_alive": "true",
                "receive_buffer_size": "-1b",
                "send_buffer_size": "-1b"
            },
            "bind_host": [],// 传输服务绑定的主机地址，默认使用transport.port或者network.host
            "connect_timeout": "30s",
            "compress": "false",
            "ping_schedule": "-1",// 使用ping来确保连接通畅，该设置是多长时间ping一次，-1表示禁用
            "connections_per_node": {
                "recovery": "2",
                "state": "1",
                "bulk": "3",
                "reg": "6",
                "ping": "1"
            },
            "tracer": {
                "include": [],
                "exclude": [
                    "internal:discovery/zen/fd*",
                    "cluster:monitor/nodes/liveness"
                ]
            },
            "type": "security4",
            "type.default": "netty4",
            "features": {
                "x-pack": "true"
            },
            "port": "9300-9400",
            "host": [],
            "publish_port": "-1",// 集群中其他节点与本节点通信使用的端口
            "tcp_no_delay": "true",
            "publish_host": [],
            "netty": {
                "receive_predictor_size": "64kb",
                "receive_predictor_max": "64kb",
                "worker_count": "64",
                "receive_predictor_min": "64kb",
                "boss_count": "1"
            }
        }

脚本配置

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
"script": {
            "allowed_contexts": [],
            "max_compilations_rate": "75/5m",
            "cache": {
                "max_size": "100",
                "expire": "0ms"
            },
            "painless": {
                "regex": {
                    "enabled": "false"
                }
            },
            "max_size_in_bytes": "65535",
            "allowed_types": []
        }

快照和备份配置

可以通过快照将索引或整个集群存储起来，在使用时需要县创建快照

操作

创建快照

支持的参数有

  • location 快照位置
  • compress 压缩
  • chunk_size 如果需要把大的文件分解成不同的快照，按照大小指定字节1GB, 10MB, 5KB, 500B，默认是null，没有限制
  • max_restore_bytes_per_sec 每个节点的恢复速度，默认40M每秒
  • max_snapshot_bytes_per_sec 每个节点生成的快照速度，默认40M每秒
  • readonly 是否只读
1
2
3
4
5
6
7
PUT /_snapshot/my_backup
{
  "type": "fs",
  "settings": {
    "location": "my_backup_location"
  }
}
查询快照
1
2
GET /_snapshot/my_backup

恢复

默认情况下，快照中的所有索引以及集群状态都会被恢复，也支持恢复部分索引

1
2
3
4
5
6
7
8
9
POST /_snapshot/my_backup/snapshot_1/_restore

{
  "indices": "index_1,index_2",
  "ignore_unavailable": true,
  "include_global_state": true,
  "rename_pattern": "index_(.+)",
  "rename_replacement": "restored_index_$1"
}

线程池配置

elasticsearch下设置有很多线程组，使用的类型也不同，包括

  • cached 无限的线程，线程无用之后默认会在5分钟之后被销毁，keep_alive决定空闲时间
  • fixed 固定大小的线程，size为数量，queue_size为队列的数量
  • scaling 动态数量的线程，在1和size之间变化
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
"thread_pool": {
            "force_merge": {
                "queue_size": "-1",
                "size": "1"
            },
            "fetch_shard_started": {
                "core": "1",
                "max": "64",
                "keep_alive": "5m"
            },
            "listener": {// 用于客户端执行时的监听线程，类型时scaling
                "queue_size": "-1",
                "size": "10"
            },
            "index": {// 索引操作，类型是fixed
                "queue_size": "200",
                "size": "32"
            },
            "refresh": {// 刷新操作，类型是scaling
                "core": "1",
                "max": "10",
                "keep_alive": "5m"
            },
            "generic": { // 通用操作，类型是cached
                "core": "4",
                "max": "128",
                "keep_alive": "30s"
            },
            "warmer": { // 索引预加载，类型是scaling
                "core": "1",
                "max": "5",
                "keep_alive": "5m"
            },
            "search": {// 搜索操作，类型是fixed
                "max_queue_size": "1000",
                "queue_size": "1000",
                "size": "49",
                "auto_queue_frame_size": "2000",
                "target_response_time": "1s",
                "min_queue_size": "1000"
            },
            "fetch_shard_store": {
                "core": "1",
                "max": "64",
                "keep_alive": "5m"
            },
            "flush": {
                "core": "1",
                "max": "5",
                "keep_alive": "5m"
            },
            "management": {
                "core": "1",
                "max": "5",
                "keep_alive": "5m"
            },
            "analyze": {
                "queue_size": "16",
                "size": "1"
            },
            "get": { // 获取操作，类型是fixed
                "queue_size": "1000",
                "size": "32"
            },
            "bulk": {// 批量操作，类型是fxed
                "queue_size": "200",
                "size": "32"
            },
            "estimated_time_interval": "200ms",
            "write": {
                "queue_size": "200",
                "size": "32"
            },
            "snapshot": { // 快照操作，类型是scaling
                "core": "1",
                "max": "5",
                "keep_alive": "5m"
            },
            "search_throttled": {
                "max_queue_size": "100",
                "queue_size": "100",
                "size": "1",
                "auto_queue_frame_size": "200",
                "target_response_time": "1s",
                "min_queue_size": "100"
            }
        }

索引配置

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
"indices": {
            "cache": {
                "cleanup_interval": "1m"
            },
            "mapping": {
                "dynamic_timeout": "30s"
            },
            "memory": {// 索引缓冲区
                "interval": "5s",
                "max_index_buffer_size": "-1",
                "shard_inactive_time": "5m",
                "index_buffer_size": "10%",// 一个节点索引缓冲区的大小
                "min_index_buffer_size": "48mb"
            },
            "breaker": {
                "request": { // 请求内存控制
                    "limit": "60%", // 请求内存默认是JVM堆内存的60%
                    "type": "memory",
                    "overhead": "1.0"
                },
                "total": { // 总内存使用大小，默认为JVM堆内存的70%
                    "limit": "70%"
                },
                "accounting": {
                    "limit": "100%",
                    "overhead": "1.0"
                },
                "fielddata": { // 列内存控制
                    "limit": "60%", // 列数据内存大小限制，默认为JVM堆内存的60%
                    "type": "memory",
                    "overhead": "1.03"// 所有列估计内存大小的乘积，默认为1.03
                },
                "type": "hierarchy"
            },
            "query": {
                "bool": {
                    "max_clause_count": "1024"
                },
                "query_string": {
                    "analyze_wildcard": "false",
                    "allowLeadingWildcard": "true"
                }
            },
            "admin": {
                "filtered_fields": "true"
            },
            "recovery": { // 索引恢复
                "recovery_activity_timeout": "1800000ms",
                "retry_delay_network": "5s",
                "internal_action_timeout": "15m",
                "retry_delay_state_sync": "500ms",
                "internal_action_long_timeout": "1800000ms",
                "max_bytes_per_sec": "40mb",
                "max_concurrent_file_chunks": "1"
            },
            "requests": {
                "cache": { // 分片请求缓存，在进行请求时，每个分片都是进行它自己内容的搜索，然后把结果返回到协调节点，在进行合并，只缓存hits.total、aggregations、suggestions，不缓存原始数据
                    "size": "1%",
                    "expire": "0ms"
                }
            },
            "store": {
                "delete": {
                    "shard": {
                        "timeout": "30s"
                    }
                }
            },
            "analysis": {
                "hunspell": {
                    "dictionary": {
                        "ignore_case": "false",
                        "lazy": "false"
                    }
                }
            },
            "queries": {
                "cache": { // 查询缓存
                    "count": "10000",
                    "size": "10%",
                    "all_segments": "false"
                }
            },
            "lifecycle": {
                "poll_interval": "10m"
            },
            "fielddata": {
                "cache": {
                    "size": "-1b" // 数据缓存大小
                }
            }
        }
相关文章

欢迎关注我的其它发布渠道