Flink 1.19.1 standalone 集群模式摆设及设置

flink 1.19起 conf/flink-conf.yaml 更改为新的 conf/config.yaml

standalone集群: dev001、dev002、dev003
config.yaml: jobmanager address 统一使用 dev001，bind-port 统一改成 0.0.0.0，taskmanager address 分别更改为dev地点host

1. dev001 config.yaml:
2. ################################################################################
3. #  Licensed to the Apache Software Foundation (ASF) under one
4. #  or more contributor license agreements.  See the NOTICE file
5. #  distributed with this work for additional information
6. #  regarding copyright ownership.  The ASF licenses this file
7. #  to you under the Apache License, Version 2.0 (the
8. #  "License"); you may not use this file except in compliance
9. #  with the License.  You may obtain a copy of the License at
10. #
11. #      http://www.apache.org/licenses/LICENSE-2.0
12. #
13. #  Unless required by applicable law or agreed to in writing, software
14. #  distributed under the License is distributed on an "AS IS" BASIS,
15. #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16. #  See the License for the specific language governing permissions and
17. #  limitations under the License.
18. ################################################################################
20. # These parameters are required for Java 17 support.
21. # They can be safely removed when using Java 8/11.
22. env:
23.   java:
24.     opts:
25.       all: --add-exports=java.base/sun.net.util=ALL-UNNAMED --add-exports=java.rmi/sun.rmi.registry=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.text=ALL-UNNAMED --add-opens=java.base/java.time=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED
27. #==============================================================================
28. # Common
29. #==============================================================================
31. jobmanager:
32.   # The host interface the JobManager will bind to. By default, this is localhost, and will prevent
33.   # the JobManager from communicating outside the machine/container it is running on.
34.   # On YARN this setting will be ignored if it is set to 'localhost', defaulting to 0.0.0.0.
35.   # On Kubernetes this setting will be ignored, defaulting to 0.0.0.0.
36.   #
37.   # To enable this, set the bind-host address to one that has access to an outside facing network
38.   # interface, such as 0.0.0.0.
39.   bind-host: 0.0.0.0
40.   rpc:
41.     # The external address of the host on which the JobManager runs and can be
42.     # reached by the TaskManagers and any clients which want to connect. This setting
43.     # is only used in Standalone mode and may be overwritten on the JobManager side
44.     # by specifying the --host <hostname> parameter of the bin/jobmanager.sh executable.
45.     # In high availability mode, if you use the bin/start-cluster.sh script and setup
46.     # the conf/masters file, this will be taken care of automatically. Yarn
47.     # automatically configure the host name based on the hostname of the node where the
48.     # JobManager runs.
49.     address: dev001
50.     # The RPC port where the JobManager is reachable.
51.     port: 6123
52.   memory:
53.     process:
54.       # The total process memory size for the JobManager.
55.       # Note this accounts for all memory usage within the JobManager process, including JVM metaspace and other overhead.
56.       size: 1600m
57.   execution:
58.     # The failover strategy, i.e., how the job computation recovers from task failures.
59.     # Only restart tasks that may have been affected by the task failure, which typically includes
60.     # downstream tasks and potentially upstream tasks if their produced data is no longer available for consumption.
61.     failover-strategy: region
63. taskmanager:
64.   # The host interface the TaskManager will bind to. By default, this is localhost, and will prevent
65.   # the TaskManager from communicating outside the machine/container it is running on.
66.   # On YARN this setting will be ignored if it is set to 'localhost', defaulting to 0.0.0.0.
67.   # On Kubernetes this setting will be ignored, defaulting to 0.0.0.0.
68.   #
69.   # To enable this, set the bind-host address to one that has access to an outside facing network
70.   # interface, such as 0.0.0.0.
71.   bind-host: 0.0.0.0
72.   # The address of the host on which the TaskManager runs and can be reached by the JobManager and
73.   # other TaskManagers. If not specified, the TaskManager will try different strategies to identify
74.   # the address.
75.   #
76.   # Note this address needs to be reachable by the JobManager and forward traffic to one of
77.   # the interfaces the TaskManager is bound to (see 'taskmanager.bind-host').
78.   #
79.   # Note also that unless all TaskManagers are running on the same machine, this address needs to be
80.   # configured separately for each TaskManager.
81.   host: dev001
82.   # The number of task slots that each TaskManager offers. Each slot runs one parallel pipeline.
83.   numberOfTaskSlots: 2
84.   memory:
85.     process:
86.       # The total process memory size for the TaskManager.
87.       #
88.       # Note this accounts for all memory usage within the TaskManager process, including JVM metaspace and other overhead.
89.       # To exclude JVM metaspace and overhead, please, use total Flink memory size instead of 'taskmanager.memory.process.size'.
90.       # It is not recommended to set both 'taskmanager.memory.process.size' and Flink memory.
91.       size: 1728m
93. parallelism:
94.   # The parallelism used for programs that did not specify and other parallelism.
95.   default: 1
97. # # The default file system scheme and authority.
98. # # By default file paths without scheme are interpreted relative to the local
99. # # root file system 'file:///'. Use this to override the default and interpret
100. # # relative paths relative to a different file system,
101. # # for example 'hdfs://mynamenode:12345'
102. # fs:
103. #   default-scheme: hdfs://mynamenode:12345
105. #==============================================================================
106. # High Availability
107. #==============================================================================
109. # high-availability:
110. #   # The high-availability mode. Possible options are 'NONE' or 'zookeeper'.
111. #   type: zookeeper
112. #   # The path where metadata for master recovery is persisted. While ZooKeeper stores
113. #   # the small ground truth for checkpoint and leader election, this location stores
114. #   # the larger objects, like persisted dataflow graphs.
115. #   #
116. #   # Must be a durable file system that is accessible from all nodes
117. #   # (like HDFS, S3, Ceph, nfs, ...)
118. #   storageDir: hdfs:///flink/ha/
119. #   zookeeper:
120. #     # The list of ZooKeeper quorum peers that coordinate the high-availability
121. #     # setup. This must be a list of the form:
122. #     # "host1:clientPort,host2:clientPort,..." (default clientPort: 2181)
123. #     quorum: localhost:2181
124. #     client:
125. #       # ACL options are based on https://zookeeper.apache.org/doc/r3.1.2/zookeeperProgrammers.html#sc_BuiltinACLSchemes
126. #       # It can be either "creator" (ZOO_CREATE_ALL_ACL) or "open" (ZOO_OPEN_ACL_UNSAFE)
127. #       # The default value is "open" and it can be changed to "creator" if ZK security is enabled
128. #       acl: open
130. #==============================================================================
131. # Fault tolerance and checkpointing
132. #==============================================================================
134. # The backend that will be used to store operator state checkpoints if
135. # checkpointing is enabled. Checkpointing is enabled when execution.checkpointing.interval > 0.
137. # # Execution checkpointing related parameters. Please refer to CheckpointConfig and ExecutionCheckpointingOptions for more details.
138. # execution:
139. #   checkpointing:
140. #     interval: 3min
141. #     externalized-checkpoint-retention: [DELETE_ON_CANCELLATION, RETAIN_ON_CANCELLATION]
142. #     max-concurrent-checkpoints: 1
143. #     min-pause: 0
144. #     mode: [EXACTLY_ONCE, AT_LEAST_ONCE]
145. #     timeout: 10min
146. #     tolerable-failed-checkpoints: 0
147. #     unaligned: false
149. # state:
150. #   backend:
151. #     # Supported backends are 'hashmap', 'rocksdb', or the
152. #     # <class-name-of-factory>.
153. #     type: hashmap
154. #     # Flag to enable/disable incremental checkpoints for backends that
155. #     # support incremental checkpoints (like the RocksDB state backend).
156. #     incremental: false
157. #   checkpoints:
158. #       # Directory for checkpoints filesystem, when using any of the default bundled
159. #       # state backends.
160. #       dir: hdfs://namenode-host:port/flink-checkpoints
161. #   savepoints:
162. #       # Default target directory for savepoints, optional.
163. #       dir: hdfs://namenode-host:port/flink-savepoints
165. #==============================================================================
166. # Rest & web frontend
167. #==============================================================================
169. rest:
170.   # The address to which the REST client will connect to
171.   address: dev001
172.   # The address that the REST & web server binds to
173.   # By default, this is localhost, which prevents the REST & web server from
174.   # being able to communicate outside of the machine/container it is running on.
175.   #
176.   # To enable this, set the bind address to one that has access to outside-facing
177.   # network interface, such as 0.0.0.0.
178.   bind-address: 0.0.0.0
179.   # # The port to which the REST client connects to. If rest.bind-port has
180.   # # not been specified, then the server will bind to this port as well.
181.   # port: 8081
182.   # # Port range for the REST and web server to bind to.
183.   # bind-port: 8080-8090
185. # web:
186. #   submit:
187. #     # Flag to specify whether job submission is enabled from the web-based
188. #     # runtime monitor. Uncomment to disable.
189. #     enable: false
190. #   cancel:
191. #     # Flag to specify whether job cancellation is enabled from the web-based
192. #     # runtime monitor. Uncomment to disable.
193. #     enable: false
195. #==============================================================================
196. # Advanced
197. #==============================================================================
199. # io:
200. #   tmp:
201. #     # Override the directories for temporary files. If not specified, the
202. #     # system-specific Java temporary directory (java.io.tmpdir property) is taken.
203. #     #
204. #     # For framework setups on Yarn, Flink will automatically pick up the
205. #     # containers' temp directories without any need for configuration.
206. #     #
207. #     # Add a delimited list for multiple directories, using the system directory
208. #     # delimiter (colon ':' on unix) or a comma, e.g.:
209. #     # /data1/tmp:/data2/tmp:/data3/tmp
210. #     #
211. #     # Note: Each directory entry is read from and written to by a different I/O
212. #     # thread. You can include the same directory multiple times in order to create
213. #     # multiple I/O threads against that directory. This is for example relevant for
214. #     # high-throughput RAIDs.
215. #     dirs: /tmp
217. # classloader:
218. #   resolve:
219. #     # The classloading resolve order. Possible values are 'child-first' (Flink's default)
220. #     # and 'parent-first' (Java's default).
221. #     #
222. #     # Child first classloading allows users to use different dependency/library
223. #     # versions in their application than those in the classpath. Switching back
224. #     # to 'parent-first' may help with debugging dependency issues.
225. #     order: child-first
227. # The amount of memory going to the network stack. These numbers usually need
228. # no tuning. Adjusting them may be necessary in case of an "Insufficient number
229. # of network buffers" error. The default min is 64MB, the default max is 1GB.
230. #
231. # taskmanager:
232. #   memory:
233. #     network:
234. #       fraction: 0.1
235. #       min: 64mb
236. #       max: 1gb
238. #==============================================================================
239. # Flink Cluster Security Configuration
240. #==============================================================================
242. # Kerberos authentication for various components - Hadoop, ZooKeeper, and connectors -
243. # may be enabled in four steps:
244. # 1. configure the local krb5.conf file
245. # 2. provide Kerberos credentials (either a keytab or a ticket cache w/ kinit)
246. # 3. make the credentials available to various JAAS login contexts
247. # 4. configure the connector to use JAAS/SASL
249. # # The below configure how Kerberos credentials are provided. A keytab will be used instead of
250. # # a ticket cache if the keytab path and principal are set.
251. # security:
252. #   kerberos:
253. #     login:
254. #       use-ticket-cache: true
255. #       keytab: /path/to/kerberos/keytab
256. #       principal: flink-user
257. #       # The configuration below defines which JAAS login contexts
258. #       contexts: Client,KafkaClient
260. #==============================================================================
261. # ZK Security Configuration
262. #==============================================================================
264. # zookeeper:
265. #   sasl:
266. #     # Below configurations are applicable if ZK ensemble is configured for security
267. #     #
268. #     # Override below configuration to provide custom ZK service name if configured
269. #     # zookeeper.sasl.service-name: zookeeper
270. #     #
271. #     # The configuration below must match one of the values set in "security.kerberos.login.contexts"
272. #     login-context-name: Client
274. #==============================================================================
275. # HistoryServer
276. #==============================================================================
278. # The HistoryServer is started and stopped via bin/historyserver.sh (start|stop)
279. #
280. # jobmanager:
281. #   archive:
282. #     fs:
283. #       # Directory to upload completed jobs to. Add this directory to the list of
284. #       # monitored directories of the HistoryServer as well (see below).
285. #       dir: hdfs:///completed-jobs/
287. # historyserver:
288. #   web:
289. #     # The address under which the web-based HistoryServer listens.
290. #     address: 0.0.0.0
291. #     # The port under which the web-based HistoryServer listens.
292. #     port: 8082
293. #   archive:
294. #     fs:
295. #       # Comma separated list of directories to monitor for completed jobs.
296. #       dir: hdfs:///completed-jobs/
297. #       # Interval in milliseconds for refreshing the monitored directories.
298. #       fs.refresh-interval: 10000
301. dev002 config.yaml:
302. ################################################################################
303. #  Licensed to the Apache Software Foundation (ASF) under one
304. #  or more contributor license agreements.  See the NOTICE file
305. #  distributed with this work for additional information
306. #  regarding copyright ownership.  The ASF licenses this file
307. #  to you under the Apache License, Version 2.0 (the
308. #  "License"); you may not use this file except in compliance
309. #  with the License.  You may obtain a copy of the License at
310. #
311. #      http://www.apache.org/licenses/LICENSE-2.0
312. #
313. #  Unless required by applicable law or agreed to in writing, software
314. #  distributed under the License is distributed on an "AS IS" BASIS,
315. #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
316. #  See the License for the specific language governing permissions and
317. #  limitations under the License.
318. ################################################################################
320. # These parameters are required for Java 17 support.
321. # They can be safely removed when using Java 8/11.
322. env:
323.   java:
324.     opts:
325.       all: --add-exports=java.base/sun.net.util=ALL-UNNAMED --add-exports=java.rmi/sun.rmi.registry=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.text=ALL-UNNAMED --add-opens=java.base/java.time=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED
327. #==============================================================================
328. # Common
329. #==============================================================================
331. jobmanager:
332.   # The host interface the JobManager will bind to. By default, this is localhost, and will prevent
333.   # the JobManager from communicating outside the machine/container it is running on.
334.   # On YARN this setting will be ignored if it is set to 'localhost', defaulting to 0.0.0.0.
335.   # On Kubernetes this setting will be ignored, defaulting to 0.0.0.0.
336.   #
337.   # To enable this, set the bind-host address to one that has access to an outside facing network
338.   # interface, such as 0.0.0.0.
339.   bind-host: 0.0.0.0
340.   rpc:
341.     # The external address of the host on which the JobManager runs and can be
342.     # reached by the TaskManagers and any clients which want to connect. This setting
343.     # is only used in Standalone mode and may be overwritten on the JobManager side
344.     # by specifying the --host <hostname> parameter of the bin/jobmanager.sh executable.
345.     # In high availability mode, if you use the bin/start-cluster.sh script and setup
346.     # the conf/masters file, this will be taken care of automatically. Yarn
347.     # automatically configure the host name based on the hostname of the node where the
348.     # JobManager runs.
349.     address: dev001
350.     # The RPC port where the JobManager is reachable.
351.     port: 6123
352.   memory:
353.     process:
354.       # The total process memory size for the JobManager.
355.       # Note this accounts for all memory usage within the JobManager process, including JVM metaspace and other overhead.
356.       size: 1600m
357.   execution:
358.     # The failover strategy, i.e., how the job computation recovers from task failures.
359.     # Only restart tasks that may have been affected by the task failure, which typically includes
360.     # downstream tasks and potentially upstream tasks if their produced data is no longer available for consumption.
361.     failover-strategy: region
363. taskmanager:
364.   # The host interface the TaskManager will bind to. By default, this is localhost, and will prevent
365.   # the TaskManager from communicating outside the machine/container it is running on.
366.   # On YARN this setting will be ignored if it is set to 'localhost', defaulting to 0.0.0.0.
367.   # On Kubernetes this setting will be ignored, defaulting to 0.0.0.0.
368.   #
369.   # To enable this, set the bind-host address to one that has access to an outside facing network
370.   # interface, such as 0.0.0.0.
371.   bind-host: 0.0.0.0
372.   # The address of the host on which the TaskManager runs and can be reached by the JobManager and
373.   # other TaskManagers. If not specified, the TaskManager will try different strategies to identify
374.   # the address.
375.   #
376.   # Note this address needs to be reachable by the JobManager and forward traffic to one of
377.   # the interfaces the TaskManager is bound to (see 'taskmanager.bind-host').
378.   #
379.   # Note also that unless all TaskManagers are running on the same machine, this address needs to be
380.   # configured separately for each TaskManager.
381.   host: dev002
382.   # The number of task slots that each TaskManager offers. Each slot runs one parallel pipeline.
383.   numberOfTaskSlots: 2
384.   memory:
385.     process:
386.       # The total process memory size for the TaskManager.
387.       #
388.       # Note this accounts for all memory usage within the TaskManager process, including JVM metaspace and other overhead.
389.       # To exclude JVM metaspace and overhead, please, use total Flink memory size instead of 'taskmanager.memory.process.size'.
390.       # It is not recommended to set both 'taskmanager.memory.process.size' and Flink memory.
391.       size: 1728m
393. parallelism:
394.   # The parallelism used for programs that did not specify and other parallelism.
395.   default: 1
397. # # The default file system scheme and authority.
398. # # By default file paths without scheme are interpreted relative to the local
399. # # root file system 'file:///'. Use this to override the default and interpret
400. # # relative paths relative to a different file system,
401. # # for example 'hdfs://mynamenode:12345'
402. # fs:
403. #   default-scheme: hdfs://mynamenode:12345
405. #==============================================================================
406. # High Availability
407. #==============================================================================
409. # high-availability:
410. #   # The high-availability mode. Possible options are 'NONE' or 'zookeeper'.
411. #   type: zookeeper
412. #   # The path where metadata for master recovery is persisted. While ZooKeeper stores
413. #   # the small ground truth for checkpoint and leader election, this location stores
414. #   # the larger objects, like persisted dataflow graphs.
415. #   #
416. #   # Must be a durable file system that is accessible from all nodes
417. #   # (like HDFS, S3, Ceph, nfs, ...)
418. #   storageDir: hdfs:///flink/ha/
419. #   zookeeper:
420. #     # The list of ZooKeeper quorum peers that coordinate the high-availability
421. #     # setup. This must be a list of the form:
422. #     # "host1:clientPort,host2:clientPort,..." (default clientPort: 2181)
423. #     quorum: localhost:2181
424. #     client:
425. #       # ACL options are based on https://zookeeper.apache.org/doc/r3.1.2/zookeeperProgrammers.html#sc_BuiltinACLSchemes
426. #       # It can be either "creator" (ZOO_CREATE_ALL_ACL) or "open" (ZOO_OPEN_ACL_UNSAFE)
427. #       # The default value is "open" and it can be changed to "creator" if ZK security is enabled
428. #       acl: open
430. #==============================================================================
431. # Fault tolerance and checkpointing
432. #==============================================================================
434. # The backend that will be used to store operator state checkpoints if
435. # checkpointing is enabled. Checkpointing is enabled when execution.checkpointing.interval > 0.
437. # # Execution checkpointing related parameters. Please refer to CheckpointConfig and ExecutionCheckpointingOptions for more details.
438. # execution:
439. #   checkpointing:
440. #     interval: 3min
441. #     externalized-checkpoint-retention: [DELETE_ON_CANCELLATION, RETAIN_ON_CANCELLATION]
442. #     max-concurrent-checkpoints: 1
443. #     min-pause: 0
444. #     mode: [EXACTLY_ONCE, AT_LEAST_ONCE]
445. #     timeout: 10min
446. #     tolerable-failed-checkpoints: 0
447. #     unaligned: false
449. # state:
450. #   backend:
451. #     # Supported backends are 'hashmap', 'rocksdb', or the
452. #     # <class-name-of-factory>.
453. #     type: hashmap
454. #     # Flag to enable/disable incremental checkpoints for backends that
455. #     # support incremental checkpoints (like the RocksDB state backend).
456. #     incremental: false
457. #   checkpoints:
458. #       # Directory for checkpoints filesystem, when using any of the default bundled
459. #       # state backends.
460. #       dir: hdfs://namenode-host:port/flink-checkpoints
461. #   savepoints:
462. #       # Default target directory for savepoints, optional.
463. #       dir: hdfs://namenode-host:port/flink-savepoints
465. #==============================================================================
466. # Rest & web frontend
467. #==============================================================================
469. rest:
470.   # The address to which the REST client will connect to
471.   address: dev002
472.   # The address that the REST & web server binds to
473.   # By default, this is localhost, which prevents the REST & web server from
474.   # being able to communicate outside of the machine/container it is running on.
475.   #
476.   # To enable this, set the bind address to one that has access to outside-facing
477.   # network interface, such as 0.0.0.0.
478.   bind-address: 0.0.0.0
479.   # # The port to which the REST client connects to. If rest.bind-port has
480.   # # not been specified, then the server will bind to this port as well.
481.   # port: 8081
482.   # # Port range for the REST and web server to bind to.
483.   # bind-port: 8080-8090
485. # web:
486. #   submit:
487. #     # Flag to specify whether job submission is enabled from the web-based
488. #     # runtime monitor. Uncomment to disable.
489. #     enable: false
490. #   cancel:
491. #     # Flag to specify whether job cancellation is enabled from the web-based
492. #     # runtime monitor. Uncomment to disable.
493. #     enable: false
495. #==============================================================================
496. # Advanced
497. #==============================================================================
499. # io:
500. #   tmp:
501. #     # Override the directories for temporary files. If not specified, the
502. #     # system-specific Java temporary directory (java.io.tmpdir property) is taken.
503. #     #
504. #     # For framework setups on Yarn, Flink will automatically pick up the
505. #     # containers' temp directories without any need for configuration.
506. #     #
507. #     # Add a delimited list for multiple directories, using the system directory
508. #     # delimiter (colon ':' on unix) or a comma, e.g.:
509. #     # /data1/tmp:/data2/tmp:/data3/tmp
510. #     #
511. #     # Note: Each directory entry is read from and written to by a different I/O
512. #     # thread. You can include the same directory multiple times in order to create
513. #     # multiple I/O threads against that directory. This is for example relevant for
514. #     # high-throughput RAIDs.
515. #     dirs: /tmp
517. # classloader:
518. #   resolve:
519. #     # The classloading resolve order. Possible values are 'child-first' (Flink's default)
520. #     # and 'parent-first' (Java's default).
521. #     #
522. #     # Child first classloading allows users to use different dependency/library
523. #     # versions in their application than those in the classpath. Switching back
524. #     # to 'parent-first' may help with debugging dependency issues.
525. #     order: child-first
527. # The amount of memory going to the network stack. These numbers usually need
528. # no tuning. Adjusting them may be necessary in case of an "Insufficient number
529. # of network buffers" error. The default min is 64MB, the default max is 1GB.
530. #
531. # taskmanager:
532. #   memory:
533. #     network:
534. #       fraction: 0.1
535. #       min: 64mb
536. #       max: 1gb
538. #==============================================================================
539. # Flink Cluster Security Configuration
540. #==============================================================================
542. # Kerberos authentication for various components - Hadoop, ZooKeeper, and connectors -
543. # may be enabled in four steps:
544. # 1. configure the local krb5.conf file
545. # 2. provide Kerberos credentials (either a keytab or a ticket cache w/ kinit)
546. # 3. make the credentials available to various JAAS login contexts
547. # 4. configure the connector to use JAAS/SASL
549. # # The below configure how Kerberos credentials are provided. A keytab will be used instead of
550. # # a ticket cache if the keytab path and principal are set.
551. # security:
552. #   kerberos:
553. #     login:
554. #       use-ticket-cache: true
555. #       keytab: /path/to/kerberos/keytab
556. #       principal: flink-user
557. #       # The configuration below defines which JAAS login contexts
558. #       contexts: Client,KafkaClient
560. #==============================================================================
561. # ZK Security Configuration
562. #==============================================================================
564. # zookeeper:
565. #   sasl:
566. #     # Below configurations are applicable if ZK ensemble is configured for security
567. #     #
568. #     # Override below configuration to provide custom ZK service name if configured
569. #     # zookeeper.sasl.service-name: zookeeper
570. #     #
571. #     # The configuration below must match one of the values set in "security.kerberos.login.contexts"
572. #     login-context-name: Client
574. #==============================================================================
575. # HistoryServer
576. #==============================================================================
578. # The HistoryServer is started and stopped via bin/historyserver.sh (start|stop)
579. #
580. # jobmanager:
581. #   archive:
582. #     fs:
583. #       # Directory to upload completed jobs to. Add this directory to the list of
584. #       # monitored directories of the HistoryServer as well (see below).
585. #       dir: hdfs:///completed-jobs/
587. # historyserver:
588. #   web:
589. #     # The address under which the web-based HistoryServer listens.
590. #     address: 0.0.0.0
591. #     # The port under which the web-based HistoryServer listens.
592. #     port: 8082
593. #   archive:
594. #     fs:
595. #       # Comma separated list of directories to monitor for completed jobs.
596. #       dir: hdfs:///completed-jobs/
597. #       # Interval in milliseconds for refreshing the monitored directories.
598. #       fs.refresh-interval: 10000
601. dev003 config.yaml:
602. ################################################################################
603. #  Licensed to the Apache Software Foundation (ASF) under one
604. #  or more contributor license agreements.  See the NOTICE file
605. #  distributed with this work for additional information
606. #  regarding copyright ownership.  The ASF licenses this file
607. #  to you under the Apache License, Version 2.0 (the
608. #  "License"); you may not use this file except in compliance
609. #  with the License.  You may obtain a copy of the License at
610. #
611. #      http://www.apache.org/licenses/LICENSE-2.0
612. #
613. #  Unless required by applicable law or agreed to in writing, software
614. #  distributed under the License is distributed on an "AS IS" BASIS,
615. #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
616. #  See the License for the specific language governing permissions and
617. #  limitations under the License.
618. ################################################################################
620. # These parameters are required for Java 17 support.
621. # They can be safely removed when using Java 8/11.
622. env:
623.   java:
624.     opts:
625.       all: --add-exports=java.base/sun.net.util=ALL-UNNAMED --add-exports=java.rmi/sun.rmi.registry=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.text=ALL-UNNAMED --add-opens=java.base/java.time=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED
627. #==============================================================================
628. # Common
629. #==============================================================================
631. jobmanager:
632.   # The host interface the JobManager will bind to. By default, this is localhost, and will prevent
633.   # the JobManager from communicating outside the machine/container it is running on.
634.   # On YARN this setting will be ignored if it is set to 'localhost', defaulting to 0.0.0.0.
635.   # On Kubernetes this setting will be ignored, defaulting to 0.0.0.0.
636.   #
637.   # To enable this, set the bind-host address to one that has access to an outside facing network
638.   # interface, such as 0.0.0.0.
639.   bind-host: 0.0.0.0
640.   rpc:
641.     # The external address of the host on which the JobManager runs and can be
642.     # reached by the TaskManagers and any clients which want to connect. This setting
643.     # is only used in Standalone mode and may be overwritten on the JobManager side
644.     # by specifying the --host <hostname> parameter of the bin/jobmanager.sh executable.
645.     # In high availability mode, if you use the bin/start-cluster.sh script and setup
646.     # the conf/masters file, this will be taken care of automatically. Yarn
647.     # automatically configure the host name based on the hostname of the node where the
648.     # JobManager runs.
649.     address: dev001
650.     # The RPC port where the JobManager is reachable.
651.     port: 6123
652.   memory:
653.     process:
654.       # The total process memory size for the JobManager.
655.       # Note this accounts for all memory usage within the JobManager process, including JVM metaspace and other overhead.
656.       size: 1600m
657.   execution:
658.     # The failover strategy, i.e., how the job computation recovers from task failures.
659.     # Only restart tasks that may have been affected by the task failure, which typically includes
660.     # downstream tasks and potentially upstream tasks if their produced data is no longer available for consumption.
661.     failover-strategy: region
663. taskmanager:
664.   # The host interface the TaskManager will bind to. By default, this is localhost, and will prevent
665.   # the TaskManager from communicating outside the machine/container it is running on.
666.   # On YARN this setting will be ignored if it is set to 'localhost', defaulting to 0.0.0.0.
667.   # On Kubernetes this setting will be ignored, defaulting to 0.0.0.0.
668.   #
669.   # To enable this, set the bind-host address to one that has access to an outside facing network
670.   # interface, such as 0.0.0.0.
671.   bind-host: 0.0.0.0
672.   # The address of the host on which the TaskManager runs and can be reached by the JobManager and
673.   # other TaskManagers. If not specified, the TaskManager will try different strategies to identify
674.   # the address.
675.   #
676.   # Note this address needs to be reachable by the JobManager and forward traffic to one of
677.   # the interfaces the TaskManager is bound to (see 'taskmanager.bind-host').
678.   #
679.   # Note also that unless all TaskManagers are running on the same machine, this address needs to be
680.   # configured separately for each TaskManager.
681.   host: dev003
682.   # The number of task slots that each TaskManager offers. Each slot runs one parallel pipeline.
683.   numberOfTaskSlots: 2
684.   memory:
685.     process:
686.       # The total process memory size for the TaskManager.
687.       #
688.       # Note this accounts for all memory usage within the TaskManager process, including JVM metaspace and other overhead.
689.       # To exclude JVM metaspace and overhead, please, use total Flink memory size instead of 'taskmanager.memory.process.size'.
690.       # It is not recommended to set both 'taskmanager.memory.process.size' and Flink memory.
691.       size: 1728m
693. parallelism:
694.   # The parallelism used for programs that did not specify and other parallelism.
695.   default: 1
697. # # The default file system scheme and authority.
698. # # By default file paths without scheme are interpreted relative to the local
699. # # root file system 'file:///'. Use this to override the default and interpret
700. # # relative paths relative to a different file system,
701. # # for example 'hdfs://mynamenode:12345'
702. # fs:
703. #   default-scheme: hdfs://mynamenode:12345
705. #==============================================================================
706. # High Availability
707. #==============================================================================
709. # high-availability:
710. #   # The high-availability mode. Possible options are 'NONE' or 'zookeeper'.
711. #   type: zookeeper
712. #   # The path where metadata for master recovery is persisted. While ZooKeeper stores
713. #   # the small ground truth for checkpoint and leader election, this location stores
714. #   # the larger objects, like persisted dataflow graphs.
715. #   #
716. #   # Must be a durable file system that is accessible from all nodes
717. #   # (like HDFS, S3, Ceph, nfs, ...)
718. #   storageDir: hdfs:///flink/ha/
719. #   zookeeper:
720. #     # The list of ZooKeeper quorum peers that coordinate the high-availability
721. #     # setup. This must be a list of the form:
722. #     # "host1:clientPort,host2:clientPort,..." (default clientPort: 2181)
723. #     quorum: localhost:2181
724. #     client:
725. #       # ACL options are based on https://zookeeper.apache.org/doc/r3.1.2/zookeeperProgrammers.html#sc_BuiltinACLSchemes
726. #       # It can be either "creator" (ZOO_CREATE_ALL_ACL) or "open" (ZOO_OPEN_ACL_UNSAFE)
727. #       # The default value is "open" and it can be changed to "creator" if ZK security is enabled
728. #       acl: open
730. #==============================================================================
731. # Fault tolerance and checkpointing
732. #==============================================================================
734. # The backend that will be used to store operator state checkpoints if
735. # checkpointing is enabled. Checkpointing is enabled when execution.checkpointing.interval > 0.
737. # # Execution checkpointing related parameters. Please refer to CheckpointConfig and ExecutionCheckpointingOptions for more details.
738. # execution:
739. #   checkpointing:
740. #     interval: 3min
741. #     externalized-checkpoint-retention: [DELETE_ON_CANCELLATION, RETAIN_ON_CANCELLATION]
742. #     max-concurrent-checkpoints: 1
743. #     min-pause: 0
744. #     mode: [EXACTLY_ONCE, AT_LEAST_ONCE]
745. #     timeout: 10min
746. #     tolerable-failed-checkpoints: 0
747. #     unaligned: false
749. # state:
750. #   backend:
751. #     # Supported backends are 'hashmap', 'rocksdb', or the
752. #     # <class-name-of-factory>.
753. #     type: hashmap
754. #     # Flag to enable/disable incremental checkpoints for backends that
755. #     # support incremental checkpoints (like the RocksDB state backend).
756. #     incremental: false
757. #   checkpoints:
758. #       # Directory for checkpoints filesystem, when using any of the default bundled
759. #       # state backends.
760. #       dir: hdfs://namenode-host:port/flink-checkpoints
761. #   savepoints:
762. #       # Default target directory for savepoints, optional.
763. #       dir: hdfs://namenode-host:port/flink-savepoints
765. #==============================================================================
766. # Rest & web frontend
767. #==============================================================================
769. rest:
770.   # The address to which the REST client will connect to
771.   address: dev003
772.   # The address that the REST & web server binds to
773.   # By default, this is localhost, which prevents the REST & web server from
774.   # being able to communicate outside of the machine/container it is running on.
775.   #
776.   # To enable this, set the bind address to one that has access to outside-facing
777.   # network interface, such as 0.0.0.0.
778.   bind-address: 0.0.0.0
779.   # # The port to which the REST client connects to. If rest.bind-port has
780.   # # not been specified, then the server will bind to this port as well.
781.   # port: 8081
782.   # # Port range for the REST and web server to bind to.
783.   # bind-port: 8080-8090
785. # web:
786. #   submit:
787. #     # Flag to specify whether job submission is enabled from the web-based
788. #     # runtime monitor. Uncomment to disable.
789. #     enable: false
790. #   cancel:
791. #     # Flag to specify whether job cancellation is enabled from the web-based
792. #     # runtime monitor. Uncomment to disable.
793. #     enable: false
795. #==============================================================================
796. # Advanced
797. #==============================================================================
799. # io:
800. #   tmp:
801. #     # Override the directories for temporary files. If not specified, the
802. #     # system-specific Java temporary directory (java.io.tmpdir property) is taken.
803. #     #
804. #     # For framework setups on Yarn, Flink will automatically pick up the
805. #     # containers' temp directories without any need for configuration.
806. #     #
807. #     # Add a delimited list for multiple directories, using the system directory
808. #     # delimiter (colon ':' on unix) or a comma, e.g.:
809. #     # /data1/tmp:/data2/tmp:/data3/tmp
810. #     #
811. #     # Note: Each directory entry is read from and written to by a different I/O
812. #     # thread. You can include the same directory multiple times in order to create
813. #     # multiple I/O threads against that directory. This is for example relevant for
814. #     # high-throughput RAIDs.
815. #     dirs: /tmp
817. # classloader:
818. #   resolve:
819. #     # The classloading resolve order. Possible values are 'child-first' (Flink's default)
820. #     # and 'parent-first' (Java's default).
821. #     #
822. #     # Child first classloading allows users to use different dependency/library
823. #     # versions in their application than those in the classpath. Switching back
824. #     # to 'parent-first' may help with debugging dependency issues.
825. #     order: child-first
827. # The amount of memory going to the network stack. These numbers usually need
828. # no tuning. Adjusting them may be necessary in case of an "Insufficient number
829. # of network buffers" error. The default min is 64MB, the default max is 1GB.
830. #
831. # taskmanager:
832. #   memory:
833. #     network:
834. #       fraction: 0.1
835. #       min: 64mb
836. #       max: 1gb
838. #==============================================================================
839. # Flink Cluster Security Configuration
840. #==============================================================================
842. # Kerberos authentication for various components - Hadoop, ZooKeeper, and connectors -
843. # may be enabled in four steps:
844. # 1. configure the local krb5.conf file
845. # 2. provide Kerberos credentials (either a keytab or a ticket cache w/ kinit)
846. # 3. make the credentials available to various JAAS login contexts
847. # 4. configure the connector to use JAAS/SASL
849. # # The below configure how Kerberos credentials are provided. A keytab will be used instead of
850. # # a ticket cache if the keytab path and principal are set.
851. # security:
852. #   kerberos:
853. #     login:
854. #       use-ticket-cache: true
855. #       keytab: /path/to/kerberos/keytab
856. #       principal: flink-user
857. #       # The configuration below defines which JAAS login contexts
858. #       contexts: Client,KafkaClient
860. #==============================================================================
861. # ZK Security Configuration
862. #==============================================================================
864. # zookeeper:
865. #   sasl:
866. #     # Below configurations are applicable if ZK ensemble is configured for security
867. #     #
868. #     # Override below configuration to provide custom ZK service name if configured
869. #     # zookeeper.sasl.service-name: zookeeper
870. #     #
871. #     # The configuration below must match one of the values set in "security.kerberos.login.contexts"
872. #     login-context-name: Client
874. #==============================================================================
875. # HistoryServer
876. #==============================================================================
878. # The HistoryServer is started and stopped via bin/historyserver.sh (start|stop)
879. #
880. # jobmanager:
881. #   archive:
882. #     fs:
883. #       # Directory to upload completed jobs to. Add this directory to the list of
884. #       # monitored directories of the HistoryServer as well (see below).
885. #       dir: hdfs:///completed-jobs/
887. # historyserver:
888. #   web:
889. #     # The address under which the web-based HistoryServer listens.
890. #     address: 0.0.0.0
891. #     # The port under which the web-based HistoryServer listens.
892. #     port: 8082
893. #   archive:
894. #     fs:
895. #       # Comma separated list of directories to monitor for completed jobs.
896. #       dir: hdfs:///completed-jobs/
897. #       # Interval in milliseconds for refreshing the monitored directories.
898. #       fs.refresh-interval: 10000

复制代码

conf/masters 及 conf/works 使用 xsync 同步分发命令 分发到各机器节点即可：

1. -- masters
2. dev001:8081
4. -- workers
5. dev001
6. dev002
7. dev003

复制代码

增补：
linux 查看port是否被占用：nestat -apn|grep 8081
linux 查看各节点flink使命：jps


免责声明：如果侵犯了您的权益，请联系站长，我们会及时删除侵权内容，谢谢合作！更多信息从访问主页：qidao123.com:ToB企服之家，中国第一个企服评测及商务社交产业平台。