Skip to content

Commit e2f88fc

Browse files
JTakyJTaky
committed
[core] Add a ZooKeeper-backed catalog lock
1 parent 5209d3a commit e2f88fc

11 files changed

Lines changed: 1349 additions & 0 deletions

File tree

‎docs/docs/maintenance/configurations.mdx‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import catalogConfigurationHtml from '@site/generated/catalog_configuration.html
99
import hiveCatalogConfigurationHtml from '@site/generated/hive_catalog_configuration.html';
1010
import hiveConnectorConfigurationHtml from '@site/generated/hive_connector_configuration.html';
1111
import jdbcCatalogConfigurationHtml from '@site/generated/jdbc_catalog_configuration.html';
12+
import zooKeeperCatalogLockConfigurationHtml from '@site/generated/zoo_keeper_catalog_lock_configuration.html';
1213
import flinkCatalogConfigurationHtml from '@site/generated/flink_catalog_configuration.html';
1314
import flinkConnectorConfigurationHtml from '@site/generated/flink_connector_configuration.html';
1415
import sparkCatalogConfigurationHtml from '@site/generated/spark_catalog_configuration.html';
@@ -66,6 +67,12 @@ Options for Jdbc catalog.
6667

6768
<ConfigTable html={jdbcCatalogConfigurationHtml} />
6869

70+
### ZooKeeperCatalogLockOptions
71+
72+
Options for ZooKeeper catalog locks.
73+
74+
<ConfigTable html={zooKeeperCatalogLockConfigurationHtml} />
75+
6976
### FlinkCatalogOptions
7077

7178
Flink catalog options for paimon.

‎docs/generated/zoo_keeper_catalog_lock_configuration.html‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
<!--
2+
Licensed to the Apache Software Foundation (ASF) under one
3+
or more contributor license agreements. See the NOTICE file
4+
distributed with this work for additional information
5+
regarding copyright ownership. The ASF licenses this file
6+
to you under the Apache License, Version 2.0 (the
7+
"License"); you may not use this file except in compliance
8+
with the License. You may obtain a copy of the License at
9+
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
12+
Unless required by applicable law or agreed to in writing,
13+
software distributed under the License is distributed on an
14+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
KIND, either express or implied. See the License for the
16+
specific language governing permissions and limitations
17+
under the License.
18+
-->
19+
<table class="configuration table table-bordered">
20+
<thead>
21+
<tr>
22+
<th class="text-left" style="width: 20%">Key</th>
23+
<th class="text-left" style="width: 15%">Default</th>
24+
<th class="text-left" style="width: 10%">Type</th>
25+
<th class="text-left" style="width: 55%">Description</th>
26+
</tr>
27+
</thead>
28+
<tbody>
29+
<tr>
30+
<td><h5>lock.zookeeper.connection-timeout</h5></td>
31+
<td style="word-wrap: break-word;">15 s</td>
32+
<td>Duration</td>
33+
<td>ZooKeeper connection timeout.</td>
34+
</tr>
35+
<tr>
36+
<td><h5>lock.zookeeper.quorum</h5></td>
37+
<td style="word-wrap: break-word;">(none)</td>
38+
<td>String</td>
39+
<td>ZooKeeper connect string, e.g. 'host1:2181,host2:2181'. Required.</td>
40+
</tr>
41+
<tr>
42+
<td><h5>lock.zookeeper.retry-base-sleep</h5></td>
43+
<td style="word-wrap: break-word;">1 s</td>
44+
<td>Duration</td>
45+
<td>Initial backoff between ZooKeeper connection retries.</td>
46+
</tr>
47+
<tr>
48+
<td><h5>lock.zookeeper.retry-max-attempts</h5></td>
49+
<td style="word-wrap: break-word;">5</td>
50+
<td>Integer</td>
51+
<td>Maximum ZooKeeper connection retry attempts.</td>
52+
</tr>
53+
<tr>
54+
<td><h5>lock.zookeeper.root</h5></td>
55+
<td style="word-wrap: break-word;">"paimon/locks"</td>
56+
<td>String</td>
57+
<td>Root znode namespace for Paimon catalog locks.</td>
58+
</tr>
59+
<tr>
60+
<td><h5>lock.zookeeper.session-timeout</h5></td>
61+
<td style="word-wrap: break-word;">30 s</td>
62+
<td>Duration</td>
63+
<td>ZooKeeper session timeout.</td>
64+
</tr>
65+
</tbody>
66+
</table>

‎paimon-core/pom.xml‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,8 +79,55 @@ under the License.
7979
</exclusions>
8080
</dependency>
8181

82+
<dependency>
83+
<groupId>org.apache.curator</groupId>
84+
<artifactId>curator-recipes</artifactId>
85+
<version>${curator.version}</version>
86+
<scope>provided</scope>
87+
</dependency>
88+
89+
<dependency>
90+
<groupId>org.apache.zookeeper</groupId>
91+
<artifactId>zookeeper</artifactId>
92+
<version>${zookeeper.version}</version>
93+
<scope>provided</scope>
94+
<exclusions>
95+
<exclusion>
96+
<groupId>org.slf4j</groupId>
97+
<artifactId>slf4j-log4j12</artifactId>
98+
</exclusion>
99+
<exclusion>
100+
<groupId>log4j</groupId>
101+
<artifactId>log4j</artifactId>
102+
</exclusion>
103+
<exclusion>
104+
<groupId>ch.qos.logback</groupId>
105+
<artifactId>logback-classic</artifactId>
106+
</exclusion>
107+
</exclusions>
108+
</dependency>
109+
110+
<!--
111+
Same nearest-declaration-wins concern as zookeeper above: hadoop-common pulls in
112+
curator-client 2.7.1 at a shallower depth than curator-recipes's own transitive
113+
curator-client, so it would otherwise win and mismatch curator-recipes/curator-framework.
114+
-->
115+
<dependency>
116+
<groupId>org.apache.curator</groupId>
117+
<artifactId>curator-client</artifactId>
118+
<version>${curator.version}</version>
119+
<scope>provided</scope>
120+
</dependency>
121+
82122
<!-- test dependencies -->
83123

124+
<dependency>
125+
<groupId>org.apache.curator</groupId>
126+
<artifactId>curator-test</artifactId>
127+
<version>${curator.version}</version>
128+
<scope>test</scope>
129+
</dependency>
130+
84131
<dependency>
85132
<groupId>org.apache.paimon</groupId>
86133
<artifactId>paimon-arrow</artifactId>

0 commit comments

Comments
 (0)