Skip to content

Commit a274834

Browse files
JunRuiLeejerry-024
JunRuiLee
authored and
jerry-024
committed
[filesystem] Isolate Jindo OSS presigning dependencies (#9856)
(cherry picked from commit de872c4)
1 parent ca2017f commit a274834

6 files changed

Lines changed: 824 additions & 56 deletions

File tree

‎paimon-filesystems/paimon-jindo/pom.xml‎

Lines changed: 112 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
<properties>
3535
<fs.oss.sdk.version>3.17.4</fs.oss.sdk.version>
3636
<jindodata.version>6.9.1</jindodata.version>
37+
<oss.plugin.directory>paimon-plugin-jindo-oss</oss.plugin.directory>
3738
</properties>
3839

3940
<repositories>
@@ -55,6 +56,12 @@
5556
<groupId>org.apache.paimon</groupId>
5657
<artifactId>paimon-oss-impl</artifactId>
5758
<version>${project.version}</version>
59+
<exclusions>
60+
<exclusion>
61+
<groupId>*</groupId>
62+
<artifactId>*</artifactId>
63+
</exclusion>
64+
</exclusions>
5865
</dependency>
5966

6067
<dependency>
@@ -160,6 +167,9 @@
160167
<groupId>org.apache.maven.plugins</groupId>
161168
<artifactId>maven-jar-plugin</artifactId>
162169
<configuration>
170+
<excludes>
171+
<exclude>org/apache/paimon/jindo/JindoBlobPresigner*.class</exclude>
172+
</excludes>
163173
<archive>
164174
<manifestEntries>
165175
<Multi-Release>true</Multi-Release>
@@ -168,34 +178,121 @@
168178
</configuration>
169179
</plugin>
170180

181+
<plugin>
182+
<groupId>org.apache.maven.plugins</groupId>
183+
<artifactId>maven-resources-plugin</artifactId>
184+
<executions>
185+
<execution>
186+
<id>copy-oss-presigner-implementation</id>
187+
<phase>prepare-package</phase>
188+
<goals>
189+
<goal>copy-resources</goal>
190+
</goals>
191+
<configuration>
192+
<outputDirectory>${project.build.outputDirectory}/${oss.plugin.directory}</outputDirectory>
193+
<resources>
194+
<resource>
195+
<directory>${project.build.outputDirectory}</directory>
196+
<includes>
197+
<include>org/apache/paimon/jindo/JindoBlobPresigner*.class</include>
198+
</includes>
199+
</resource>
200+
</resources>
201+
</configuration>
202+
</execution>
203+
</executions>
204+
</plugin>
205+
206+
<plugin>
207+
<groupId>org.apache.maven.plugins</groupId>
208+
<artifactId>maven-dependency-plugin</artifactId>
209+
<executions>
210+
<execution>
211+
<id>unpack-oss-runtime</id>
212+
<phase>prepare-package</phase>
213+
<goals>
214+
<goal>unpack-dependencies</goal>
215+
</goals>
216+
<configuration>
217+
<includeScope>runtime</includeScope>
218+
<excludeArtifactIds>paimon-oss-impl,slf4j-api,jsr305</excludeArtifactIds>
219+
<outputDirectory>${project.build.outputDirectory}/${oss.plugin.directory}</outputDirectory>
220+
<excludes>META-INF/*.SF,META-INF/*.DSA,META-INF/*.RSA,META-INF/maven/**</excludes>
221+
</configuration>
222+
</execution>
223+
<execution>
224+
<id>unpack-oss-presigner</id>
225+
<phase>prepare-package</phase>
226+
<goals>
227+
<goal>unpack</goal>
228+
</goals>
229+
<configuration>
230+
<artifactItems>
231+
<artifactItem>
232+
<groupId>org.apache.paimon</groupId>
233+
<artifactId>paimon-oss-impl</artifactId>
234+
<version>${project.version}</version>
235+
<overWrite>true</overWrite>
236+
<outputDirectory>${project.build.outputDirectory}/${oss.plugin.directory}</outputDirectory>
237+
<includes>org/apache/paimon/oss/OSSBlobPresigner*.class</includes>
238+
</artifactItem>
239+
</artifactItems>
240+
</configuration>
241+
</execution>
242+
</executions>
243+
</plugin>
244+
245+
<plugin>
246+
<groupId>org.apache.maven.plugins</groupId>
247+
<artifactId>maven-surefire-plugin</artifactId>
248+
<executions>
249+
<execution>
250+
<id>integration-tests</id>
251+
<configuration>
252+
<systemPropertyVariables>
253+
<jindo.plugin.jar>${project.build.directory}/${project.build.finalName}.jar</jindo.plugin.jar>
254+
</systemPropertyVariables>
255+
</configuration>
256+
</execution>
257+
</executions>
258+
</plugin>
259+
171260
<plugin>
172261
<groupId>org.apache.maven.plugins</groupId>
173262
<artifactId>maven-shade-plugin</artifactId>
174263
<configuration>
175264
<artifactSet>
176265
<includes combine.children="append">
177-
<include>org.apache.paimon:paimon-oss-impl</include>
178-
<include>com.aliyun.oss:aliyun-sdk-oss</include>
179-
<include>com.aliyun:*</include>
180-
<include>com.google.code.gson:gson</include>
181-
<include>commons-codec:commons-codec</include>
182-
<include>commons-logging:commons-logging</include>
183-
<include>io.opentracing:*</include>
184-
<include>org.apache.httpcomponents:*</include>
185-
<include>org.codehaus.jettison:jettison</include>
186-
<include>org.ini4j:ini4j</include>
187-
<include>org.jdom:jdom2</include>
188-
<include>stax:stax-api</include>
266+
<include>*:*</include>
189267
</includes>
268+
<excludes>
269+
<exclude>org.slf4j:slf4j-api</exclude>
270+
<exclude>com.google.code.findbugs:jsr305</exclude>
271+
</excludes>
190272
</artifactSet>
191273
<filters>
192274
<filter>
193-
<artifact>org.apache.paimon:paimon-oss-impl</artifact>
275+
<artifact>*:*</artifact>
194276
<includes>
195-
<include>org/apache/paimon/oss/OSSBlobPresigner.class</include>
196-
<include>META-INF/versions/11/**</include>
277+
<include>org/apache/paimon/jindo/**</include>
278+
<include>${oss.plugin.directory}/**</include>
279+
<include>META-INF/MANIFEST.MF</include>
280+
<include>META-INF/LICENSE*</include>
281+
<include>META-INF/licenses/LICENSE.jdom</include>
282+
<include>META-INF/licenses/LICENSE.jaxb</include>
283+
<include>META-INF/NOTICE*</include>
284+
<include>META-INF/DEPENDENCIES</include>
285+
<include>META-INF/maven/**</include>
286+
<include>META-INF/services/org.apache.paimon.fs.FileIOLoader</include>
287+
<include>META-INF/versions/11/javax/xml/bind/**</include>
197288
</includes>
198289
</filter>
290+
<filter>
291+
<artifact>org.apache.paimon:paimon-oss-impl</artifact>
292+
<excludes>
293+
<exclude>META-INF/NOTICE*</exclude>
294+
</excludes>
295+
</filter>
199296
</filters>
200297
</configuration>
201298
</plugin>

‎paimon-filesystems/paimon-jindo/src/main/java/org/apache/paimon/jindo/JindoBlobPresigner.java‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.paimon.jindo;
20+
21+
import org.apache.paimon.data.BlobDescriptor;
22+
import org.apache.paimon.fs.Path;
23+
import org.apache.paimon.options.Options;
24+
import org.apache.paimon.oss.OSSBlobPresigner;
25+
import org.apache.paimon.utils.StringUtils;
26+
27+
import com.aliyun.oss.OSSClient;
28+
import com.aliyun.oss.OSSClientBuilder;
29+
30+
import java.io.IOException;
31+
import java.time.Duration;
32+
33+
/** OSS presigning implementation loaded from the private plugin directory. */
34+
public class JindoBlobPresigner implements JindoFileIO.BlobPresigner {
35+
36+
private OSSClient client;
37+
38+
public JindoBlobPresigner() {}
39+
40+
JindoBlobPresigner(OSSClient client) {
41+
this.client = client;
42+
}
43+
44+
@Override
45+
public void configure(Options options) {
46+
client = createBlobClient(options);
47+
}
48+
49+
@Override
50+
public String create(Path tableRoot, BlobDescriptor descriptor, Duration validity)
51+
throws IOException {
52+
return OSSBlobPresigner.create(client, tableRoot, descriptor, validity);
53+
}
54+
55+
@Override
56+
public void close() {
57+
client.shutdown();
58+
}
59+
60+
static OSSClient createBlobClient(Options options) {
61+
String endpoint = options.get("fs.oss.endpoint");
62+
if (!endpoint.contains("://")) {
63+
endpoint = "https://" + endpoint;
64+
}
65+
String securityToken = options.get("fs.oss.securityToken");
66+
OSSClientBuilder builder = new OSSClientBuilder();
67+
OSSClient client =
68+
(OSSClient)
69+
(StringUtils.isNullOrWhitespaceOnly(securityToken)
70+
? builder.build(
71+
endpoint,
72+
options.get("fs.oss.accessKeyId"),
73+
options.get("fs.oss.accessKeySecret"))
74+
: builder.build(
75+
endpoint,
76+
options.get("fs.oss.accessKeyId"),
77+
options.get("fs.oss.accessKeySecret"),
78+
securityToken));
79+
String region = options.get("fs.oss.region");
80+
if (!StringUtils.isNullOrWhitespaceOnly(region)) {
81+
client.setRegion(region);
82+
}
83+
return client;
84+
}
85+
}

0 commit comments

Comments
 (0)