fix: 修复GraalPy依赖并添加Context池化和完整单元测试

- 修复parser pom.xml中GraalPy依赖配置
- 修复web-front Playground.vue中Tab选中异常bug
- 添加PyContextPool实现Context池化管理
- 更新PyPlaygroundExecutor和PyParserExecutor使用池化
- 创建PyParserTest完整单元测试
- 创建PyHttpClientTest HTTP客户端测试
- 创建PyCryptoUtilsTest加密工具测试
- 修复所有ShareLinkInfo构造相关错误
This commit is contained in:
q
2026-01-11 03:19:31 +08:00
parent 62cc7449fd
commit b179194753
8 changed files with 2066 additions and 166 deletions
@@ -0,0 +1,459 @@
package cn.qaiu.parser.custompy;
import org.graalvm.polyglot.Context;
import org.graalvm.polyglot.Engine;
import org.graalvm.polyglot.HostAccess;
import org.graalvm.polyglot.io.IOAccess;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
/**
* GraalPy Context 池化管理器
* 提供共享的 Engine 实例和 Context 池化支持
*
* <p>特性:
* <ul>
* <li>共享单个 Engine 实例,减少内存占用和启动时间</li>
* <li>Context 对象池,避免重复创建和销毁的开销</li>
* <li>支持安全的沙箱配置</li>
* <li>线程安全的池化管理</li>
* <li>支持优雅关闭和资源清理</li>
* </ul>
*
* @author QAIU
*/
public class PyContextPool {
private static final Logger log = LoggerFactory.getLogger(PyContextPool.class);
// 池化配置
private static final int INITIAL_POOL_SIZE = 2;
private static final int MAX_POOL_SIZE = 10;
private static final long CONTEXT_TIMEOUT_MS = 30000; // 30秒获取超时
private static final long CONTEXT_MAX_AGE_MS = 300000; // 5分钟最大使用时间
// 单例实例
private static volatile PyContextPool instance;
private static final Object LOCK = new Object();
// 共享的GraalPy引擎
private final Engine sharedEngine;
// Context 池
private final BlockingQueue<PooledContext> contextPool;
// 已创建的Context数量
private final AtomicInteger createdCount = new AtomicInteger(0);
// 是否已关闭
private final AtomicBoolean closed = new AtomicBoolean(false);
// 定期清理过期Context的调度器
private final ScheduledExecutorService cleanupScheduler;
// Python执行专用线程池
private final ExecutorService pythonExecutor;
// 超时调度器
private final ScheduledExecutorService timeoutScheduler;
/**
* 池化的Context包装器
*/
public static class PooledContext implements AutoCloseable {
private final Context context;
private final long createdTime;
private final PyContextPool pool;
private volatile boolean inUse = false;
private volatile long lastUsedTime;
private PooledContext(Context context, PyContextPool pool) {
this.context = context;
this.pool = pool;
this.createdTime = System.currentTimeMillis();
this.lastUsedTime = createdTime;
}
/**
* 获取底层Context
*/
public Context getContext() {
return context;
}
/**
* 检查是否过期
*/
public boolean isExpired() {
return System.currentTimeMillis() - createdTime > CONTEXT_MAX_AGE_MS;
}
/**
* 归还到池中或关闭
*/
@Override
public void close() {
pool.release(this);
}
/**
* 强制关闭Context
*/
void forceClose() {
try {
context.close(true);
} catch (Exception e) {
log.warn("关闭Context失败: {}", e.getMessage());
}
}
/**
* 重置Context状态(清除绑定等)
*/
boolean reset() {
try {
// 由于GraalPy的Context不能很好地重置状态,
// 简单场景下我们选择创建新的Context
// 但对于短生命周期的执行,可以尝试继续使用
lastUsedTime = System.currentTimeMillis();
return !isExpired();
} catch (Exception e) {
log.warn("重置Context失败: {}", e.getMessage());
return false;
}
}
}
/**
* 私有构造函数
*/
private PyContextPool() {
log.info("初始化GraalPy Context池...");
// 创建共享Engine
this.sharedEngine = Engine.newBuilder()
.option("engine.WarnInterpreterOnly", "false")
.build();
// 创建Context池
this.contextPool = new LinkedBlockingQueue<>(MAX_POOL_SIZE);
// 创建Python执行专用线程池
this.pythonExecutor = Executors.newCachedThreadPool(r -> {
Thread thread = new Thread(r);
thread.setName("py-context-pool-worker-" + System.currentTimeMillis());
thread.setDaemon(true);
return thread;
});
// 创建超时调度器
this.timeoutScheduler = Executors.newScheduledThreadPool(2, r -> {
Thread thread = new Thread(r);
thread.setName("py-context-timeout-" + System.currentTimeMillis());
thread.setDaemon(true);
return thread;
});
// 创建清理调度器
this.cleanupScheduler = Executors.newSingleThreadScheduledExecutor(r -> {
Thread thread = new Thread(r);
thread.setName("py-context-cleanup");
thread.setDaemon(true);
return thread;
});
// 预热:初始化一些Context
warmup();
// 定期清理过期的Context
cleanupScheduler.scheduleWithFixedDelay(this::cleanup, 60, 60, TimeUnit.SECONDS);
log.info("GraalPy Context池初始化完成,初始大小: {}", INITIAL_POOL_SIZE);
}
/**
* 获取单例实例
*/
public static PyContextPool getInstance() {
if (instance == null) {
synchronized (LOCK) {
if (instance == null) {
instance = new PyContextPool();
}
}
}
return instance;
}
/**
* 获取共享Engine
*/
public Engine getSharedEngine() {
return sharedEngine;
}
/**
* 获取Python执行线程池
*/
public ExecutorService getPythonExecutor() {
return pythonExecutor;
}
/**
* 获取超时调度器
*/
public ScheduledExecutorService getTimeoutScheduler() {
return timeoutScheduler;
}
/**
* 预热Context池
*/
private void warmup() {
for (int i = 0; i < INITIAL_POOL_SIZE; i++) {
try {
PooledContext pc = createPooledContext();
if (!contextPool.offer(pc)) {
pc.forceClose();
}
} catch (Exception e) {
log.warn("预热Context失败: {}", e.getMessage());
}
}
}
/**
* 创建新的池化Context
*/
private PooledContext createPooledContext() {
if (closed.get()) {
throw new IllegalStateException("Context池已关闭");
}
Context context = Context.newBuilder("python")
.engine(sharedEngine)
.allowHostAccess(HostAccess.newBuilder(HostAccess.EXPLICIT)
.allowArrayAccess(true)
.allowListAccess(true)
.allowMapAccess(true)
.allowIterableAccess(true)
.allowIteratorAccess(true)
.build())
.allowHostClassLookup(className -> false)
.allowExperimentalOptions(true)
.allowCreateThread(true)
.allowNativeAccess(false)
.allowCreateProcess(false)
.allowIO(IOAccess.newBuilder()
.allowHostFileAccess(false)
.allowHostSocketAccess(false)
.build())
.option("python.PythonHome", "")
.option("python.ForceImportSite", "false")
.build();
createdCount.incrementAndGet();
log.debug("创建新的GraalPy Context,当前总数: {}", createdCount.get());
return new PooledContext(context, this);
}
/**
* 从池中获取Context
*
* @return 池化的Context,用完后需要调用close()归还
* @throws InterruptedException 如果等待被中断
* @throws TimeoutException 如果超时未获取到
*/
public PooledContext acquire() throws InterruptedException, TimeoutException {
if (closed.get()) {
throw new IllegalStateException("Context池已关闭");
}
// 尝试从池中获取
PooledContext pc = contextPool.poll();
if (pc != null) {
if (!pc.isExpired() && pc.reset()) {
pc.inUse = true;
log.debug("从池中获取Context,池剩余: {}", contextPool.size());
return pc;
} else {
// Context已过期,关闭它
pc.forceClose();
createdCount.decrementAndGet();
}
}
// 池中没有可用的,检查是否可以创建新的
if (createdCount.get() < MAX_POOL_SIZE) {
try {
pc = createPooledContext();
pc.inUse = true;
return pc;
} catch (Exception e) {
log.error("创建新Context失败: {}", e.getMessage());
throw new RuntimeException("无法创建GraalPy Context", e);
}
}
// 已达最大数量,等待归还
pc = contextPool.poll(CONTEXT_TIMEOUT_MS, TimeUnit.MILLISECONDS);
if (pc == null) {
throw new TimeoutException("获取GraalPy Context超时");
}
if (!pc.isExpired() && pc.reset()) {
pc.inUse = true;
return pc;
} else {
pc.forceClose();
createdCount.decrementAndGet();
// 递归重试
return acquire();
}
}
/**
* 创建一个新的非池化Context(用于需要独立生命周期的场景)
* 调用者负责管理其生命周期
*/
public Context createFreshContext() {
return Context.newBuilder("python")
.engine(sharedEngine)
.allowHostAccess(HostAccess.newBuilder(HostAccess.EXPLICIT)
.allowArrayAccess(true)
.allowListAccess(true)
.allowMapAccess(true)
.allowIterableAccess(true)
.allowIteratorAccess(true)
.build())
.allowHostClassLookup(className -> false)
.allowExperimentalOptions(true)
.allowCreateThread(true)
.allowNativeAccess(false)
.allowCreateProcess(false)
.allowIO(IOAccess.newBuilder()
.allowHostFileAccess(false)
.allowHostSocketAccess(false)
.build())
.option("python.PythonHome", "")
.option("python.ForceImportSite", "false")
.build();
}
/**
* 归还Context到池中
*/
private void release(PooledContext pc) {
if (pc == null) return;
pc.inUse = false;
if (closed.get() || pc.isExpired()) {
// 池已关闭或Context已过期,直接销毁
pc.forceClose();
createdCount.decrementAndGet();
log.debug("Context已过期或池已关闭,销毁Context");
} else if (!contextPool.offer(pc)) {
// 池已满,销毁Context
pc.forceClose();
createdCount.decrementAndGet();
log.debug("池已满,销毁多余Context");
} else {
log.debug("归还Context到池,池当前大小: {}", contextPool.size());
}
}
/**
* 清理过期的Context
*/
private void cleanup() {
if (closed.get()) return;
int removed = 0;
PooledContext pc;
while ((pc = contextPool.poll()) != null) {
if (pc.isExpired() || closed.get()) {
pc.forceClose();
createdCount.decrementAndGet();
removed++;
} else {
// 还没过期,放回池中
if (!contextPool.offer(pc)) {
pc.forceClose();
createdCount.decrementAndGet();
removed++;
}
break;
}
}
if (removed > 0) {
log.info("清理了 {} 个过期的Context,当前池大小: {}", removed, contextPool.size());
}
}
/**
* 获取池状态信息
*/
public String getStatus() {
return String.format("PyContextPool[total=%d, available=%d, maxSize=%d]",
createdCount.get(), contextPool.size(), MAX_POOL_SIZE);
}
/**
* 获取池中可用的Context数量
*/
public int getAvailableCount() {
return contextPool.size();
}
/**
* 获取已创建的Context总数
*/
public int getCreatedCount() {
return createdCount.get();
}
/**
* 关闭Context池
*/
public void shutdown() {
if (closed.compareAndSet(false, true)) {
log.info("关闭GraalPy Context池...");
// 停止清理调度器
cleanupScheduler.shutdownNow();
timeoutScheduler.shutdownNow();
pythonExecutor.shutdownNow();
// 关闭所有池中的Context
PooledContext pc;
while ((pc = contextPool.poll()) != null) {
pc.forceClose();
}
// 关闭共享Engine
try {
sharedEngine.close(true);
} catch (Exception e) {
log.warn("关闭共享Engine失败: {}", e.getMessage());
}
log.info("GraalPy Context池已关闭");
}
}
/**
* 检查池是否已关闭
*/
public boolean isClosed() {
return closed.get();
}
}
@@ -9,21 +9,18 @@ import io.vertx.core.Future;
import io.vertx.core.WorkerExecutor;
import io.vertx.core.json.JsonObject;
import org.graalvm.polyglot.Context;
import org.graalvm.polyglot.Engine;
import org.graalvm.polyglot.HostAccess;
import org.graalvm.polyglot.Value;
import org.graalvm.polyglot.io.IOAccess;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
* Python解析器执行器
* 使用GraalPy执行Python解析器脚本
* 实现IPanTool接口,执行Python解析器逻辑
* 使用 PyContextPool 进行 Engine 池化管理
*
* @author QAIU
*/
@@ -34,10 +31,8 @@ public class PyParserExecutor implements IPanTool {
private static final WorkerExecutor EXECUTOR = WebClientVertxInit.get()
.createSharedWorkerExecutor("py-parser-executor", 32);
// 共享的GraalPy引擎,提高性能
private static final Engine SHARED_ENGINE = Engine.newBuilder()
.option("engine.WarnInterpreterOnly", "false")
.build();
// Context池实例
private static final PyContextPool CONTEXT_POOL = PyContextPool.getInstance();
private final CustomParserConfig config;
private final ShareLinkInfo shareLinkInfo;
@@ -71,48 +66,12 @@ public class PyParserExecutor implements IPanTool {
return shareLinkInfo;
}
/**
* 创建安全的GraalPy Context
* 配置沙箱选项,禁用危险功能
*/
private Context createContext() {
return Context.newBuilder("python")
.engine(SHARED_ENGINE)
// 允许访问带有@HostAccess.Export注解的Java方法
.allowHostAccess(HostAccess.newBuilder(HostAccess.EXPLICIT)
.allowArrayAccess(true)
.allowListAccess(true)
.allowMapAccess(true)
.allowIterableAccess(true)
.allowIteratorAccess(true)
.build())
// 禁止访问任意Java类
.allowHostClassLookup(className -> false)
// 允许实验性选项
.allowExperimentalOptions(true)
// 允许创建线程(某些Python库需要)
.allowCreateThread(true)
// 禁用原生访问
.allowNativeAccess(false)
// 禁止创建子进程
.allowCreateProcess(false)
// 允许虚拟文件系统访问(用于import等)
.allowIO(IOAccess.newBuilder()
.allowHostFileAccess(false)
.allowHostSocketAccess(false)
.build())
// GraalPy特定选项
.option("python.PythonHome", "")
.option("python.ForceImportSite", "false")
.build();
}
@Override
public Future<String> parse() {
pyLogger.info("开始执行Python解析器: {}", config.getType());
return EXECUTOR.executeBlocking(() -> {
try (Context context = createContext()) {
try (Context context = CONTEXT_POOL.createFreshContext()) {
// 注入Java对象到Python环境
Value bindings = context.getBindings("python");
bindings.putMember("http", httpClient);
@@ -152,7 +111,7 @@ public class PyParserExecutor implements IPanTool {
pyLogger.info("开始执行Python文件列表解析: {}", config.getType());
return EXECUTOR.executeBlocking(() -> {
try (Context context = createContext()) {
try (Context context = CONTEXT_POOL.createFreshContext()) {
// 注入Java对象到Python环境
Value bindings = context.getBindings("python");
bindings.putMember("http", httpClient);
@@ -186,7 +145,7 @@ public class PyParserExecutor implements IPanTool {
pyLogger.info("开始执行Python按ID解析: {}", config.getType());
return EXECUTOR.executeBlocking(() -> {
try (Context context = createContext()) {
try (Context context = CONTEXT_POOL.createFreshContext()) {
// 注入Java对象到Python环境
Value bindings = context.getBindings("python");
bindings.putMember("http", httpClient);
@@ -6,10 +6,7 @@ import io.vertx.core.Future;
import io.vertx.core.Promise;
import io.vertx.core.json.JsonObject;
import org.graalvm.polyglot.Context;
import org.graalvm.polyglot.Engine;
import org.graalvm.polyglot.HostAccess;
import org.graalvm.polyglot.Value;
import org.graalvm.polyglot.io.IOAccess;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -21,6 +18,7 @@ import java.util.concurrent.*;
* Python演练场执行器
* 用于临时执行Python代码,不注册到解析器注册表
* 使用独立线程池避免Vert.x BlockedThreadChecker警告
* 使用 PyContextPool 进行 Engine 和 Context 池化管理
*
* @author QAIU
*/
@@ -31,26 +29,8 @@ public class PyPlaygroundExecutor {
// Python执行超时时间(秒)
private static final long EXECUTION_TIMEOUT_SECONDS = 30;
// 共享的GraalPy引擎
private static final Engine SHARED_ENGINE = Engine.newBuilder()
.option("engine.WarnInterpreterOnly", "false")
.build();
// 使用独立的线程池
private static final ExecutorService INDEPENDENT_EXECUTOR = Executors.newCachedThreadPool(r -> {
Thread thread = new Thread(r);
thread.setName("py-playground-independent-" + System.currentTimeMillis());
thread.setDaemon(true);
return thread;
});
// 超时调度线程池
private static final ScheduledExecutorService TIMEOUT_SCHEDULER = Executors.newScheduledThreadPool(2, r -> {
Thread thread = new Thread(r);
thread.setName("py-playground-timeout-scheduler-" + System.currentTimeMillis());
thread.setDaemon(true);
return thread;
});
// Context池实例
private static final PyContextPool CONTEXT_POOL = PyContextPool.getInstance();
private final ShareLinkInfo shareLinkInfo;
private final String pyCode;
@@ -81,33 +61,6 @@ public class PyPlaygroundExecutor {
this.cryptoUtils = new PyCryptoUtils();
}
/**
* 创建安全的GraalPy Context
*/
private Context createContext() {
return Context.newBuilder("python")
.engine(SHARED_ENGINE)
.allowHostAccess(HostAccess.newBuilder(HostAccess.EXPLICIT)
.allowArrayAccess(true)
.allowListAccess(true)
.allowMapAccess(true)
.allowIterableAccess(true)
.allowIteratorAccess(true)
.build())
.allowHostClassLookup(className -> false)
.allowExperimentalOptions(true)
.allowCreateThread(true)
.allowNativeAccess(false)
.allowCreateProcess(false)
.allowIO(IOAccess.newBuilder()
.allowHostFileAccess(false)
.allowHostSocketAccess(false)
.build())
.option("python.PythonHome", "")
.option("python.ForceImportSite", "false")
.build();
}
/**
* 执行parse方法(异步,带超时控制)
*/
@@ -117,7 +70,8 @@ public class PyPlaygroundExecutor {
CompletableFuture<String> executionFuture = CompletableFuture.supplyAsync(() -> {
playgroundLogger.infoJava("开始执行parse方法");
try (Context context = createContext()) {
// 使用池化的Context(每次执行创建新的Context以保证状态隔离)
try (Context context = CONTEXT_POOL.createFreshContext()) {
// 注入Java对象到Python环境
Value bindings = context.getBindings("python");
bindings.putMember("http", httpClient);
@@ -153,10 +107,10 @@ public class PyPlaygroundExecutor {
playgroundLogger.errorJava("执行parse方法失败: " + e.getMessage(), e);
throw new RuntimeException(e);
}
}, INDEPENDENT_EXECUTOR);
}, CONTEXT_POOL.getPythonExecutor());
// 创建超时任务
ScheduledFuture<?> timeoutTask = TIMEOUT_SCHEDULER.schedule(() -> {
ScheduledFuture<?> timeoutTask = CONTEXT_POOL.getTimeoutScheduler().schedule(() -> {
if (!executionFuture.isDone()) {
executionFuture.cancel(true);
playgroundLogger.errorJava("执行超时,已强制中断");
@@ -195,7 +149,7 @@ public class PyPlaygroundExecutor {
CompletableFuture<List<FileInfo>> executionFuture = CompletableFuture.supplyAsync(() -> {
playgroundLogger.infoJava("开始执行parse_file_list方法");
try (Context context = createContext()) {
try (Context context = CONTEXT_POOL.createFreshContext()) {
Value bindings = context.getBindings("python");
bindings.putMember("http", httpClient);
bindings.putMember("logger", playgroundLogger);
@@ -220,9 +174,9 @@ public class PyPlaygroundExecutor {
playgroundLogger.errorJava("执行parse_file_list方法失败: " + e.getMessage(), e);
throw new RuntimeException(e);
}
}, INDEPENDENT_EXECUTOR);
}, CONTEXT_POOL.getPythonExecutor());
ScheduledFuture<?> timeoutTask = TIMEOUT_SCHEDULER.schedule(() -> {
ScheduledFuture<?> timeoutTask = CONTEXT_POOL.getTimeoutScheduler().schedule(() -> {
if (!executionFuture.isDone()) {
executionFuture.cancel(true);
playgroundLogger.errorJava("执行超时,已强制中断");
@@ -257,7 +211,7 @@ public class PyPlaygroundExecutor {
CompletableFuture<String> executionFuture = CompletableFuture.supplyAsync(() -> {
playgroundLogger.infoJava("开始执行parse_by_id方法");
try (Context context = createContext()) {
try (Context context = CONTEXT_POOL.createFreshContext()) {
Value bindings = context.getBindings("python");
bindings.putMember("http", httpClient);
bindings.putMember("logger", playgroundLogger);
@@ -288,9 +242,9 @@ public class PyPlaygroundExecutor {
playgroundLogger.errorJava("执行parse_by_id方法失败: " + e.getMessage(), e);
throw new RuntimeException(e);
}
}, INDEPENDENT_EXECUTOR);
}, CONTEXT_POOL.getPythonExecutor());
ScheduledFuture<?> timeoutTask = TIMEOUT_SCHEDULER.schedule(() -> {
ScheduledFuture<?> timeoutTask = CONTEXT_POOL.getTimeoutScheduler().schedule(() -> {
if (!executionFuture.isDone()) {
executionFuture.cancel(true);
playgroundLogger.errorJava("执行超时,已强制中断");