ahdoawhfo/netdisk-fast-download
1
0
Fork 0
mirror of https://github.com/qaiu/netdisk-fast-download.git synced 2026-01-12 17:34:12 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add GraalPy Python parser support

Browse Source
This commit is contained in:
q
2026-01-11 02:40:33 +08:00
parent e17fb99de4
commit 62cc7449fd
21 changed files with 4167 additions and 183 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@@ -5,7 +5,7 @@
<p align="center">
<a href="https://github.com/qaiu/netdisk-fast-download/actions/workflows/maven.yml"><img src="https://img.shields.io/github/actions/workflow/status/qaiu/netdisk-fast-download/maven.yml?branch=v0.1.9b8a&style=flat"></a>
<a href="https://www.oracle.com/cn/java/technologies/downloads"><img src="https://img.shields.io/badge/jdk-%3E%3D17-blue"></a>
<a href="https://vertx-china.github.io"><img src="https://img.shields.io/badge/vert.x-4.5.22-blue?style=flat"></a>
<a href="https://vertx-china.github.io"><img src="https://img.shields.io/badge/vert.x-4.5.23-blue?style=flat"></a>
<a href="https://raw.githubusercontent.com/qaiu/netdisk-fast-download/master/LICENSE"><img src="https://img.shields.io/github/license/qaiu/netdisk-fast-download?style=flat"></a>
<a href="https://github.com/qaiu/netdisk-fast-download/releases/"><img src="https://img.shields.io/github/v/release/qaiu/netdisk-fast-download?style=flat"></a>
</p>

17
parser/pom.xml
View File

@@ -59,7 +59,7 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<!-- Versions -->
<vertx.version>4.5.22</vertx.version>
<vertx.version>4.5.23</vertx.version>
<org.reflections.version>0.10.2</org.reflections.version>
<lombok.version>1.18.38</lombok.version>
<slf4j.version>2.0.5</slf4j.version>
@@ -67,6 +67,8 @@
<jackson.version>2.14.2</jackson.version>
<logback.version>1.5.19</logback.version>
<junit.version>4.13.2</junit.version>
<!-- GraalPy -->
<graalpy.version>24.1.1</graalpy.version>
</properties>
<dependencies>
@@ -105,6 +107,19 @@
<scope>compile</scope>
</dependency>
<!-- GraalPy Python Runtime -->
<dependency>
<groupId>org.graalvm.polyglot</groupId>
<artifactId>polyglot</artifactId>
<version>${graalpy.version}</version>
</dependency>
<dependency>
<groupId>org.graalvm.polyglot</groupId>
<artifactId>python</artifactId>
<version>${graalpy.version}</version>
<type>pom</type>
</dependency>
<!-- Compression (Brotli) -->
<dependency>
<groupId>org.brotli</groupId>

4
parser/src/main/java/cn/qaiu/parser/ParserCreate.java
View File

@@ -4,6 +4,7 @@ import cn.qaiu.entity.ShareLinkInfo;
import cn.qaiu.parser.custom.CustomParserConfig;
import cn.qaiu.parser.custom.CustomParserRegistry;
import cn.qaiu.parser.customjs.JsParserExecutor;
import cn.qaiu.parser.custompy.PyParserExecutor;
import org.apache.commons.lang3.StringUtils;
@@ -155,6 +156,9 @@ public class ParserCreate {
// 检查是否为JavaScript解析器
if (customParserConfig.isJsParser()) {
return new JsParserExecutor(shareLinkInfo, customParserConfig);
} else if (customParserConfig.isPyParser()) {
// Python解析器
return new PyParserExecutor(shareLinkInfo, customParserConfig);
} else {
// Java实现的解析器
try {

74
parser/src/main/java/cn/qaiu/parser/custom/CustomParserConfig.java
View File

@@ -53,11 +53,26 @@ public class CustomParserConfig {
*/
private final String jsCode;
/**
* Python代码（用于Python解析器）
*/
private final String pyCode;
/**
* 是否为JavaScript解析器
*/
private final boolean isJsParser;
/**
* 是否为Python解析器
*/
private final boolean isPyParser;
/**
* 脚本语言类型：javascript, python
*/
private final String language;
/**
* 元数据信息（从脚本注释中解析）
*/
@@ -71,7 +86,10 @@ public class CustomParserConfig {
this.panDomain = builder.panDomain;
this.matchPattern = builder.matchPattern;
this.jsCode = builder.jsCode;
this.pyCode = builder.pyCode;
this.isJsParser = builder.isJsParser;
this.isPyParser = builder.isPyParser;
this.language = builder.language;
this.metadata = builder.metadata;
}
@@ -103,10 +121,22 @@ public class CustomParserConfig {
return jsCode;
}
public String getPyCode() {
return pyCode;
}
public boolean isJsParser() {
return isJsParser;
}
public boolean isPyParser() {
return isPyParser;
}
public String getLanguage() {
return language;
}
public Map<String, String> getMetadata() {
return metadata;
}
@@ -134,7 +164,10 @@ public class CustomParserConfig {
private String panDomain;
private Pattern matchPattern;
private String jsCode;
private String pyCode;
private boolean isJsParser;
private boolean isPyParser;
private String language;
private Map<String, String> metadata;
/**
@@ -211,12 +244,45 @@ public class CustomParserConfig {
return this;
}
/**
* 设置Python代码（用于Python解析器）
* @param pyCode Python代码
*/
public Builder pyCode(String pyCode) {
this.pyCode = pyCode;
return this;
}
/**
* 设置是否为JavaScript解析器
* @param isJsParser 是否为JavaScript解析器
*/
public Builder isJsParser(boolean isJsParser) {
this.isJsParser = isJsParser;
if (isJsParser) {
this.language = "javascript";
}
return this;
}
/**
* 设置是否为Python解析器
* @param isPyParser 是否为Python解析器
*/
public Builder isPyParser(boolean isPyParser) {
this.isPyParser = isPyParser;
if (isPyParser) {
this.language = "python";
}
return this;
}
/**
* 设置脚本语言类型
* @param language 语言类型：javascript, python
*/
public Builder language(String language) {
this.language = language;
return this;
}
@@ -246,6 +312,11 @@ public class CustomParserConfig {
if (jsCode == null || jsCode.trim().isEmpty()) {
throw new IllegalArgumentException("JavaScript解析器的jsCode不能为空");
}
} else if (isPyParser) {
// 如果是Python解析器，验证pyCode
if (pyCode == null || pyCode.trim().isEmpty()) {
throw new IllegalArgumentException("Python解析器的pyCode不能为空");
}
} else {
// 如果是Java解析器，验证toolClass
if (toolClass == null) {
@@ -288,7 +359,10 @@ public class CustomParserConfig {
", panDomain='" + panDomain + '\'' +
", matchPattern=" + (matchPattern != null ? matchPattern.pattern() : "null") +
", jsCode=" + (jsCode != null ? "[JavaScript代码]" : "null") +
", pyCode=" + (pyCode != null ? "[Python代码]" : "null") +
", isJsParser=" + isJsParser +
", isPyParser=" + isPyParser +
", language='" + language + '\'' +
", metadata=" + metadata +
'}';
}

111
parser/src/main/java/cn/qaiu/parser/custom/CustomParserRegistry.java
View File

@@ -6,6 +6,8 @@ import org.slf4j.LoggerFactory;
import cn.qaiu.parser.PanDomainTemplate;
import cn.qaiu.parser.customjs.JsScriptLoader;
import cn.qaiu.parser.customjs.JsScriptMetadataParser;
import cn.qaiu.parser.custompy.PyScriptLoader;
import cn.qaiu.parser.custompy.PyScriptMetadataParser;
import java.util.List;
import java.util.Map;
@@ -82,6 +84,24 @@ public class CustomParserRegistry {
register(config);
}
/**
* 注册Python解析器
*
* @param config Python解析器配置
* @throws IllegalArgumentException 如果type已存在或与内置解析器冲突
*/
public static void registerPy(CustomParserConfig config) {
if (config == null) {
throw new IllegalArgumentException("config不能为空");
}
if (!config.isPyParser()) {
throw new IllegalArgumentException("config必须是Python解析器配置");
}
register(config);
}
/**
* 从JavaScript代码字符串注册解析器
*
@@ -139,6 +159,63 @@ public class CustomParserRegistry {
}
}
/**
* 从Python代码字符串注册解析器
*
* @param pyCode Python代码
* @throws IllegalArgumentException 如果解析失败
*/
public static void registerPyFromCode(String pyCode) {
if (pyCode == null || pyCode.trim().isEmpty()) {
throw new IllegalArgumentException("Python代码不能为空");
}
try {
CustomParserConfig config = PyScriptMetadataParser.parseScript(pyCode);
registerPy(config);
} catch (Exception e) {
throw new IllegalArgumentException("解析Python代码失败: " + e.getMessage(), e);
}
}
/**
* 从文件注册Python解析器
*
* @param filePath 文件路径
* @throws IllegalArgumentException 如果文件不存在或解析失败
*/
public static void registerPyFromFile(String filePath) {
if (filePath == null || filePath.trim().isEmpty()) {
throw new IllegalArgumentException("文件路径不能为空");
}
try {
CustomParserConfig config = PyScriptLoader.loadFromFile(filePath);
registerPy(config);
} catch (Exception e) {
throw new IllegalArgumentException("从文件加载Python解析器失败: " + e.getMessage(), e);
}
}
/**
* 从资源文件注册Python解析器
*
* @param resourcePath 资源路径
* @throws IllegalArgumentException 如果资源不存在或解析失败
*/
public static void registerPyFromResource(String resourcePath) {
if (resourcePath == null || resourcePath.trim().isEmpty()) {
throw new IllegalArgumentException("资源路径不能为空");
}
try {
CustomParserConfig config = PyScriptLoader.loadFromResource(resourcePath);
registerPy(config);
} catch (Exception e) {
throw new IllegalArgumentException("从资源加载Python解析器失败: " + e.getMessage(), e);
}
}
/**
* 自动加载所有JavaScript脚本
*/
@@ -165,6 +242,40 @@ public class CustomParserRegistry {
}
}
/**
* 自动加载所有Python脚本
*/
public static void autoLoadPyScripts() {
try {
List<CustomParserConfig> configs = PyScriptLoader.loadAllScripts();
int successCount = 0;
int failCount = 0;
for (CustomParserConfig config : configs) {
try {
registerPy(config);
successCount++;
} catch (Exception e) {
log.error("加载Python脚本失败: {}", config.getType(), e);
failCount++;
}
}
log.info("自动加载Python脚本完成: 成功 {} 个，失败 {} 个", successCount, failCount);
} catch (Exception e) {
log.error("自动加载Python脚本时发生异常", e);
}
}
/**
* 自动加载所有脚本（JavaScript和Python）
*/
public static void autoLoadAllScripts() {
autoLoadJsScripts();
autoLoadPyScripts();
}
/**
* 注销自定义解析器
*

381
parser/src/main/java/cn/qaiu/parser/custompy/PyCryptoUtils.java Normal file
View File

@@ -0,0 +1,381 @@
package cn.qaiu.parser.custompy;
import org.graalvm.polyglot.HostAccess;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.crypto.Cipher;
import javax.crypto.spec.IvParameterSpec;
import javax.crypto.spec.SecretKeySpec;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.util.Base64;
/**
* Python加密工具类
* 为Python脚本提供常用的加密解密功能
*
* @author QAIU
*/
public class PyCryptoUtils {
private static final Logger log = LoggerFactory.getLogger(PyCryptoUtils.class);
// ==================== MD5 ====================
/**
* MD5加密（返回32位小写）
* @param data 待加密数据
* @return MD5值（32位小写）
*/
@HostAccess.Export
public String md5(String data) {
if (data == null) {
return null;
}
try {
MessageDigest md = MessageDigest.getInstance("MD5");
byte[] digest = md.digest(data.getBytes(StandardCharsets.UTF_8));
return bytesToHex(digest);
} catch (Exception e) {
log.error("MD5加密失败", e);
throw new RuntimeException("MD5加密失败: " + e.getMessage(), e);
}
}
/**
* MD5加密（返回16位小写，取中间16位）
* @param data 待加密数据
* @return MD5值（16位小写）
*/
@HostAccess.Export
public String md5_16(String data) {
String md5 = md5(data);
return md5 != null ? md5.substring(8, 24) : null;
}
// ==================== SHA ====================
/**
* SHA-1加密
* @param data 待加密数据
* @return SHA-1值（小写）
*/
@HostAccess.Export
public String sha1(String data) {
return sha(data, "SHA-1");
}
/**
* SHA-256加密
* @param data 待加密数据
* @return SHA-256值（小写）
*/
@HostAccess.Export
public String sha256(String data) {
return sha(data, "SHA-256");
}
/**
* SHA-512加密
* @param data 待加密数据
* @return SHA-512值（小写）
*/
@HostAccess.Export
public String sha512(String data) {
return sha(data, "SHA-512");
}
private String sha(String data, String algorithm) {
if (data == null) {
return null;
}
try {
MessageDigest md = MessageDigest.getInstance(algorithm);
byte[] digest = md.digest(data.getBytes(StandardCharsets.UTF_8));
return bytesToHex(digest);
} catch (Exception e) {
log.error(algorithm + "加密失败", e);
throw new RuntimeException(algorithm + "加密失败: " + e.getMessage(), e);
}
}
// ==================== Base64 ====================
/**
* Base64编码
* @param data 待编码数据
* @return Base64字符串
*/
@HostAccess.Export
public String base64_encode(String data) {
if (data == null) {
return null;
}
return Base64.getEncoder().encodeToString(data.getBytes(StandardCharsets.UTF_8));
}
/**
* Base64编码（字节数组）
* @param data 待编码字节数组
* @return Base64字符串
*/
@HostAccess.Export
public String base64_encode_bytes(byte[] data) {
if (data == null) {
return null;
}
return Base64.getEncoder().encodeToString(data);
}
/**
* Base64解码
* @param data Base64字符串
* @return 解码后的字符串
*/
@HostAccess.Export
public String base64_decode(String data) {
if (data == null) {
return null;
}
try {
byte[] decoded = Base64.getDecoder().decode(data);
return new String(decoded, StandardCharsets.UTF_8);
} catch (Exception e) {
log.error("Base64解码失败", e);
throw new RuntimeException("Base64解码失败: " + e.getMessage(), e);
}
}
/**
* Base64解码（返回字节数组）
* @param data Base64字符串
* @return 解码后的字节数组
*/
@HostAccess.Export
public byte[] base64_decode_bytes(String data) {
if (data == null) {
return null;
}
try {
return Base64.getDecoder().decode(data);
} catch (Exception e) {
log.error("Base64解码失败", e);
throw new RuntimeException("Base64解码失败: " + e.getMessage(), e);
}
}
/**
* URL安全的Base64编码
* @param data 待编码数据
* @return URL安全的Base64字符串
*/
@HostAccess.Export
public String base64_url_encode(String data) {
if (data == null) {
return null;
}
return Base64.getUrlEncoder().encodeToString(data.getBytes(StandardCharsets.UTF_8));
}
/**
* URL安全的Base64解码
* @param data URL安全的Base64字符串
* @return 解码后的字符串
*/
@HostAccess.Export
public String base64_url_decode(String data) {
if (data == null) {
return null;
}
try {
byte[] decoded = Base64.getUrlDecoder().decode(data);
return new String(decoded, StandardCharsets.UTF_8);
} catch (Exception e) {
log.error("Base64 URL解码失败", e);
throw new RuntimeException("Base64 URL解码失败: " + e.getMessage(), e);
}
}
// ==================== AES ====================
/**
* AES加密（ECB模式，PKCS5Padding）
* @param data 待加密数据
* @param key 密钥（16/24/32字节）
* @return Base64编码的密文
*/
@HostAccess.Export
public String aes_encrypt_ecb(String data, String key) {
if (data == null || key == null) {
return null;
}
try {
SecretKeySpec secretKey = new SecretKeySpec(padKey(key), "AES");
Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5Padding");
cipher.init(Cipher.ENCRYPT_MODE, secretKey);
byte[] encrypted = cipher.doFinal(data.getBytes(StandardCharsets.UTF_8));
return Base64.getEncoder().encodeToString(encrypted);
} catch (Exception e) {
log.error("AES ECB加密失败", e);
throw new RuntimeException("AES ECB加密失败: " + e.getMessage(), e);
}
}
/**
* AES解密（ECB模式，PKCS5Padding）
* @param data Base64编码的密文
* @param key 密钥（16/24/32字节）
* @return 明文
*/
@HostAccess.Export
public String aes_decrypt_ecb(String data, String key) {
if (data == null || key == null) {
return null;
}
try {
SecretKeySpec secretKey = new SecretKeySpec(padKey(key), "AES");
Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5Padding");
cipher.init(Cipher.DECRYPT_MODE, secretKey);
byte[] decrypted = cipher.doFinal(Base64.getDecoder().decode(data));
return new String(decrypted, StandardCharsets.UTF_8);
} catch (Exception e) {
log.error("AES ECB解密失败", e);
throw new RuntimeException("AES ECB解密失败: " + e.getMessage(), e);
}
}
/**
* AES加密（CBC模式，PKCS5Padding）
* @param data 待加密数据
* @param key 密钥（16/24/32字节）
* @param iv 初始向量（16字节）
* @return Base64编码的密文
*/
@HostAccess.Export
public String aes_encrypt_cbc(String data, String key, String iv) {
if (data == null || key == null || iv == null) {
return null;
}
try {
SecretKeySpec secretKey = new SecretKeySpec(padKey(key), "AES");
IvParameterSpec ivSpec = new IvParameterSpec(padIv(iv));
Cipher cipher = Cipher.getInstance("AES/CBC/PKCS5Padding");
cipher.init(Cipher.ENCRYPT_MODE, secretKey, ivSpec);
byte[] encrypted = cipher.doFinal(data.getBytes(StandardCharsets.UTF_8));
return Base64.getEncoder().encodeToString(encrypted);
} catch (Exception e) {
log.error("AES CBC加密失败", e);
throw new RuntimeException("AES CBC加密失败: " + e.getMessage(), e);
}
}
/**
* AES解密（CBC模式，PKCS5Padding）
* @param data Base64编码的密文
* @param key 密钥（16/24/32字节）
* @param iv 初始向量（16字节）
* @return 明文
*/
@HostAccess.Export
public String aes_decrypt_cbc(String data, String key, String iv) {
if (data == null || key == null || iv == null) {
return null;
}
try {
SecretKeySpec secretKey = new SecretKeySpec(padKey(key), "AES");
IvParameterSpec ivSpec = new IvParameterSpec(padIv(iv));
Cipher cipher = Cipher.getInstance("AES/CBC/PKCS5Padding");
cipher.init(Cipher.DECRYPT_MODE, secretKey, ivSpec);
byte[] decrypted = cipher.doFinal(Base64.getDecoder().decode(data));
return new String(decrypted, StandardCharsets.UTF_8);
} catch (Exception e) {
log.error("AES CBC解密失败", e);
throw new RuntimeException("AES CBC解密失败: " + e.getMessage(), e);
}
}
// ==================== Hex ====================
/**
* 字节数组转十六进制字符串
* @param bytes 字节数组
* @return 十六进制字符串（小写）
*/
@HostAccess.Export
public String bytes_to_hex(byte[] bytes) {
return bytesToHex(bytes);
}
/**
* 十六进制字符串转字节数组
* @param hex 十六进制字符串
* @return 字节数组
*/
@HostAccess.Export
public byte[] hex_to_bytes(String hex) {
if (hex == null || hex.length() % 2 != 0) {
return null;
}
int len = hex.length();
byte[] data = new byte[len / 2];
for (int i = 0; i < len; i += 2) {
data[i / 2] = (byte) ((Character.digit(hex.charAt(i), 16) << 4)
+ Character.digit(hex.charAt(i + 1), 16));
}
return data;
}
// ==================== 工具方法 ====================
private static String bytesToHex(byte[] bytes) {
if (bytes == null) {
return null;
}
StringBuilder sb = new StringBuilder();
for (byte b : bytes) {
sb.append(String.format("%02x", b));
}
return sb.toString();
}
/**
* 将密钥填充到16/24/32字节
*/
private byte[] padKey(String key) {
byte[] keyBytes = key.getBytes(StandardCharsets.UTF_8);
int len = keyBytes.length;
// 根据密钥长度决定填充到16/24/32字节
int targetLen;
if (len <= 16) {
targetLen = 16;
} else if (len <= 24) {
targetLen = 24;
} else {
targetLen = 32;
}
if (len == targetLen) {
return keyBytes;
}
byte[] paddedKey = new byte[targetLen];
System.arraycopy(keyBytes, 0, paddedKey, 0, Math.min(len, targetLen));
return paddedKey;
}
/**
* 将IV填充到16字节
*/
private byte[] padIv(String iv) {
byte[] ivBytes = iv.getBytes(StandardCharsets.UTF_8);
if (ivBytes.length == 16) {
return ivBytes;
}
byte[] paddedIv = new byte[16];
System.arraycopy(ivBytes, 0, paddedIv, 0, Math.min(ivBytes.length, 16));
return paddedIv;
}
}

649
parser/src/main/java/cn/qaiu/parser/custompy/PyHttpClient.java Normal file
View File

@@ -0,0 +1,649 @@
package cn.qaiu.parser.custompy;
import cn.qaiu.WebClientVertxInit;
import cn.qaiu.util.HttpResponseHelper;
import io.vertx.core.Future;
import io.vertx.core.MultiMap;
import io.vertx.core.Promise;
import io.vertx.core.buffer.Buffer;
import io.vertx.core.json.JsonObject;
import io.vertx.core.net.ProxyOptions;
import io.vertx.core.net.ProxyType;
import io.vertx.ext.web.client.HttpRequest;
import io.vertx.ext.web.client.HttpResponse;
import io.vertx.ext.web.client.WebClient;
import io.vertx.ext.web.client.WebClientOptions;
import io.vertx.ext.web.client.WebClientSession;
import io.vertx.ext.web.multipart.MultipartForm;
import org.apache.commons.lang3.StringUtils;
import org.graalvm.polyglot.HostAccess;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.net.InetAddress;
import java.net.URI;
import java.net.URLDecoder;
import java.net.URLEncoder;
import java.net.UnknownHostException;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.regex.Pattern;
/**
* Python HTTP客户端封装
* 为Python脚本提供类似requests库的HTTP请求功能
* 基于Vert.x WebClient实现，提供同步API风格
*
* @author QAIU
*/
public class PyHttpClient {
private static final Logger log = LoggerFactory.getLogger(PyHttpClient.class);
private final WebClient client;
private final WebClientSession clientSession;
private MultiMap headers;
private int timeoutSeconds = 30; // 默认超时时间30秒
// SSRF防护：内网IP正则表达式
private static final Pattern PRIVATE_IP_PATTERN = Pattern.compile(
"^(127\\..*|10\\..*|172\\.(1[6-9]|2[0-9]|3[01])\\..*|192\\.168\\..*|169\\.254\\..*|::1|[fF][cCdD].*)"
);
// SSRF防护：危险域名黑名单
private static final String[] DANGEROUS_HOSTS = {
"localhost",
"169.254.169.254", // AWS/阿里云等云服务元数据API
"metadata.google.internal", // GCP元数据
"100.100.100.200" // 阿里云元数据
};
public PyHttpClient() {
this.client = WebClient.create(WebClientVertxInit.get(), new WebClientOptions());
this.clientSession = WebClientSession.create(client);
this.headers = MultiMap.caseInsensitiveMultiMap();
initDefaultHeaders();
}
/**
* 带代理配置的构造函数
* @param proxyConfig 代理配置JsonObject，包含type、host、port、username、password
*/
public PyHttpClient(JsonObject proxyConfig) {
if (proxyConfig != null && proxyConfig.containsKey("type")) {
ProxyOptions proxyOptions = new ProxyOptions()
.setType(ProxyType.valueOf(proxyConfig.getString("type").toUpperCase()))
.setHost(proxyConfig.getString("host"))
.setPort(proxyConfig.getInteger("port"));
if (StringUtils.isNotEmpty(proxyConfig.getString("username"))) {
proxyOptions.setUsername(proxyConfig.getString("username"));
}
if (StringUtils.isNotEmpty(proxyConfig.getString("password"))) {
proxyOptions.setPassword(proxyConfig.getString("password"));
}
this.client = WebClient.create(WebClientVertxInit.get(),
new WebClientOptions()
.setUserAgentEnabled(false)
.setProxyOptions(proxyOptions));
this.clientSession = WebClientSession.create(client);
} else {
this.client = WebClient.create(WebClientVertxInit.get());
this.clientSession = WebClientSession.create(client);
}
this.headers = MultiMap.caseInsensitiveMultiMap();
initDefaultHeaders();
}
private void initDefaultHeaders() {
// 设置默认的Accept-Encoding头以支持压缩响应
this.headers.set("Accept-Encoding", "gzip, deflate, br, zstd");
// 设置默认的User-Agent头
this.headers.set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36 Edg/140.0.0.0");
// 设置默认的Accept-Language头
this.headers.set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6");
}
/**
* 验证URL安全性（SSRF防护）- 仅拦截明显的内网攻击
* @param url 待验证的URL
* @throws SecurityException 如果URL不安全
*/
private void validateUrlSecurity(String url) {
try {
URI uri = new URI(url);
String host = uri.getHost();
if (host == null) {
log.debug("URL没有host信息: {}", url);
return;
}
String lowerHost = host.toLowerCase();
// 1. 检查明确的危险域名（云服务元数据API等）
for (String dangerous : DANGEROUS_HOSTS) {
if (lowerHost.equals(dangerous)) {
log.warn("🔒 安全拦截: 尝试访问云服务元数据API - {}", host);
throw new SecurityException("🔒 安全拦截: 禁止访问云服务元数据API");
}
}
// 2. 如果host是IP地址格式，检查是否为内网IP
if (isIpAddress(lowerHost)) {
if (PRIVATE_IP_PATTERN.matcher(lowerHost).find()) {
log.warn("🔒 安全拦截: 尝试访问内网IP - {}", host);
throw new SecurityException("🔒 安全拦截: 禁止访问内网IP地址");
}
}
// 3. 对于域名，尝试解析IP（但不因解析失败而拦截）
if (!isIpAddress(lowerHost)) {
try {
InetAddress addr = InetAddress.getByName(host);
String ip = addr.getHostAddress();
if (PRIVATE_IP_PATTERN.matcher(ip).find()) {
log.warn("🔒 安全拦截: 域名解析到内网IP - {} -> {}", host, ip);
throw new SecurityException("🔒 安全拦截: 该域名指向内网地址");
}
} catch (UnknownHostException e) {
log.debug("DNS解析失败，允许继续: {}", host);
}
}
log.debug("URL安全检查通过: {}", url);
} catch (SecurityException e) {
throw e;
} catch (Exception e) {
log.debug("URL验证异常，允许继续: {}", url, e);
}
}
/**
* 判断字符串是否为IP地址格式
*/
private boolean isIpAddress(String host) {
return host.matches("^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}$") || host.contains(":");
}
/**
* 发起GET请求
* @param url 请求URL
* @return HTTP响应
*/
@HostAccess.Export
public PyHttpResponse get(String url) {
validateUrlSecurity(url);
return executeRequest(() -> {
HttpRequest<Buffer> request = client.getAbs(url);
if (!headers.isEmpty()) {
request.putHeaders(headers);
}
return request.send();
});
}
/**
* 发起GET请求并跟随重定向
* @param url 请求URL
* @return HTTP响应
*/
@HostAccess.Export
public PyHttpResponse get_with_redirect(String url) {
validateUrlSecurity(url);
return executeRequest(() -> {
HttpRequest<Buffer> request = client.getAbs(url);
if (!headers.isEmpty()) {
request.putHeaders(headers);
}
request.followRedirects(true);
return request.send();
});
}
/**
* 发起GET请求但不跟随重定向（用于获取Location头）
* @param url 请求URL
* @return HTTP响应
*/
@HostAccess.Export
public PyHttpResponse get_no_redirect(String url) {
validateUrlSecurity(url);
return executeRequest(() -> {
HttpRequest<Buffer> request = client.getAbs(url);
if (!headers.isEmpty()) {
request.putHeaders(headers);
}
request.followRedirects(false);
return request.send();
});
}
/**
* 发起POST请求
* @param url 请求URL
* @param data 请求数据（支持String、Map）
* @return HTTP响应
*/
@HostAccess.Export
public PyHttpResponse post(String url, Object data) {
validateUrlSecurity(url);
return executeRequest(() -> {
HttpRequest<Buffer> request = client.postAbs(url);
if (!headers.isEmpty()) {
request.putHeaders(headers);
}
if (data != null) {
if (data instanceof String) {
return request.sendBuffer(Buffer.buffer((String) data));
} else if (data instanceof Map) {
@SuppressWarnings("unchecked")
Map<String, String> mapData = (Map<String, String>) data;
return request.sendForm(MultiMap.caseInsensitiveMultiMap().addAll(mapData));
} else {
return request.sendJson(data);
}
} else {
return request.send();
}
});
}
/**
* 发起POST请求（JSON数据）
* @param url 请求URL
* @param jsonData JSON字符串或Map
* @return HTTP响应
*/
@HostAccess.Export
public PyHttpResponse post_json(String url, Object jsonData) {
validateUrlSecurity(url);
return executeRequest(() -> {
HttpRequest<Buffer> request = client.postAbs(url);
if (!headers.isEmpty()) {
request.putHeaders(headers);
}
headers.set("Content-Type", "application/json");
if (jsonData instanceof String) {
return request.sendBuffer(Buffer.buffer((String) jsonData));
} else {
return request.sendJson(jsonData);
}
});
}
/**
* 发起PUT请求
* @param url 请求URL
* @param data 请求数据
* @return HTTP响应
*/
@HostAccess.Export
public PyHttpResponse put(String url, Object data) {
validateUrlSecurity(url);
return executeRequest(() -> {
HttpRequest<Buffer> request = client.putAbs(url);
if (!headers.isEmpty()) {
request.putHeaders(headers);
}
if (data != null) {
if (data instanceof String) {
return request.sendBuffer(Buffer.buffer((String) data));
} else if (data instanceof Map) {
@SuppressWarnings("unchecked")
Map<String, String> mapData = (Map<String, String>) data;
return request.sendForm(MultiMap.caseInsensitiveMultiMap().addAll(mapData));
} else {
return request.sendJson(data);
}
} else {
return request.send();
}
});
}
/**
* 发起DELETE请求
* @param url 请求URL
* @return HTTP响应
*/
@HostAccess.Export
public PyHttpResponse delete(String url) {
validateUrlSecurity(url);
return executeRequest(() -> {
HttpRequest<Buffer> request = client.deleteAbs(url);
if (!headers.isEmpty()) {
request.putHeaders(headers);
}
return request.send();
});
}
/**
* 发起PATCH请求
* @param url 请求URL
* @param data 请求数据
* @return HTTP响应
*/
@HostAccess.Export
public PyHttpResponse patch(String url, Object data) {
validateUrlSecurity(url);
return executeRequest(() -> {
HttpRequest<Buffer> request = client.patchAbs(url);
if (!headers.isEmpty()) {
request.putHeaders(headers);
}
if (data != null) {
if (data instanceof String) {
return request.sendBuffer(Buffer.buffer((String) data));
} else if (data instanceof Map) {
@SuppressWarnings("unchecked")
Map<String, String> mapData = (Map<String, String>) data;
return request.sendForm(MultiMap.caseInsensitiveMultiMap().addAll(mapData));
} else {
return request.sendJson(data);
}
} else {
return request.send();
}
});
}
/**
* 设置请求头
* @param name 头名称
* @param value 头值
* @return 当前客户端实例（支持链式调用）
*/
@HostAccess.Export
public PyHttpClient put_header(String name, String value) {
if (name != null && value != null) {
headers.set(name, value);
}
return this;
}
/**
* 批量设置请求头
* @param headersMap 请求头Map
* @return 当前客户端实例（支持链式调用）
*/
@HostAccess.Export
public PyHttpClient put_headers(Map<String, String> headersMap) {
if (headersMap != null) {
for (Map.Entry<String, String> entry : headersMap.entrySet()) {
if (entry.getKey() != null && entry.getValue() != null) {
headers.set(entry.getKey(), entry.getValue());
}
}
}
return this;
}
/**
* 删除指定请求头
* @param name 头名称
* @return 当前客户端实例（支持链式调用）
*/
@HostAccess.Export
public PyHttpClient remove_header(String name) {
if (name != null) {
headers.remove(name);
}
return this;
}
/**
* 清空所有请求头（保留默认头）
* @return 当前客户端实例（支持链式调用）
*/
@HostAccess.Export
public PyHttpClient clear_headers() {
headers.clear();
initDefaultHeaders();
return this;
}
/**
* 获取所有请求头
* @return 请求头Map
*/
@HostAccess.Export
public Map<String, String> get_headers() {
Map<String, String> result = new HashMap<>();
for (String name : headers.names()) {
result.put(name, headers.get(name));
}
return result;
}
/**
* 设置请求超时时间
* @param seconds 超时时间（秒）
* @return 当前客户端实例（支持链式调用）
*/
@HostAccess.Export
public PyHttpClient set_timeout(int seconds) {
if (seconds > 0) {
this.timeoutSeconds = seconds;
}
return this;
}
/**
* URL编码
* @param str 要编码的字符串
* @return 编码后的字符串
*/
@HostAccess.Export
public static String url_encode(String str) {
if (str == null) {
return null;
}
try {
return URLEncoder.encode(str, StandardCharsets.UTF_8.name());
} catch (Exception e) {
log.error("URL编码失败", e);
return str;
}
}
/**
* URL解码
* @param str 要解码的字符串
* @return 解码后的字符串
*/
@HostAccess.Export
public static String url_decode(String str) {
if (str == null) {
return null;
}
try {
return URLDecoder.decode(str, StandardCharsets.UTF_8.name());
} catch (Exception e) {
log.error("URL解码失败", e);
return str;
}
}
/**
* 执行HTTP请求（同步）
*/
private PyHttpResponse executeRequest(RequestExecutor executor) {
try {
Promise<HttpResponse<Buffer>> promise = Promise.promise();
Future<HttpResponse<Buffer>> future = executor.execute();
future.onComplete(result -> {
if (result.succeeded()) {
promise.complete(result.result());
} else {
promise.fail(result.cause());
}
}).onFailure(Throwable::printStackTrace);
// 等待响应完成（使用配置的超时时间）
HttpResponse<Buffer> response = promise.future().toCompletionStage()
.toCompletableFuture()
.get(timeoutSeconds, TimeUnit.SECONDS);
return new PyHttpResponse(response);
} catch (TimeoutException e) {
String errorMsg = "HTTP请求超时（" + timeoutSeconds + "秒）";
log.error(errorMsg, e);
throw new RuntimeException(errorMsg, e);
} catch (Exception e) {
String errorMsg = e.getMessage();
if (errorMsg == null || errorMsg.trim().isEmpty()) {
errorMsg = e.getClass().getSimpleName();
if (e.getCause() != null && e.getCause().getMessage() != null) {
errorMsg += ": " + e.getCause().getMessage();
}
}
log.error("HTTP请求执行失败: " + errorMsg, e);
throw new RuntimeException("HTTP请求执行失败: " + errorMsg, e);
}
}
/**
* 请求执行器接口
*/
@FunctionalInterface
private interface RequestExecutor {
Future<HttpResponse<Buffer>> execute();
}
/**
* Python HTTP响应封装
*/
public static class PyHttpResponse {
private final HttpResponse<Buffer> response;
public PyHttpResponse(HttpResponse<Buffer> response) {
this.response = response;
}
/**
* 获取响应体（字符串）
* @return 响应体字符串
*/
@HostAccess.Export
public String text() {
return HttpResponseHelper.asText(response);
}
/**
* 获取响应体（字符串）- 别名
*/
@HostAccess.Export
public String body() {
return text();
}
/**
* 解析JSON响应
* @return JSON对象的Map表示
*/
@HostAccess.Export
public Object json() {
try {
JsonObject jsonObject = HttpResponseHelper.asJson(response);
if (jsonObject == null || jsonObject.isEmpty()) {
return null;
}
return jsonObject.getMap();
} catch (Exception e) {
log.error("解析JSON响应失败", e);
throw new RuntimeException("解析JSON响应失败: " + e.getMessage(), e);
}
}
/**
* 获取HTTP状态码
* @return 状态码
*/
@HostAccess.Export
public int status_code() {
return response.statusCode();
}
/**
* 获取响应头
* @param name 头名称
* @return 头值
*/
@HostAccess.Export
public String header(String name) {
return response.getHeader(name);
}
/**
* 获取所有响应头
* @return 响应头Map
*/
@HostAccess.Export
public Map<String, String> headers() {
MultiMap responseHeaders = response.headers();
Map<String, String> result = new HashMap<>();
for (String name : responseHeaders.names()) {
result.put(name, responseHeaders.get(name));
}
return result;
}
/**
* 检查请求是否成功
* @return true表示成功（2xx状态码）
*/
@HostAccess.Export
public boolean ok() {
int status = status_code();
return status >= 200 && status < 300;
}
/**
* 获取响应体字节数组
* @return 响应体字节数组
*/
@HostAccess.Export
public byte[] content() {
Buffer buffer = response.body();
if (buffer == null) {
return new byte[0];
}
return buffer.getBytes();
}
/**
* 获取响应体大小
* @return 响应体大小（字节）
*/
@HostAccess.Export
public long content_length() {
Buffer buffer = response.body();
if (buffer == null) {
return 0;
}
return buffer.length();
}
/**
* 获取原始响应对象
*/
public HttpResponse<Buffer> getOriginalResponse() {
return response;
}
}
}

139
parser/src/main/java/cn/qaiu/parser/custompy/PyLogger.java Normal file
View File

@@ -0,0 +1,139 @@
package cn.qaiu.parser.custompy;
import org.graalvm.polyglot.HostAccess;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Python日志封装
* 为Python脚本提供日志功能
*
* @author QAIU
*/
public class PyLogger {
private final Logger logger;
private final String prefix;
public PyLogger(String name) {
this.logger = LoggerFactory.getLogger(name);
this.prefix = "[" + name + "] ";
}
public PyLogger(Class<?> clazz) {
this.logger = LoggerFactory.getLogger(clazz);
this.prefix = "[" + clazz.getSimpleName() + "] ";
}
/**
* 调试日志
* @param message 日志消息
*/
@HostAccess.Export
public void debug(String message) {
logger.debug(prefix + message);
}
/**
* 调试日志（带参数）
* @param message 日志消息模板
* @param args 参数
*/
@HostAccess.Export
public void debug(String message, Object... args) {
logger.debug(prefix + message, args);
}
/**
* 信息日志
* @param message 日志消息
*/
@HostAccess.Export
public void info(String message) {
logger.info(prefix + message);
}
/**
* 信息日志（带参数）
* @param message 日志消息模板
* @param args 参数
*/
@HostAccess.Export
public void info(String message, Object... args) {
logger.info(prefix + message, args);
}
/**
* 警告日志
* @param message 日志消息
*/
@HostAccess.Export
public void warn(String message) {
logger.warn(prefix + message);
}
/**
* 警告日志（带参数）
* @param message 日志消息模板
* @param args 参数
*/
@HostAccess.Export
public void warn(String message, Object... args) {
logger.warn(prefix + message, args);
}
/**
* 错误日志
* @param message 日志消息
*/
@HostAccess.Export
public void error(String message) {
logger.error(prefix + message);
}
/**
* 错误日志（带参数）
* @param message 日志消息模板
* @param args 参数
*/
@HostAccess.Export
public void error(String message, Object... args) {
logger.error(prefix + message, args);
}
/**
* 错误日志（带异常）
* @param message 日志消息
* @param throwable 异常对象
*/
@HostAccess.Export
public void error(String message, Throwable throwable) {
logger.error(prefix + message, throwable);
}
/**
* 检查是否启用调试级别日志
* @return true表示启用，false表示不启用
*/
@HostAccess.Export
public boolean isDebugEnabled() {
return logger.isDebugEnabled();
}
/**
* 检查是否启用信息级别日志
* @return true表示启用，false表示不启用
*/
@HostAccess.Export
public boolean isInfoEnabled() {
return logger.isInfoEnabled();
}
/**
* 获取原始Logger对象
* @return Logger对象
*/
public Logger getOriginalLogger() {
return logger;
}
}

307
parser/src/main/java/cn/qaiu/parser/custompy/PyParserExecutor.java Normal file
View File

@@ -0,0 +1,307 @@
package cn.qaiu.parser.custompy;
import cn.qaiu.WebClientVertxInit;
import cn.qaiu.entity.FileInfo;
import cn.qaiu.entity.ShareLinkInfo;
import cn.qaiu.parser.IPanTool;
import cn.qaiu.parser.custom.CustomParserConfig;
import io.vertx.core.Future;
import io.vertx.core.WorkerExecutor;
import io.vertx.core.json.JsonObject;
import org.graalvm.polyglot.Context;
import org.graalvm.polyglot.Engine;
import org.graalvm.polyglot.HostAccess;
import org.graalvm.polyglot.Value;
import org.graalvm.polyglot.io.IOAccess;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
* Python解析器执行器
* 使用GraalPy执行Python解析器脚本
* 实现IPanTool接口，执行Python解析器逻辑
*
* @author QAIU
*/
public class PyParserExecutor implements IPanTool {
private static final Logger log = LoggerFactory.getLogger(PyParserExecutor.class);
private static final WorkerExecutor EXECUTOR = WebClientVertxInit.get()
.createSharedWorkerExecutor("py-parser-executor", 32);
// 共享的GraalPy引擎，提高性能
private static final Engine SHARED_ENGINE = Engine.newBuilder()
.option("engine.WarnInterpreterOnly", "false")
.build();
private final CustomParserConfig config;
private final ShareLinkInfo shareLinkInfo;
private final PyHttpClient httpClient;
private final PyLogger pyLogger;
private final PyShareLinkInfoWrapper shareLinkInfoWrapper;
private final PyCryptoUtils cryptoUtils;
public PyParserExecutor(ShareLinkInfo shareLinkInfo, CustomParserConfig config) {
this.config = config;
this.shareLinkInfo = shareLinkInfo;
// 检查是否有代理配置
JsonObject proxyConfig = null;
if (shareLinkInfo.getOtherParam().containsKey("proxy")) {
proxyConfig = (JsonObject) shareLinkInfo.getOtherParam().get("proxy");
}
this.httpClient = new PyHttpClient(proxyConfig);
this.pyLogger = new PyLogger("PyParser-" + config.getType());
this.shareLinkInfoWrapper = new PyShareLinkInfoWrapper(shareLinkInfo);
this.cryptoUtils = new PyCryptoUtils();
}
/**
* 获取ShareLinkInfo对象
* @return ShareLinkInfo对象
*/
@Override
public ShareLinkInfo getShareLinkInfo() {
return shareLinkInfo;
}
/**
* 创建安全的GraalPy Context
* 配置沙箱选项，禁用危险功能
*/
private Context createContext() {
return Context.newBuilder("python")
.engine(SHARED_ENGINE)
// 允许访问带有@HostAccess.Export注解的Java方法
.allowHostAccess(HostAccess.newBuilder(HostAccess.EXPLICIT)
.allowArrayAccess(true)
.allowListAccess(true)
.allowMapAccess(true)
.allowIterableAccess(true)
.allowIteratorAccess(true)
.build())
// 禁止访问任意Java类
.allowHostClassLookup(className -> false)
// 允许实验性选项
.allowExperimentalOptions(true)
// 允许创建线程（某些Python库需要）
.allowCreateThread(true)
// 禁用原生访问
.allowNativeAccess(false)
// 禁止创建子进程
.allowCreateProcess(false)
// 允许虚拟文件系统访问（用于import等）
.allowIO(IOAccess.newBuilder()
.allowHostFileAccess(false)
.allowHostSocketAccess(false)
.build())
// GraalPy特定选项
.option("python.PythonHome", "")
.option("python.ForceImportSite", "false")
.build();
}
@Override
public Future<String> parse() {
pyLogger.info("开始执行Python解析器: {}", config.getType());
return EXECUTOR.executeBlocking(() -> {
try (Context context = createContext()) {
// 注入Java对象到Python环境
Value bindings = context.getBindings("python");
bindings.putMember("http", httpClient);
bindings.putMember("logger", pyLogger);
bindings.putMember("share_link_info", shareLinkInfoWrapper);
bindings.putMember("crypto", cryptoUtils);
// 执行Python代码
context.eval("python", config.getPyCode());
// 调用parse函数
Value parseFunc = bindings.getMember("parse");
if (parseFunc == null || !parseFunc.canExecute()) {
throw new RuntimeException("Python代码中未找到parse函数");
}
Value result = parseFunc.execute(shareLinkInfoWrapper, httpClient, pyLogger);
if (result.isString()) {
String downloadUrl = result.asString();
pyLogger.info("解析成功: {}", downloadUrl);
return downloadUrl;
} else {
pyLogger.error("parse方法返回值类型错误，期望String，实际: {}",
result.getMetaObject().toString());
throw new RuntimeException("parse方法返回值类型错误");
}
} catch (Exception e) {
pyLogger.error("Python解析器执行失败: {}", e.getMessage());
throw new RuntimeException("Python解析器执行失败: " + e.getMessage(), e);
}
});
}
@Override
public Future<List<FileInfo>> parseFileList() {
pyLogger.info("开始执行Python文件列表解析: {}", config.getType());
return EXECUTOR.executeBlocking(() -> {
try (Context context = createContext()) {
// 注入Java对象到Python环境
Value bindings = context.getBindings("python");
bindings.putMember("http", httpClient);
bindings.putMember("logger", pyLogger);
bindings.putMember("share_link_info", shareLinkInfoWrapper);
bindings.putMember("crypto", cryptoUtils);
// 执行Python代码
context.eval("python", config.getPyCode());
// 调用parseFileList函数
Value parseFileListFunc = bindings.getMember("parse_file_list");
if (parseFileListFunc == null || !parseFileListFunc.canExecute()) {
throw new RuntimeException("Python代码中未找到parse_file_list函数");
}
Value result = parseFileListFunc.execute(shareLinkInfoWrapper, httpClient, pyLogger);
List<FileInfo> fileList = convertToFileInfoList(result);
pyLogger.info("文件列表解析成功，共 {} 个文件", fileList.size());
return fileList;
} catch (Exception e) {
pyLogger.error("Python文件列表解析失败: {}", e.getMessage());
throw new RuntimeException("Python文件列表解析失败: " + e.getMessage(), e);
}
});
}
@Override
public Future<String> parseById() {
pyLogger.info("开始执行Python按ID解析: {}", config.getType());
return EXECUTOR.executeBlocking(() -> {
try (Context context = createContext()) {
// 注入Java对象到Python环境
Value bindings = context.getBindings("python");
bindings.putMember("http", httpClient);
bindings.putMember("logger", pyLogger);
bindings.putMember("share_link_info", shareLinkInfoWrapper);
bindings.putMember("crypto", cryptoUtils);
// 执行Python代码
context.eval("python", config.getPyCode());
// 调用parseById函数
Value parseByIdFunc = bindings.getMember("parse_by_id");
if (parseByIdFunc == null || !parseByIdFunc.canExecute()) {
throw new RuntimeException("Python代码中未找到parse_by_id函数");
}
Value result = parseByIdFunc.execute(shareLinkInfoWrapper, httpClient, pyLogger);
if (result.isString()) {
String downloadUrl = result.asString();
pyLogger.info("按ID解析成功: {}", downloadUrl);
return downloadUrl;
} else {
pyLogger.error("parse_by_id方法返回值类型错误，期望String，实际: {}",
result.getMetaObject().toString());
throw new RuntimeException("parse_by_id方法返回值类型错误");
}
} catch (Exception e) {
pyLogger.error("Python按ID解析失败: {}", e.getMessage());
throw new RuntimeException("Python按ID解析失败: " + e.getMessage(), e);
}
});
}
/**
* 将Python列表转换为FileInfo列表
*/
private List<FileInfo> convertToFileInfoList(Value result) {
List<FileInfo> fileList = new ArrayList<>();
if (result.hasArrayElements()) {
long size = result.getArraySize();
for (long i = 0; i < size; i++) {
Value item = result.getArrayElement(i);
FileInfo fileInfo = convertToFileInfo(item);
if (fileInfo != null) {
fileList.add(fileInfo);
}
}
}
return fileList;
}
/**
* 将Python字典转换为FileInfo
*/
private FileInfo convertToFileInfo(Value item) {
try {
FileInfo fileInfo = new FileInfo();
if (item.hasMember("file_name") || item.hasMember("fileName")) {
Value val = item.hasMember("file_name") ? item.getMember("file_name") : item.getMember("fileName");
if (val != null && !val.isNull()) {
fileInfo.setFileName(val.asString());
}
}
if (item.hasMember("file_id") || item.hasMember("fileId")) {
Value val = item.hasMember("file_id") ? item.getMember("file_id") : item.getMember("fileId");
if (val != null && !val.isNull()) {
fileInfo.setFileId(val.asString());
}
}
if (item.hasMember("file_type") || item.hasMember("fileType")) {
Value val = item.hasMember("file_type") ? item.getMember("file_type") : item.getMember("fileType");
if (val != null && !val.isNull()) {
fileInfo.setFileType(val.asString());
}
}
if (item.hasMember("size")) {
Value val = item.getMember("size");
if (val != null && !val.isNull() && val.isNumber()) {
fileInfo.setSize(val.asLong());
}
}
if (item.hasMember("size_str") || item.hasMember("sizeStr")) {
Value val = item.hasMember("size_str") ? item.getMember("size_str") : item.getMember("sizeStr");
if (val != null && !val.isNull()) {
fileInfo.setSizeStr(val.asString());
}
}
if (item.hasMember("create_time") || item.hasMember("createTime")) {
Value val = item.hasMember("create_time") ? item.getMember("create_time") : item.getMember("createTime");
if (val != null && !val.isNull()) {
fileInfo.setCreateTime(val.asString());
}
}
if (item.hasMember("pan_type") || item.hasMember("panType")) {
Value val = item.hasMember("pan_type") ? item.getMember("pan_type") : item.getMember("panType");
if (val != null && !val.isNull()) {
fileInfo.setPanType(val.asString());
}
}
if (item.hasMember("parser_url") || item.hasMember("parserUrl")) {
Value val = item.hasMember("parser_url") ? item.getMember("parser_url") : item.getMember("parserUrl");
if (val != null && !val.isNull()) {
fileInfo.setParserUrl(val.asString());
}
}
return fileInfo;
} catch (Exception e) {
pyLogger.error("转换FileInfo对象失败", e);
return null;
}
}
}

397
parser/src/main/java/cn/qaiu/parser/custompy/PyPlaygroundExecutor.java Normal file
View File

@@ -0,0 +1,397 @@
package cn.qaiu.parser.custompy;
import cn.qaiu.entity.FileInfo;
import cn.qaiu.entity.ShareLinkInfo;
import io.vertx.core.Future;
import io.vertx.core.Promise;
import io.vertx.core.json.JsonObject;
import org.graalvm.polyglot.Context;
import org.graalvm.polyglot.Engine;
import org.graalvm.polyglot.HostAccess;
import org.graalvm.polyglot.Value;
import org.graalvm.polyglot.io.IOAccess;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.*;
/**
* Python演练场执行器
* 用于临时执行Python代码，不注册到解析器注册表
* 使用独立线程池避免Vert.x BlockedThreadChecker警告
*
* @author QAIU
*/
public class PyPlaygroundExecutor {
private static final Logger log = LoggerFactory.getLogger(PyPlaygroundExecutor.class);
// Python执行超时时间（秒）
private static final long EXECUTION_TIMEOUT_SECONDS = 30;
// 共享的GraalPy引擎
private static final Engine SHARED_ENGINE = Engine.newBuilder()
.option("engine.WarnInterpreterOnly", "false")
.build();
// 使用独立的线程池
private static final ExecutorService INDEPENDENT_EXECUTOR = Executors.newCachedThreadPool(r -> {
Thread thread = new Thread(r);
thread.setName("py-playground-independent-" + System.currentTimeMillis());
thread.setDaemon(true);
return thread;
});
// 超时调度线程池
private static final ScheduledExecutorService TIMEOUT_SCHEDULER = Executors.newScheduledThreadPool(2, r -> {
Thread thread = new Thread(r);
thread.setName("py-playground-timeout-scheduler-" + System.currentTimeMillis());
thread.setDaemon(true);
return thread;
});
private final ShareLinkInfo shareLinkInfo;
private final String pyCode;
private final PyHttpClient httpClient;
private final PyPlaygroundLogger playgroundLogger;
private final PyShareLinkInfoWrapper shareLinkInfoWrapper;
private final PyCryptoUtils cryptoUtils;
/**
* 创建演练场执行器
*
* @param shareLinkInfo 分享链接信息
* @param pyCode Python代码
*/
public PyPlaygroundExecutor(ShareLinkInfo shareLinkInfo, String pyCode) {
this.shareLinkInfo = shareLinkInfo;
this.pyCode = pyCode;
// 检查是否有代理配置
JsonObject proxyConfig = null;
if (shareLinkInfo.getOtherParam().containsKey("proxy")) {
proxyConfig = (JsonObject) shareLinkInfo.getOtherParam().get("proxy");
}
this.httpClient = new PyHttpClient(proxyConfig);
this.playgroundLogger = new PyPlaygroundLogger();
this.shareLinkInfoWrapper = new PyShareLinkInfoWrapper(shareLinkInfo);
this.cryptoUtils = new PyCryptoUtils();
}
/**
* 创建安全的GraalPy Context
*/
private Context createContext() {
return Context.newBuilder("python")
.engine(SHARED_ENGINE)
.allowHostAccess(HostAccess.newBuilder(HostAccess.EXPLICIT)
.allowArrayAccess(true)
.allowListAccess(true)
.allowMapAccess(true)
.allowIterableAccess(true)
.allowIteratorAccess(true)
.build())
.allowHostClassLookup(className -> false)
.allowExperimentalOptions(true)
.allowCreateThread(true)
.allowNativeAccess(false)
.allowCreateProcess(false)
.allowIO(IOAccess.newBuilder()
.allowHostFileAccess(false)
.allowHostSocketAccess(false)
.build())
.option("python.PythonHome", "")
.option("python.ForceImportSite", "false")
.build();
}
/**
* 执行parse方法（异步，带超时控制）
*/
public Future<String> executeParseAsync() {
Promise<String> promise = Promise.promise();
CompletableFuture<String> executionFuture = CompletableFuture.supplyAsync(() -> {
playgroundLogger.infoJava("开始执行parse方法");
try (Context context = createContext()) {
// 注入Java对象到Python环境
Value bindings = context.getBindings("python");
bindings.putMember("http", httpClient);
bindings.putMember("logger", playgroundLogger);
bindings.putMember("share_link_info", shareLinkInfoWrapper);
bindings.putMember("crypto", cryptoUtils);
// 执行Python代码
playgroundLogger.debugJava("执行Python代码");
context.eval("python", pyCode);
// 调用parse函数
Value parseFunc = bindings.getMember("parse");
if (parseFunc == null || !parseFunc.canExecute()) {
playgroundLogger.errorJava("Python代码中未找到parse函数");
throw new RuntimeException("Python代码中未找到parse函数");
}
playgroundLogger.debugJava("调用parse函数");
Value result = parseFunc.execute(shareLinkInfoWrapper, httpClient, playgroundLogger);
if (result.isString()) {
String downloadUrl = result.asString();
playgroundLogger.infoJava("解析成功，返回结果: " + downloadUrl);
return downloadUrl;
} else {
String errorMsg = "parse方法返回值类型错误，期望String，实际: " +
(result.isNull() ? "null" : result.getMetaObject().toString());
playgroundLogger.errorJava(errorMsg);
throw new RuntimeException(errorMsg);
}
} catch (Exception e) {
playgroundLogger.errorJava("执行parse方法失败: " + e.getMessage(), e);
throw new RuntimeException(e);
}
}, INDEPENDENT_EXECUTOR);
// 创建超时任务
ScheduledFuture<?> timeoutTask = TIMEOUT_SCHEDULER.schedule(() -> {
if (!executionFuture.isDone()) {
executionFuture.cancel(true);
playgroundLogger.errorJava("执行超时，已强制中断");
log.warn("Python执行超时，已强制取消");
}
}, EXECUTION_TIMEOUT_SECONDS, TimeUnit.SECONDS);
// 处理执行结果
executionFuture.whenComplete((result, error) -> {
timeoutTask.cancel(false);
if (error != null) {
if (error instanceof CancellationException) {
String timeoutMsg = "Python执行超时（超过" + EXECUTION_TIMEOUT_SECONDS + "秒），已强制中断";
playgroundLogger.errorJava(timeoutMsg);
log.error(timeoutMsg);
promise.fail(new RuntimeException(timeoutMsg));
} else {
Throwable cause = error.getCause();
promise.fail(cause != null ? cause : error);
}
} else {
promise.complete(result);
}
});
return promise.future();
}
/**
* 执行parseFileList方法（异步，带超时控制）
*/
public Future<List<FileInfo>> executeParseFileListAsync() {
Promise<List<FileInfo>> promise = Promise.promise();
CompletableFuture<List<FileInfo>> executionFuture = CompletableFuture.supplyAsync(() -> {
playgroundLogger.infoJava("开始执行parse_file_list方法");
try (Context context = createContext()) {
Value bindings = context.getBindings("python");
bindings.putMember("http", httpClient);
bindings.putMember("logger", playgroundLogger);
bindings.putMember("share_link_info", shareLinkInfoWrapper);
bindings.putMember("crypto", cryptoUtils);
context.eval("python", pyCode);
Value parseFileListFunc = bindings.getMember("parse_file_list");
if (parseFileListFunc == null || !parseFileListFunc.canExecute()) {
playgroundLogger.errorJava("Python代码中未找到parse_file_list函数");
throw new RuntimeException("Python代码中未找到parse_file_list函数");
}
playgroundLogger.debugJava("调用parse_file_list函数");
Value result = parseFileListFunc.execute(shareLinkInfoWrapper, httpClient, playgroundLogger);
List<FileInfo> fileList = convertToFileInfoList(result);
playgroundLogger.infoJava("文件列表解析成功，共 " + fileList.size() + " 个文件");
return fileList;
} catch (Exception e) {
playgroundLogger.errorJava("执行parse_file_list方法失败: " + e.getMessage(), e);
throw new RuntimeException(e);
}
}, INDEPENDENT_EXECUTOR);
ScheduledFuture<?> timeoutTask = TIMEOUT_SCHEDULER.schedule(() -> {
if (!executionFuture.isDone()) {
executionFuture.cancel(true);
playgroundLogger.errorJava("执行超时，已强制中断");
}
}, EXECUTION_TIMEOUT_SECONDS, TimeUnit.SECONDS);
executionFuture.whenComplete((result, error) -> {
timeoutTask.cancel(false);
if (error != null) {
if (error instanceof CancellationException) {
String timeoutMsg = "Python执行超时（超过" + EXECUTION_TIMEOUT_SECONDS + "秒），已强制中断";
promise.fail(new RuntimeException(timeoutMsg));
} else {
Throwable cause = error.getCause();
promise.fail(cause != null ? cause : error);
}
} else {
promise.complete(result);
}
});
return promise.future();
}
/**
* 执行parseById方法（异步，带超时控制）
*/
public Future<String> executeParseByIdAsync() {
Promise<String> promise = Promise.promise();
CompletableFuture<String> executionFuture = CompletableFuture.supplyAsync(() -> {
playgroundLogger.infoJava("开始执行parse_by_id方法");
try (Context context = createContext()) {
Value bindings = context.getBindings("python");
bindings.putMember("http", httpClient);
bindings.putMember("logger", playgroundLogger);
bindings.putMember("share_link_info", shareLinkInfoWrapper);
bindings.putMember("crypto", cryptoUtils);
context.eval("python", pyCode);
Value parseByIdFunc = bindings.getMember("parse_by_id");
if (parseByIdFunc == null || !parseByIdFunc.canExecute()) {
playgroundLogger.errorJava("Python代码中未找到parse_by_id函数");
throw new RuntimeException("Python代码中未找到parse_by_id函数");
}
playgroundLogger.debugJava("调用parse_by_id函数");
Value result = parseByIdFunc.execute(shareLinkInfoWrapper, httpClient, playgroundLogger);
if (result.isString()) {
String downloadUrl = result.asString();
playgroundLogger.infoJava("按ID解析成功，返回结果: " + downloadUrl);
return downloadUrl;
} else {
String errorMsg = "parse_by_id方法返回值类型错误";
playgroundLogger.errorJava(errorMsg);
throw new RuntimeException(errorMsg);
}
} catch (Exception e) {
playgroundLogger.errorJava("执行parse_by_id方法失败: " + e.getMessage(), e);
throw new RuntimeException(e);
}
}, INDEPENDENT_EXECUTOR);
ScheduledFuture<?> timeoutTask = TIMEOUT_SCHEDULER.schedule(() -> {
if (!executionFuture.isDone()) {
executionFuture.cancel(true);
playgroundLogger.errorJava("执行超时，已强制中断");
}
}, EXECUTION_TIMEOUT_SECONDS, TimeUnit.SECONDS);
executionFuture.whenComplete((result, error) -> {
timeoutTask.cancel(false);
if (error != null) {
if (error instanceof CancellationException) {
String timeoutMsg = "Python执行超时（超过" + EXECUTION_TIMEOUT_SECONDS + "秒），已强制中断";
promise.fail(new RuntimeException(timeoutMsg));
} else {
Throwable cause = error.getCause();
promise.fail(cause != null ? cause : error);
}
} else {
promise.complete(result);
}
});
return promise.future();
}
/**
* 获取日志列表
*/
public List<PyPlaygroundLogger.LogEntry> getLogs() {
return playgroundLogger.getLogs();
}
/**
* 将Python列表转换为FileInfo列表
*/
private List<FileInfo> convertToFileInfoList(Value result) {
List<FileInfo> fileList = new ArrayList<>();
if (result.hasArrayElements()) {
long size = result.getArraySize();
for (long i = 0; i < size; i++) {
Value item = result.getArrayElement(i);
FileInfo fileInfo = convertToFileInfo(item);
if (fileInfo != null) {
fileList.add(fileInfo);
}
}
}
return fileList;
}
/**
* 将Python字典转换为FileInfo
*/
private FileInfo convertToFileInfo(Value item) {
try {
FileInfo fileInfo = new FileInfo();
if (item.hasMember("file_name") || item.hasMember("fileName")) {
Value val = item.hasMember("file_name") ? item.getMember("file_name") : item.getMember("fileName");
if (val != null && !val.isNull()) {
fileInfo.setFileName(val.asString());
}
}
if (item.hasMember("file_id") || item.hasMember("fileId")) {
Value val = item.hasMember("file_id") ? item.getMember("file_id") : item.getMember("fileId");
if (val != null && !val.isNull()) {
fileInfo.setFileId(val.asString());
}
}
if (item.hasMember("file_type") || item.hasMember("fileType")) {
Value val = item.hasMember("file_type") ? item.getMember("file_type") : item.getMember("fileType");
if (val != null && !val.isNull()) {
fileInfo.setFileType(val.asString());
}
}
if (item.hasMember("size")) {
Value val = item.getMember("size");
if (val != null && !val.isNull() && val.isNumber()) {
fileInfo.setSize(val.asLong());
}
}
if (item.hasMember("pan_type") || item.hasMember("panType")) {
Value val = item.hasMember("pan_type") ? item.getMember("pan_type") : item.getMember("panType");
if (val != null && !val.isNull()) {
fileInfo.setPanType(val.asString());
}
}
if (item.hasMember("parser_url") || item.hasMember("parserUrl")) {
Value val = item.hasMember("parser_url") ? item.getMember("parser_url") : item.getMember("parserUrl");
if (val != null && !val.isNull()) {
fileInfo.setParserUrl(val.asString());
}
}
return fileInfo;
} catch (Exception e) {
playgroundLogger.errorJava("转换FileInfo对象失败: " + e.getMessage());
return null;
}
}
}

183
parser/src/main/java/cn/qaiu/parser/custompy/PyPlaygroundLogger.java Normal file
View File

@@ -0,0 +1,183 @@
package cn.qaiu.parser.custompy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
/**
* Python演练场日志封装
* 收集日志信息用于前端显示
*
* @author QAIU
*/
public class PyPlaygroundLogger extends PyLogger {
private static final Logger log = LoggerFactory.getLogger(PyPlaygroundLogger.class);
private final List<LogEntry> logs = new ArrayList<>();
public PyPlaygroundLogger() {
super("PyPlayground");
}
@Override
public void debug(String message) {
super.debug(message);
addLog("DEBUG", message);
}
@Override
public void debug(String message, Object... args) {
super.debug(message, args);
addLog("DEBUG", formatMessage(message, args));
}
@Override
public void info(String message) {
super.info(message);
addLog("INFO", message);
}
@Override
public void info(String message, Object... args) {
super.info(message, args);
addLog("INFO", formatMessage(message, args));
}
@Override
public void warn(String message) {
super.warn(message);
addLog("WARN", message);
}
@Override
public void warn(String message, Object... args) {
super.warn(message, args);
addLog("WARN", formatMessage(message, args));
}
@Override
public void error(String message) {
super.error(message);
addLog("ERROR", message);
}
@Override
public void error(String message, Object... args) {
super.error(message, args);
addLog("ERROR", formatMessage(message, args));
}
@Override
public void error(String message, Throwable throwable) {
super.error(message, throwable);
addLog("ERROR", message + " - " + throwable.getMessage());
}
/**
* 添加Java内部日志（不在Python脚本中调用）
*/
public void infoJava(String message) {
log.info("[PyPlayground] " + message);
addLog("INFO", "[Java] " + message, "java");
}
public void debugJava(String message) {
log.debug("[PyPlayground] " + message);
addLog("DEBUG", "[Java] " + message, "java");
}
public void errorJava(String message) {
log.error("[PyPlayground] " + message);
addLog("ERROR", "[Java] " + message, "java");
}
public void errorJava(String message, Throwable throwable) {
log.error("[PyPlayground] " + message, throwable);
addLog("ERROR", "[Java] " + message + " - " + throwable.getMessage(), "java");
}
private void addLog(String level, String message) {
addLog(level, message, "python");
}
private void addLog(String level, String message, String source) {
logs.add(new LogEntry(level, message, System.currentTimeMillis(), source));
}
private String formatMessage(String message, Object... args) {
if (args == null || args.length == 0) {
return message;
}
// 简单的占位符替换
String result = message;
for (Object arg : args) {
int index = result.indexOf("{}");
if (index >= 0) {
result = result.substring(0, index) + (arg != null ? arg.toString() : "null") + result.substring(index + 2);
}
}
return result;
}
/**
* 获取所有日志
*/
public List<LogEntry> getLogs() {
return new ArrayList<>(logs);
}
/**
* 清空日志
*/
public void clearLogs() {
logs.clear();
}
/**
* 获取日志数量
*/
public int size() {
return logs.size();
}
/**
* 日志条目
*/
public static class LogEntry {
private final String level;
private final String message;
private final long timestamp;
private final String source;
public LogEntry(String level, String message, long timestamp) {
this(level, message, timestamp, "python");
}
public LogEntry(String level, String message, long timestamp, String source) {
this.level = level;
this.message = message;
this.timestamp = timestamp;
this.source = source;
}
public String getLevel() {
return level;
}
public String getMessage() {
return message;
}
public long getTimestamp() {
return timestamp;
}
public String getSource() {
return source;
}
}
}

334
parser/src/main/java/cn/qaiu/parser/custompy/PyScriptLoader.java Normal file
View File

@@ -0,0 +1,334 @@
package cn.qaiu.parser.custompy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import cn.qaiu.parser.custom.CustomParserConfig;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.List;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
import java.util.stream.Stream;
/**
* Python脚本加载器
* 自动加载资源目录和外部目录的Python脚本文件
*
* @author QAIU
*/
public class PyScriptLoader {
private static final Logger log = LoggerFactory.getLogger(PyScriptLoader.class);
private static final String RESOURCE_PATH = "custom-parsers/py";
private static final String EXTERNAL_PATH = "./custom-parsers/py";
// 系统属性配置的外部目录路径
private static final String EXTERNAL_PATH_PROPERTY = "parser.custom-parsers.py.path";
/**
* 加载所有Python脚本
* @return 解析器配置列表
*/
public static List<CustomParserConfig> loadAllScripts() {
List<CustomParserConfig> configs = new ArrayList<>();
// 1. 加载资源目录下的Python文件
try {
List<CustomParserConfig> resourceConfigs = loadFromResources();
configs.addAll(resourceConfigs);
log.info("从资源目录加载了 {} 个Python解析器", resourceConfigs.size());
} catch (Exception e) {
log.warn("从资源目录加载Python脚本失败", e);
}
// 2. 加载外部目录下的Python文件
try {
List<CustomParserConfig> externalConfigs = loadFromExternal();
configs.addAll(externalConfigs);
log.info("从外部目录加载了 {} 个Python解析器", externalConfigs.size());
} catch (Exception e) {
log.warn("从外部目录加载Python脚本失败", e);
}
log.info("总共加载了 {} 个Python解析器", configs.size());
return configs;
}
/**
* 从资源目录加载Python脚本
*/
private static List<CustomParserConfig> loadFromResources() {
List<CustomParserConfig> configs = new ArrayList<>();
try {
List<String> resourceFiles = getResourceFileList();
resourceFiles.sort(String::compareTo);
for (String resourceFile : resourceFiles) {
try {
InputStream inputStream = PyScriptLoader.class.getClassLoader()
.getResourceAsStream(resourceFile);
if (inputStream != null) {
String pyCode = new String(inputStream.readAllBytes(), StandardCharsets.UTF_8);
CustomParserConfig config = PyScriptMetadataParser.parseScript(pyCode);
configs.add(config);
String fileName = resourceFile.substring(resourceFile.lastIndexOf('/') + 1);
log.debug("从资源目录加载Python脚本: {}", fileName);
}
} catch (Exception e) {
log.warn("加载资源脚本失败: {}", resourceFile, e);
}
}
} catch (Exception e) {
log.error("从资源目录加载脚本时发生异常", e);
}
return configs;
}
/**
* 获取资源目录中的Python文件列表
*/
private static List<String> getResourceFileList() {
List<String> resourceFiles = new ArrayList<>();
try {
java.net.URL resourceUrl = PyScriptLoader.class.getClassLoader()
.getResource(RESOURCE_PATH);
if (resourceUrl != null) {
String protocol = resourceUrl.getProtocol();
if ("jar".equals(protocol)) {
resourceFiles = getJarResourceFiles(resourceUrl);
} else if ("file".equals(protocol)) {
resourceFiles = getFileSystemResourceFiles(resourceUrl);
}
}
} catch (Exception e) {
log.debug("获取资源文件列表失败", e);
}
return resourceFiles;
}
/**
* 获取JAR包内的Python资源文件列表
*/
private static List<String> getJarResourceFiles(java.net.URL jarUrl) {
List<String> resourceFiles = new ArrayList<>();
try {
String jarPath = jarUrl.getPath().substring(5, jarUrl.getPath().indexOf("!"));
JarFile jarFile = new JarFile(jarPath);
Enumeration<JarEntry> entries = jarFile.entries();
while (entries.hasMoreElements()) {
JarEntry entry = entries.nextElement();
String entryName = entry.getName();
if (entryName.startsWith(RESOURCE_PATH + "/") &&
entryName.endsWith(".py") &&
!isExcludedFile(entryName.substring(entryName.lastIndexOf('/') + 1))) {
resourceFiles.add(entryName);
}
}
jarFile.close();
} catch (Exception e) {
log.debug("解析JAR包资源文件失败", e);
}
return resourceFiles;
}
/**
* 获取文件系统中的Python资源文件列表
*/
private static List<String> getFileSystemResourceFiles(java.net.URL fileUrl) {
List<String> resourceFiles = new ArrayList<>();
try {
java.io.File resourceDir = new java.io.File(fileUrl.getPath());
if (resourceDir.exists() && resourceDir.isDirectory()) {
java.io.File[] files = resourceDir.listFiles();
if (files != null) {
for (java.io.File file : files) {
if (file.isFile() && file.getName().endsWith(".py") &&
!isExcludedFile(file.getName())) {
resourceFiles.add(RESOURCE_PATH + "/" + file.getName());
}
}
}
}
} catch (Exception e) {
log.debug("解析文件系统资源文件失败", e);
}
return resourceFiles;
}
/**
* 从外部目录加载Python脚本
*/
private static List<CustomParserConfig> loadFromExternal() {
List<CustomParserConfig> configs = new ArrayList<>();
try {
String externalPath = getExternalPath();
Path externalDir = Paths.get(externalPath);
if (!Files.exists(externalDir) || !Files.isDirectory(externalDir)) {
log.debug("外部目录 {} 不存在或不是目录", externalPath);
return configs;
}
try (Stream<Path> paths = Files.walk(externalDir)) {
paths.filter(Files::isRegularFile)
.filter(path -> path.toString().endsWith(".py"))
.filter(path -> !isExcludedFile(path.getFileName().toString()))
.forEach(path -> {
try {
String pyCode = Files.readString(path, StandardCharsets.UTF_8);
CustomParserConfig config = PyScriptMetadataParser.parseScript(pyCode);
configs.add(config);
log.debug("从外部目录加载Python脚本: {}", path.getFileName());
} catch (Exception e) {
log.warn("加载外部脚本失败: {}", path.getFileName(), e);
}
});
}
} catch (Exception e) {
log.error("从外部目录加载脚本时发生异常", e);
}
return configs;
}
/**
* 获取外部目录路径
*/
private static String getExternalPath() {
// 1. 检查系统属性
String systemProperty = System.getProperty(EXTERNAL_PATH_PROPERTY);
if (systemProperty != null && !systemProperty.trim().isEmpty()) {
log.debug("使用系统属性配置的Python外部目录: {}", systemProperty);
return systemProperty;
}
// 2. 检查环境变量
String envVariable = System.getenv("PARSER_CUSTOM_PARSERS_PY_PATH");
if (envVariable != null && !envVariable.trim().isEmpty()) {
log.debug("使用环境变量配置的Python外部目录: {}", envVariable);
return envVariable;
}
// 3. 使用默认路径
log.debug("使用默认Python外部目录: {}", EXTERNAL_PATH);
return EXTERNAL_PATH;
}
/**
* 从指定文件加载Python脚本
* @param filePath 文件路径
* @return 解析器配置
*/
public static CustomParserConfig loadFromFile(String filePath) {
try {
Path path = Paths.get(filePath);
if (!Files.exists(path)) {
throw new IllegalArgumentException("文件不存在: " + filePath);
}
String pyCode = Files.readString(path, StandardCharsets.UTF_8);
return PyScriptMetadataParser.parseScript(pyCode);
} catch (IOException e) {
throw new RuntimeException("读取文件失败: " + filePath, e);
}
}
/**
* 从指定资源路径加载Python脚本
* @param resourcePath 资源路径
* @return 解析器配置
*/
public static CustomParserConfig loadFromResource(String resourcePath) {
try {
InputStream inputStream = PyScriptLoader.class.getClassLoader()
.getResourceAsStream(resourcePath);
if (inputStream == null) {
throw new IllegalArgumentException("资源文件不存在: " + resourcePath);
}
String pyCode = new String(inputStream.readAllBytes(), StandardCharsets.UTF_8);
return PyScriptMetadataParser.parseScript(pyCode);
} catch (IOException e) {
throw new RuntimeException("读取资源文件失败: " + resourcePath, e);
}
}
/**
* 检查外部目录是否存在
*/
public static boolean isExternalDirectoryExists() {
Path externalDir = Paths.get(EXTERNAL_PATH);
return Files.exists(externalDir) && Files.isDirectory(externalDir);
}
/**
* 创建外部目录
*/
public static boolean createExternalDirectory() {
try {
Path externalDir = Paths.get(EXTERNAL_PATH);
Files.createDirectories(externalDir);
log.info("创建Python外部目录成功: {}", EXTERNAL_PATH);
return true;
} catch (IOException e) {
log.error("创建Python外部目录失败: {}", EXTERNAL_PATH, e);
return false;
}
}
/**
* 获取外部目录路径
*/
public static String getExternalDirectoryPath() {
return EXTERNAL_PATH;
}
/**
* 获取资源目录路径
*/
public static String getResourceDirectoryPath() {
return RESOURCE_PATH;
}
/**
* 检查文件是否应该被排除
*/
private static boolean isExcludedFile(String fileName) {
return fileName.equals("types.pyi") ||
fileName.equals("__init__.py") ||
fileName.equals("README.md") ||
fileName.contains("_test.") ||
fileName.contains("_spec.") ||
fileName.startsWith("test_");
}
}

188
parser/src/main/java/cn/qaiu/parser/custompy/PyScriptMetadataParser.java Normal file
View File

@@ -0,0 +1,188 @@
package cn.qaiu.parser.custompy;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import cn.qaiu.parser.custom.CustomParserConfig;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Python脚本元数据解析器
* 解析类油猴格式的元数据注释（Python风格）
*
* @author QAIU
*/
public class PyScriptMetadataParser {
private static final Logger log = LoggerFactory.getLogger(PyScriptMetadataParser.class);
// 元数据块匹配正则（Python注释风格）
// 支持 # ==UserScript== 格式
private static final Pattern METADATA_BLOCK_PATTERN = Pattern.compile(
"#\\s*==UserScript==\\s*(.*?)\\s*#\\s*==/UserScript==",
Pattern.DOTALL
);
// 元数据行匹配正则
private static final Pattern METADATA_LINE_PATTERN = Pattern.compile(
"#\\s*@(\\w+)\\s+(.*)"
);
/**
* 解析Python脚本，提取元数据并构建CustomParserConfig
*
* @param pyCode Python代码
* @return CustomParserConfig配置对象
* @throws IllegalArgumentException 如果解析失败或缺少必填字段
*/
public static CustomParserConfig parseScript(String pyCode) {
if (StringUtils.isBlank(pyCode)) {
throw new IllegalArgumentException("Python代码不能为空");
}
// 1. 提取元数据块
Map<String, String> metadata = extractMetadata(pyCode);
// 2. 验证必填字段
validateRequiredFields(metadata);
// 3. 构建CustomParserConfig
return buildConfig(metadata, pyCode);
}
/**
* 提取元数据
*/
private static Map<String, String> extractMetadata(String pyCode) {
Map<String, String> metadata = new HashMap<>();
Matcher blockMatcher = METADATA_BLOCK_PATTERN.matcher(pyCode);
if (!blockMatcher.find()) {
throw new IllegalArgumentException("未找到元数据块，请确保包含 # ==UserScript== ... # ==/UserScript== 格式的注释");
}
String metadataBlock = blockMatcher.group(1);
Matcher lineMatcher = METADATA_LINE_PATTERN.matcher(metadataBlock);
while (lineMatcher.find()) {
String key = lineMatcher.group(1).toLowerCase();
String value = lineMatcher.group(2).trim();
metadata.put(key, value);
}
log.debug("解析到Python脚本元数据: {}", metadata);
return metadata;
}
/**
* 验证必填字段
*/
private static void validateRequiredFields(Map<String, String> metadata) {
if (!metadata.containsKey("name")) {
throw new IllegalArgumentException("缺少必填字段 @name");
}
if (!metadata.containsKey("type")) {
throw new IllegalArgumentException("缺少必填字段 @type");
}
if (!metadata.containsKey("displayname")) {
throw new IllegalArgumentException("缺少必填字段 @displayName");
}
if (!metadata.containsKey("match")) {
throw new IllegalArgumentException("缺少必填字段 @match");
}
// 验证match字段包含KEY命名捕获组
String matchPattern = metadata.get("match");
if (!matchPattern.contains("(?P<KEY>") && !matchPattern.contains("(?<KEY>")) {
throw new IllegalArgumentException("@match 正则表达式必须包含命名捕获组 KEY（Python格式: (?P<KEY>...) 或 Java格式: (?<KEY>...)）");
}
}
/**
* 构建CustomParserConfig
*/
private static CustomParserConfig buildConfig(Map<String, String> metadata, String pyCode) {
CustomParserConfig.Builder builder = CustomParserConfig.builder()
.type(metadata.get("type"))
.displayName(metadata.get("displayname"))
.isPyParser(true)
.pyCode(pyCode)
.language("python")
.metadata(metadata);
// 设置匹配正则（将Python风格的(?P<KEY>...)转换为Java风格的(?<KEY>...)）
String matchPattern = metadata.get("match");
if (StringUtils.isNotBlank(matchPattern)) {
// 将Python命名捕获组转换为Java格式
matchPattern = matchPattern.replace("(?P<", "(?<");
builder.matchPattern(matchPattern);
}
return builder.build();
}
/**
* 检查Python代码是否包含有效的元数据块
*
* @param pyCode Python代码
* @return true表示包含有效元数据，false表示不包含
*/
public static boolean hasValidMetadata(String pyCode) {
if (StringUtils.isBlank(pyCode)) {
return false;
}
try {
Map<String, String> metadata = extractMetadata(pyCode);
return metadata.containsKey("name") &&
metadata.containsKey("type") &&
metadata.containsKey("displayname") &&
metadata.containsKey("match");
} catch (Exception e) {
return false;
}
}
/**
* 获取脚本类型（不验证必填字段）
*
* @param pyCode Python代码
* @return 脚本类型，如果无法提取则返回null
*/
public static String getScriptType(String pyCode) {
if (StringUtils.isBlank(pyCode)) {
return null;
}
try {
Map<String, String> metadata = extractMetadata(pyCode);
return metadata.get("type");
} catch (Exception e) {
return null;
}
}
/**
* 获取脚本显示名称（不验证必填字段）
*
* @param pyCode Python代码
* @return 显示名称，如果无法提取则返回null
*/
public static String getScriptDisplayName(String pyCode) {
if (StringUtils.isBlank(pyCode)) {
return null;
}
try {
Map<String, String> metadata = extractMetadata(pyCode);
return metadata.get("displayname");
} catch (Exception e) {
return null;
}
}
}

262
parser/src/main/java/cn/qaiu/parser/custompy/PyShareLinkInfoWrapper.java Normal file
View File

@@ -0,0 +1,262 @@
package cn.qaiu.parser.custompy;
import cn.qaiu.entity.ShareLinkInfo;
import org.graalvm.polyglot.HostAccess;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Map;
/**
* ShareLinkInfo的Python包装器
* 为Python脚本提供ShareLinkInfo对象的访问接口
*
* @author QAIU
*/
public class PyShareLinkInfoWrapper {
private static final Logger log = LoggerFactory.getLogger(PyShareLinkInfoWrapper.class);
private final ShareLinkInfo shareLinkInfo;
public PyShareLinkInfoWrapper(ShareLinkInfo shareLinkInfo) {
this.shareLinkInfo = shareLinkInfo;
}
/**
* 获取分享URL
* @return 分享URL
*/
@HostAccess.Export
public String getShareUrl() {
return shareLinkInfo.getShareUrl();
}
/**
* Python风格方法名 - 获取分享URL
*/
@HostAccess.Export
public String get_share_url() {
return getShareUrl();
}
/**
* 获取分享Key
* @return 分享Key
*/
@HostAccess.Export
public String getShareKey() {
return shareLinkInfo.getShareKey();
}
/**
* Python风格方法名 - 获取分享Key
*/
@HostAccess.Export
public String get_share_key() {
return getShareKey();
}
/**
* 获取分享密码
* @return 分享密码
*/
@HostAccess.Export
public String getSharePassword() {
return shareLinkInfo.getSharePassword();
}
/**
* Python风格方法名 - 获取分享密码
*/
@HostAccess.Export
public String get_share_password() {
return getSharePassword();
}
/**
* 获取网盘类型
* @return 网盘类型
*/
@HostAccess.Export
public String getType() {
return shareLinkInfo.getType();
}
/**
* Python风格方法名 - 获取网盘类型
*/
@HostAccess.Export
public String get_type() {
return getType();
}
/**
* 获取网盘名称
* @return 网盘名称
*/
@HostAccess.Export
public String getPanName() {
return shareLinkInfo.getPanName();
}
/**
* Python风格方法名 - 获取网盘名称
*/
@HostAccess.Export
public String get_pan_name() {
return getPanName();
}
/**
* 获取其他参数
* @param key 参数键
* @return 参数值
*/
@HostAccess.Export
public Object getOtherParam(String key) {
if (key == null) {
return null;
}
return shareLinkInfo.getOtherParam().get(key);
}
/**
* Python风格方法名 - 获取其他参数
*/
@HostAccess.Export
public Object get_other_param(String key) {
return getOtherParam(key);
}
/**
* 获取所有其他参数
* @return 参数Map
*/
@HostAccess.Export
public Map<String, Object> getAllOtherParams() {
return shareLinkInfo.getOtherParam();
}
/**
* Python风格方法名 - 获取所有其他参数
*/
@HostAccess.Export
public Map<String, Object> get_all_other_params() {
return getAllOtherParams();
}
/**
* 检查是否包含指定参数
* @param key 参数键
* @return true表示包含，false表示不包含
*/
@HostAccess.Export
public boolean hasOtherParam(String key) {
if (key == null) {
return false;
}
return shareLinkInfo.getOtherParam().containsKey(key);
}
/**
* Python风格方法名 - 检查是否包含指定参数
*/
@HostAccess.Export
public boolean has_other_param(String key) {
return hasOtherParam(key);
}
/**
* 获取其他参数的字符串值
* @param key 参数键
* @return 参数值（字符串形式）
*/
@HostAccess.Export
public String getOtherParamAsString(String key) {
Object value = getOtherParam(key);
return value != null ? value.toString() : null;
}
/**
* Python风格方法名 - 获取其他参数的字符串值
*/
@HostAccess.Export
public String get_other_param_as_string(String key) {
return getOtherParamAsString(key);
}
/**
* 获取其他参数的整数值
* @param key 参数键
* @return 参数值（整数形式）
*/
@HostAccess.Export
public Integer getOtherParamAsInteger(String key) {
Object value = getOtherParam(key);
if (value instanceof Integer) {
return (Integer) value;
} else if (value instanceof Number) {
return ((Number) value).intValue();
} else if (value instanceof String) {
try {
return Integer.parseInt((String) value);
} catch (NumberFormatException e) {
log.warn("无法将参数 {} 转换为整数: {}", key, value);
return null;
}
}
return null;
}
/**
* Python风格方法名 - 获取其他参数的整数值
*/
@HostAccess.Export
public Integer get_other_param_as_integer(String key) {
return getOtherParamAsInteger(key);
}
/**
* 获取其他参数的布尔值
* @param key 参数键
* @return 参数值（布尔形式）
*/
@HostAccess.Export
public Boolean getOtherParamAsBoolean(String key) {
Object value = getOtherParam(key);
if (value instanceof Boolean) {
return (Boolean) value;
} else if (value instanceof String) {
return Boolean.parseBoolean((String) value);
}
return null;
}
/**
* Python风格方法名 - 获取其他参数的布尔值
*/
@HostAccess.Export
public Boolean get_other_param_as_boolean(String key) {
return getOtherParamAsBoolean(key);
}
/**
* 获取原始的ShareLinkInfo对象
* @return ShareLinkInfo对象
*/
public ShareLinkInfo getOriginalShareLinkInfo() {
return shareLinkInfo;
}
@Override
public String toString() {
return "PyShareLinkInfoWrapper{" +
"shareUrl='" + getShareUrl() + '\'' +
", shareKey='" + getShareKey() + '\'' +
", sharePassword='" + getSharePassword() + '\'' +
", type='" + getType() + '\'' +
", panName='" + getPanName() + '\'' +
'}';
}
}

137
parser/src/main/resources/custom-parsers/py/example_parser.py Normal file
View File

@@ -0,0 +1,137 @@
# ==UserScript==
# @name 示例Python解析器
# @type example_py_parser
# @displayName 示例网盘(Python)
# @match https?://example\.com/s/(?P<KEY>\w+)(?:\?pwd=(?P<PWD>\w+))?
# @description Python解析器示例，展示如何编写Python网盘解析器
# @author QAIU
# @version 1.0.0
# ==/UserScript==
"""
Python解析器示例
可用的全局对象:
- http: HTTP客户端 (PyHttpClient)
- logger: 日志对象 (PyLogger)
- share_link_info: 分享信息 (PyShareLinkInfoWrapper)
- crypto: 加密工具 (PyCryptoUtils)
必须实现的函数:
- parse(share_link_info, http, logger): 解析下载链接，返回下载URL字符串
可选实现的函数:
- parse_file_list(share_link_info, http, logger): 解析文件列表，返回文件信息列表
- parse_by_id(share_link_info, http, logger): 根据文件ID解析下载链接
"""
def parse(share_link_info, http, logger):
"""
解析分享链接，获取直链下载地址
参数:
share_link_info: 分享信息对象
- get_share_url(): 获取分享URL
- get_share_key(): 获取分享Key
- get_share_password(): 获取分享密码
- get_type(): 获取网盘类型
http: HTTP客户端
- get(url): GET请求
- post(url, data): POST请求
- put_header(name, value): 设置请求头
- set_timeout(seconds): 设置超时时间
logger: 日志对象
- info(msg): 信息日志
- debug(msg): 调试日志
- warn(msg): 警告日志
- error(msg): 错误日志
返回:
str: 直链下载地址
"""
# 获取分享信息
share_url = share_link_info.get_share_url()
share_key = share_link_info.get_share_key()
share_password = share_link_info.get_share_password()
logger.info(f"开始解析: {share_url}")
logger.info(f"分享Key: {share_key}")
# 设置请求头
http.put_header("Referer", share_url)
# 发起GET请求获取页面内容
response = http.get(share_url)
if not response.ok():
logger.error(f"请求失败: {response.status_code()}")
raise Exception(f"请求失败: {response.status_code()}")
html = response.text()
logger.debug(f"响应长度: {len(html)}")
# 示例：从响应中提取下载链接
# 实际解析逻辑根据具体网盘API实现
# 演示使用加密工具
# md5_hash = crypto.md5(share_key)
# logger.info(f"MD5: {md5_hash}")
# 返回模拟的下载链接
return f"https://example.com/download/{share_key}"
def parse_file_list(share_link_info, http, logger):
"""
解析文件列表
返回:
list: 文件信息列表，每个元素是字典，包含:
- file_name: 文件名
- file_id: 文件ID
- file_type: 文件类型
- size: 文件大小(字节)
- pan_type: 网盘类型
- parser_url: 解析URL
"""
share_url = share_link_info.get_share_url()
share_key = share_link_info.get_share_key()
logger.info(f"获取文件列表: {share_url}")
# 示例返回
return [
{
"file_name": "示例文件1.txt",
"file_id": "file_001",
"file_type": "file",
"size": 1024,
"pan_type": "example_py_parser",
"parser_url": f"/parser?type=example_py_parser&key={share_key}&fileId=file_001"
},
{
"file_name": "示例文件2.zip",
"file_id": "file_002",
"file_type": "file",
"size": 2048,
"pan_type": "example_py_parser",
"parser_url": f"/parser?type=example_py_parser&key={share_key}&fileId=file_002"
}
]
def parse_by_id(share_link_info, http, logger):
"""
根据文件ID解析下载链接
返回:
str: 直链下载地址
"""
file_id = share_link_info.get_other_param("fileId")
share_key = share_link_info.get_share_key()
logger.info(f"按ID解析: fileId={file_id}, shareKey={share_key}")
# 返回模拟的下载链接
return f"https://example.com/download/{share_key}/{file_id}"

339
parser/src/main/resources/py/types.pyi Normal file
View File

@@ -0,0 +1,339 @@
"""
NFD Python解析器类型存根文件
提供IDE自动补全和类型检查支持
"""
from typing import Dict, List, Optional, Any
class PyShareLinkInfoWrapper:
"""分享链接信息包装器"""
def get_share_url(self) -> str:
"""获取分享URL"""
...
def get_share_key(self) -> str:
"""获取分享Key"""
...
def get_share_password(self) -> Optional[str]:
"""获取分享密码"""
...
def get_type(self) -> str:
"""获取网盘类型"""
...
def get_pan_name(self) -> str:
"""获取网盘名称"""
...
def get_other_param(self, key: str) -> Optional[Any]:
"""获取其他参数"""
...
def get_all_other_params(self) -> Dict[str, Any]:
"""获取所有其他参数"""
...
def has_other_param(self, key: str) -> bool:
"""检查是否包含指定参数"""
...
def get_other_param_as_string(self, key: str) -> Optional[str]:
"""获取其他参数的字符串值"""
...
def get_other_param_as_integer(self, key: str) -> Optional[int]:
"""获取其他参数的整数值"""
...
def get_other_param_as_boolean(self, key: str) -> Optional[bool]:
"""获取其他参数的布尔值"""
...
class PyHttpResponse:
"""HTTP响应封装"""
def text(self) -> str:
"""获取响应体文本"""
...
def body(self) -> str:
"""获取响应体文本（别名）"""
...
def json(self) -> Optional[Dict[str, Any]]:
"""解析JSON响应"""
...
def status_code(self) -> int:
"""获取HTTP状态码"""
...
def header(self, name: str) -> Optional[str]:
"""获取响应头"""
...
def headers(self) -> Dict[str, str]:
"""获取所有响应头"""
...
def ok(self) -> bool:
"""检查请求是否成功（2xx状态码）"""
...
def content(self) -> bytes:
"""获取响应体字节数组"""
...
def content_length(self) -> int:
"""获取响应体大小"""
...
class PyHttpClient:
"""HTTP客户端"""
def get(self, url: str) -> PyHttpResponse:
"""发起GET请求"""
...
def get_with_redirect(self, url: str) -> PyHttpResponse:
"""发起GET请求并跟随重定向"""
...
def get_no_redirect(self, url: str) -> PyHttpResponse:
"""发起GET请求但不跟随重定向"""
...
def post(self, url: str, data: Any = None) -> PyHttpResponse:
"""发起POST请求"""
...
def post_json(self, url: str, json_data: Any = None) -> PyHttpResponse:
"""发起POST请求（JSON数据）"""
...
def put(self, url: str, data: Any = None) -> PyHttpResponse:
"""发起PUT请求"""
...
def delete(self, url: str) -> PyHttpResponse:
"""发起DELETE请求"""
...
def patch(self, url: str, data: Any = None) -> PyHttpResponse:
"""发起PATCH请求"""
...
def put_header(self, name: str, value: str) -> 'PyHttpClient':
"""设置请求头"""
...
def put_headers(self, headers: Dict[str, str]) -> 'PyHttpClient':
"""批量设置请求头"""
...
def remove_header(self, name: str) -> 'PyHttpClient':
"""删除指定请求头"""
...
def clear_headers(self) -> 'PyHttpClient':
"""清空所有请求头"""
...
def get_headers(self) -> Dict[str, str]:
"""获取所有请求头"""
...
def set_timeout(self, seconds: int) -> 'PyHttpClient':
"""设置请求超时时间"""
...
@staticmethod
def url_encode(string: str) -> str:
"""URL编码"""
...
@staticmethod
def url_decode(string: str) -> str:
"""URL解码"""
...
class PyLogger:
"""日志记录器"""
def debug(self, message: str, *args) -> None:
"""调试日志"""
...
def info(self, message: str, *args) -> None:
"""信息日志"""
...
def warn(self, message: str, *args) -> None:
"""警告日志"""
...
def error(self, message: str, *args) -> None:
"""错误日志"""
...
def is_debug_enabled(self) -> bool:
"""检查是否启用调试级别日志"""
...
def is_info_enabled(self) -> bool:
"""检查是否启用信息级别日志"""
...
class PyCryptoUtils:
"""加密工具类"""
def md5(self, data: str) -> str:
"""MD5加密（32位小写）"""
...
def md5_16(self, data: str) -> str:
"""MD5加密（16位小写）"""
...
def sha1(self, data: str) -> str:
"""SHA-1加密"""
...
def sha256(self, data: str) -> str:
"""SHA-256加密"""
...
def sha512(self, data: str) -> str:
"""SHA-512加密"""
...
def base64_encode(self, data: str) -> str:
"""Base64编码"""
...
def base64_encode_bytes(self, data: bytes) -> str:
"""Base64编码（字节数组）"""
...
def base64_decode(self, data: str) -> str:
"""Base64解码"""
...
def base64_decode_bytes(self, data: str) -> bytes:
"""Base64解码（返回字节数组）"""
...
def base64_url_encode(self, data: str) -> str:
"""URL安全的Base64编码"""
...
def base64_url_decode(self, data: str) -> str:
"""URL安全的Base64解码"""
...
def aes_encrypt_ecb(self, data: str, key: str) -> str:
"""AES加密（ECB模式）"""
...
def aes_decrypt_ecb(self, data: str, key: str) -> str:
"""AES解密（ECB模式）"""
...
def aes_encrypt_cbc(self, data: str, key: str, iv: str) -> str:
"""AES加密（CBC模式）"""
...
def aes_decrypt_cbc(self, data: str, key: str, iv: str) -> str:
"""AES解密（CBC模式）"""
...
def bytes_to_hex(self, data: bytes) -> str:
"""字节数组转十六进制"""
...
def hex_to_bytes(self, hex_string: str) -> bytes:
"""十六进制转字节数组"""
...
# 全局变量类型声明
http: PyHttpClient
logger: PyLogger
share_link_info: PyShareLinkInfoWrapper
crypto: PyCryptoUtils
class FileInfo:
"""文件信息"""
file_name: str
file_id: str
file_type: str
size: int
size_str: str
create_time: str
update_time: str
create_by: str
download_count: int
file_icon: str
pan_type: str
parser_url: str
preview_url: str
def parse(share_link_info: PyShareLinkInfoWrapper, http: PyHttpClient, logger: PyLogger) -> str:
"""
解析分享链接，获取直链下载地址
这是必须实现的主要解析函数
Args:
share_link_info: 分享链接信息
http: HTTP客户端
logger: 日志记录器
Returns:
直链下载地址
"""
...
def parse_file_list(share_link_info: PyShareLinkInfoWrapper, http: PyHttpClient, logger: PyLogger) -> List[Dict[str, Any]]:
"""
解析文件列表
可选实现，用于支持目录分享
Args:
share_link_info: 分享链接信息
http: HTTP客户端
logger: 日志记录器
Returns:
文件信息列表
"""
...
def parse_by_id(share_link_info: PyShareLinkInfoWrapper, http: PyHttpClient, logger: PyLogger) -> str:
"""
根据文件ID解析下载链接
可选实现，用于支持按文件ID解析
Args:
share_link_info: 分享链接信息
http: HTTP客户端
logger: 日志记录器
Returns:
直链下载地址
"""
...

2
pom.xml
View File

@@ -25,7 +25,7 @@
<packageDirectory>${project.basedir}/web-service/target/package</packageDirectory>
<vertx.version>4.5.22</vertx.version>
<vertx.version>4.5.23</vertx.version>
<org.reflections.version>0.10.2</org.reflections.version>
<lombok.version>1.18.38</lombok.version>
<slf4j.version>2.0.5</slf4j.version>

51
web-front/src/utils/playgroundApi.js
View File

@@ -37,20 +37,23 @@ export const playgroundApi = {
},
/**
* 测试执行JavaScript代码
* @param {string} jsCode - JavaScript代码
* 测试执行JavaScript/Python代码
* @param {string} code - 代码
* @param {string} shareUrl - 分享链接
* @param {string} pwd - 密码（可选）
* @param {string} method - 测试方法：parse/parseFileList/parseById
* @param {string} language - 语言类型：javascript/python
* @returns {Promise} 测试结果
*/
async testScript(jsCode, shareUrl, pwd = '', method = 'parse') {
async testScript(code, shareUrl, pwd = '', method = 'parse', language = 'javascript') {
try {
const response = await axiosInstance.post('/v2/playground/test', {
jsCode,
jsCode: code, // 兼容后端旧字段名
code,
shareUrl,
pwd,
method
method,
language
});
// 框架会自动包装成JsonResult，需要从data字段获取
if (response.data && response.data.data) {
@@ -83,6 +86,21 @@ export const playgroundApi = {
}
},
/**
* 获取types.pyi文件内容（Python类型提示）
* @returns {Promise<string>} types.pyi内容
*/
async getTypesPyi() {
try {
const response = await axiosInstance.get('/v2/playground/types.pyi', {
responseType: 'text'
});
return response.data;
} catch (error) {
throw new Error(error.response?.data?.error || error.message || '获取types.pyi失败');
}
},
/**
* 获取解析器列表
*/
@@ -106,10 +124,16 @@ export const playgroundApi = {
/**
* 保存解析器
* @param {string} code - 代码
* @param {string} language - 语言类型：javascript/python
*/
async saveParser(jsCode) {
async saveParser(code, language = 'javascript') {
try {
const response = await axiosInstance.post('/v2/playground/parsers', { jsCode });
const response = await axiosInstance.post('/v2/playground/parsers', {
jsCode: code, // 兼容后端旧字段名
code,
language
});
// 框架会自动包装成JsonResult
if (response.data && response.data.data) {
return {
@@ -132,10 +156,19 @@ export const playgroundApi = {
/**
* 更新解析器
* @param {number} id - 解析器ID
* @param {string} code - 代码
* @param {boolean} enabled - 是否启用
* @param {string} language - 语言类型：javascript/python
*/
async updateParser(id, jsCode, enabled = true) {
async updateParser(id, code, enabled = true, language = 'javascript') {
try {
const response = await axiosInstance.put(`/v2/playground/parsers/${id}`, { jsCode, enabled });
const response = await axiosInstance.put(`/v2/playground/parsers/${id}`, {
jsCode: code, // 兼容后端旧字段名
code,
enabled,
language
});
return response.data;
} catch (error) {
throw new Error(error.response?.data?.error || error.message || '更新解析器失败');

226
web-front/src/views/Playground.vue
View File

@@ -76,7 +76,7 @@
</el-link>
</el-breadcrumb-item>
<el-breadcrumb-item>脚本解析器演练场 <span style="color: var(--el-text-color-secondary); font-size: 12px;">
JavaScript (ES5)
{{ currentFileLanguageDisplay }}
</span></el-breadcrumb-item>
</el-breadcrumb>
</div>
@@ -771,6 +771,19 @@
:rules="newFileFormRules"
:label-width="isMobile ? '80px' : '100px'"
>
<el-form-item label="开发语言" prop="language">
<el-radio-group v-model="newFileForm.language">
<el-radio label="javascript">
<el-icon style="margin-right: 4px;"><Document /></el-icon>
JavaScript (ES5)
</el-radio>
<el-radio label="python">
<el-icon style="margin-right: 4px;"><Grape /></el-icon>
Python (GraalPy)
</el-radio>
</el-radio-group>
<div class="form-tip">选择解析器开发语言</div>
</el-form-item>
<el-form-item label="解析器名" prop="name">
<el-input
v-model="newFileForm.name"
@@ -879,6 +892,24 @@ export default {
return files.value.find(f => f.id === activeFileId.value) || files.value[0];
});
// 当前文件语言类型（用于显示）
const currentFileLanguageDisplay = computed(() => {
const file = activeFile.value;
if (!file) return 'JavaScript (ES5)';
// 优先使用文件的language属性
if (file.language === 'python') {
return 'Python (GraalPy)';
}
// 根据文件扩展名判断
if (file.name && file.name.endsWith('.py')) {
return 'Python (GraalPy)';
}
return 'JavaScript (ES5)';
});
// 当前编辑的代码（绑定到活动文件）
const isFileChanging = ref(false); // 标记是否正在切换文件
const currentCode = computed({
@@ -902,7 +933,8 @@ export default {
name: '',
identifier: '',
author: '',
match: ''
match: '',
language: 'javascript'
});
const newFileFormRules = {
name: [
@@ -910,6 +942,9 @@ export default {
],
identifier: [
{ required: true, message: '请输入标识', trigger: 'blur' }
],
language: [
{ required: true, message: '请选择开发语言', trigger: 'change' }
]
};
const newFileFormRef = ref(null);
@@ -1398,12 +1433,22 @@ function parseById(shareLinkInfo, http, logger) {
isFileChanging.value = true;
activeFileId.value = fileId;
saveAllFilesToStorage();
// 获取切换后的文件
const newFile = files.value.find(f => f.id === fileId);
// 等待编辑器更新
nextTick(() => {
if (editorRef.value && editorRef.value.getEditor) {
const editor = editorRef.value.getEditor();
if (editor) {
editor.focus();
// 更新编辑器语言模式
if (newFile) {
const language = newFile.language || getLanguageFromFile(newFile.name);
updateEditorLanguage(language);
}
}
}
// 切换完成后，取消标记
@@ -1436,13 +1481,14 @@ function parseById(shareLinkInfo, http, logger) {
name: '',
identifier: '',
author: '',
match: ''
match: '',
language: 'javascript'
};
newFileDialogVisible.value = true;
};
// 生成模板代码
const generateTemplate = (name, identifier, author, match) => {
// 生成JavaScript模板代码
const generateJsTemplate = (name, identifier, author, match) => {
const type = identifier.toLowerCase().replace(/[^a-z0-9]/g, '_');
const displayName = name;
const description = `使用JavaScript实现的${name}解析器`;
@@ -1498,6 +1544,83 @@ function parseFileList(shareLinkInfo, http, logger) {
}`;
};
// 生成Python模板代码
const generatePyTemplate = (name, identifier, author, match) => {
const type = identifier.toLowerCase().replace(/[^a-z0-9]/g, '_');
const displayName = name;
const description = `使用Python实现的${name}解析器`;
return `# ==UserScript==
# @name ${name}
# @type ${type}
# @displayName ${displayName}
# @description ${description}
# @match ${match || 'https?://example.com/s/(?<KEY>\\w+)'}
# @author ${author || 'yourname'}
# @version 1.0.0
# ==/UserScript==
"""
${name}解析器 - Python实现
使用GraalPy运行，提供与JavaScript解析器相同的功能
"""
def parse(share_link_info, http, logger):
"""
解析单个文件下载链接
Args:
share_link_info: 分享链接信息对象
http: HTTP客户端
logger: 日志记录器
Returns:
str: 直链下载地址
"""
url = share_link_info.get_share_url()
logger.info(f"开始解析: {url}")
response = http.get(url)
if not response.ok():
raise Exception(f"请求失败: {response.status_code()}")
html = response.text()
# 这里添加你的解析逻辑
# 例如：使用正则表达式提取下载链接
return "https://example.com/download/file.zip"
def parse_file_list(share_link_info, http, logger):
"""
解析文件列表（可选）
Args:
share_link_info: 分享链接信息对象
http: HTTP客户端
logger: 日志记录器
Returns:
list: 文件信息列表
"""
dir_id = share_link_info.get_other_param("dirId") or "0"
logger.info(f"解析文件列表，目录ID: {dir_id}")
# 这里添加你的文件列表解析逻辑
file_list = []
return file_list
`;
};
// 生成模板代码（根据语言选择）
const generateTemplate = (name, identifier, author, match, language = 'javascript') => {
if (language === 'python') {
return generatePyTemplate(name, identifier, author, match);
}
return generateJsTemplate(name, identifier, author, match);
};
// 创建新文件
const createNewFile = async () => {
if (!newFileFormRef.value) return;
@@ -1505,10 +1628,17 @@ function parseFileList(shareLinkInfo, http, logger) {
await newFileFormRef.value.validate((valid) => {
if (!valid) return;
const language = newFileForm.value.language || 'javascript';
const isPython = language === 'python';
const fileExt = isPython ? '.py' : '.js';
// 使用解析器名称作为文件名
const fileName = newFileForm.value.name.endsWith('.js')
? newFileForm.value.name
: newFileForm.value.name + '.js';
let fileName = newFileForm.value.name;
if (!fileName.endsWith(fileExt)) {
// 移除可能的错误扩展名
fileName = fileName.replace(/\.(js|py)$/i, '');
fileName = fileName + fileExt;
}
// 检查文件名是否已存在
if (files.value.some(f => f.name === fileName)) {
@@ -1518,10 +1648,11 @@ function parseFileList(shareLinkInfo, http, logger) {
// 生成模板代码
const template = generateTemplate(
newFileForm.value.name,
newFileForm.value.name.replace(/\.(js|py)$/i, ''),
newFileForm.value.identifier,
newFileForm.value.author,
newFileForm.value.match
newFileForm.value.match,
language
);
// 创建新文件
@@ -1530,6 +1661,7 @@ function parseFileList(shareLinkInfo, http, logger) {
id: 'file' + fileIdCounter.value,
name: fileName,
content: template,
language: language,
modified: false
};
@@ -1538,7 +1670,10 @@ function parseFileList(shareLinkInfo, http, logger) {
newFileDialogVisible.value = false;
saveAllFilesToStorage();
ElMessage.success('文件创建成功');
// 更新编辑器语言模式
updateEditorLanguage(language);
ElMessage.success(`${isPython ? 'Python' : 'JavaScript'}文件创建成功`);
// 等待编辑器更新后聚焦
nextTick(() => {
@@ -1573,6 +1708,31 @@ function parseFileList(shareLinkInfo, http, logger) {
}
};
// 更新编辑器语言模式
const updateEditorLanguage = (language) => {
if (editorRef.value && editorRef.value.getEditor) {
const editor = editorRef.value.getEditor();
if (editor) {
const model = editor.getModel();
if (model) {
const monaco = window.monaco || editorRef.value.monaco;
if (monaco) {
const langId = language === 'python' ? 'python' : 'javascript';
monaco.editor.setModelLanguage(model, langId);
}
}
}
}
};
// 根据文件扩展名获取语言类型
const getLanguageFromFile = (fileName) => {
if (fileName && fileName.endsWith('.py')) {
return 'python';
}
return 'javascript';
};
// IDE功能：切换自动换行
const toggleWordWrap = () => {
wordWrapEnabled.value = !wordWrapEnabled.value;
@@ -1708,8 +1868,13 @@ function parseFileList(shareLinkInfo, http, logger) {
// 执行测试
const executeTest = async () => {
const codeToTest = currentCode.value;
// 获取当前文件的语言类型
const currentLanguage = activeFile.value?.language || getLanguageFromFile(activeFile.value?.name) || 'javascript';
const isPython = currentLanguage === 'python';
if (!codeToTest.trim()) {
ElMessage.warning('请先输入JavaScript代码');
ElMessage.warning(`请先输入${isPython ? 'Python' : 'JavaScript'}代码`);
return;
}
@@ -1718,7 +1883,8 @@ function parseFileList(shareLinkInfo, http, logger) {
return;
}
// 检查代码中是否包含潜在的危险模式
// 检查代码中是否包含潜在的危险模式（仅针对JavaScript）
if (!isPython) {
const dangerousPatterns = [
{ pattern: /while\s*\(\s*true\s*\)/gi, message: '检测到 while(true) 无限循环' },
{ pattern: /for\s*\(\s*;\s*;\s*\)/gi, message: '检测到 for(;;) 无限循环' },
@@ -1744,6 +1910,34 @@ function parseFileList(shareLinkInfo, http, logger) {
break;
}
}
}
// Python 无限循环检查
if (isPython) {
const pythonDangerousPatterns = [
{ pattern: /while\s+True\s*:/gi, message: '检测到 while True: 无限循环' }
];
for (const { pattern, message } of pythonDangerousPatterns) {
if (pattern.test(codeToTest)) {
const confirmed = await ElMessageBox.confirm(
`⚠️ ${message}\n\n这可能导致脚本无法停止并占用服务器资源。\n\n建议修改代码，添加合理的循环退出条件。\n\n确定要继续执行吗？`,
'危险代码警告',
{
confirmButtonText: '我知道风险，继续执行',
cancelButtonText: '取消',
type: 'warning',
dangerouslyUseHTMLString: true
}
).catch(() => false);
if (!confirmed) {
return;
}
break;
}
}
}
testing.value = true;
testResult.value = null;
@@ -1754,7 +1948,8 @@ function parseFileList(shareLinkInfo, http, logger) {
codeToTest, // 使用当前活动文件的代码
testParams.value.shareUrl,
testParams.value.pwd,
testParams.value.method
testParams.value.method,
currentLanguage // 传递语言类型
);
console.log('测试结果:', result);
@@ -2286,6 +2481,7 @@ curl "${baseUrl}/json/parser?url=${encodeURIComponent(exampleUrl)}"</pre>
files,
activeFileId,
activeFile,
currentFileLanguageDisplay,
handleFileChange,
removeFile,
// 新建文件
@@ -2303,6 +2499,8 @@ curl "${baseUrl}/json/parser?url=${encodeURIComponent(exampleUrl)}"</pre>
exportCurrentFile,
undo,
redo,
updateEditorLanguage,
getLanguageFromFile,
// 加载和认证
loading,
loadProgress,

317
web-service/src/main/java/cn/qaiu/lz/web/controller/PlaygroundApi.java
View File

@@ -9,6 +9,9 @@ import cn.qaiu.parser.custom.CustomParserRegistry;
import cn.qaiu.parser.customjs.JsPlaygroundExecutor;
import cn.qaiu.parser.customjs.JsPlaygroundLogger;
import cn.qaiu.parser.customjs.JsScriptMetadataParser;
import cn.qaiu.parser.custompy.PyPlaygroundExecutor;
import cn.qaiu.parser.custompy.PyPlaygroundLogger;
import cn.qaiu.parser.custompy.PyScriptMetadataParser;
import cn.qaiu.vx.core.annotaions.RouteHandler;
import cn.qaiu.vx.core.annotaions.RouteMapping;
import cn.qaiu.vx.core.enums.RouteMethod;
@@ -155,7 +158,7 @@ public class PlaygroundApi {
}
/**
* 测试执行JavaScript代码
* 测试执行JavaScript/Python代码
*
* @param ctx 路由上下文
* @return 测试结果
@@ -176,25 +179,38 @@ public class PlaygroundApi {
try {
JsonObject body = ctx.body().asJsonObject();
String jsCode = body.getString("jsCode");
String code = body.getString("jsCode"); // 兼容旧字段名
if (StringUtils.isBlank(code)) {
code = body.getString("code"); // 也支持新字段名
}
String shareUrl = body.getString("shareUrl");
String pwd = body.getString("pwd");
String method = body.getString("method", "parse");
String language = body.getString("language", "javascript").toLowerCase();
// 参数验证
if (StringUtils.isBlank(jsCode)) {
if (StringUtils.isBlank(code)) {
promise.complete(JsonObject.mapFrom(PlaygroundTestResp.builder()
.success(false)
.error("JavaScript代码不能为空")
.error("代码不能为空")
.build()));
return promise.future();
}
// 验证语言类型
if (!"javascript".equals(language) && !"python".equals(language)) {
promise.complete(JsonObject.mapFrom(PlaygroundTestResp.builder()
.success(false)
.error("不支持的语言类型: " + language + "，仅支持 javascript 或 python")
.build()));
return promise.future();
}
// 代码长度验证
if (jsCode.length() > MAX_CODE_LENGTH) {
if (code.length() > MAX_CODE_LENGTH) {
promise.complete(JsonObject.mapFrom(PlaygroundTestResp.builder()
.success(false)
.error("代码长度超过限制（最大128KB），当前长度: " + jsCode.length() + " 字节")
.error("代码长度超过限制（最大128KB），当前长度: " + code.length() + " 字节")
.build()));
return promise.future();
}
@@ -207,10 +223,16 @@ public class PlaygroundApi {
return promise.future();
}
// ===== 新增：验证URL匹配 =====
// ===== 验证URL匹配（根据语言类型选择解析器） =====
try {
var config = JsScriptMetadataParser.parseScript(jsCode);
Pattern matchPattern = config.getMatchPattern();
Pattern matchPattern;
if ("python".equals(language)) {
var config = PyScriptMetadataParser.parseScript(code);
matchPattern = config.getMatchPattern();
} else {
var config = JsScriptMetadataParser.parseScript(code);
matchPattern = config.getMatchPattern();
}
if (matchPattern != null) {
Matcher matcher = matchPattern.matcher(shareUrl);
@@ -242,6 +264,30 @@ public class PlaygroundApi {
return promise.future();
}
// 根据语言类型执行代码
final String finalCode = code;
if ("python".equals(language)) {
executePythonTest(promise, finalCode, shareUrl, pwd, method);
} else {
executeJavaScriptTest(promise, finalCode, shareUrl, pwd, method);
}
} catch (Exception e) {
log.error("解析请求参数失败", e);
promise.complete(JsonObject.mapFrom(PlaygroundTestResp.builder()
.success(false)
.error("解析请求参数失败: " + e.getMessage())
.stackTrace(getStackTrace(e))
.build()));
}
return promise.future();
}
/**
* 执行JavaScript测试
*/
private void executeJavaScriptTest(Promise<JsonObject> promise, String jsCode, String shareUrl, String pwd, String method) {
long startTime = System.currentTimeMillis();
try {
@@ -269,12 +315,12 @@ public class PlaygroundApi {
break;
default:
promise.fail(new IllegalArgumentException("未知的方法类型: " + method));
return promise.future();
return;
}
// 异步处理执行结果
executionFuture.onSuccess(result -> {
log.debug("执行成功，结果类型: {}, 结果值: {}",
log.debug("JavaScript执行成功，结果类型: {}, 结果值: {}",
result != null ? result.getClass().getSimpleName() : "null",
result);
@@ -287,13 +333,12 @@ public class PlaygroundApi {
.level(entry.getLevel())
.message(entry.getMessage())
.timestamp(entry.getTimestamp())
.source(entry.getSource()) // 使用日志条目的来源标识
.source(entry.getSource())
.build())
.collect(Collectors.toList());
long executionTime = System.currentTimeMillis() - startTime;
// 构建响应
PlaygroundTestResp response = PlaygroundTestResp.builder()
.success(true)
.result(result)
@@ -302,23 +347,22 @@ public class PlaygroundApi {
.build();
JsonObject jsonResponse = JsonObject.mapFrom(response);
log.debug("测试成功响应: {}", jsonResponse.encodePrettily());
log.debug("JavaScript测试成功响应: {}", jsonResponse.encodePrettily());
promise.complete(jsonResponse);
}).onFailure(e -> {
long executionTime = System.currentTimeMillis() - startTime;
String errorMessage = e.getMessage();
String stackTrace = getStackTrace(e);
log.error("演练场执行失败", e);
log.error("JavaScript演练场执行失败", e);
// 尝试获取已有的日志
List<JsPlaygroundLogger.LogEntry> logEntries = executor.getLogs();
List<PlaygroundTestResp.LogEntry> respLogs = logEntries.stream()
.map(entry -> PlaygroundTestResp.LogEntry.builder()
.level(entry.getLevel())
.message(entry.getMessage())
.timestamp(entry.getTimestamp())
.source(entry.getSource()) // 使用日志条目的来源标识
.source(entry.getSource())
.build())
.collect(Collectors.toList());
@@ -338,7 +382,7 @@ public class PlaygroundApi {
String errorMessage = e.getMessage();
String stackTrace = getStackTrace(e);
log.error("演练场初始化失败", e);
log.error("JavaScript演练场初始化失败", e);
PlaygroundTestResp response = PlaygroundTestResp.builder()
.success(false)
@@ -350,16 +394,118 @@ public class PlaygroundApi {
promise.complete(JsonObject.mapFrom(response));
}
} catch (Exception e) {
log.error("解析请求参数失败", e);
promise.complete(JsonObject.mapFrom(PlaygroundTestResp.builder()
.success(false)
.error("解析请求参数失败: " + e.getMessage())
.stackTrace(getStackTrace(e))
.build()));
}
return promise.future();
/**
* 执行Python测试
*/
private void executePythonTest(Promise<JsonObject> promise, String pyCode, String shareUrl, String pwd, String method) {
long startTime = System.currentTimeMillis();
try {
// 创建ShareLinkInfo
ParserCreate parserCreate = ParserCreate.fromShareUrl(shareUrl);
if (StringUtils.isNotBlank(pwd)) {
parserCreate.setShareLinkInfoPwd(pwd);
}
ShareLinkInfo shareLinkInfo = parserCreate.getShareLinkInfo();
// 创建Python演练场执行器
PyPlaygroundExecutor executor = new PyPlaygroundExecutor(shareLinkInfo, pyCode);
// 根据方法类型选择执行，并异步处理结果
Future<Object> executionFuture;
switch (method) {
case "parse":
executionFuture = executor.executeParseAsync().map(r -> (Object) r);
break;
case "parseFileList":
executionFuture = executor.executeParseFileListAsync().map(r -> (Object) r);
break;
case "parseById":
executionFuture = executor.executeParseByIdAsync().map(r -> (Object) r);
break;
default:
promise.fail(new IllegalArgumentException("未知的方法类型: " + method));
return;
}
// 异步处理执行结果
executionFuture.onSuccess(result -> {
log.debug("Python执行成功，结果类型: {}, 结果值: {}",
result != null ? result.getClass().getSimpleName() : "null",
result);
// 获取日志
List<PyPlaygroundLogger.LogEntry> logEntries = executor.getLogs();
log.debug("获取到 {} 条日志记录", logEntries.size());
List<PlaygroundTestResp.LogEntry> respLogs = logEntries.stream()
.map(entry -> PlaygroundTestResp.LogEntry.builder()
.level(entry.getLevel())
.message(entry.getMessage())
.timestamp(entry.getTimestamp())
.source(entry.getSource())
.build())
.collect(Collectors.toList());
long executionTime = System.currentTimeMillis() - startTime;
PlaygroundTestResp response = PlaygroundTestResp.builder()
.success(true)
.result(result)
.logs(respLogs)
.executionTime(executionTime)
.build();
JsonObject jsonResponse = JsonObject.mapFrom(response);
log.debug("Python测试成功响应: {}", jsonResponse.encodePrettily());
promise.complete(jsonResponse);
}).onFailure(e -> {
long executionTime = System.currentTimeMillis() - startTime;
String errorMessage = e.getMessage();
String stackTrace = getStackTrace(e);
log.error("Python演练场执行失败", e);
List<PyPlaygroundLogger.LogEntry> logEntries = executor.getLogs();
List<PlaygroundTestResp.LogEntry> respLogs = logEntries.stream()
.map(entry -> PlaygroundTestResp.LogEntry.builder()
.level(entry.getLevel())
.message(entry.getMessage())
.timestamp(entry.getTimestamp())
.source(entry.getSource())
.build())
.collect(Collectors.toList());
PlaygroundTestResp response = PlaygroundTestResp.builder()
.success(false)
.error(errorMessage)
.stackTrace(stackTrace)
.executionTime(executionTime)
.logs(respLogs)
.build();
promise.complete(JsonObject.mapFrom(response));
});
} catch (Exception e) {
long executionTime = System.currentTimeMillis() - startTime;
String errorMessage = e.getMessage();
String stackTrace = getStackTrace(e);
log.error("Python演练场初始化失败", e);
PlaygroundTestResp response = PlaygroundTestResp.builder()
.success(false)
.error(errorMessage)
.stackTrace(stackTrace)
.executionTime(executionTime)
.logs(new ArrayList<>())
.build();
promise.complete(JsonObject.mapFrom(response));
}
}
/**
@@ -403,6 +549,47 @@ public class PlaygroundApi {
}
}
/**
* 获取types.pyi文件内容（Python类型提示）
*
* @param ctx 路由上下文
* @param response HTTP响应
*/
@RouteMapping(value = "/types.pyi", method = RouteMethod.GET)
public void getTypesPyi(RoutingContext ctx, HttpServerResponse response) {
// 检查是否启用
if (!checkEnabled()) {
ResponseUtil.fireJsonResultResponse(response, JsonResult.error("演练场功能已禁用"));
return;
}
// 权限检查
if (!checkAuth(ctx)) {
ResponseUtil.fireJsonResultResponse(response, JsonResult.error("未授权访问"));
return;
}
try (InputStream inputStream = getClass().getClassLoader()
.getResourceAsStream("py/types.pyi")) {
if (inputStream == null) {
ResponseUtil.fireJsonResultResponse(response, JsonResult.error("types.pyi文件不存在"));
return;
}
String content = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))
.lines()
.collect(Collectors.joining("\n"));
response.putHeader("Content-Type", "text/x-python; charset=utf-8")
.end(content);
} catch (Exception e) {
log.error("读取types.pyi失败", e);
ResponseUtil.fireJsonResultResponse(response, JsonResult.error("读取types.pyi失败: " + e.getMessage()));
}
}
/**
* 获取解析器列表
*/
@@ -439,22 +626,39 @@ public class PlaygroundApi {
try {
JsonObject body = ctx.body().asJsonObject();
String jsCode = body.getString("jsCode");
String code = body.getString("jsCode"); // 兼容旧字段名
if (StringUtils.isBlank(code)) {
code = body.getString("code"); // 也支持新字段名
}
String language = body.getString("language", "javascript").toLowerCase();
if (StringUtils.isBlank(jsCode)) {
promise.complete(JsonResult.error("JavaScript代码不能为空").toJsonObject());
if (StringUtils.isBlank(code)) {
promise.complete(JsonResult.error("代码不能为空").toJsonObject());
return promise.future();
}
// 验证语言类型
if (!"javascript".equals(language) && !"python".equals(language)) {
promise.complete(JsonResult.error("不支持的语言类型: " + language + "，仅支持 javascript 或 python").toJsonObject());
return promise.future();
}
// 代码长度验证
if (jsCode.length() > MAX_CODE_LENGTH) {
promise.complete(JsonResult.error("代码长度超过限制（最大128KB），当前长度: " + jsCode.length() + " 字节").toJsonObject());
if (code.length() > MAX_CODE_LENGTH) {
promise.complete(JsonResult.error("代码长度超过限制（最大128KB），当前长度: " + code.length() + " 字节").toJsonObject());
return promise.future();
}
// 解析元数据
// 根据语言类型解析元数据
final String finalCode = code;
try {
var config = JsScriptMetadataParser.parseScript(jsCode);
cn.qaiu.parser.custom.CustomParserConfig config;
if ("python".equals(language)) {
config = PyScriptMetadataParser.parseScript(finalCode);
} else {
config = JsScriptMetadataParser.parseScript(finalCode);
}
String type = config.getType();
String displayName = config.getDisplayName();
String name = config.getMetadata().get("name");
@@ -462,6 +666,7 @@ public class PlaygroundApi {
String author = config.getMetadata().get("author");
String version = config.getMetadata().get("version");
String matchPattern = config.getMatchPattern() != null ? config.getMatchPattern().pattern() : null;
final boolean isPython = "python".equals(language);
// 检查数量限制
dbService.getPlaygroundParserCount().onSuccess(count -> {
@@ -498,15 +703,20 @@ public class PlaygroundApi {
parser.put("author", author);
parser.put("version", version);
parser.put("matchPattern", matchPattern);
parser.put("jsCode", jsCode);
parser.put("jsCode", finalCode); // 兼容旧字段名存储
parser.put("language", isPython ? "python" : "javascript");
parser.put("ip", getClientIp(ctx.request()));
parser.put("enabled", true);
dbService.savePlaygroundParser(parser).onSuccess(result -> {
// 保存成功后，立即注册到解析器系统
try {
if (isPython) {
CustomParserRegistry.registerPy(config);
} else {
CustomParserRegistry.register(config);
log.info("已注册演练场解析器: {} ({})", displayName, type);
}
log.info("已注册演练场{}解析器: {} ({})", isPython ? "Python" : "JavaScript", displayName, type);
promise.complete(JsonResult.success("保存并注册成功").toJsonObject());
} catch (Exception e) {
log.error("注册解析器失败", e);
@@ -559,16 +769,33 @@ public class PlaygroundApi {
try {
JsonObject body = ctx.body().asJsonObject();
String jsCode = body.getString("jsCode");
String code = body.getString("jsCode"); // 兼容旧字段名
if (StringUtils.isBlank(code)) {
code = body.getString("code"); // 也支持新字段名
}
String language = body.getString("language", "javascript").toLowerCase();
if (StringUtils.isBlank(jsCode)) {
promise.complete(JsonResult.error("JavaScript代码不能为空").toJsonObject());
if (StringUtils.isBlank(code)) {
promise.complete(JsonResult.error("代码不能为空").toJsonObject());
return promise.future();
}
// 解析元数据
// 验证语言类型
if (!"javascript".equals(language) && !"python".equals(language)) {
promise.complete(JsonResult.error("不支持的语言类型: " + language + "，仅支持 javascript 或 python").toJsonObject());
return promise.future();
}
// 根据语言类型解析元数据
final String finalCode = code;
try {
var config = JsScriptMetadataParser.parseScript(jsCode);
cn.qaiu.parser.custom.CustomParserConfig config;
if ("python".equals(language)) {
config = PyScriptMetadataParser.parseScript(finalCode);
} else {
config = JsScriptMetadataParser.parseScript(finalCode);
}
String type = config.getType();
String displayName = config.getDisplayName();
String name = config.getMetadata().get("name");
@@ -577,6 +804,7 @@ public class PlaygroundApi {
String version = config.getMetadata().get("version");
String matchPattern = config.getMatchPattern() != null ? config.getMatchPattern().pattern() : null;
boolean enabled = body.getBoolean("enabled", true);
final boolean isPython = "python".equals(language);
JsonObject parser = new JsonObject();
parser.put("name", name);
@@ -585,7 +813,8 @@ public class PlaygroundApi {
parser.put("author", author);
parser.put("version", version);
parser.put("matchPattern", matchPattern);
parser.put("jsCode", jsCode);
parser.put("jsCode", finalCode); // 兼容旧字段名存储
parser.put("language", isPython ? "python" : "javascript");
parser.put("enabled", enabled);
dbService.updatePlaygroundParser(id, parser).onSuccess(result -> {
@@ -595,8 +824,12 @@ public class PlaygroundApi {
// 先注销旧的（如果存在）
CustomParserRegistry.unregister(type);
// 重新注册新的
if (isPython) {
CustomParserRegistry.registerPy(config);
} else {
CustomParserRegistry.register(config);
log.info("已重新注册演练场解析器: {} ({})", displayName, type);
}
log.info("已重新注册演练场{}解析器: {} ({})", isPython ? "Python" : "JavaScript", displayName, type);
} else {
// 禁用时注销
CustomParserRegistry.unregister(type);

2
web-service/src/main/resources/app.yml
View File

@@ -1,4 +1,4 @@
# 要激活的配置: app-配置名称.yml
active: dev
active: local
# 控制台输出的版权文字
copyright: QAIU