From 31c6a611930cf6791c8edc028a4515f1df362efc Mon Sep 17 00:00:00 2001 From: q Date: Sun, 11 Jan 2026 02:40:33 +0800 Subject: [PATCH] feat: add GraalPy Python parser support --- README.md | 2 +- parser/pom.xml | 17 +- .../java/cn/qaiu/parser/ParserCreate.java | 4 + .../parser/custom/CustomParserConfig.java | 74 ++ .../parser/custom/CustomParserRegistry.java | 111 +++ .../qaiu/parser/custompy/PyCryptoUtils.java | 381 ++++++++++ .../cn/qaiu/parser/custompy/PyHttpClient.java | 649 ++++++++++++++++++ .../cn/qaiu/parser/custompy/PyLogger.java | 139 ++++ .../parser/custompy/PyParserExecutor.java | 307 +++++++++ .../parser/custompy/PyPlaygroundExecutor.java | 397 +++++++++++ .../parser/custompy/PyPlaygroundLogger.java | 183 +++++ .../qaiu/parser/custompy/PyScriptLoader.java | 334 +++++++++ .../custompy/PyScriptMetadataParser.java | 188 +++++ .../custompy/PyShareLinkInfoWrapper.java | 262 +++++++ .../custom-parsers/py/example_parser.py | 137 ++++ parser/src/main/resources/py/types.pyi | 339 +++++++++ pom.xml | 2 +- web-front/src/utils/playgroundApi.js | 51 +- web-front/src/views/Playground.vue | 268 +++++++- .../qaiu/lz/web/controller/PlaygroundApi.java | 503 ++++++++++---- web-service/src/main/resources/app.yml | 2 +- 21 files changed, 4167 insertions(+), 183 deletions(-) create mode 100644 parser/src/main/java/cn/qaiu/parser/custompy/PyCryptoUtils.java create mode 100644 parser/src/main/java/cn/qaiu/parser/custompy/PyHttpClient.java create mode 100644 parser/src/main/java/cn/qaiu/parser/custompy/PyLogger.java create mode 100644 parser/src/main/java/cn/qaiu/parser/custompy/PyParserExecutor.java create mode 100644 parser/src/main/java/cn/qaiu/parser/custompy/PyPlaygroundExecutor.java create mode 100644 parser/src/main/java/cn/qaiu/parser/custompy/PyPlaygroundLogger.java create mode 100644 parser/src/main/java/cn/qaiu/parser/custompy/PyScriptLoader.java create mode 100644 parser/src/main/java/cn/qaiu/parser/custompy/PyScriptMetadataParser.java create mode 100644 parser/src/main/java/cn/qaiu/parser/custompy/PyShareLinkInfoWrapper.java create mode 100644 parser/src/main/resources/custom-parsers/py/example_parser.py create mode 100644 parser/src/main/resources/py/types.pyi diff --git a/README.md b/README.md index db145c5..2fb43f8 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@

- +

diff --git a/parser/pom.xml b/parser/pom.xml index 5733122..bc9b0f8 100644 --- a/parser/pom.xml +++ b/parser/pom.xml @@ -59,7 +59,7 @@ UTF-8 - 4.5.22 + 4.5.23 0.10.2 1.18.38 2.0.5 @@ -67,6 +67,8 @@ 2.14.2 1.5.19 4.13.2 + + 24.1.1 @@ -105,6 +107,19 @@ compile + + + org.graalvm.polyglot + polyglot + ${graalpy.version} + + + org.graalvm.polyglot + python + ${graalpy.version} + pom + + org.brotli diff --git a/parser/src/main/java/cn/qaiu/parser/ParserCreate.java b/parser/src/main/java/cn/qaiu/parser/ParserCreate.java index 71f0e13..7b2c6c3 100644 --- a/parser/src/main/java/cn/qaiu/parser/ParserCreate.java +++ b/parser/src/main/java/cn/qaiu/parser/ParserCreate.java @@ -4,6 +4,7 @@ import cn.qaiu.entity.ShareLinkInfo; import cn.qaiu.parser.custom.CustomParserConfig; import cn.qaiu.parser.custom.CustomParserRegistry; import cn.qaiu.parser.customjs.JsParserExecutor; +import cn.qaiu.parser.custompy.PyParserExecutor; import org.apache.commons.lang3.StringUtils; @@ -155,6 +156,9 @@ public class ParserCreate { // 检查是否为JavaScript解析器 if (customParserConfig.isJsParser()) { return new JsParserExecutor(shareLinkInfo, customParserConfig); + } else if (customParserConfig.isPyParser()) { + // Python解析器 + return new PyParserExecutor(shareLinkInfo, customParserConfig); } else { // Java实现的解析器 try { diff --git a/parser/src/main/java/cn/qaiu/parser/custom/CustomParserConfig.java b/parser/src/main/java/cn/qaiu/parser/custom/CustomParserConfig.java index 54ec474..a9a32a7 100644 --- a/parser/src/main/java/cn/qaiu/parser/custom/CustomParserConfig.java +++ b/parser/src/main/java/cn/qaiu/parser/custom/CustomParserConfig.java @@ -53,11 +53,26 @@ public class CustomParserConfig { */ private final String jsCode; + /** + * Python代码（用于Python解析器） + */ + private final String pyCode; + /** * 是否为JavaScript解析器 */ private final boolean isJsParser; + /** + * 是否为Python解析器 + */ + private final boolean isPyParser; + + /** + * 脚本语言类型：javascript, python + */ + private final String language; + /** * 元数据信息（从脚本注释中解析） */ @@ -71,7 +86,10 @@ public class CustomParserConfig { this.panDomain = builder.panDomain; this.matchPattern = builder.matchPattern; this.jsCode = builder.jsCode; + this.pyCode = builder.pyCode; this.isJsParser = builder.isJsParser; + this.isPyParser = builder.isPyParser; + this.language = builder.language; this.metadata = builder.metadata; } @@ -103,10 +121,22 @@ public class CustomParserConfig { return jsCode; } + public String getPyCode() { + return pyCode; + } + public boolean isJsParser() { return isJsParser; } + public boolean isPyParser() { + return isPyParser; + } + + public String getLanguage() { + return language; + } + public Map getMetadata() { return metadata; } @@ -134,7 +164,10 @@ public class CustomParserConfig { private String panDomain; private Pattern matchPattern; private String jsCode; + private String pyCode; private boolean isJsParser; + private boolean isPyParser; + private String language; private Map metadata; /** @@ -211,12 +244,45 @@ public class CustomParserConfig { return this; } + /** + * 设置Python代码（用于Python解析器） + * @param pyCode Python代码 + */ + public Builder pyCode(String pyCode) { + this.pyCode = pyCode; + return this; + } + /** * 设置是否为JavaScript解析器 * @param isJsParser 是否为JavaScript解析器 */ public Builder isJsParser(boolean isJsParser) { this.isJsParser = isJsParser; + if (isJsParser) { + this.language = "javascript"; + } + return this; + } + + /** + * 设置是否为Python解析器 + * @param isPyParser 是否为Python解析器 + */ + public Builder isPyParser(boolean isPyParser) { + this.isPyParser = isPyParser; + if (isPyParser) { + this.language = "python"; + } + return this; + } + + /** + * 设置脚本语言类型 + * @param language 语言类型：javascript, python + */ + public Builder language(String language) { + this.language = language; return this; } @@ -246,6 +312,11 @@ public class CustomParserConfig { if (jsCode == null || jsCode.trim().isEmpty()) { throw new IllegalArgumentException("JavaScript解析器的jsCode不能为空"); } + } else if (isPyParser) { + // 如果是Python解析器，验证pyCode + if (pyCode == null || pyCode.trim().isEmpty()) { + throw new IllegalArgumentException("Python解析器的pyCode不能为空"); + } } else { // 如果是Java解析器，验证toolClass if (toolClass == null) { @@ -288,7 +359,10 @@ public class CustomParserConfig { ", panDomain='" + panDomain + '\'' + ", matchPattern=" + (matchPattern != null ? matchPattern.pattern() : "null") + ", jsCode=" + (jsCode != null ? "[JavaScript代码]" : "null") + + ", pyCode=" + (pyCode != null ? "[Python代码]" : "null") + ", isJsParser=" + isJsParser + + ", isPyParser=" + isPyParser + + ", language='" + language + '\'' + ", metadata=" + metadata + '}'; } diff --git a/parser/src/main/java/cn/qaiu/parser/custom/CustomParserRegistry.java b/parser/src/main/java/cn/qaiu/parser/custom/CustomParserRegistry.java index a978ed3..3e9aa03 100644 --- a/parser/src/main/java/cn/qaiu/parser/custom/CustomParserRegistry.java +++ b/parser/src/main/java/cn/qaiu/parser/custom/CustomParserRegistry.java @@ -6,6 +6,8 @@ import org.slf4j.LoggerFactory; import cn.qaiu.parser.PanDomainTemplate; import cn.qaiu.parser.customjs.JsScriptLoader; import cn.qaiu.parser.customjs.JsScriptMetadataParser; +import cn.qaiu.parser.custompy.PyScriptLoader; +import cn.qaiu.parser.custompy.PyScriptMetadataParser; import java.util.List; import java.util.Map; @@ -82,6 +84,24 @@ public class CustomParserRegistry { register(config); } + /** + * 注册Python解析器 + * + * @param config Python解析器配置 + * @throws IllegalArgumentException 如果type已存在或与内置解析器冲突 + */ + public static void registerPy(CustomParserConfig config) { + if (config == null) { + throw new IllegalArgumentException("config不能为空"); + } + + if (!config.isPyParser()) { + throw new IllegalArgumentException("config必须是Python解析器配置"); + } + + register(config); + } + /** * 从JavaScript代码字符串注册解析器 * @@ -139,6 +159,63 @@ public class CustomParserRegistry { } } + /** + * 从Python代码字符串注册解析器 + * + * @param pyCode Python代码 + * @throws IllegalArgumentException 如果解析失败 + */ + public static void registerPyFromCode(String pyCode) { + if (pyCode == null || pyCode.trim().isEmpty()) { + throw new IllegalArgumentException("Python代码不能为空"); + } + + try { + CustomParserConfig config = PyScriptMetadataParser.parseScript(pyCode); + registerPy(config); + } catch (Exception e) { + throw new IllegalArgumentException("解析Python代码失败: " + e.getMessage(), e); + } + } + + /** + * 从文件注册Python解析器 + * + * @param filePath 文件路径 + * @throws IllegalArgumentException 如果文件不存在或解析失败 + */ + public static void registerPyFromFile(String filePath) { + if (filePath == null || filePath.trim().isEmpty()) { + throw new IllegalArgumentException("文件路径不能为空"); + } + + try { + CustomParserConfig config = PyScriptLoader.loadFromFile(filePath); + registerPy(config); + } catch (Exception e) { + throw new IllegalArgumentException("从文件加载Python解析器失败: " + e.getMessage(), e); + } + } + + /** + * 从资源文件注册Python解析器 + * + * @param resourcePath 资源路径 + * @throws IllegalArgumentException 如果资源不存在或解析失败 + */ + public static void registerPyFromResource(String resourcePath) { + if (resourcePath == null || resourcePath.trim().isEmpty()) { + throw new IllegalArgumentException("资源路径不能为空"); + } + + try { + CustomParserConfig config = PyScriptLoader.loadFromResource(resourcePath); + registerPy(config); + } catch (Exception e) { + throw new IllegalArgumentException("从资源加载Python解析器失败: " + e.getMessage(), e); + } + } + /** * 自动加载所有JavaScript脚本 */ @@ -165,6 +242,40 @@ public class CustomParserRegistry { } } + /** + * 自动加载所有Python脚本 + */ + public static void autoLoadPyScripts() { + try { + List configs = PyScriptLoader.loadAllScripts(); + int successCount = 0; + int failCount = 0; + + for (CustomParserConfig config : configs) { + try { + registerPy(config); + successCount++; + } catch (Exception e) { + log.error("加载Python脚本失败: {}", config.getType(), e); + failCount++; + } + } + + log.info("自动加载Python脚本完成: 成功 {} 个，失败 {} 个", successCount, failCount); + + } catch (Exception e) { + log.error("自动加载Python脚本时发生异常", e); + } + } + + /** + * 自动加载所有脚本（JavaScript和Python） + */ + public static void autoLoadAllScripts() { + autoLoadJsScripts(); + autoLoadPyScripts(); + } + /** * 注销自定义解析器 * diff --git a/parser/src/main/java/cn/qaiu/parser/custompy/PyCryptoUtils.java b/parser/src/main/java/cn/qaiu/parser/custompy/PyCryptoUtils.java new file mode 100644 index 0000000..c0918a4 --- /dev/null +++ b/parser/src/main/java/cn/qaiu/parser/custompy/PyCryptoUtils.java @@ -0,0 +1,381 @@ +package cn.qaiu.parser.custompy; + +import org.graalvm.polyglot.HostAccess; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.crypto.Cipher; +import javax.crypto.spec.IvParameterSpec; +import javax.crypto.spec.SecretKeySpec; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.util.Base64; + +/** + * Python加密工具类 + * 为Python脚本提供常用的加密解密功能 + * + * @author QAIU + */ +public class PyCryptoUtils { + + private static final Logger log = LoggerFactory.getLogger(PyCryptoUtils.class); + + // ==================== MD5 ==================== + + /** + * MD5加密（返回32位小写） + * @param data 待加密数据 + * @return MD5值（32位小写） + */ + @HostAccess.Export + public String md5(String data) { + if (data == null) { + return null; + } + try { + MessageDigest md = MessageDigest.getInstance("MD5"); + byte[] digest = md.digest(data.getBytes(StandardCharsets.UTF_8)); + return bytesToHex(digest); + } catch (Exception e) { + log.error("MD5加密失败", e); + throw new RuntimeException("MD5加密失败: " + e.getMessage(), e); + } + } + + /** + * MD5加密（返回16位小写，取中间16位） + * @param data 待加密数据 + * @return MD5值（16位小写） + */ + @HostAccess.Export + public String md5_16(String data) { + String md5 = md5(data); + return md5 != null ? md5.substring(8, 24) : null; + } + + // ==================== SHA ==================== + + /** + * SHA-1加密 + * @param data 待加密数据 + * @return SHA-1值（小写） + */ + @HostAccess.Export + public String sha1(String data) { + return sha(data, "SHA-1"); + } + + /** + * SHA-256加密 + * @param data 待加密数据 + * @return SHA-256值（小写） + */ + @HostAccess.Export + public String sha256(String data) { + return sha(data, "SHA-256"); + } + + /** + * SHA-512加密 + * @param data 待加密数据 + * @return SHA-512值（小写） + */ + @HostAccess.Export + public String sha512(String data) { + return sha(data, "SHA-512"); + } + + private String sha(String data, String algorithm) { + if (data == null) { + return null; + } + try { + MessageDigest md = MessageDigest.getInstance(algorithm); + byte[] digest = md.digest(data.getBytes(StandardCharsets.UTF_8)); + return bytesToHex(digest); + } catch (Exception e) { + log.error(algorithm + "加密失败", e); + throw new RuntimeException(algorithm + "加密失败: " + e.getMessage(), e); + } + } + + // ==================== Base64 ==================== + + /** + * Base64编码 + * @param data 待编码数据 + * @return Base64字符串 + */ + @HostAccess.Export + public String base64_encode(String data) { + if (data == null) { + return null; + } + return Base64.getEncoder().encodeToString(data.getBytes(StandardCharsets.UTF_8)); + } + + /** + * Base64编码（字节数组） + * @param data 待编码字节数组 + * @return Base64字符串 + */ + @HostAccess.Export + public String base64_encode_bytes(byte[] data) { + if (data == null) { + return null; + } + return Base64.getEncoder().encodeToString(data); + } + + /** + * Base64解码 + * @param data Base64字符串 + * @return 解码后的字符串 + */ + @HostAccess.Export + public String base64_decode(String data) { + if (data == null) { + return null; + } + try { + byte[] decoded = Base64.getDecoder().decode(data); + return new String(decoded, StandardCharsets.UTF_8); + } catch (Exception e) { + log.error("Base64解码失败", e); + throw new RuntimeException("Base64解码失败: " + e.getMessage(), e); + } + } + + /** + * Base64解码（返回字节数组） + * @param data Base64字符串 + * @return 解码后的字节数组 + */ + @HostAccess.Export + public byte[] base64_decode_bytes(String data) { + if (data == null) { + return null; + } + try { + return Base64.getDecoder().decode(data); + } catch (Exception e) { + log.error("Base64解码失败", e); + throw new RuntimeException("Base64解码失败: " + e.getMessage(), e); + } + } + + /** + * URL安全的Base64编码 + * @param data 待编码数据 + * @return URL安全的Base64字符串 + */ + @HostAccess.Export + public String base64_url_encode(String data) { + if (data == null) { + return null; + } + return Base64.getUrlEncoder().encodeToString(data.getBytes(StandardCharsets.UTF_8)); + } + + /** + * URL安全的Base64解码 + * @param data URL安全的Base64字符串 + * @return 解码后的字符串 + */ + @HostAccess.Export + public String base64_url_decode(String data) { + if (data == null) { + return null; + } + try { + byte[] decoded = Base64.getUrlDecoder().decode(data); + return new String(decoded, StandardCharsets.UTF_8); + } catch (Exception e) { + log.error("Base64 URL解码失败", e); + throw new RuntimeException("Base64 URL解码失败: " + e.getMessage(), e); + } + } + + // ==================== AES ==================== + + /** + * AES加密（ECB模式，PKCS5Padding） + * @param data 待加密数据 + * @param key 密钥（16/24/32字节） + * @return Base64编码的密文 + */ + @HostAccess.Export + public String aes_encrypt_ecb(String data, String key) { + if (data == null || key == null) { + return null; + } + try { + SecretKeySpec secretKey = new SecretKeySpec(padKey(key), "AES"); + Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5Padding"); + cipher.init(Cipher.ENCRYPT_MODE, secretKey); + byte[] encrypted = cipher.doFinal(data.getBytes(StandardCharsets.UTF_8)); + return Base64.getEncoder().encodeToString(encrypted); + } catch (Exception e) { + log.error("AES ECB加密失败", e); + throw new RuntimeException("AES ECB加密失败: " + e.getMessage(), e); + } + } + + /** + * AES解密（ECB模式，PKCS5Padding） + * @param data Base64编码的密文 + * @param key 密钥（16/24/32字节） + * @return 明文 + */ + @HostAccess.Export + public String aes_decrypt_ecb(String data, String key) { + if (data == null || key == null) { + return null; + } + try { + SecretKeySpec secretKey = new SecretKeySpec(padKey(key), "AES"); + Cipher cipher = Cipher.getInstance("AES/ECB/PKCS5Padding"); + cipher.init(Cipher.DECRYPT_MODE, secretKey); + byte[] decrypted = cipher.doFinal(Base64.getDecoder().decode(data)); + return new String(decrypted, StandardCharsets.UTF_8); + } catch (Exception e) { + log.error("AES ECB解密失败", e); + throw new RuntimeException("AES ECB解密失败: " + e.getMessage(), e); + } + } + + /** + * AES加密（CBC模式，PKCS5Padding） + * @param data 待加密数据 + * @param key 密钥（16/24/32字节） + * @param iv 初始向量（16字节） + * @return Base64编码的密文 + */ + @HostAccess.Export + public String aes_encrypt_cbc(String data, String key, String iv) { + if (data == null || key == null || iv == null) { + return null; + } + try { + SecretKeySpec secretKey = new SecretKeySpec(padKey(key), "AES"); + IvParameterSpec ivSpec = new IvParameterSpec(padIv(iv)); + Cipher cipher = Cipher.getInstance("AES/CBC/PKCS5Padding"); + cipher.init(Cipher.ENCRYPT_MODE, secretKey, ivSpec); + byte[] encrypted = cipher.doFinal(data.getBytes(StandardCharsets.UTF_8)); + return Base64.getEncoder().encodeToString(encrypted); + } catch (Exception e) { + log.error("AES CBC加密失败", e); + throw new RuntimeException("AES CBC加密失败: " + e.getMessage(), e); + } + } + + /** + * AES解密（CBC模式，PKCS5Padding） + * @param data Base64编码的密文 + * @param key 密钥（16/24/32字节） + * @param iv 初始向量（16字节） + * @return 明文 + */ + @HostAccess.Export + public String aes_decrypt_cbc(String data, String key, String iv) { + if (data == null || key == null || iv == null) { + return null; + } + try { + SecretKeySpec secretKey = new SecretKeySpec(padKey(key), "AES"); + IvParameterSpec ivSpec = new IvParameterSpec(padIv(iv)); + Cipher cipher = Cipher.getInstance("AES/CBC/PKCS5Padding"); + cipher.init(Cipher.DECRYPT_MODE, secretKey, ivSpec); + byte[] decrypted = cipher.doFinal(Base64.getDecoder().decode(data)); + return new String(decrypted, StandardCharsets.UTF_8); + } catch (Exception e) { + log.error("AES CBC解密失败", e); + throw new RuntimeException("AES CBC解密失败: " + e.getMessage(), e); + } + } + + // ==================== Hex ==================== + + /** + * 字节数组转十六进制字符串 + * @param bytes 字节数组 + * @return 十六进制字符串（小写） + */ + @HostAccess.Export + public String bytes_to_hex(byte[] bytes) { + return bytesToHex(bytes); + } + + /** + * 十六进制字符串转字节数组 + * @param hex 十六进制字符串 + * @return 字节数组 + */ + @HostAccess.Export + public byte[] hex_to_bytes(String hex) { + if (hex == null || hex.length() % 2 != 0) { + return null; + } + int len = hex.length(); + byte[] data = new byte[len / 2]; + for (int i = 0; i < len; i += 2) { + data[i / 2] = (byte) ((Character.digit(hex.charAt(i), 16) << 4) + + Character.digit(hex.charAt(i + 1), 16)); + } + return data; + } + + // ==================== 工具方法 ==================== + + private static String bytesToHex(byte[] bytes) { + if (bytes == null) { + return null; + } + StringBuilder sb = new StringBuilder(); + for (byte b : bytes) { + sb.append(String.format("%02x", b)); + } + return sb.toString(); + } + + /** + * 将密钥填充到16/24/32字节 + */ + private byte[] padKey(String key) { + byte[] keyBytes = key.getBytes(StandardCharsets.UTF_8); + int len = keyBytes.length; + + // 根据密钥长度决定填充到16/24/32字节 + int targetLen; + if (len <= 16) { + targetLen = 16; + } else if (len <= 24) { + targetLen = 24; + } else { + targetLen = 32; + } + + if (len == targetLen) { + return keyBytes; + } + + byte[] paddedKey = new byte[targetLen]; + System.arraycopy(keyBytes, 0, paddedKey, 0, Math.min(len, targetLen)); + return paddedKey; + } + + /** + * 将IV填充到16字节 + */ + private byte[] padIv(String iv) { + byte[] ivBytes = iv.getBytes(StandardCharsets.UTF_8); + if (ivBytes.length == 16) { + return ivBytes; + } + + byte[] paddedIv = new byte[16]; + System.arraycopy(ivBytes, 0, paddedIv, 0, Math.min(ivBytes.length, 16)); + return paddedIv; + } +} diff --git a/parser/src/main/java/cn/qaiu/parser/custompy/PyHttpClient.java b/parser/src/main/java/cn/qaiu/parser/custompy/PyHttpClient.java new file mode 100644 index 0000000..1d9d1e5 --- /dev/null +++ b/parser/src/main/java/cn/qaiu/parser/custompy/PyHttpClient.java @@ -0,0 +1,649 @@ +package cn.qaiu.parser.custompy; + +import cn.qaiu.WebClientVertxInit; +import cn.qaiu.util.HttpResponseHelper; +import io.vertx.core.Future; +import io.vertx.core.MultiMap; +import io.vertx.core.Promise; +import io.vertx.core.buffer.Buffer; +import io.vertx.core.json.JsonObject; +import io.vertx.core.net.ProxyOptions; +import io.vertx.core.net.ProxyType; +import io.vertx.ext.web.client.HttpRequest; +import io.vertx.ext.web.client.HttpResponse; +import io.vertx.ext.web.client.WebClient; +import io.vertx.ext.web.client.WebClientOptions; +import io.vertx.ext.web.client.WebClientSession; +import io.vertx.ext.web.multipart.MultipartForm; +import org.apache.commons.lang3.StringUtils; +import org.graalvm.polyglot.HostAccess; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.InetAddress; +import java.net.URI; +import java.net.URLDecoder; +import java.net.URLEncoder; +import java.net.UnknownHostException; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.regex.Pattern; + +/** + * Python HTTP客户端封装 + * 为Python脚本提供类似requests库的HTTP请求功能 + * 基于Vert.x WebClient实现，提供同步API风格 + * + * @author QAIU + */ +public class PyHttpClient { + + private static final Logger log = LoggerFactory.getLogger(PyHttpClient.class); + + private final WebClient client; + private final WebClientSession clientSession; + private MultiMap headers; + private int timeoutSeconds = 30; // 默认超时时间30秒 + + // SSRF防护：内网IP正则表达式 + private static final Pattern PRIVATE_IP_PATTERN = Pattern.compile( + "^(127\\..*|10\\..*|172\\.(1[6-9]|2[0-9]|3[01])\\..*|192\\.168\\..*|169\\.254\\..*|::1|[fF][cCdD].*)" + ); + + // SSRF防护：危险域名黑名单 + private static final String[] DANGEROUS_HOSTS = { + "localhost", + "169.254.169.254", // AWS/阿里云等云服务元数据API + "metadata.google.internal", // GCP元数据 + "100.100.100.200" // 阿里云元数据 + }; + + public PyHttpClient() { + this.client = WebClient.create(WebClientVertxInit.get(), new WebClientOptions()); + this.clientSession = WebClientSession.create(client); + this.headers = MultiMap.caseInsensitiveMultiMap(); + initDefaultHeaders(); + } + + /** + * 带代理配置的构造函数 + * @param proxyConfig 代理配置JsonObject，包含type、host、port、username、password + */ + public PyHttpClient(JsonObject proxyConfig) { + if (proxyConfig != null && proxyConfig.containsKey("type")) { + ProxyOptions proxyOptions = new ProxyOptions() + .setType(ProxyType.valueOf(proxyConfig.getString("type").toUpperCase())) + .setHost(proxyConfig.getString("host")) + .setPort(proxyConfig.getInteger("port")); + + if (StringUtils.isNotEmpty(proxyConfig.getString("username"))) { + proxyOptions.setUsername(proxyConfig.getString("username")); + } + if (StringUtils.isNotEmpty(proxyConfig.getString("password"))) { + proxyOptions.setPassword(proxyConfig.getString("password")); + } + + this.client = WebClient.create(WebClientVertxInit.get(), + new WebClientOptions() + .setUserAgentEnabled(false) + .setProxyOptions(proxyOptions)); + this.clientSession = WebClientSession.create(client); + } else { + this.client = WebClient.create(WebClientVertxInit.get()); + this.clientSession = WebClientSession.create(client); + } + this.headers = MultiMap.caseInsensitiveMultiMap(); + initDefaultHeaders(); + } + + private void initDefaultHeaders() { + // 设置默认的Accept-Encoding头以支持压缩响应 + this.headers.set("Accept-Encoding", "gzip, deflate, br, zstd"); + // 设置默认的User-Agent头 + this.headers.set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36 Edg/140.0.0.0"); + // 设置默认的Accept-Language头 + this.headers.set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6"); + } + + /** + * 验证URL安全性（SSRF防护）- 仅拦截明显的内网攻击 + * @param url 待验证的URL + * @throws SecurityException 如果URL不安全 + */ + private void validateUrlSecurity(String url) { + try { + URI uri = new URI(url); + String host = uri.getHost(); + + if (host == null) { + log.debug("URL没有host信息: {}", url); + return; + } + + String lowerHost = host.toLowerCase(); + + // 1. 检查明确的危险域名（云服务元数据API等） + for (String dangerous : DANGEROUS_HOSTS) { + if (lowerHost.equals(dangerous)) { + log.warn("🔒 安全拦截: 尝试访问云服务元数据API - {}", host); + throw new SecurityException("🔒 安全拦截: 禁止访问云服务元数据API"); + } + } + + // 2. 如果host是IP地址格式，检查是否为内网IP + if (isIpAddress(lowerHost)) { + if (PRIVATE_IP_PATTERN.matcher(lowerHost).find()) { + log.warn("🔒 安全拦截: 尝试访问内网IP - {}", host); + throw new SecurityException("🔒 安全拦截: 禁止访问内网IP地址"); + } + } + + // 3. 对于域名，尝试解析IP（但不因解析失败而拦截） + if (!isIpAddress(lowerHost)) { + try { + InetAddress addr = InetAddress.getByName(host); + String ip = addr.getHostAddress(); + + if (PRIVATE_IP_PATTERN.matcher(ip).find()) { + log.warn("🔒 安全拦截: 域名解析到内网IP - {} -> {}", host, ip); + throw new SecurityException("🔒 安全拦截: 该域名指向内网地址"); + } + } catch (UnknownHostException e) { + log.debug("DNS解析失败，允许继续: {}", host); + } + } + + log.debug("URL安全检查通过: {}", url); + + } catch (SecurityException e) { + throw e; + } catch (Exception e) { + log.debug("URL验证异常，允许继续: {}", url, e); + } + } + + /** + * 判断字符串是否为IP地址格式 + */ + private boolean isIpAddress(String host) { + return host.matches("^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}$") || host.contains(":"); + } + + /** + * 发起GET请求 + * @param url 请求URL + * @return HTTP响应 + */ + @HostAccess.Export + public PyHttpResponse get(String url) { + validateUrlSecurity(url); + return executeRequest(() -> { + HttpRequest request = client.getAbs(url); + if (!headers.isEmpty()) { + request.putHeaders(headers); + } + return request.send(); + }); + } + + /** + * 发起GET请求并跟随重定向 + * @param url 请求URL + * @return HTTP响应 + */ + @HostAccess.Export + public PyHttpResponse get_with_redirect(String url) { + validateUrlSecurity(url); + return executeRequest(() -> { + HttpRequest request = client.getAbs(url); + if (!headers.isEmpty()) { + request.putHeaders(headers); + } + request.followRedirects(true); + return request.send(); + }); + } + + /** + * 发起GET请求但不跟随重定向（用于获取Location头） + * @param url 请求URL + * @return HTTP响应 + */ + @HostAccess.Export + public PyHttpResponse get_no_redirect(String url) { + validateUrlSecurity(url); + return executeRequest(() -> { + HttpRequest request = client.getAbs(url); + if (!headers.isEmpty()) { + request.putHeaders(headers); + } + request.followRedirects(false); + return request.send(); + }); + } + + /** + * 发起POST请求 + * @param url 请求URL + * @param data 请求数据（支持String、Map） + * @return HTTP响应 + */ + @HostAccess.Export + public PyHttpResponse post(String url, Object data) { + validateUrlSecurity(url); + return executeRequest(() -> { + HttpRequest request = client.postAbs(url); + if (!headers.isEmpty()) { + request.putHeaders(headers); + } + + if (data != null) { + if (data instanceof String) { + return request.sendBuffer(Buffer.buffer((String) data)); + } else if (data instanceof Map) { + @SuppressWarnings("unchecked") + Map mapData = (Map) data; + return request.sendForm(MultiMap.caseInsensitiveMultiMap().addAll(mapData)); + } else { + return request.sendJson(data); + } + } else { + return request.send(); + } + }); + } + + /** + * 发起POST请求（JSON数据） + * @param url 请求URL + * @param jsonData JSON字符串或Map + * @return HTTP响应 + */ + @HostAccess.Export + public PyHttpResponse post_json(String url, Object jsonData) { + validateUrlSecurity(url); + return executeRequest(() -> { + HttpRequest request = client.postAbs(url); + if (!headers.isEmpty()) { + request.putHeaders(headers); + } + headers.set("Content-Type", "application/json"); + + if (jsonData instanceof String) { + return request.sendBuffer(Buffer.buffer((String) jsonData)); + } else { + return request.sendJson(jsonData); + } + }); + } + + /** + * 发起PUT请求 + * @param url 请求URL + * @param data 请求数据 + * @return HTTP响应 + */ + @HostAccess.Export + public PyHttpResponse put(String url, Object data) { + validateUrlSecurity(url); + return executeRequest(() -> { + HttpRequest request = client.putAbs(url); + if (!headers.isEmpty()) { + request.putHeaders(headers); + } + + if (data != null) { + if (data instanceof String) { + return request.sendBuffer(Buffer.buffer((String) data)); + } else if (data instanceof Map) { + @SuppressWarnings("unchecked") + Map mapData = (Map) data; + return request.sendForm(MultiMap.caseInsensitiveMultiMap().addAll(mapData)); + } else { + return request.sendJson(data); + } + } else { + return request.send(); + } + }); + } + + /** + * 发起DELETE请求 + * @param url 请求URL + * @return HTTP响应 + */ + @HostAccess.Export + public PyHttpResponse delete(String url) { + validateUrlSecurity(url); + return executeRequest(() -> { + HttpRequest request = client.deleteAbs(url); + if (!headers.isEmpty()) { + request.putHeaders(headers); + } + return request.send(); + }); + } + + /** + * 发起PATCH请求 + * @param url 请求URL + * @param data 请求数据 + * @return HTTP响应 + */ + @HostAccess.Export + public PyHttpResponse patch(String url, Object data) { + validateUrlSecurity(url); + return executeRequest(() -> { + HttpRequest request = client.patchAbs(url); + if (!headers.isEmpty()) { + request.putHeaders(headers); + } + + if (data != null) { + if (data instanceof String) { + return request.sendBuffer(Buffer.buffer((String) data)); + } else if (data instanceof Map) { + @SuppressWarnings("unchecked") + Map mapData = (Map) data; + return request.sendForm(MultiMap.caseInsensitiveMultiMap().addAll(mapData)); + } else { + return request.sendJson(data); + } + } else { + return request.send(); + } + }); + } + + /** + * 设置请求头 + * @param name 头名称 + * @param value 头值 + * @return 当前客户端实例（支持链式调用） + */ + @HostAccess.Export + public PyHttpClient put_header(String name, String value) { + if (name != null && value != null) { + headers.set(name, value); + } + return this; + } + + /** + * 批量设置请求头 + * @param headersMap 请求头Map + * @return 当前客户端实例（支持链式调用） + */ + @HostAccess.Export + public PyHttpClient put_headers(Map headersMap) { + if (headersMap != null) { + for (Map.Entry entry : headersMap.entrySet()) { + if (entry.getKey() != null && entry.getValue() != null) { + headers.set(entry.getKey(), entry.getValue()); + } + } + } + return this; + } + + /** + * 删除指定请求头 + * @param name 头名称 + * @return 当前客户端实例（支持链式调用） + */ + @HostAccess.Export + public PyHttpClient remove_header(String name) { + if (name != null) { + headers.remove(name); + } + return this; + } + + /** + * 清空所有请求头（保留默认头） + * @return 当前客户端实例（支持链式调用） + */ + @HostAccess.Export + public PyHttpClient clear_headers() { + headers.clear(); + initDefaultHeaders(); + return this; + } + + /** + * 获取所有请求头 + * @return 请求头Map + */ + @HostAccess.Export + public Map get_headers() { + Map result = new HashMap<>(); + for (String name : headers.names()) { + result.put(name, headers.get(name)); + } + return result; + } + + /** + * 设置请求超时时间 + * @param seconds 超时时间（秒） + * @return 当前客户端实例（支持链式调用） + */ + @HostAccess.Export + public PyHttpClient set_timeout(int seconds) { + if (seconds > 0) { + this.timeoutSeconds = seconds; + } + return this; + } + + /** + * URL编码 + * @param str 要编码的字符串 + * @return 编码后的字符串 + */ + @HostAccess.Export + public static String url_encode(String str) { + if (str == null) { + return null; + } + try { + return URLEncoder.encode(str, StandardCharsets.UTF_8.name()); + } catch (Exception e) { + log.error("URL编码失败", e); + return str; + } + } + + /** + * URL解码 + * @param str 要解码的字符串 + * @return 解码后的字符串 + */ + @HostAccess.Export + public static String url_decode(String str) { + if (str == null) { + return null; + } + try { + return URLDecoder.decode(str, StandardCharsets.UTF_8.name()); + } catch (Exception e) { + log.error("URL解码失败", e); + return str; + } + } + + /** + * 执行HTTP请求（同步） + */ + private PyHttpResponse executeRequest(RequestExecutor executor) { + try { + Promise> promise = Promise.promise(); + Future> future = executor.execute(); + + future.onComplete(result -> { + if (result.succeeded()) { + promise.complete(result.result()); + } else { + promise.fail(result.cause()); + } + }).onFailure(Throwable::printStackTrace); + + // 等待响应完成（使用配置的超时时间） + HttpResponse response = promise.future().toCompletionStage() + .toCompletableFuture() + .get(timeoutSeconds, TimeUnit.SECONDS); + + return new PyHttpResponse(response); + + } catch (TimeoutException e) { + String errorMsg = "HTTP请求超时（" + timeoutSeconds + "秒）"; + log.error(errorMsg, e); + throw new RuntimeException(errorMsg, e); + } catch (Exception e) { + String errorMsg = e.getMessage(); + if (errorMsg == null || errorMsg.trim().isEmpty()) { + errorMsg = e.getClass().getSimpleName(); + if (e.getCause() != null && e.getCause().getMessage() != null) { + errorMsg += ": " + e.getCause().getMessage(); + } + } + log.error("HTTP请求执行失败: " + errorMsg, e); + throw new RuntimeException("HTTP请求执行失败: " + errorMsg, e); + } + } + + /** + * 请求执行器接口 + */ + @FunctionalInterface + private interface RequestExecutor { + Future> execute(); + } + + /** + * Python HTTP响应封装 + */ + public static class PyHttpResponse { + + private final HttpResponse response; + + public PyHttpResponse(HttpResponse response) { + this.response = response; + } + + /** + * 获取响应体（字符串） + * @return 响应体字符串 + */ + @HostAccess.Export + public String text() { + return HttpResponseHelper.asText(response); + } + + /** + * 获取响应体（字符串）- 别名 + */ + @HostAccess.Export + public String body() { + return text(); + } + + /** + * 解析JSON响应 + * @return JSON对象的Map表示 + */ + @HostAccess.Export + public Object json() { + try { + JsonObject jsonObject = HttpResponseHelper.asJson(response); + if (jsonObject == null || jsonObject.isEmpty()) { + return null; + } + return jsonObject.getMap(); + } catch (Exception e) { + log.error("解析JSON响应失败", e); + throw new RuntimeException("解析JSON响应失败: " + e.getMessage(), e); + } + } + + /** + * 获取HTTP状态码 + * @return 状态码 + */ + @HostAccess.Export + public int status_code() { + return response.statusCode(); + } + + /** + * 获取响应头 + * @param name 头名称 + * @return 头值 + */ + @HostAccess.Export + public String header(String name) { + return response.getHeader(name); + } + + /** + * 获取所有响应头 + * @return 响应头Map + */ + @HostAccess.Export + public Map headers() { + MultiMap responseHeaders = response.headers(); + Map result = new HashMap<>(); + for (String name : responseHeaders.names()) { + result.put(name, responseHeaders.get(name)); + } + return result; + } + + /** + * 检查请求是否成功 + * @return true表示成功（2xx状态码） + */ + @HostAccess.Export + public boolean ok() { + int status = status_code(); + return status >= 200 && status < 300; + } + + /** + * 获取响应体字节数组 + * @return 响应体字节数组 + */ + @HostAccess.Export + public byte[] content() { + Buffer buffer = response.body(); + if (buffer == null) { + return new byte[0]; + } + return buffer.getBytes(); + } + + /** + * 获取响应体大小 + * @return 响应体大小（字节） + */ + @HostAccess.Export + public long content_length() { + Buffer buffer = response.body(); + if (buffer == null) { + return 0; + } + return buffer.length(); + } + + /** + * 获取原始响应对象 + */ + public HttpResponse getOriginalResponse() { + return response; + } + } +} diff --git a/parser/src/main/java/cn/qaiu/parser/custompy/PyLogger.java b/parser/src/main/java/cn/qaiu/parser/custompy/PyLogger.java new file mode 100644 index 0000000..a02f70a --- /dev/null +++ b/parser/src/main/java/cn/qaiu/parser/custompy/PyLogger.java @@ -0,0 +1,139 @@ +package cn.qaiu.parser.custompy; + +import org.graalvm.polyglot.HostAccess; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Python日志封装 + * 为Python脚本提供日志功能 + * + * @author QAIU + */ +public class PyLogger { + + private final Logger logger; + private final String prefix; + + public PyLogger(String name) { + this.logger = LoggerFactory.getLogger(name); + this.prefix = "[" + name + "] "; + } + + public PyLogger(Class clazz) { + this.logger = LoggerFactory.getLogger(clazz); + this.prefix = "[" + clazz.getSimpleName() + "] "; + } + + /** + * 调试日志 + * @param message 日志消息 + */ + @HostAccess.Export + public void debug(String message) { + logger.debug(prefix + message); + } + + /** + * 调试日志（带参数） + * @param message 日志消息模板 + * @param args 参数 + */ + @HostAccess.Export + public void debug(String message, Object... args) { + logger.debug(prefix + message, args); + } + + /** + * 信息日志 + * @param message 日志消息 + */ + @HostAccess.Export + public void info(String message) { + logger.info(prefix + message); + } + + /** + * 信息日志（带参数） + * @param message 日志消息模板 + * @param args 参数 + */ + @HostAccess.Export + public void info(String message, Object... args) { + logger.info(prefix + message, args); + } + + /** + * 警告日志 + * @param message 日志消息 + */ + @HostAccess.Export + public void warn(String message) { + logger.warn(prefix + message); + } + + /** + * 警告日志（带参数） + * @param message 日志消息模板 + * @param args 参数 + */ + @HostAccess.Export + public void warn(String message, Object... args) { + logger.warn(prefix + message, args); + } + + /** + * 错误日志 + * @param message 日志消息 + */ + @HostAccess.Export + public void error(String message) { + logger.error(prefix + message); + } + + /** + * 错误日志（带参数） + * @param message 日志消息模板 + * @param args 参数 + */ + @HostAccess.Export + public void error(String message, Object... args) { + logger.error(prefix + message, args); + } + + /** + * 错误日志（带异常） + * @param message 日志消息 + * @param throwable 异常对象 + */ + @HostAccess.Export + public void error(String message, Throwable throwable) { + logger.error(prefix + message, throwable); + } + + /** + * 检查是否启用调试级别日志 + * @return true表示启用，false表示不启用 + */ + @HostAccess.Export + public boolean isDebugEnabled() { + return logger.isDebugEnabled(); + } + + /** + * 检查是否启用信息级别日志 + * @return true表示启用，false表示不启用 + */ + @HostAccess.Export + public boolean isInfoEnabled() { + return logger.isInfoEnabled(); + } + + /** + * 获取原始Logger对象 + * @return Logger对象 + */ + public Logger getOriginalLogger() { + return logger; + } +} diff --git a/parser/src/main/java/cn/qaiu/parser/custompy/PyParserExecutor.java b/parser/src/main/java/cn/qaiu/parser/custompy/PyParserExecutor.java new file mode 100644 index 0000000..8d1ae89 --- /dev/null +++ b/parser/src/main/java/cn/qaiu/parser/custompy/PyParserExecutor.java @@ -0,0 +1,307 @@ +package cn.qaiu.parser.custompy; + +import cn.qaiu.WebClientVertxInit; +import cn.qaiu.entity.FileInfo; +import cn.qaiu.entity.ShareLinkInfo; +import cn.qaiu.parser.IPanTool; +import cn.qaiu.parser.custom.CustomParserConfig; +import io.vertx.core.Future; +import io.vertx.core.WorkerExecutor; +import io.vertx.core.json.JsonObject; +import org.graalvm.polyglot.Context; +import org.graalvm.polyglot.Engine; +import org.graalvm.polyglot.HostAccess; +import org.graalvm.polyglot.Value; +import org.graalvm.polyglot.io.IOAccess; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Python解析器执行器 + * 使用GraalPy执行Python解析器脚本 + * 实现IPanTool接口，执行Python解析器逻辑 + * + * @author QAIU + */ +public class PyParserExecutor implements IPanTool { + + private static final Logger log = LoggerFactory.getLogger(PyParserExecutor.class); + + private static final WorkerExecutor EXECUTOR = WebClientVertxInit.get() + .createSharedWorkerExecutor("py-parser-executor", 32); + + // 共享的GraalPy引擎，提高性能 + private static final Engine SHARED_ENGINE = Engine.newBuilder() + .option("engine.WarnInterpreterOnly", "false") + .build(); + + private final CustomParserConfig config; + private final ShareLinkInfo shareLinkInfo; + private final PyHttpClient httpClient; + private final PyLogger pyLogger; + private final PyShareLinkInfoWrapper shareLinkInfoWrapper; + private final PyCryptoUtils cryptoUtils; + + public PyParserExecutor(ShareLinkInfo shareLinkInfo, CustomParserConfig config) { + this.config = config; + this.shareLinkInfo = shareLinkInfo; + + // 检查是否有代理配置 + JsonObject proxyConfig = null; + if (shareLinkInfo.getOtherParam().containsKey("proxy")) { + proxyConfig = (JsonObject) shareLinkInfo.getOtherParam().get("proxy"); + } + + this.httpClient = new PyHttpClient(proxyConfig); + this.pyLogger = new PyLogger("PyParser-" + config.getType()); + this.shareLinkInfoWrapper = new PyShareLinkInfoWrapper(shareLinkInfo); + this.cryptoUtils = new PyCryptoUtils(); + } + + /** + * 获取ShareLinkInfo对象 + * @return ShareLinkInfo对象 + */ + @Override + public ShareLinkInfo getShareLinkInfo() { + return shareLinkInfo; + } + + /** + * 创建安全的GraalPy Context + * 配置沙箱选项，禁用危险功能 + */ + private Context createContext() { + return Context.newBuilder("python") + .engine(SHARED_ENGINE) + // 允许访问带有@HostAccess.Export注解的Java方法 + .allowHostAccess(HostAccess.newBuilder(HostAccess.EXPLICIT) + .allowArrayAccess(true) + .allowListAccess(true) + .allowMapAccess(true) + .allowIterableAccess(true) + .allowIteratorAccess(true) + .build()) + // 禁止访问任意Java类 + .allowHostClassLookup(className -> false) + // 允许实验性选项 + .allowExperimentalOptions(true) + // 允许创建线程（某些Python库需要） + .allowCreateThread(true) + // 禁用原生访问 + .allowNativeAccess(false) + // 禁止创建子进程 + .allowCreateProcess(false) + // 允许虚拟文件系统访问（用于import等） + .allowIO(IOAccess.newBuilder() + .allowHostFileAccess(false) + .allowHostSocketAccess(false) + .build()) + // GraalPy特定选项 + .option("python.PythonHome", "") + .option("python.ForceImportSite", "false") + .build(); + } + + @Override + public Future parse() { + pyLogger.info("开始执行Python解析器: {}", config.getType()); + + return EXECUTOR.executeBlocking(() -> { + try (Context context = createContext()) { + // 注入Java对象到Python环境 + Value bindings = context.getBindings("python"); + bindings.putMember("http", httpClient); + bindings.putMember("logger", pyLogger); + bindings.putMember("share_link_info", shareLinkInfoWrapper); + bindings.putMember("crypto", cryptoUtils); + + // 执行Python代码 + context.eval("python", config.getPyCode()); + + // 调用parse函数 + Value parseFunc = bindings.getMember("parse"); + if (parseFunc == null || !parseFunc.canExecute()) { + throw new RuntimeException("Python代码中未找到parse函数"); + } + + Value result = parseFunc.execute(shareLinkInfoWrapper, httpClient, pyLogger); + + if (result.isString()) { + String downloadUrl = result.asString(); + pyLogger.info("解析成功: {}", downloadUrl); + return downloadUrl; + } else { + pyLogger.error("parse方法返回值类型错误，期望String，实际: {}", + result.getMetaObject().toString()); + throw new RuntimeException("parse方法返回值类型错误"); + } + } catch (Exception e) { + pyLogger.error("Python解析器执行失败: {}", e.getMessage()); + throw new RuntimeException("Python解析器执行失败: " + e.getMessage(), e); + } + }); + } + + @Override + public Future> parseFileList() { + pyLogger.info("开始执行Python文件列表解析: {}", config.getType()); + + return EXECUTOR.executeBlocking(() -> { + try (Context context = createContext()) { + // 注入Java对象到Python环境 + Value bindings = context.getBindings("python"); + bindings.putMember("http", httpClient); + bindings.putMember("logger", pyLogger); + bindings.putMember("share_link_info", shareLinkInfoWrapper); + bindings.putMember("crypto", cryptoUtils); + + // 执行Python代码 + context.eval("python", config.getPyCode()); + + // 调用parseFileList函数 + Value parseFileListFunc = bindings.getMember("parse_file_list"); + if (parseFileListFunc == null || !parseFileListFunc.canExecute()) { + throw new RuntimeException("Python代码中未找到parse_file_list函数"); + } + + Value result = parseFileListFunc.execute(shareLinkInfoWrapper, httpClient, pyLogger); + + List fileList = convertToFileInfoList(result); + pyLogger.info("文件列表解析成功，共 {} 个文件", fileList.size()); + return fileList; + } catch (Exception e) { + pyLogger.error("Python文件列表解析失败: {}", e.getMessage()); + throw new RuntimeException("Python文件列表解析失败: " + e.getMessage(), e); + } + }); + } + + @Override + public Future parseById() { + pyLogger.info("开始执行Python按ID解析: {}", config.getType()); + + return EXECUTOR.executeBlocking(() -> { + try (Context context = createContext()) { + // 注入Java对象到Python环境 + Value bindings = context.getBindings("python"); + bindings.putMember("http", httpClient); + bindings.putMember("logger", pyLogger); + bindings.putMember("share_link_info", shareLinkInfoWrapper); + bindings.putMember("crypto", cryptoUtils); + + // 执行Python代码 + context.eval("python", config.getPyCode()); + + // 调用parseById函数 + Value parseByIdFunc = bindings.getMember("parse_by_id"); + if (parseByIdFunc == null || !parseByIdFunc.canExecute()) { + throw new RuntimeException("Python代码中未找到parse_by_id函数"); + } + + Value result = parseByIdFunc.execute(shareLinkInfoWrapper, httpClient, pyLogger); + + if (result.isString()) { + String downloadUrl = result.asString(); + pyLogger.info("按ID解析成功: {}", downloadUrl); + return downloadUrl; + } else { + pyLogger.error("parse_by_id方法返回值类型错误，期望String，实际: {}", + result.getMetaObject().toString()); + throw new RuntimeException("parse_by_id方法返回值类型错误"); + } + } catch (Exception e) { + pyLogger.error("Python按ID解析失败: {}", e.getMessage()); + throw new RuntimeException("Python按ID解析失败: " + e.getMessage(), e); + } + }); + } + + /** + * 将Python列表转换为FileInfo列表 + */ + private List convertToFileInfoList(Value result) { + List fileList = new ArrayList<>(); + + if (result.hasArrayElements()) { + long size = result.getArraySize(); + for (long i = 0; i < size; i++) { + Value item = result.getArrayElement(i); + FileInfo fileInfo = convertToFileInfo(item); + if (fileInfo != null) { + fileList.add(fileInfo); + } + } + } + + return fileList; + } + + /** + * 将Python字典转换为FileInfo + */ + private FileInfo convertToFileInfo(Value item) { + try { + FileInfo fileInfo = new FileInfo(); + + if (item.hasMember("file_name") || item.hasMember("fileName")) { + Value val = item.hasMember("file_name") ? item.getMember("file_name") : item.getMember("fileName"); + if (val != null && !val.isNull()) { + fileInfo.setFileName(val.asString()); + } + } + if (item.hasMember("file_id") || item.hasMember("fileId")) { + Value val = item.hasMember("file_id") ? item.getMember("file_id") : item.getMember("fileId"); + if (val != null && !val.isNull()) { + fileInfo.setFileId(val.asString()); + } + } + if (item.hasMember("file_type") || item.hasMember("fileType")) { + Value val = item.hasMember("file_type") ? item.getMember("file_type") : item.getMember("fileType"); + if (val != null && !val.isNull()) { + fileInfo.setFileType(val.asString()); + } + } + if (item.hasMember("size")) { + Value val = item.getMember("size"); + if (val != null && !val.isNull() && val.isNumber()) { + fileInfo.setSize(val.asLong()); + } + } + if (item.hasMember("size_str") || item.hasMember("sizeStr")) { + Value val = item.hasMember("size_str") ? item.getMember("size_str") : item.getMember("sizeStr"); + if (val != null && !val.isNull()) { + fileInfo.setSizeStr(val.asString()); + } + } + if (item.hasMember("create_time") || item.hasMember("createTime")) { + Value val = item.hasMember("create_time") ? item.getMember("create_time") : item.getMember("createTime"); + if (val != null && !val.isNull()) { + fileInfo.setCreateTime(val.asString()); + } + } + if (item.hasMember("pan_type") || item.hasMember("panType")) { + Value val = item.hasMember("pan_type") ? item.getMember("pan_type") : item.getMember("panType"); + if (val != null && !val.isNull()) { + fileInfo.setPanType(val.asString()); + } + } + if (item.hasMember("parser_url") || item.hasMember("parserUrl")) { + Value val = item.hasMember("parser_url") ? item.getMember("parser_url") : item.getMember("parserUrl"); + if (val != null && !val.isNull()) { + fileInfo.setParserUrl(val.asString()); + } + } + + return fileInfo; + + } catch (Exception e) { + pyLogger.error("转换FileInfo对象失败", e); + return null; + } + } +} diff --git a/parser/src/main/java/cn/qaiu/parser/custompy/PyPlaygroundExecutor.java b/parser/src/main/java/cn/qaiu/parser/custompy/PyPlaygroundExecutor.java new file mode 100644 index 0000000..98cbe3f --- /dev/null +++ b/parser/src/main/java/cn/qaiu/parser/custompy/PyPlaygroundExecutor.java @@ -0,0 +1,397 @@ +package cn.qaiu.parser.custompy; + +import cn.qaiu.entity.FileInfo; +import cn.qaiu.entity.ShareLinkInfo; +import io.vertx.core.Future; +import io.vertx.core.Promise; +import io.vertx.core.json.JsonObject; +import org.graalvm.polyglot.Context; +import org.graalvm.polyglot.Engine; +import org.graalvm.polyglot.HostAccess; +import org.graalvm.polyglot.Value; +import org.graalvm.polyglot.io.IOAccess; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.*; + +/** + * Python演练场执行器 + * 用于临时执行Python代码，不注册到解析器注册表 + * 使用独立线程池避免Vert.x BlockedThreadChecker警告 + * + * @author QAIU + */ +public class PyPlaygroundExecutor { + + private static final Logger log = LoggerFactory.getLogger(PyPlaygroundExecutor.class); + + // Python执行超时时间（秒） + private static final long EXECUTION_TIMEOUT_SECONDS = 30; + + // 共享的GraalPy引擎 + private static final Engine SHARED_ENGINE = Engine.newBuilder() + .option("engine.WarnInterpreterOnly", "false") + .build(); + + // 使用独立的线程池 + private static final ExecutorService INDEPENDENT_EXECUTOR = Executors.newCachedThreadPool(r -> { + Thread thread = new Thread(r); + thread.setName("py-playground-independent-" + System.currentTimeMillis()); + thread.setDaemon(true); + return thread; + }); + + // 超时调度线程池 + private static final ScheduledExecutorService TIMEOUT_SCHEDULER = Executors.newScheduledThreadPool(2, r -> { + Thread thread = new Thread(r); + thread.setName("py-playground-timeout-scheduler-" + System.currentTimeMillis()); + thread.setDaemon(true); + return thread; + }); + + private final ShareLinkInfo shareLinkInfo; + private final String pyCode; + private final PyHttpClient httpClient; + private final PyPlaygroundLogger playgroundLogger; + private final PyShareLinkInfoWrapper shareLinkInfoWrapper; + private final PyCryptoUtils cryptoUtils; + + /** + * 创建演练场执行器 + * + * @param shareLinkInfo 分享链接信息 + * @param pyCode Python代码 + */ + public PyPlaygroundExecutor(ShareLinkInfo shareLinkInfo, String pyCode) { + this.shareLinkInfo = shareLinkInfo; + this.pyCode = pyCode; + + // 检查是否有代理配置 + JsonObject proxyConfig = null; + if (shareLinkInfo.getOtherParam().containsKey("proxy")) { + proxyConfig = (JsonObject) shareLinkInfo.getOtherParam().get("proxy"); + } + + this.httpClient = new PyHttpClient(proxyConfig); + this.playgroundLogger = new PyPlaygroundLogger(); + this.shareLinkInfoWrapper = new PyShareLinkInfoWrapper(shareLinkInfo); + this.cryptoUtils = new PyCryptoUtils(); + } + + /** + * 创建安全的GraalPy Context + */ + private Context createContext() { + return Context.newBuilder("python") + .engine(SHARED_ENGINE) + .allowHostAccess(HostAccess.newBuilder(HostAccess.EXPLICIT) + .allowArrayAccess(true) + .allowListAccess(true) + .allowMapAccess(true) + .allowIterableAccess(true) + .allowIteratorAccess(true) + .build()) + .allowHostClassLookup(className -> false) + .allowExperimentalOptions(true) + .allowCreateThread(true) + .allowNativeAccess(false) + .allowCreateProcess(false) + .allowIO(IOAccess.newBuilder() + .allowHostFileAccess(false) + .allowHostSocketAccess(false) + .build()) + .option("python.PythonHome", "") + .option("python.ForceImportSite", "false") + .build(); + } + + /** + * 执行parse方法（异步，带超时控制） + */ + public Future executeParseAsync() { + Promise promise = Promise.promise(); + + CompletableFuture executionFuture = CompletableFuture.supplyAsync(() -> { + playgroundLogger.infoJava("开始执行parse方法"); + + try (Context context = createContext()) { + // 注入Java对象到Python环境 + Value bindings = context.getBindings("python"); + bindings.putMember("http", httpClient); + bindings.putMember("logger", playgroundLogger); + bindings.putMember("share_link_info", shareLinkInfoWrapper); + bindings.putMember("crypto", cryptoUtils); + + // 执行Python代码 + playgroundLogger.debugJava("执行Python代码"); + context.eval("python", pyCode); + + // 调用parse函数 + Value parseFunc = bindings.getMember("parse"); + if (parseFunc == null || !parseFunc.canExecute()) { + playgroundLogger.errorJava("Python代码中未找到parse函数"); + throw new RuntimeException("Python代码中未找到parse函数"); + } + + playgroundLogger.debugJava("调用parse函数"); + Value result = parseFunc.execute(shareLinkInfoWrapper, httpClient, playgroundLogger); + + if (result.isString()) { + String downloadUrl = result.asString(); + playgroundLogger.infoJava("解析成功，返回结果: " + downloadUrl); + return downloadUrl; + } else { + String errorMsg = "parse方法返回值类型错误，期望String，实际: " + + (result.isNull() ? "null" : result.getMetaObject().toString()); + playgroundLogger.errorJava(errorMsg); + throw new RuntimeException(errorMsg); + } + } catch (Exception e) { + playgroundLogger.errorJava("执行parse方法失败: " + e.getMessage(), e); + throw new RuntimeException(e); + } + }, INDEPENDENT_EXECUTOR); + + // 创建超时任务 + ScheduledFuture timeoutTask = TIMEOUT_SCHEDULER.schedule(() -> { + if (!executionFuture.isDone()) { + executionFuture.cancel(true); + playgroundLogger.errorJava("执行超时，已强制中断"); + log.warn("Python执行超时，已强制取消"); + } + }, EXECUTION_TIMEOUT_SECONDS, TimeUnit.SECONDS); + + // 处理执行结果 + executionFuture.whenComplete((result, error) -> { + timeoutTask.cancel(false); + + if (error != null) { + if (error instanceof CancellationException) { + String timeoutMsg = "Python执行超时（超过" + EXECUTION_TIMEOUT_SECONDS + "秒），已强制中断"; + playgroundLogger.errorJava(timeoutMsg); + log.error(timeoutMsg); + promise.fail(new RuntimeException(timeoutMsg)); + } else { + Throwable cause = error.getCause(); + promise.fail(cause != null ? cause : error); + } + } else { + promise.complete(result); + } + }); + + return promise.future(); + } + + /** + * 执行parseFileList方法（异步，带超时控制） + */ + public Future> executeParseFileListAsync() { + Promise> promise = Promise.promise(); + + CompletableFuture> executionFuture = CompletableFuture.supplyAsync(() -> { + playgroundLogger.infoJava("开始执行parse_file_list方法"); + + try (Context context = createContext()) { + Value bindings = context.getBindings("python"); + bindings.putMember("http", httpClient); + bindings.putMember("logger", playgroundLogger); + bindings.putMember("share_link_info", shareLinkInfoWrapper); + bindings.putMember("crypto", cryptoUtils); + + context.eval("python", pyCode); + + Value parseFileListFunc = bindings.getMember("parse_file_list"); + if (parseFileListFunc == null || !parseFileListFunc.canExecute()) { + playgroundLogger.errorJava("Python代码中未找到parse_file_list函数"); + throw new RuntimeException("Python代码中未找到parse_file_list函数"); + } + + playgroundLogger.debugJava("调用parse_file_list函数"); + Value result = parseFileListFunc.execute(shareLinkInfoWrapper, httpClient, playgroundLogger); + + List fileList = convertToFileInfoList(result); + playgroundLogger.infoJava("文件列表解析成功，共 " + fileList.size() + " 个文件"); + return fileList; + } catch (Exception e) { + playgroundLogger.errorJava("执行parse_file_list方法失败: " + e.getMessage(), e); + throw new RuntimeException(e); + } + }, INDEPENDENT_EXECUTOR); + + ScheduledFuture timeoutTask = TIMEOUT_SCHEDULER.schedule(() -> { + if (!executionFuture.isDone()) { + executionFuture.cancel(true); + playgroundLogger.errorJava("执行超时，已强制中断"); + } + }, EXECUTION_TIMEOUT_SECONDS, TimeUnit.SECONDS); + + executionFuture.whenComplete((result, error) -> { + timeoutTask.cancel(false); + + if (error != null) { + if (error instanceof CancellationException) { + String timeoutMsg = "Python执行超时（超过" + EXECUTION_TIMEOUT_SECONDS + "秒），已强制中断"; + promise.fail(new RuntimeException(timeoutMsg)); + } else { + Throwable cause = error.getCause(); + promise.fail(cause != null ? cause : error); + } + } else { + promise.complete(result); + } + }); + + return promise.future(); + } + + /** + * 执行parseById方法（异步，带超时控制） + */ + public Future executeParseByIdAsync() { + Promise promise = Promise.promise(); + + CompletableFuture executionFuture = CompletableFuture.supplyAsync(() -> { + playgroundLogger.infoJava("开始执行parse_by_id方法"); + + try (Context context = createContext()) { + Value bindings = context.getBindings("python"); + bindings.putMember("http", httpClient); + bindings.putMember("logger", playgroundLogger); + bindings.putMember("share_link_info", shareLinkInfoWrapper); + bindings.putMember("crypto", cryptoUtils); + + context.eval("python", pyCode); + + Value parseByIdFunc = bindings.getMember("parse_by_id"); + if (parseByIdFunc == null || !parseByIdFunc.canExecute()) { + playgroundLogger.errorJava("Python代码中未找到parse_by_id函数"); + throw new RuntimeException("Python代码中未找到parse_by_id函数"); + } + + playgroundLogger.debugJava("调用parse_by_id函数"); + Value result = parseByIdFunc.execute(shareLinkInfoWrapper, httpClient, playgroundLogger); + + if (result.isString()) { + String downloadUrl = result.asString(); + playgroundLogger.infoJava("按ID解析成功，返回结果: " + downloadUrl); + return downloadUrl; + } else { + String errorMsg = "parse_by_id方法返回值类型错误"; + playgroundLogger.errorJava(errorMsg); + throw new RuntimeException(errorMsg); + } + } catch (Exception e) { + playgroundLogger.errorJava("执行parse_by_id方法失败: " + e.getMessage(), e); + throw new RuntimeException(e); + } + }, INDEPENDENT_EXECUTOR); + + ScheduledFuture timeoutTask = TIMEOUT_SCHEDULER.schedule(() -> { + if (!executionFuture.isDone()) { + executionFuture.cancel(true); + playgroundLogger.errorJava("执行超时，已强制中断"); + } + }, EXECUTION_TIMEOUT_SECONDS, TimeUnit.SECONDS); + + executionFuture.whenComplete((result, error) -> { + timeoutTask.cancel(false); + + if (error != null) { + if (error instanceof CancellationException) { + String timeoutMsg = "Python执行超时（超过" + EXECUTION_TIMEOUT_SECONDS + "秒），已强制中断"; + promise.fail(new RuntimeException(timeoutMsg)); + } else { + Throwable cause = error.getCause(); + promise.fail(cause != null ? cause : error); + } + } else { + promise.complete(result); + } + }); + + return promise.future(); + } + + /** + * 获取日志列表 + */ + public List getLogs() { + return playgroundLogger.getLogs(); + } + + /** + * 将Python列表转换为FileInfo列表 + */ + private List convertToFileInfoList(Value result) { + List fileList = new ArrayList<>(); + + if (result.hasArrayElements()) { + long size = result.getArraySize(); + for (long i = 0; i < size; i++) { + Value item = result.getArrayElement(i); + FileInfo fileInfo = convertToFileInfo(item); + if (fileInfo != null) { + fileList.add(fileInfo); + } + } + } + + return fileList; + } + + /** + * 将Python字典转换为FileInfo + */ + private FileInfo convertToFileInfo(Value item) { + try { + FileInfo fileInfo = new FileInfo(); + + if (item.hasMember("file_name") || item.hasMember("fileName")) { + Value val = item.hasMember("file_name") ? item.getMember("file_name") : item.getMember("fileName"); + if (val != null && !val.isNull()) { + fileInfo.setFileName(val.asString()); + } + } + if (item.hasMember("file_id") || item.hasMember("fileId")) { + Value val = item.hasMember("file_id") ? item.getMember("file_id") : item.getMember("fileId"); + if (val != null && !val.isNull()) { + fileInfo.setFileId(val.asString()); + } + } + if (item.hasMember("file_type") || item.hasMember("fileType")) { + Value val = item.hasMember("file_type") ? item.getMember("file_type") : item.getMember("fileType"); + if (val != null && !val.isNull()) { + fileInfo.setFileType(val.asString()); + } + } + if (item.hasMember("size")) { + Value val = item.getMember("size"); + if (val != null && !val.isNull() && val.isNumber()) { + fileInfo.setSize(val.asLong()); + } + } + if (item.hasMember("pan_type") || item.hasMember("panType")) { + Value val = item.hasMember("pan_type") ? item.getMember("pan_type") : item.getMember("panType"); + if (val != null && !val.isNull()) { + fileInfo.setPanType(val.asString()); + } + } + if (item.hasMember("parser_url") || item.hasMember("parserUrl")) { + Value val = item.hasMember("parser_url") ? item.getMember("parser_url") : item.getMember("parserUrl"); + if (val != null && !val.isNull()) { + fileInfo.setParserUrl(val.asString()); + } + } + + return fileInfo; + + } catch (Exception e) { + playgroundLogger.errorJava("转换FileInfo对象失败: " + e.getMessage()); + return null; + } + } +} diff --git a/parser/src/main/java/cn/qaiu/parser/custompy/PyPlaygroundLogger.java b/parser/src/main/java/cn/qaiu/parser/custompy/PyPlaygroundLogger.java new file mode 100644 index 0000000..141e427 --- /dev/null +++ b/parser/src/main/java/cn/qaiu/parser/custompy/PyPlaygroundLogger.java @@ -0,0 +1,183 @@ +package cn.qaiu.parser.custompy; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +/** + * Python演练场日志封装 + * 收集日志信息用于前端显示 + * + * @author QAIU + */ +public class PyPlaygroundLogger extends PyLogger { + + private static final Logger log = LoggerFactory.getLogger(PyPlaygroundLogger.class); + + private final List logs = new ArrayList<>(); + + public PyPlaygroundLogger() { + super("PyPlayground"); + } + + @Override + public void debug(String message) { + super.debug(message); + addLog("DEBUG", message); + } + + @Override + public void debug(String message, Object... args) { + super.debug(message, args); + addLog("DEBUG", formatMessage(message, args)); + } + + @Override + public void info(String message) { + super.info(message); + addLog("INFO", message); + } + + @Override + public void info(String message, Object... args) { + super.info(message, args); + addLog("INFO", formatMessage(message, args)); + } + + @Override + public void warn(String message) { + super.warn(message); + addLog("WARN", message); + } + + @Override + public void warn(String message, Object... args) { + super.warn(message, args); + addLog("WARN", formatMessage(message, args)); + } + + @Override + public void error(String message) { + super.error(message); + addLog("ERROR", message); + } + + @Override + public void error(String message, Object... args) { + super.error(message, args); + addLog("ERROR", formatMessage(message, args)); + } + + @Override + public void error(String message, Throwable throwable) { + super.error(message, throwable); + addLog("ERROR", message + " - " + throwable.getMessage()); + } + + /** + * 添加Java内部日志（不在Python脚本中调用） + */ + public void infoJava(String message) { + log.info("[PyPlayground] " + message); + addLog("INFO", "[Java] " + message, "java"); + } + + public void debugJava(String message) { + log.debug("[PyPlayground] " + message); + addLog("DEBUG", "[Java] " + message, "java"); + } + + public void errorJava(String message) { + log.error("[PyPlayground] " + message); + addLog("ERROR", "[Java] " + message, "java"); + } + + public void errorJava(String message, Throwable throwable) { + log.error("[PyPlayground] " + message, throwable); + addLog("ERROR", "[Java] " + message + " - " + throwable.getMessage(), "java"); + } + + private void addLog(String level, String message) { + addLog(level, message, "python"); + } + + private void addLog(String level, String message, String source) { + logs.add(new LogEntry(level, message, System.currentTimeMillis(), source)); + } + + private String formatMessage(String message, Object... args) { + if (args == null || args.length == 0) { + return message; + } + + // 简单的占位符替换 + String result = message; + for (Object arg : args) { + int index = result.indexOf("{}"); + if (index >= 0) { + result = result.substring(0, index) + (arg != null ? arg.toString() : "null") + result.substring(index + 2); + } + } + return result; + } + + /** + * 获取所有日志 + */ + public List getLogs() { + return new ArrayList<>(logs); + } + + /** + * 清空日志 + */ + public void clearLogs() { + logs.clear(); + } + + /** + * 获取日志数量 + */ + public int size() { + return logs.size(); + } + + /** + * 日志条目 + */ + public static class LogEntry { + private final String level; + private final String message; + private final long timestamp; + private final String source; + + public LogEntry(String level, String message, long timestamp) { + this(level, message, timestamp, "python"); + } + + public LogEntry(String level, String message, long timestamp, String source) { + this.level = level; + this.message = message; + this.timestamp = timestamp; + this.source = source; + } + + public String getLevel() { + return level; + } + + public String getMessage() { + return message; + } + + public long getTimestamp() { + return timestamp; + } + + public String getSource() { + return source; + } + } +} diff --git a/parser/src/main/java/cn/qaiu/parser/custompy/PyScriptLoader.java b/parser/src/main/java/cn/qaiu/parser/custompy/PyScriptLoader.java new file mode 100644 index 0000000..1919831 --- /dev/null +++ b/parser/src/main/java/cn/qaiu/parser/custompy/PyScriptLoader.java @@ -0,0 +1,334 @@ +package cn.qaiu.parser.custompy; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import cn.qaiu.parser.custom.CustomParserConfig; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Enumeration; +import java.util.List; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.stream.Stream; + +/** + * Python脚本加载器 + * 自动加载资源目录和外部目录的Python脚本文件 + * + * @author QAIU + */ +public class PyScriptLoader { + + private static final Logger log = LoggerFactory.getLogger(PyScriptLoader.class); + + private static final String RESOURCE_PATH = "custom-parsers/py"; + private static final String EXTERNAL_PATH = "./custom-parsers/py"; + + // 系统属性配置的外部目录路径 + private static final String EXTERNAL_PATH_PROPERTY = "parser.custom-parsers.py.path"; + + /** + * 加载所有Python脚本 + * @return 解析器配置列表 + */ + public static List loadAllScripts() { + List configs = new ArrayList<>(); + + // 1. 加载资源目录下的Python文件 + try { + List resourceConfigs = loadFromResources(); + configs.addAll(resourceConfigs); + log.info("从资源目录加载了 {} 个Python解析器", resourceConfigs.size()); + } catch (Exception e) { + log.warn("从资源目录加载Python脚本失败", e); + } + + // 2. 加载外部目录下的Python文件 + try { + List externalConfigs = loadFromExternal(); + configs.addAll(externalConfigs); + log.info("从外部目录加载了 {} 个Python解析器", externalConfigs.size()); + } catch (Exception e) { + log.warn("从外部目录加载Python脚本失败", e); + } + + log.info("总共加载了 {} 个Python解析器", configs.size()); + return configs; + } + + /** + * 从资源目录加载Python脚本 + */ + private static List loadFromResources() { + List configs = new ArrayList<>(); + + try { + List resourceFiles = getResourceFileList(); + resourceFiles.sort(String::compareTo); + + for (String resourceFile : resourceFiles) { + try { + InputStream inputStream = PyScriptLoader.class.getClassLoader() + .getResourceAsStream(resourceFile); + + if (inputStream != null) { + String pyCode = new String(inputStream.readAllBytes(), StandardCharsets.UTF_8); + CustomParserConfig config = PyScriptMetadataParser.parseScript(pyCode); + configs.add(config); + + String fileName = resourceFile.substring(resourceFile.lastIndexOf('/') + 1); + log.debug("从资源目录加载Python脚本: {}", fileName); + } + } catch (Exception e) { + log.warn("加载资源脚本失败: {}", resourceFile, e); + } + } + + } catch (Exception e) { + log.error("从资源目录加载脚本时发生异常", e); + } + + return configs; + } + + /** + * 获取资源目录中的Python文件列表 + */ + private static List getResourceFileList() { + List resourceFiles = new ArrayList<>(); + + try { + java.net.URL resourceUrl = PyScriptLoader.class.getClassLoader() + .getResource(RESOURCE_PATH); + + if (resourceUrl != null) { + String protocol = resourceUrl.getProtocol(); + + if ("jar".equals(protocol)) { + resourceFiles = getJarResourceFiles(resourceUrl); + } else if ("file".equals(protocol)) { + resourceFiles = getFileSystemResourceFiles(resourceUrl); + } + } + } catch (Exception e) { + log.debug("获取资源文件列表失败", e); + } + + return resourceFiles; + } + + /** + * 获取JAR包内的Python资源文件列表 + */ + private static List getJarResourceFiles(java.net.URL jarUrl) { + List resourceFiles = new ArrayList<>(); + + try { + String jarPath = jarUrl.getPath().substring(5, jarUrl.getPath().indexOf("!")); + JarFile jarFile = new JarFile(jarPath); + + Enumeration entries = jarFile.entries(); + while (entries.hasMoreElements()) { + JarEntry entry = entries.nextElement(); + String entryName = entry.getName(); + + if (entryName.startsWith(RESOURCE_PATH + "/") && + entryName.endsWith(".py") && + !isExcludedFile(entryName.substring(entryName.lastIndexOf('/') + 1))) { + resourceFiles.add(entryName); + } + } + + jarFile.close(); + } catch (Exception e) { + log.debug("解析JAR包资源文件失败", e); + } + + return resourceFiles; + } + + /** + * 获取文件系统中的Python资源文件列表 + */ + private static List getFileSystemResourceFiles(java.net.URL fileUrl) { + List resourceFiles = new ArrayList<>(); + + try { + java.io.File resourceDir = new java.io.File(fileUrl.getPath()); + if (resourceDir.exists() && resourceDir.isDirectory()) { + java.io.File[] files = resourceDir.listFiles(); + if (files != null) { + for (java.io.File file : files) { + if (file.isFile() && file.getName().endsWith(".py") && + !isExcludedFile(file.getName())) { + resourceFiles.add(RESOURCE_PATH + "/" + file.getName()); + } + } + } + } + } catch (Exception e) { + log.debug("解析文件系统资源文件失败", e); + } + + return resourceFiles; + } + + /** + * 从外部目录加载Python脚本 + */ + private static List loadFromExternal() { + List configs = new ArrayList<>(); + + try { + String externalPath = getExternalPath(); + Path externalDir = Paths.get(externalPath); + + if (!Files.exists(externalDir) || !Files.isDirectory(externalDir)) { + log.debug("外部目录 {} 不存在或不是目录", externalPath); + return configs; + } + + try (Stream paths = Files.walk(externalDir)) { + paths.filter(Files::isRegularFile) + .filter(path -> path.toString().endsWith(".py")) + .filter(path -> !isExcludedFile(path.getFileName().toString())) + .forEach(path -> { + try { + String pyCode = Files.readString(path, StandardCharsets.UTF_8); + CustomParserConfig config = PyScriptMetadataParser.parseScript(pyCode); + configs.add(config); + log.debug("从外部目录加载Python脚本: {}", path.getFileName()); + } catch (Exception e) { + log.warn("加载外部脚本失败: {}", path.getFileName(), e); + } + }); + } + + } catch (Exception e) { + log.error("从外部目录加载脚本时发生异常", e); + } + + return configs; + } + + /** + * 获取外部目录路径 + */ + private static String getExternalPath() { + // 1. 检查系统属性 + String systemProperty = System.getProperty(EXTERNAL_PATH_PROPERTY); + if (systemProperty != null && !systemProperty.trim().isEmpty()) { + log.debug("使用系统属性配置的Python外部目录: {}", systemProperty); + return systemProperty; + } + + // 2. 检查环境变量 + String envVariable = System.getenv("PARSER_CUSTOM_PARSERS_PY_PATH"); + if (envVariable != null && !envVariable.trim().isEmpty()) { + log.debug("使用环境变量配置的Python外部目录: {}", envVariable); + return envVariable; + } + + // 3. 使用默认路径 + log.debug("使用默认Python外部目录: {}", EXTERNAL_PATH); + return EXTERNAL_PATH; + } + + /** + * 从指定文件加载Python脚本 + * @param filePath 文件路径 + * @return 解析器配置 + */ + public static CustomParserConfig loadFromFile(String filePath) { + try { + Path path = Paths.get(filePath); + if (!Files.exists(path)) { + throw new IllegalArgumentException("文件不存在: " + filePath); + } + + String pyCode = Files.readString(path, StandardCharsets.UTF_8); + return PyScriptMetadataParser.parseScript(pyCode); + + } catch (IOException e) { + throw new RuntimeException("读取文件失败: " + filePath, e); + } + } + + /** + * 从指定资源路径加载Python脚本 + * @param resourcePath 资源路径 + * @return 解析器配置 + */ + public static CustomParserConfig loadFromResource(String resourcePath) { + try { + InputStream inputStream = PyScriptLoader.class.getClassLoader() + .getResourceAsStream(resourcePath); + + if (inputStream == null) { + throw new IllegalArgumentException("资源文件不存在: " + resourcePath); + } + + String pyCode = new String(inputStream.readAllBytes(), StandardCharsets.UTF_8); + return PyScriptMetadataParser.parseScript(pyCode); + + } catch (IOException e) { + throw new RuntimeException("读取资源文件失败: " + resourcePath, e); + } + } + + /** + * 检查外部目录是否存在 + */ + public static boolean isExternalDirectoryExists() { + Path externalDir = Paths.get(EXTERNAL_PATH); + return Files.exists(externalDir) && Files.isDirectory(externalDir); + } + + /** + * 创建外部目录 + */ + public static boolean createExternalDirectory() { + try { + Path externalDir = Paths.get(EXTERNAL_PATH); + Files.createDirectories(externalDir); + log.info("创建Python外部目录成功: {}", EXTERNAL_PATH); + return true; + } catch (IOException e) { + log.error("创建Python外部目录失败: {}", EXTERNAL_PATH, e); + return false; + } + } + + /** + * 获取外部目录路径 + */ + public static String getExternalDirectoryPath() { + return EXTERNAL_PATH; + } + + /** + * 获取资源目录路径 + */ + public static String getResourceDirectoryPath() { + return RESOURCE_PATH; + } + + /** + * 检查文件是否应该被排除 + */ + private static boolean isExcludedFile(String fileName) { + return fileName.equals("types.pyi") || + fileName.equals("__init__.py") || + fileName.equals("README.md") || + fileName.contains("_test.") || + fileName.contains("_spec.") || + fileName.startsWith("test_"); + } +} diff --git a/parser/src/main/java/cn/qaiu/parser/custompy/PyScriptMetadataParser.java b/parser/src/main/java/cn/qaiu/parser/custompy/PyScriptMetadataParser.java new file mode 100644 index 0000000..495fcdf --- /dev/null +++ b/parser/src/main/java/cn/qaiu/parser/custompy/PyScriptMetadataParser.java @@ -0,0 +1,188 @@ +package cn.qaiu.parser.custompy; + +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import cn.qaiu.parser.custom.CustomParserConfig; + +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Python脚本元数据解析器 + * 解析类油猴格式的元数据注释（Python风格） + * + * @author QAIU + */ +public class PyScriptMetadataParser { + + private static final Logger log = LoggerFactory.getLogger(PyScriptMetadataParser.class); + + // 元数据块匹配正则（Python注释风格） + // 支持 # ==UserScript== 格式 + private static final Pattern METADATA_BLOCK_PATTERN = Pattern.compile( + "#\\s*==UserScript==\\s*(.*?)\\s*#\\s*==/UserScript==", + Pattern.DOTALL + ); + + // 元数据行匹配正则 + private static final Pattern METADATA_LINE_PATTERN = Pattern.compile( + "#\\s*@(\\w+)\\s+(.*)" + ); + + /** + * 解析Python脚本，提取元数据并构建CustomParserConfig + * + * @param pyCode Python代码 + * @return CustomParserConfig配置对象 + * @throws IllegalArgumentException 如果解析失败或缺少必填字段 + */ + public static CustomParserConfig parseScript(String pyCode) { + if (StringUtils.isBlank(pyCode)) { + throw new IllegalArgumentException("Python代码不能为空"); + } + + // 1. 提取元数据块 + Map metadata = extractMetadata(pyCode); + + // 2. 验证必填字段 + validateRequiredFields(metadata); + + // 3. 构建CustomParserConfig + return buildConfig(metadata, pyCode); + } + + /** + * 提取元数据 + */ + private static Map extractMetadata(String pyCode) { + Map metadata = new HashMap<>(); + + Matcher blockMatcher = METADATA_BLOCK_PATTERN.matcher(pyCode); + if (!blockMatcher.find()) { + throw new IllegalArgumentException("未找到元数据块，请确保包含 # ==UserScript== ... # ==/UserScript== 格式的注释"); + } + + String metadataBlock = blockMatcher.group(1); + Matcher lineMatcher = METADATA_LINE_PATTERN.matcher(metadataBlock); + + while (lineMatcher.find()) { + String key = lineMatcher.group(1).toLowerCase(); + String value = lineMatcher.group(2).trim(); + metadata.put(key, value); + } + + log.debug("解析到Python脚本元数据: {}", metadata); + return metadata; + } + + /** + * 验证必填字段 + */ + private static void validateRequiredFields(Map metadata) { + if (!metadata.containsKey("name")) { + throw new IllegalArgumentException("缺少必填字段 @name"); + } + if (!metadata.containsKey("type")) { + throw new IllegalArgumentException("缺少必填字段 @type"); + } + if (!metadata.containsKey("displayname")) { + throw new IllegalArgumentException("缺少必填字段 @displayName"); + } + if (!metadata.containsKey("match")) { + throw new IllegalArgumentException("缺少必填字段 @match"); + } + + // 验证match字段包含KEY命名捕获组 + String matchPattern = metadata.get("match"); + if (!matchPattern.contains("(?P") && !matchPattern.contains("(?")) { + throw new IllegalArgumentException("@match 正则表达式必须包含命名捕获组 KEY（Python格式: (?P...) 或 Java格式: (?...)）"); + } + } + + /** + * 构建CustomParserConfig + */ + private static CustomParserConfig buildConfig(Map metadata, String pyCode) { + CustomParserConfig.Builder builder = CustomParserConfig.builder() + .type(metadata.get("type")) + .displayName(metadata.get("displayname")) + .isPyParser(true) + .pyCode(pyCode) + .language("python") + .metadata(metadata); + + // 设置匹配正则（将Python风格的(?P...)转换为Java风格的(?...)） + String matchPattern = metadata.get("match"); + if (StringUtils.isNotBlank(matchPattern)) { + // 将Python命名捕获组转换为Java格式 + matchPattern = matchPattern.replace("(?P<", "(?<"); + builder.matchPattern(matchPattern); + } + + return builder.build(); + } + + /** + * 检查Python代码是否包含有效的元数据块 + * + * @param pyCode Python代码 + * @return true表示包含有效元数据，false表示不包含 + */ + public static boolean hasValidMetadata(String pyCode) { + if (StringUtils.isBlank(pyCode)) { + return false; + } + + try { + Map metadata = extractMetadata(pyCode); + return metadata.containsKey("name") && + metadata.containsKey("type") && + metadata.containsKey("displayname") && + metadata.containsKey("match"); + } catch (Exception e) { + return false; + } + } + + /** + * 获取脚本类型（不验证必填字段） + * + * @param pyCode Python代码 + * @return 脚本类型，如果无法提取则返回null + */ + public static String getScriptType(String pyCode) { + if (StringUtils.isBlank(pyCode)) { + return null; + } + + try { + Map metadata = extractMetadata(pyCode); + return metadata.get("type"); + } catch (Exception e) { + return null; + } + } + + /** + * 获取脚本显示名称（不验证必填字段） + * + * @param pyCode Python代码 + * @return 显示名称，如果无法提取则返回null + */ + public static String getScriptDisplayName(String pyCode) { + if (StringUtils.isBlank(pyCode)) { + return null; + } + + try { + Map metadata = extractMetadata(pyCode); + return metadata.get("displayname"); + } catch (Exception e) { + return null; + } + } +} diff --git a/parser/src/main/java/cn/qaiu/parser/custompy/PyShareLinkInfoWrapper.java b/parser/src/main/java/cn/qaiu/parser/custompy/PyShareLinkInfoWrapper.java new file mode 100644 index 0000000..84d9f05 --- /dev/null +++ b/parser/src/main/java/cn/qaiu/parser/custompy/PyShareLinkInfoWrapper.java @@ -0,0 +1,262 @@ +package cn.qaiu.parser.custompy; + +import cn.qaiu.entity.ShareLinkInfo; +import org.graalvm.polyglot.HostAccess; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; + +/** + * ShareLinkInfo的Python包装器 + * 为Python脚本提供ShareLinkInfo对象的访问接口 + * + * @author QAIU + */ +public class PyShareLinkInfoWrapper { + + private static final Logger log = LoggerFactory.getLogger(PyShareLinkInfoWrapper.class); + + private final ShareLinkInfo shareLinkInfo; + + public PyShareLinkInfoWrapper(ShareLinkInfo shareLinkInfo) { + this.shareLinkInfo = shareLinkInfo; + } + + /** + * 获取分享URL + * @return 分享URL + */ + @HostAccess.Export + public String getShareUrl() { + return shareLinkInfo.getShareUrl(); + } + + /** + * Python风格方法名 - 获取分享URL + */ + @HostAccess.Export + public String get_share_url() { + return getShareUrl(); + } + + /** + * 获取分享Key + * @return 分享Key + */ + @HostAccess.Export + public String getShareKey() { + return shareLinkInfo.getShareKey(); + } + + /** + * Python风格方法名 - 获取分享Key + */ + @HostAccess.Export + public String get_share_key() { + return getShareKey(); + } + + /** + * 获取分享密码 + * @return 分享密码 + */ + @HostAccess.Export + public String getSharePassword() { + return shareLinkInfo.getSharePassword(); + } + + /** + * Python风格方法名 - 获取分享密码 + */ + @HostAccess.Export + public String get_share_password() { + return getSharePassword(); + } + + /** + * 获取网盘类型 + * @return 网盘类型 + */ + @HostAccess.Export + public String getType() { + return shareLinkInfo.getType(); + } + + /** + * Python风格方法名 - 获取网盘类型 + */ + @HostAccess.Export + public String get_type() { + return getType(); + } + + /** + * 获取网盘名称 + * @return 网盘名称 + */ + @HostAccess.Export + public String getPanName() { + return shareLinkInfo.getPanName(); + } + + /** + * Python风格方法名 - 获取网盘名称 + */ + @HostAccess.Export + public String get_pan_name() { + return getPanName(); + } + + /** + * 获取其他参数 + * @param key 参数键 + * @return 参数值 + */ + @HostAccess.Export + public Object getOtherParam(String key) { + if (key == null) { + return null; + } + return shareLinkInfo.getOtherParam().get(key); + } + + /** + * Python风格方法名 - 获取其他参数 + */ + @HostAccess.Export + public Object get_other_param(String key) { + return getOtherParam(key); + } + + /** + * 获取所有其他参数 + * @return 参数Map + */ + @HostAccess.Export + public Map getAllOtherParams() { + return shareLinkInfo.getOtherParam(); + } + + /** + * Python风格方法名 - 获取所有其他参数 + */ + @HostAccess.Export + public Map get_all_other_params() { + return getAllOtherParams(); + } + + /** + * 检查是否包含指定参数 + * @param key 参数键 + * @return true表示包含，false表示不包含 + */ + @HostAccess.Export + public boolean hasOtherParam(String key) { + if (key == null) { + return false; + } + return shareLinkInfo.getOtherParam().containsKey(key); + } + + /** + * Python风格方法名 - 检查是否包含指定参数 + */ + @HostAccess.Export + public boolean has_other_param(String key) { + return hasOtherParam(key); + } + + /** + * 获取其他参数的字符串值 + * @param key 参数键 + * @return 参数值（字符串形式） + */ + @HostAccess.Export + public String getOtherParamAsString(String key) { + Object value = getOtherParam(key); + return value != null ? value.toString() : null; + } + + /** + * Python风格方法名 - 获取其他参数的字符串值 + */ + @HostAccess.Export + public String get_other_param_as_string(String key) { + return getOtherParamAsString(key); + } + + /** + * 获取其他参数的整数值 + * @param key 参数键 + * @return 参数值（整数形式） + */ + @HostAccess.Export + public Integer getOtherParamAsInteger(String key) { + Object value = getOtherParam(key); + if (value instanceof Integer) { + return (Integer) value; + } else if (value instanceof Number) { + return ((Number) value).intValue(); + } else if (value instanceof String) { + try { + return Integer.parseInt((String) value); + } catch (NumberFormatException e) { + log.warn("无法将参数 {} 转换为整数: {}", key, value); + return null; + } + } + return null; + } + + /** + * Python风格方法名 - 获取其他参数的整数值 + */ + @HostAccess.Export + public Integer get_other_param_as_integer(String key) { + return getOtherParamAsInteger(key); + } + + /** + * 获取其他参数的布尔值 + * @param key 参数键 + * @return 参数值（布尔形式） + */ + @HostAccess.Export + public Boolean getOtherParamAsBoolean(String key) { + Object value = getOtherParam(key); + if (value instanceof Boolean) { + return (Boolean) value; + } else if (value instanceof String) { + return Boolean.parseBoolean((String) value); + } + return null; + } + + /** + * Python风格方法名 - 获取其他参数的布尔值 + */ + @HostAccess.Export + public Boolean get_other_param_as_boolean(String key) { + return getOtherParamAsBoolean(key); + } + + /** + * 获取原始的ShareLinkInfo对象 + * @return ShareLinkInfo对象 + */ + public ShareLinkInfo getOriginalShareLinkInfo() { + return shareLinkInfo; + } + + @Override + public String toString() { + return "PyShareLinkInfoWrapper{" + + "shareUrl='" + getShareUrl() + '\'' + + ", shareKey='" + getShareKey() + '\'' + + ", sharePassword='" + getSharePassword() + '\'' + + ", type='" + getType() + '\'' + + ", panName='" + getPanName() + '\'' + + '}'; + } +} diff --git a/parser/src/main/resources/custom-parsers/py/example_parser.py b/parser/src/main/resources/custom-parsers/py/example_parser.py new file mode 100644 index 0000000..25c52dd --- /dev/null +++ b/parser/src/main/resources/custom-parsers/py/example_parser.py @@ -0,0 +1,137 @@ +# ==UserScript== +# @name 示例Python解析器 +# @type example_py_parser +# @displayName 示例网盘(Python) +# @match https?://example\.com/s/(?P\w+)(?:\?pwd=(?P\w+))? +# @description Python解析器示例，展示如何编写Python网盘解析器 +# @author QAIU +# @version 1.0.0 +# ==/UserScript== + +""" +Python解析器示例 + +可用的全局对象: +- http: HTTP客户端 (PyHttpClient) +- logger: 日志对象 (PyLogger) +- share_link_info: 分享信息 (PyShareLinkInfoWrapper) +- crypto: 加密工具 (PyCryptoUtils) + +必须实现的函数: +- parse(share_link_info, http, logger): 解析下载链接，返回下载URL字符串 + +可选实现的函数: +- parse_file_list(share_link_info, http, logger): 解析文件列表，返回文件信息列表 +- parse_by_id(share_link_info, http, logger): 根据文件ID解析下载链接 +""" + + +def parse(share_link_info, http, logger): + """ + 解析分享链接，获取直链下载地址 + + 参数: + share_link_info: 分享信息对象 + - get_share_url(): 获取分享URL + - get_share_key(): 获取分享Key + - get_share_password(): 获取分享密码 + - get_type(): 获取网盘类型 + http: HTTP客户端 + - get(url): GET请求 + - post(url, data): POST请求 + - put_header(name, value): 设置请求头 + - set_timeout(seconds): 设置超时时间 + logger: 日志对象 + - info(msg): 信息日志 + - debug(msg): 调试日志 + - warn(msg): 警告日志 + - error(msg): 错误日志 + + 返回: + str: 直链下载地址 + """ + # 获取分享信息 + share_url = share_link_info.get_share_url() + share_key = share_link_info.get_share_key() + share_password = share_link_info.get_share_password() + + logger.info(f"开始解析: {share_url}") + logger.info(f"分享Key: {share_key}") + + # 设置请求头 + http.put_header("Referer", share_url) + + # 发起GET请求获取页面内容 + response = http.get(share_url) + + if not response.ok(): + logger.error(f"请求失败: {response.status_code()}") + raise Exception(f"请求失败: {response.status_code()}") + + html = response.text() + logger.debug(f"响应长度: {len(html)}") + + # 示例：从响应中提取下载链接 + # 实际解析逻辑根据具体网盘API实现 + + # 演示使用加密工具 + # md5_hash = crypto.md5(share_key) + # logger.info(f"MD5: {md5_hash}") + + # 返回模拟的下载链接 + return f"https://example.com/download/{share_key}" + + +def parse_file_list(share_link_info, http, logger): + """ + 解析文件列表 + + 返回: + list: 文件信息列表，每个元素是字典，包含: + - file_name: 文件名 + - file_id: 文件ID + - file_type: 文件类型 + - size: 文件大小(字节) + - pan_type: 网盘类型 + - parser_url: 解析URL + """ + share_url = share_link_info.get_share_url() + share_key = share_link_info.get_share_key() + + logger.info(f"获取文件列表: {share_url}") + + # 示例返回 + return [ + { + "file_name": "示例文件1.txt", + "file_id": "file_001", + "file_type": "file", + "size": 1024, + "pan_type": "example_py_parser", + "parser_url": f"/parser?type=example_py_parser&key={share_key}&fileId=file_001" + }, + { + "file_name": "示例文件2.zip", + "file_id": "file_002", + "file_type": "file", + "size": 2048, + "pan_type": "example_py_parser", + "parser_url": f"/parser?type=example_py_parser&key={share_key}&fileId=file_002" + } + ] + + +def parse_by_id(share_link_info, http, logger): + """ + 根据文件ID解析下载链接 + + 返回: + str: 直链下载地址 + """ + file_id = share_link_info.get_other_param("fileId") + share_key = share_link_info.get_share_key() + + logger.info(f"按ID解析: fileId={file_id}, shareKey={share_key}") + + # 返回模拟的下载链接 + return f"https://example.com/download/{share_key}/{file_id}" diff --git a/parser/src/main/resources/py/types.pyi b/parser/src/main/resources/py/types.pyi new file mode 100644 index 0000000..57907d2 --- /dev/null +++ b/parser/src/main/resources/py/types.pyi @@ -0,0 +1,339 @@ +""" +NFD Python解析器类型存根文件 +提供IDE自动补全和类型检查支持 +""" + +from typing import Dict, List, Optional, Any + + +class PyShareLinkInfoWrapper: + """分享链接信息包装器""" + + def get_share_url(self) -> str: + """获取分享URL""" + ... + + def get_share_key(self) -> str: + """获取分享Key""" + ... + + def get_share_password(self) -> Optional[str]: + """获取分享密码""" + ... + + def get_type(self) -> str: + """获取网盘类型""" + ... + + def get_pan_name(self) -> str: + """获取网盘名称""" + ... + + def get_other_param(self, key: str) -> Optional[Any]: + """获取其他参数""" + ... + + def get_all_other_params(self) -> Dict[str, Any]: + """获取所有其他参数""" + ... + + def has_other_param(self, key: str) -> bool: + """检查是否包含指定参数""" + ... + + def get_other_param_as_string(self, key: str) -> Optional[str]: + """获取其他参数的字符串值""" + ... + + def get_other_param_as_integer(self, key: str) -> Optional[int]: + """获取其他参数的整数值""" + ... + + def get_other_param_as_boolean(self, key: str) -> Optional[bool]: + """获取其他参数的布尔值""" + ... + + +class PyHttpResponse: + """HTTP响应封装""" + + def text(self) -> str: + """获取响应体文本""" + ... + + def body(self) -> str: + """获取响应体文本（别名）""" + ... + + def json(self) -> Optional[Dict[str, Any]]: + """解析JSON响应""" + ... + + def status_code(self) -> int: + """获取HTTP状态码""" + ... + + def header(self, name: str) -> Optional[str]: + """获取响应头""" + ... + + def headers(self) -> Dict[str, str]: + """获取所有响应头""" + ... + + def ok(self) -> bool: + """检查请求是否成功（2xx状态码）""" + ... + + def content(self) -> bytes: + """获取响应体字节数组""" + ... + + def content_length(self) -> int: + """获取响应体大小""" + ... + + +class PyHttpClient: + """HTTP客户端""" + + def get(self, url: str) -> PyHttpResponse: + """发起GET请求""" + ... + + def get_with_redirect(self, url: str) -> PyHttpResponse: + """发起GET请求并跟随重定向""" + ... + + def get_no_redirect(self, url: str) -> PyHttpResponse: + """发起GET请求但不跟随重定向""" + ... + + def post(self, url: str, data: Any = None) -> PyHttpResponse: + """发起POST请求""" + ... + + def post_json(self, url: str, json_data: Any = None) -> PyHttpResponse: + """发起POST请求（JSON数据）""" + ... + + def put(self, url: str, data: Any = None) -> PyHttpResponse: + """发起PUT请求""" + ... + + def delete(self, url: str) -> PyHttpResponse: + """发起DELETE请求""" + ... + + def patch(self, url: str, data: Any = None) -> PyHttpResponse: + """发起PATCH请求""" + ... + + def put_header(self, name: str, value: str) -> 'PyHttpClient': + """设置请求头""" + ... + + def put_headers(self, headers: Dict[str, str]) -> 'PyHttpClient': + """批量设置请求头""" + ... + + def remove_header(self, name: str) -> 'PyHttpClient': + """删除指定请求头""" + ... + + def clear_headers(self) -> 'PyHttpClient': + """清空所有请求头""" + ... + + def get_headers(self) -> Dict[str, str]: + """获取所有请求头""" + ... + + def set_timeout(self, seconds: int) -> 'PyHttpClient': + """设置请求超时时间""" + ... + + @staticmethod + def url_encode(string: str) -> str: + """URL编码""" + ... + + @staticmethod + def url_decode(string: str) -> str: + """URL解码""" + ... + + +class PyLogger: + """日志记录器""" + + def debug(self, message: str, *args) -> None: + """调试日志""" + ... + + def info(self, message: str, *args) -> None: + """信息日志""" + ... + + def warn(self, message: str, *args) -> None: + """警告日志""" + ... + + def error(self, message: str, *args) -> None: + """错误日志""" + ... + + def is_debug_enabled(self) -> bool: + """检查是否启用调试级别日志""" + ... + + def is_info_enabled(self) -> bool: + """检查是否启用信息级别日志""" + ... + + +class PyCryptoUtils: + """加密工具类""" + + def md5(self, data: str) -> str: + """MD5加密（32位小写）""" + ... + + def md5_16(self, data: str) -> str: + """MD5加密（16位小写）""" + ... + + def sha1(self, data: str) -> str: + """SHA-1加密""" + ... + + def sha256(self, data: str) -> str: + """SHA-256加密""" + ... + + def sha512(self, data: str) -> str: + """SHA-512加密""" + ... + + def base64_encode(self, data: str) -> str: + """Base64编码""" + ... + + def base64_encode_bytes(self, data: bytes) -> str: + """Base64编码（字节数组）""" + ... + + def base64_decode(self, data: str) -> str: + """Base64解码""" + ... + + def base64_decode_bytes(self, data: str) -> bytes: + """Base64解码（返回字节数组）""" + ... + + def base64_url_encode(self, data: str) -> str: + """URL安全的Base64编码""" + ... + + def base64_url_decode(self, data: str) -> str: + """URL安全的Base64解码""" + ... + + def aes_encrypt_ecb(self, data: str, key: str) -> str: + """AES加密（ECB模式）""" + ... + + def aes_decrypt_ecb(self, data: str, key: str) -> str: + """AES解密（ECB模式）""" + ... + + def aes_encrypt_cbc(self, data: str, key: str, iv: str) -> str: + """AES加密（CBC模式）""" + ... + + def aes_decrypt_cbc(self, data: str, key: str, iv: str) -> str: + """AES解密（CBC模式）""" + ... + + def bytes_to_hex(self, data: bytes) -> str: + """字节数组转十六进制""" + ... + + def hex_to_bytes(self, hex_string: str) -> bytes: + """十六进制转字节数组""" + ... + + +# 全局变量类型声明 +http: PyHttpClient +logger: PyLogger +share_link_info: PyShareLinkInfoWrapper +crypto: PyCryptoUtils + + +class FileInfo: + """文件信息""" + file_name: str + file_id: str + file_type: str + size: int + size_str: str + create_time: str + update_time: str + create_by: str + download_count: int + file_icon: str + pan_type: str + parser_url: str + preview_url: str + + +def parse(share_link_info: PyShareLinkInfoWrapper, http: PyHttpClient, logger: PyLogger) -> str: + """ + 解析分享链接，获取直链下载地址 + + 这是必须实现的主要解析函数 + + Args: + share_link_info: 分享链接信息 + http: HTTP客户端 + logger: 日志记录器 + + Returns: + 直链下载地址 + """ + ... + + +def parse_file_list(share_link_info: PyShareLinkInfoWrapper, http: PyHttpClient, logger: PyLogger) -> List[Dict[str, Any]]: + """ + 解析文件列表 + + 可选实现，用于支持目录分享 + + Args: + share_link_info: 分享链接信息 + http: HTTP客户端 + logger: 日志记录器 + + Returns: + 文件信息列表 + """ + ... + + +def parse_by_id(share_link_info: PyShareLinkInfoWrapper, http: PyHttpClient, logger: PyLogger) -> str: + """ + 根据文件ID解析下载链接 + + 可选实现，用于支持按文件ID解析 + + Args: + share_link_info: 分享链接信息 + http: HTTP客户端 + logger: 日志记录器 + + Returns: + 直链下载地址 + """ + ... diff --git a/pom.xml b/pom.xml index ccb34b7..e0de19b 100644 --- a/pom.xml +++ b/pom.xml @@ -25,7 +25,7 @@ ${project.basedir}/web-service/target/package - 4.5.22 + 4.5.23 0.10.2 1.18.38 2.0.5 diff --git a/web-front/src/utils/playgroundApi.js b/web-front/src/utils/playgroundApi.js index 566f87f..c90fe65 100644 --- a/web-front/src/utils/playgroundApi.js +++ b/web-front/src/utils/playgroundApi.js @@ -37,20 +37,23 @@ export const playgroundApi = { }, /** - * 测试执行JavaScript代码 - * @param {string} jsCode - JavaScript代码 + * 测试执行JavaScript/Python代码 + * @param {string} code - 代码 * @param {string} shareUrl - 分享链接 * @param {string} pwd - 密码（可选） * @param {string} method - 测试方法：parse/parseFileList/parseById + * @param {string} language - 语言类型：javascript/python * @returns {Promise} 测试结果 */ - async testScript(jsCode, shareUrl, pwd = '', method = 'parse') { + async testScript(code, shareUrl, pwd = '', method = 'parse', language = 'javascript') { try { const response = await axiosInstance.post('/v2/playground/test', { - jsCode, + jsCode: code, // 兼容后端旧字段名 + code, shareUrl, pwd, - method + method, + language }); // 框架会自动包装成JsonResult，需要从data字段获取 if (response.data && response.data.data) { @@ -83,6 +86,21 @@ export const playgroundApi = { } }, + /** + * 获取types.pyi文件内容（Python类型提示） + * @returns {Promise} types.pyi内容 + */ + async getTypesPyi() { + try { + const response = await axiosInstance.get('/v2/playground/types.pyi', { + responseType: 'text' + }); + return response.data; + } catch (error) { + throw new Error(error.response?.data?.error || error.message || '获取types.pyi失败'); + } + }, + /** * 获取解析器列表 */ @@ -106,10 +124,16 @@ export const playgroundApi = { /** * 保存解析器 + * @param {string} code - 代码 + * @param {string} language - 语言类型：javascript/python */ - async saveParser(jsCode) { + async saveParser(code, language = 'javascript') { try { - const response = await axiosInstance.post('/v2/playground/parsers', { jsCode }); + const response = await axiosInstance.post('/v2/playground/parsers', { + jsCode: code, // 兼容后端旧字段名 + code, + language + }); // 框架会自动包装成JsonResult if (response.data && response.data.data) { return { @@ -132,10 +156,19 @@ export const playgroundApi = { /** * 更新解析器 + * @param {number} id - 解析器ID + * @param {string} code - 代码 + * @param {boolean} enabled - 是否启用 + * @param {string} language - 语言类型：javascript/python */ - async updateParser(id, jsCode, enabled = true) { + async updateParser(id, code, enabled = true, language = 'javascript') { try { - const response = await axiosInstance.put(`/v2/playground/parsers/${id}`, { jsCode, enabled }); + const response = await axiosInstance.put(`/v2/playground/parsers/${id}`, { + jsCode: code, // 兼容后端旧字段名 + code, + enabled, + language + }); return response.data; } catch (error) { throw new Error(error.response?.data?.error || error.message || '更新解析器失败'); diff --git a/web-front/src/views/Playground.vue b/web-front/src/views/Playground.vue index 4f35206..d883c98 100644 --- a/web-front/src/views/Playground.vue +++ b/web-front/src/views/Playground.vue @@ -76,7 +76,7 @@ 脚本解析器演练场 - JavaScript (ES5) + {{ currentFileLanguageDisplay }} @@ -771,6 +771,19 @@ :rules="newFileFormRules" :label-width="isMobile ? '80px' : '100px'" > + + + + + JavaScript (ES5) + + + + Python (GraalPy) + + +
选择解析器开发语言
+ f.id === activeFileId.value) || files.value[0]; }); + // 当前文件语言类型（用于显示） + const currentFileLanguageDisplay = computed(() => { + const file = activeFile.value; + if (!file) return 'JavaScript (ES5)'; + + // 优先使用文件的language属性 + if (file.language === 'python') { + return 'Python (GraalPy)'; + } + + // 根据文件扩展名判断 + if (file.name && file.name.endsWith('.py')) { + return 'Python (GraalPy)'; + } + + return 'JavaScript (ES5)'; + }); + // 当前编辑的代码（绑定到活动文件） const isFileChanging = ref(false); // 标记是否正在切换文件 const currentCode = computed({ @@ -902,7 +933,8 @@ export default { name: '', identifier: '', author: '', - match: '' + match: '', + language: 'javascript' }); const newFileFormRules = { name: [ @@ -910,6 +942,9 @@ export default { ], identifier: [ { required: true, message: '请输入标识', trigger: 'blur' } + ], + language: [ + { required: true, message: '请选择开发语言', trigger: 'change' } ] }; const newFileFormRef = ref(null); @@ -1398,12 +1433,22 @@ function parseById(shareLinkInfo, http, logger) { isFileChanging.value = true; activeFileId.value = fileId; saveAllFilesToStorage(); + + // 获取切换后的文件 + const newFile = files.value.find(f => f.id === fileId); + // 等待编辑器更新 nextTick(() => { if (editorRef.value && editorRef.value.getEditor) { const editor = editorRef.value.getEditor(); if (editor) { editor.focus(); + + // 更新编辑器语言模式 + if (newFile) { + const language = newFile.language || getLanguageFromFile(newFile.name); + updateEditorLanguage(language); + } } } // 切换完成后，取消标记 @@ -1436,13 +1481,14 @@ function parseById(shareLinkInfo, http, logger) { name: '', identifier: '', author: '', - match: '' + match: '', + language: 'javascript' }; newFileDialogVisible.value = true; }; - // 生成模板代码 - const generateTemplate = (name, identifier, author, match) => { + // 生成JavaScript模板代码 + const generateJsTemplate = (name, identifier, author, match) => { const type = identifier.toLowerCase().replace(/[^a-z0-9]/g, '_'); const displayName = name; const description = `使用JavaScript实现的${name}解析器`; @@ -1498,6 +1544,83 @@ function parseFileList(shareLinkInfo, http, logger) { }`; }; + // 生成Python模板代码 + const generatePyTemplate = (name, identifier, author, match) => { + const type = identifier.toLowerCase().replace(/[^a-z0-9]/g, '_'); + const displayName = name; + const description = `使用Python实现的${name}解析器`; + + return `# ==UserScript== +# @name ${name} +# @type ${type} +# @displayName ${displayName} +# @description ${description} +# @match ${match || 'https?://example.com/s/(?\\w+)'} +# @author ${author || 'yourname'} +# @version 1.0.0 +# ==/UserScript== + +""" +${name}解析器 - Python实现 +使用GraalPy运行，提供与JavaScript解析器相同的功能 +""" + +def parse(share_link_info, http, logger): + """ + 解析单个文件下载链接 + + Args: + share_link_info: 分享链接信息对象 + http: HTTP客户端 + logger: 日志记录器 + + Returns: + str: 直链下载地址 + """ + url = share_link_info.get_share_url() + logger.info(f"开始解析: {url}") + + response = http.get(url) + if not response.ok(): + raise Exception(f"请求失败: {response.status_code()}") + + html = response.text() + # 这里添加你的解析逻辑 + # 例如：使用正则表达式提取下载链接 + + return "https://example.com/download/file.zip" + + +def parse_file_list(share_link_info, http, logger): + """ + 解析文件列表（可选） + + Args: + share_link_info: 分享链接信息对象 + http: HTTP客户端 + logger: 日志记录器 + + Returns: + list: 文件信息列表 + """ + dir_id = share_link_info.get_other_param("dirId") or "0" + logger.info(f"解析文件列表，目录ID: {dir_id}") + + # 这里添加你的文件列表解析逻辑 + file_list = [] + + return file_list +`; + }; + + // 生成模板代码（根据语言选择） + const generateTemplate = (name, identifier, author, match, language = 'javascript') => { + if (language === 'python') { + return generatePyTemplate(name, identifier, author, match); + } + return generateJsTemplate(name, identifier, author, match); + }; + // 创建新文件 const createNewFile = async () => { if (!newFileFormRef.value) return; @@ -1505,10 +1628,17 @@ function parseFileList(shareLinkInfo, http, logger) { await newFileFormRef.value.validate((valid) => { if (!valid) return; + const language = newFileForm.value.language || 'javascript'; + const isPython = language === 'python'; + const fileExt = isPython ? '.py' : '.js'; + // 使用解析器名称作为文件名 - const fileName = newFileForm.value.name.endsWith('.js') - ? newFileForm.value.name - : newFileForm.value.name + '.js'; + let fileName = newFileForm.value.name; + if (!fileName.endsWith(fileExt)) { + // 移除可能的错误扩展名 + fileName = fileName.replace(/\.(js|py)$/i, ''); + fileName = fileName + fileExt; + } // 检查文件名是否已存在 if (files.value.some(f => f.name === fileName)) { @@ -1518,10 +1648,11 @@ function parseFileList(shareLinkInfo, http, logger) { // 生成模板代码 const template = generateTemplate( - newFileForm.value.name, + newFileForm.value.name.replace(/\.(js|py)$/i, ''), newFileForm.value.identifier, newFileForm.value.author, - newFileForm.value.match + newFileForm.value.match, + language ); // 创建新文件 @@ -1530,6 +1661,7 @@ function parseFileList(shareLinkInfo, http, logger) { id: 'file' + fileIdCounter.value, name: fileName, content: template, + language: language, modified: false }; @@ -1538,7 +1670,10 @@ function parseFileList(shareLinkInfo, http, logger) { newFileDialogVisible.value = false; saveAllFilesToStorage(); - ElMessage.success('文件创建成功'); + // 更新编辑器语言模式 + updateEditorLanguage(language); + + ElMessage.success(`${isPython ? 'Python' : 'JavaScript'}文件创建成功`); // 等待编辑器更新后聚焦 nextTick(() => { @@ -1573,6 +1708,31 @@ function parseFileList(shareLinkInfo, http, logger) { } }; + // 更新编辑器语言模式 + const updateEditorLanguage = (language) => { + if (editorRef.value && editorRef.value.getEditor) { + const editor = editorRef.value.getEditor(); + if (editor) { + const model = editor.getModel(); + if (model) { + const monaco = window.monaco || editorRef.value.monaco; + if (monaco) { + const langId = language === 'python' ? 'python' : 'javascript'; + monaco.editor.setModelLanguage(model, langId); + } + } + } + } + }; + + // 根据文件扩展名获取语言类型 + const getLanguageFromFile = (fileName) => { + if (fileName && fileName.endsWith('.py')) { + return 'python'; + } + return 'javascript'; + }; + // IDE功能：切换自动换行 const toggleWordWrap = () => { wordWrapEnabled.value = !wordWrapEnabled.value; @@ -1708,8 +1868,13 @@ function parseFileList(shareLinkInfo, http, logger) { // 执行测试 const executeTest = async () => { const codeToTest = currentCode.value; + + // 获取当前文件的语言类型 + const currentLanguage = activeFile.value?.language || getLanguageFromFile(activeFile.value?.name) || 'javascript'; + const isPython = currentLanguage === 'python'; + if (!codeToTest.trim()) { - ElMessage.warning('请先输入JavaScript代码'); + ElMessage.warning(`请先输入${isPython ? 'Python' : 'JavaScript'}代码`); return; } @@ -1718,30 +1883,59 @@ function parseFileList(shareLinkInfo, http, logger) { return; } - // 检查代码中是否包含潜在的危险模式 - const dangerousPatterns = [ - { pattern: /while\s*\(\s*true\s*\)/gi, message: '检测到 while(true) 无限循环' }, - { pattern: /for\s*\(\s*;\s*;\s*\)/gi, message: '检测到 for(;;) 无限循环' }, - { pattern: /for\s*\(\s*var\s+\w+\s*=\s*\d+\s*;\s*true\s*;/gi, message: '检测到可能的无限循环' } - ]; - - for (const { pattern, message } of dangerousPatterns) { - if (pattern.test(codeToTest)) { - const confirmed = await ElMessageBox.confirm( - `⚠️ ${message}\n\n这可能导致脚本无法停止并占用服务器资源。\n\n建议修改代码，添加合理的循环退出条件。\n\n确定要继续执行吗？`, - '危险代码警告', - { - confirmButtonText: '我知道风险，继续执行', - cancelButtonText: '取消', - type: 'warning', - dangerouslyUseHTMLString: true + // 检查代码中是否包含潜在的危险模式（仅针对JavaScript） + if (!isPython) { + const dangerousPatterns = [ + { pattern: /while\s*\(\s*true\s*\)/gi, message: '检测到 while(true) 无限循环' }, + { pattern: /for\s*\(\s*;\s*;\s*\)/gi, message: '检测到 for(;;) 无限循环' }, + { pattern: /for\s*\(\s*var\s+\w+\s*=\s*\d+\s*;\s*true\s*;/gi, message: '检测到可能的无限循环' } + ]; + + for (const { pattern, message } of dangerousPatterns) { + if (pattern.test(codeToTest)) { + const confirmed = await ElMessageBox.confirm( + `⚠️ ${message}\n\n这可能导致脚本无法停止并占用服务器资源。\n\n建议修改代码，添加合理的循环退出条件。\n\n确定要继续执行吗？`, + '危险代码警告', + { + confirmButtonText: '我知道风险，继续执行', + cancelButtonText: '取消', + type: 'warning', + dangerouslyUseHTMLString: true + } + ).catch(() => false); + + if (!confirmed) { + return; } - ).catch(() => false); - - if (!confirmed) { - return; + break; + } + } + } + + // Python 无限循环检查 + if (isPython) { + const pythonDangerousPatterns = [ + { pattern: /while\s+True\s*:/gi, message: '检测到 while True: 无限循环' } + ]; + + for (const { pattern, message } of pythonDangerousPatterns) { + if (pattern.test(codeToTest)) { + const confirmed = await ElMessageBox.confirm( + `⚠️ ${message}\n\n这可能导致脚本无法停止并占用服务器资源。\n\n建议修改代码，添加合理的循环退出条件。\n\n确定要继续执行吗？`, + '危险代码警告', + { + confirmButtonText: '我知道风险，继续执行', + cancelButtonText: '取消', + type: 'warning', + dangerouslyUseHTMLString: true + } + ).catch(() => false); + + if (!confirmed) { + return; + } + break; } - break; } } @@ -1754,7 +1948,8 @@ function parseFileList(shareLinkInfo, http, logger) { codeToTest, // 使用当前活动文件的代码 testParams.value.shareUrl, testParams.value.pwd, - testParams.value.method + testParams.value.method, + currentLanguage // 传递语言类型 ); console.log('测试结果:', result); @@ -2286,6 +2481,7 @@ curl "${baseUrl}/json/parser?url=${encodeURIComponent(exampleUrl)}" files, activeFileId, activeFile, + currentFileLanguageDisplay, handleFileChange, removeFile, // 新建文件 @@ -2303,6 +2499,8 @@ curl "${baseUrl}/json/parser?url=${encodeURIComponent(exampleUrl)}" exportCurrentFile, undo, redo, + updateEditorLanguage, + getLanguageFromFile, // 加载和认证 loading, loadProgress, diff --git a/web-service/src/main/java/cn/qaiu/lz/web/controller/PlaygroundApi.java b/web-service/src/main/java/cn/qaiu/lz/web/controller/PlaygroundApi.java index bf9f794..cc43612 100644 --- a/web-service/src/main/java/cn/qaiu/lz/web/controller/PlaygroundApi.java +++ b/web-service/src/main/java/cn/qaiu/lz/web/controller/PlaygroundApi.java @@ -9,6 +9,9 @@ import cn.qaiu.parser.custom.CustomParserRegistry; import cn.qaiu.parser.customjs.JsPlaygroundExecutor; import cn.qaiu.parser.customjs.JsPlaygroundLogger; import cn.qaiu.parser.customjs.JsScriptMetadataParser; +import cn.qaiu.parser.custompy.PyPlaygroundExecutor; +import cn.qaiu.parser.custompy.PyPlaygroundLogger; +import cn.qaiu.parser.custompy.PyScriptMetadataParser; import cn.qaiu.vx.core.annotaions.RouteHandler; import cn.qaiu.vx.core.annotaions.RouteMapping; import cn.qaiu.vx.core.enums.RouteMethod; @@ -155,7 +158,7 @@ public class PlaygroundApi { } /** - * 测试执行JavaScript代码 + * 测试执行JavaScript/Python代码 * * @param ctx 路由上下文 * @return 测试结果 @@ -176,25 +179,38 @@ public class PlaygroundApi { try { JsonObject body = ctx.body().asJsonObject(); - String jsCode = body.getString("jsCode"); + String code = body.getString("jsCode"); // 兼容旧字段名 + if (StringUtils.isBlank(code)) { + code = body.getString("code"); // 也支持新字段名 + } String shareUrl = body.getString("shareUrl"); String pwd = body.getString("pwd"); String method = body.getString("method", "parse"); + String language = body.getString("language", "javascript").toLowerCase(); // 参数验证 - if (StringUtils.isBlank(jsCode)) { + if (StringUtils.isBlank(code)) { promise.complete(JsonObject.mapFrom(PlaygroundTestResp.builder() .success(false) - .error("JavaScript代码不能为空") + .error("代码不能为空") + .build())); + return promise.future(); + } + + // 验证语言类型 + if (!"javascript".equals(language) && !"python".equals(language)) { + promise.complete(JsonObject.mapFrom(PlaygroundTestResp.builder() + .success(false) + .error("不支持的语言类型: " + language + "，仅支持 javascript 或 python") .build())); return promise.future(); } // 代码长度验证 - if (jsCode.length() > MAX_CODE_LENGTH) { + if (code.length() > MAX_CODE_LENGTH) { promise.complete(JsonObject.mapFrom(PlaygroundTestResp.builder() .success(false) - .error("代码长度超过限制（最大128KB），当前长度: " + jsCode.length() + " 字节") + .error("代码长度超过限制（最大128KB），当前长度: " + code.length() + " 字节") .build())); return promise.future(); } @@ -207,10 +223,16 @@ public class PlaygroundApi { return promise.future(); } - // ===== 新增：验证URL匹配 ===== + // ===== 验证URL匹配（根据语言类型选择解析器） ===== try { - var config = JsScriptMetadataParser.parseScript(jsCode); - Pattern matchPattern = config.getMatchPattern(); + Pattern matchPattern; + if ("python".equals(language)) { + var config = PyScriptMetadataParser.parseScript(code); + matchPattern = config.getMatchPattern(); + } else { + var config = JsScriptMetadataParser.parseScript(code); + matchPattern = config.getMatchPattern(); + } if (matchPattern != null) { Matcher matcher = matchPattern.matcher(shareUrl); @@ -241,115 +263,15 @@ public class PlaygroundApi { .build())); return promise.future(); } - - long startTime = System.currentTimeMillis(); - - try { - // 创建ShareLinkInfo - ParserCreate parserCreate = ParserCreate.fromShareUrl(shareUrl); - if (StringUtils.isNotBlank(pwd)) { - parserCreate.setShareLinkInfoPwd(pwd); - } - ShareLinkInfo shareLinkInfo = parserCreate.getShareLinkInfo(); - - // 创建演练场执行器 - JsPlaygroundExecutor executor = new JsPlaygroundExecutor(shareLinkInfo, jsCode); - - // 根据方法类型选择执行，并异步处理结果 - Future executionFuture; - switch (method) { - case "parse": - executionFuture = executor.executeParseAsync().map(r -> (Object) r); - break; - case "parseFileList": - executionFuture = executor.executeParseFileListAsync().map(r -> (Object) r); - break; - case "parseById": - executionFuture = executor.executeParseByIdAsync().map(r -> (Object) r); - break; - default: - promise.fail(new IllegalArgumentException("未知的方法类型: " + method)); - return promise.future(); - } - - // 异步处理执行结果 - executionFuture.onSuccess(result -> { - log.debug("执行成功，结果类型: {}, 结果值: {}", - result != null ? result.getClass().getSimpleName() : "null", - result); - - // 获取日志 - List logEntries = executor.getLogs(); - log.debug("获取到 {} 条日志记录", logEntries.size()); - - List respLogs = logEntries.stream() - .map(entry -> PlaygroundTestResp.LogEntry.builder() - .level(entry.getLevel()) - .message(entry.getMessage()) - .timestamp(entry.getTimestamp()) - .source(entry.getSource()) // 使用日志条目的来源标识 - .build()) - .collect(Collectors.toList()); - - long executionTime = System.currentTimeMillis() - startTime; - - // 构建响应 - PlaygroundTestResp response = PlaygroundTestResp.builder() - .success(true) - .result(result) - .logs(respLogs) - .executionTime(executionTime) - .build(); - - JsonObject jsonResponse = JsonObject.mapFrom(response); - log.debug("测试成功响应: {}", jsonResponse.encodePrettily()); - promise.complete(jsonResponse); - }).onFailure(e -> { - long executionTime = System.currentTimeMillis() - startTime; - String errorMessage = e.getMessage(); - String stackTrace = getStackTrace(e); - - log.error("演练场执行失败", e); - - // 尝试获取已有的日志 - List logEntries = executor.getLogs(); - List respLogs = logEntries.stream() - .map(entry -> PlaygroundTestResp.LogEntry.builder() - .level(entry.getLevel()) - .message(entry.getMessage()) - .timestamp(entry.getTimestamp()) - .source(entry.getSource()) // 使用日志条目的来源标识 - .build()) - .collect(Collectors.toList()); - - PlaygroundTestResp response = PlaygroundTestResp.builder() - .success(false) - .error(errorMessage) - .stackTrace(stackTrace) - .executionTime(executionTime) - .logs(respLogs) - .build(); - - promise.complete(JsonObject.mapFrom(response)); - }); - - } catch (Exception e) { - long executionTime = System.currentTimeMillis() - startTime; - String errorMessage = e.getMessage(); - String stackTrace = getStackTrace(e); - - log.error("演练场初始化失败", e); - - PlaygroundTestResp response = PlaygroundTestResp.builder() - .success(false) - .error(errorMessage) - .stackTrace(stackTrace) - .executionTime(executionTime) - .logs(new ArrayList<>()) - .build(); - - promise.complete(JsonObject.mapFrom(response)); + + // 根据语言类型执行代码 + final String finalCode = code; + if ("python".equals(language)) { + executePythonTest(promise, finalCode, shareUrl, pwd, method); + } else { + executeJavaScriptTest(promise, finalCode, shareUrl, pwd, method); } + } catch (Exception e) { log.error("解析请求参数失败", e); promise.complete(JsonObject.mapFrom(PlaygroundTestResp.builder() @@ -361,6 +283,230 @@ public class PlaygroundApi { return promise.future(); } + + /** + * 执行JavaScript测试 + */ + private void executeJavaScriptTest(Promise promise, String jsCode, String shareUrl, String pwd, String method) { + long startTime = System.currentTimeMillis(); + + try { + // 创建ShareLinkInfo + ParserCreate parserCreate = ParserCreate.fromShareUrl(shareUrl); + if (StringUtils.isNotBlank(pwd)) { + parserCreate.setShareLinkInfoPwd(pwd); + } + ShareLinkInfo shareLinkInfo = parserCreate.getShareLinkInfo(); + + // 创建演练场执行器 + JsPlaygroundExecutor executor = new JsPlaygroundExecutor(shareLinkInfo, jsCode); + + // 根据方法类型选择执行，并异步处理结果 + Future executionFuture; + switch (method) { + case "parse": + executionFuture = executor.executeParseAsync().map(r -> (Object) r); + break; + case "parseFileList": + executionFuture = executor.executeParseFileListAsync().map(r -> (Object) r); + break; + case "parseById": + executionFuture = executor.executeParseByIdAsync().map(r -> (Object) r); + break; + default: + promise.fail(new IllegalArgumentException("未知的方法类型: " + method)); + return; + } + + // 异步处理执行结果 + executionFuture.onSuccess(result -> { + log.debug("JavaScript执行成功，结果类型: {}, 结果值: {}", + result != null ? result.getClass().getSimpleName() : "null", + result); + + // 获取日志 + List logEntries = executor.getLogs(); + log.debug("获取到 {} 条日志记录", logEntries.size()); + + List respLogs = logEntries.stream() + .map(entry -> PlaygroundTestResp.LogEntry.builder() + .level(entry.getLevel()) + .message(entry.getMessage()) + .timestamp(entry.getTimestamp()) + .source(entry.getSource()) + .build()) + .collect(Collectors.toList()); + + long executionTime = System.currentTimeMillis() - startTime; + + PlaygroundTestResp response = PlaygroundTestResp.builder() + .success(true) + .result(result) + .logs(respLogs) + .executionTime(executionTime) + .build(); + + JsonObject jsonResponse = JsonObject.mapFrom(response); + log.debug("JavaScript测试成功响应: {}", jsonResponse.encodePrettily()); + promise.complete(jsonResponse); + }).onFailure(e -> { + long executionTime = System.currentTimeMillis() - startTime; + String errorMessage = e.getMessage(); + String stackTrace = getStackTrace(e); + + log.error("JavaScript演练场执行失败", e); + + List logEntries = executor.getLogs(); + List respLogs = logEntries.stream() + .map(entry -> PlaygroundTestResp.LogEntry.builder() + .level(entry.getLevel()) + .message(entry.getMessage()) + .timestamp(entry.getTimestamp()) + .source(entry.getSource()) + .build()) + .collect(Collectors.toList()); + + PlaygroundTestResp response = PlaygroundTestResp.builder() + .success(false) + .error(errorMessage) + .stackTrace(stackTrace) + .executionTime(executionTime) + .logs(respLogs) + .build(); + + promise.complete(JsonObject.mapFrom(response)); + }); + + } catch (Exception e) { + long executionTime = System.currentTimeMillis() - startTime; + String errorMessage = e.getMessage(); + String stackTrace = getStackTrace(e); + + log.error("JavaScript演练场初始化失败", e); + + PlaygroundTestResp response = PlaygroundTestResp.builder() + .success(false) + .error(errorMessage) + .stackTrace(stackTrace) + .executionTime(executionTime) + .logs(new ArrayList<>()) + .build(); + + promise.complete(JsonObject.mapFrom(response)); + } + } + + /** + * 执行Python测试 + */ + private void executePythonTest(Promise promise, String pyCode, String shareUrl, String pwd, String method) { + long startTime = System.currentTimeMillis(); + + try { + // 创建ShareLinkInfo + ParserCreate parserCreate = ParserCreate.fromShareUrl(shareUrl); + if (StringUtils.isNotBlank(pwd)) { + parserCreate.setShareLinkInfoPwd(pwd); + } + ShareLinkInfo shareLinkInfo = parserCreate.getShareLinkInfo(); + + // 创建Python演练场执行器 + PyPlaygroundExecutor executor = new PyPlaygroundExecutor(shareLinkInfo, pyCode); + + // 根据方法类型选择执行，并异步处理结果 + Future executionFuture; + switch (method) { + case "parse": + executionFuture = executor.executeParseAsync().map(r -> (Object) r); + break; + case "parseFileList": + executionFuture = executor.executeParseFileListAsync().map(r -> (Object) r); + break; + case "parseById": + executionFuture = executor.executeParseByIdAsync().map(r -> (Object) r); + break; + default: + promise.fail(new IllegalArgumentException("未知的方法类型: " + method)); + return; + } + + // 异步处理执行结果 + executionFuture.onSuccess(result -> { + log.debug("Python执行成功，结果类型: {}, 结果值: {}", + result != null ? result.getClass().getSimpleName() : "null", + result); + + // 获取日志 + List logEntries = executor.getLogs(); + log.debug("获取到 {} 条日志记录", logEntries.size()); + + List respLogs = logEntries.stream() + .map(entry -> PlaygroundTestResp.LogEntry.builder() + .level(entry.getLevel()) + .message(entry.getMessage()) + .timestamp(entry.getTimestamp()) + .source(entry.getSource()) + .build()) + .collect(Collectors.toList()); + + long executionTime = System.currentTimeMillis() - startTime; + + PlaygroundTestResp response = PlaygroundTestResp.builder() + .success(true) + .result(result) + .logs(respLogs) + .executionTime(executionTime) + .build(); + + JsonObject jsonResponse = JsonObject.mapFrom(response); + log.debug("Python测试成功响应: {}", jsonResponse.encodePrettily()); + promise.complete(jsonResponse); + }).onFailure(e -> { + long executionTime = System.currentTimeMillis() - startTime; + String errorMessage = e.getMessage(); + String stackTrace = getStackTrace(e); + + log.error("Python演练场执行失败", e); + + List logEntries = executor.getLogs(); + List respLogs = logEntries.stream() + .map(entry -> PlaygroundTestResp.LogEntry.builder() + .level(entry.getLevel()) + .message(entry.getMessage()) + .timestamp(entry.getTimestamp()) + .source(entry.getSource()) + .build()) + .collect(Collectors.toList()); + + PlaygroundTestResp response = PlaygroundTestResp.builder() + .success(false) + .error(errorMessage) + .stackTrace(stackTrace) + .executionTime(executionTime) + .logs(respLogs) + .build(); + + promise.complete(JsonObject.mapFrom(response)); + }); + + } catch (Exception e) { + long executionTime = System.currentTimeMillis() - startTime; + String errorMessage = e.getMessage(); + String stackTrace = getStackTrace(e); + + log.error("Python演练场初始化失败", e); + + PlaygroundTestResp response = PlaygroundTestResp.builder() + .success(false) + .error(errorMessage) + .stackTrace(stackTrace) + .executionTime(executionTime) + .logs(new ArrayList<>()) + .build(); + + promise.complete(JsonObject.mapFrom(response)); + } + } /** * 获取types.js文件内容 @@ -402,6 +548,47 @@ public class PlaygroundApi { ResponseUtil.fireJsonResultResponse(response, JsonResult.error("读取types.js失败: " + e.getMessage())); } } + + /** + * 获取types.pyi文件内容（Python类型提示） + * + * @param ctx 路由上下文 + * @param response HTTP响应 + */ + @RouteMapping(value = "/types.pyi", method = RouteMethod.GET) + public void getTypesPyi(RoutingContext ctx, HttpServerResponse response) { + // 检查是否启用 + if (!checkEnabled()) { + ResponseUtil.fireJsonResultResponse(response, JsonResult.error("演练场功能已禁用")); + return; + } + + // 权限检查 + if (!checkAuth(ctx)) { + ResponseUtil.fireJsonResultResponse(response, JsonResult.error("未授权访问")); + return; + } + + try (InputStream inputStream = getClass().getClassLoader() + .getResourceAsStream("py/types.pyi")) { + + if (inputStream == null) { + ResponseUtil.fireJsonResultResponse(response, JsonResult.error("types.pyi文件不存在")); + return; + } + + String content = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8)) + .lines() + .collect(Collectors.joining("\n")); + + response.putHeader("Content-Type", "text/x-python; charset=utf-8") + .end(content); + + } catch (Exception e) { + log.error("读取types.pyi失败", e); + ResponseUtil.fireJsonResultResponse(response, JsonResult.error("读取types.pyi失败: " + e.getMessage())); + } + } /** * 获取解析器列表 @@ -439,22 +626,39 @@ public class PlaygroundApi { try { JsonObject body = ctx.body().asJsonObject(); - String jsCode = body.getString("jsCode"); + String code = body.getString("jsCode"); // 兼容旧字段名 + if (StringUtils.isBlank(code)) { + code = body.getString("code"); // 也支持新字段名 + } + String language = body.getString("language", "javascript").toLowerCase(); - if (StringUtils.isBlank(jsCode)) { - promise.complete(JsonResult.error("JavaScript代码不能为空").toJsonObject()); + if (StringUtils.isBlank(code)) { + promise.complete(JsonResult.error("代码不能为空").toJsonObject()); + return promise.future(); + } + + // 验证语言类型 + if (!"javascript".equals(language) && !"python".equals(language)) { + promise.complete(JsonResult.error("不支持的语言类型: " + language + "，仅支持 javascript 或 python").toJsonObject()); return promise.future(); } // 代码长度验证 - if (jsCode.length() > MAX_CODE_LENGTH) { - promise.complete(JsonResult.error("代码长度超过限制（最大128KB），当前长度: " + jsCode.length() + " 字节").toJsonObject()); + if (code.length() > MAX_CODE_LENGTH) { + promise.complete(JsonResult.error("代码长度超过限制（最大128KB），当前长度: " + code.length() + " 字节").toJsonObject()); return promise.future(); } - // 解析元数据 + // 根据语言类型解析元数据 + final String finalCode = code; try { - var config = JsScriptMetadataParser.parseScript(jsCode); + cn.qaiu.parser.custom.CustomParserConfig config; + if ("python".equals(language)) { + config = PyScriptMetadataParser.parseScript(finalCode); + } else { + config = JsScriptMetadataParser.parseScript(finalCode); + } + String type = config.getType(); String displayName = config.getDisplayName(); String name = config.getMetadata().get("name"); @@ -462,6 +666,7 @@ public class PlaygroundApi { String author = config.getMetadata().get("author"); String version = config.getMetadata().get("version"); String matchPattern = config.getMatchPattern() != null ? config.getMatchPattern().pattern() : null; + final boolean isPython = "python".equals(language); // 检查数量限制 dbService.getPlaygroundParserCount().onSuccess(count -> { @@ -498,15 +703,20 @@ public class PlaygroundApi { parser.put("author", author); parser.put("version", version); parser.put("matchPattern", matchPattern); - parser.put("jsCode", jsCode); + parser.put("jsCode", finalCode); // 兼容旧字段名存储 + parser.put("language", isPython ? "python" : "javascript"); parser.put("ip", getClientIp(ctx.request())); parser.put("enabled", true); dbService.savePlaygroundParser(parser).onSuccess(result -> { // 保存成功后，立即注册到解析器系统 try { - CustomParserRegistry.register(config); - log.info("已注册演练场解析器: {} ({})", displayName, type); + if (isPython) { + CustomParserRegistry.registerPy(config); + } else { + CustomParserRegistry.register(config); + } + log.info("已注册演练场{}解析器: {} ({})", isPython ? "Python" : "JavaScript", displayName, type); promise.complete(JsonResult.success("保存并注册成功").toJsonObject()); } catch (Exception e) { log.error("注册解析器失败", e); @@ -559,16 +769,33 @@ public class PlaygroundApi { try { JsonObject body = ctx.body().asJsonObject(); - String jsCode = body.getString("jsCode"); + String code = body.getString("jsCode"); // 兼容旧字段名 + if (StringUtils.isBlank(code)) { + code = body.getString("code"); // 也支持新字段名 + } + String language = body.getString("language", "javascript").toLowerCase(); - if (StringUtils.isBlank(jsCode)) { - promise.complete(JsonResult.error("JavaScript代码不能为空").toJsonObject()); + if (StringUtils.isBlank(code)) { + promise.complete(JsonResult.error("代码不能为空").toJsonObject()); + return promise.future(); + } + + // 验证语言类型 + if (!"javascript".equals(language) && !"python".equals(language)) { + promise.complete(JsonResult.error("不支持的语言类型: " + language + "，仅支持 javascript 或 python").toJsonObject()); return promise.future(); } - // 解析元数据 + // 根据语言类型解析元数据 + final String finalCode = code; try { - var config = JsScriptMetadataParser.parseScript(jsCode); + cn.qaiu.parser.custom.CustomParserConfig config; + if ("python".equals(language)) { + config = PyScriptMetadataParser.parseScript(finalCode); + } else { + config = JsScriptMetadataParser.parseScript(finalCode); + } + String type = config.getType(); String displayName = config.getDisplayName(); String name = config.getMetadata().get("name"); @@ -577,6 +804,7 @@ public class PlaygroundApi { String version = config.getMetadata().get("version"); String matchPattern = config.getMatchPattern() != null ? config.getMatchPattern().pattern() : null; boolean enabled = body.getBoolean("enabled", true); + final boolean isPython = "python".equals(language); JsonObject parser = new JsonObject(); parser.put("name", name); @@ -585,7 +813,8 @@ public class PlaygroundApi { parser.put("author", author); parser.put("version", version); parser.put("matchPattern", matchPattern); - parser.put("jsCode", jsCode); + parser.put("jsCode", finalCode); // 兼容旧字段名存储 + parser.put("language", isPython ? "python" : "javascript"); parser.put("enabled", enabled); dbService.updatePlaygroundParser(id, parser).onSuccess(result -> { @@ -595,8 +824,12 @@ public class PlaygroundApi { // 先注销旧的（如果存在） CustomParserRegistry.unregister(type); // 重新注册新的 - CustomParserRegistry.register(config); - log.info("已重新注册演练场解析器: {} ({})", displayName, type); + if (isPython) { + CustomParserRegistry.registerPy(config); + } else { + CustomParserRegistry.register(config); + } + log.info("已重新注册演练场{}解析器: {} ({})", isPython ? "Python" : "JavaScript", displayName, type); } else { // 禁用时注销 CustomParserRegistry.unregister(type); diff --git a/web-service/src/main/resources/app.yml b/web-service/src/main/resources/app.yml index 3947d90..c4a7112 100644 --- a/web-service/src/main/resources/app.yml +++ b/web-service/src/main/resources/app.yml @@ -1,4 +1,4 @@ # 要激活的配置: app-配置名称.yml -active: dev +active: local # 控制台输出的版权文字 copyright: QAIU