diff --git a/parser/README.md b/parser/README.md new file mode 100644 index 0000000..dabd156 --- /dev/null +++ b/parser/README.md @@ -0,0 +1,66 @@ +# parser + +NFD 解析器模块：聚合各类网盘/分享页解析，统一输出文件列表与下载信息，供上层下载器使用。 + +- 语言：Java 17 +- 构建：Maven +- 模块版本：10.1.9 + +## 依赖（Maven Central） +- Maven（无需额外仓库配置）： +```xml + + cn.qaiu + parser + 10.1.9 + +``` +- Gradle Groovy DSL： +```groovy +dependencies { + implementation 'cn.qaiu:parser:10.1.9' +} +``` +- Gradle Kotlin DSL： +```kotlin +dependencies { + implementation("cn.qaiu:parser:10.1.9") +} +``` + +## 核心 API 速览 +- WebClientVertxInit：注入/获取 Vert.x 实例（内部 HTTP 客户端依赖）。 +- ParserCreate：从分享链接或类型构建解析器；生成短链 path。 +- IPanTool：统一解析接口（parse、parseFileList、parseById）。 + +## 使用示例（极简） +```java +Vertx vx = Vertx.vertx(); +WebClientVertxInit.init(vx); +IPanTool tool = ParserCreate.fromShareUrl("https://www.ilanzou.com/s/xxxx").createTool(); +List list = tool.parseFileList().toCompletionStage().toCompletableFuture().join(); +``` +完整示例与调试脚本见 parser/doc/README.md。 + +## 快速开始 +- 环境：JDK >= 17，Maven >= 3.9 +- 构建/安装： +``` +mvn -pl parser -am clean package +mvn -pl parser -am install +``` +- 测试： +``` +mvn -pl parser test +``` + +## 文档 +开发者请阅读 parser/doc/README.md（含解析约定、示例、IDEA `.http` 调试）。 + +## 目录 +- src/main/java/cn/qaiu/entity：通用实体（如 FileInfo） +- src/main/java/cn/qaiu/parser：解析框架 & 各站点实现（impl） +- src/test/java：单测与示例 + +## 许可证 +MIT License diff --git a/parser/doc/README.md b/parser/doc/README.md new file mode 100644 index 0000000..5f7bfc1 --- /dev/null +++ b/parser/doc/README.md @@ -0,0 +1,282 @@ +# parser 开发文档 + +面向开发者的解析器实现说明：约定、数据映射、HTTP 调试与示例代码。 + +- 语言/构建：Java 17 / Maven +- 关键接口：cn.qaiu.parser.IPanTool（返回 Future>），各站点位于 parser/src/main/java/cn/qaiu/parser/impl +- 数据模型：cn.qaiu.entity.FileInfo（统一对外文件项） + +--- + +## 0. 快速调用示例（最小可运行） +```java +import cn.qaiu.WebClientVertxInit; +import cn.qaiu.entity.FileInfo; +import cn.qaiu.parser.IPanTool; +import cn.qaiu.parser.ParserCreate; +import io.vertx.core.Vertx; +import java.util.List; + +public class ParserQuickStart { + public static void main(String[] args) { + // 1) 初始化 Vert.x（parser 内部 WebClient 依赖它） + Vertx vertx = Vertx.vertx(); + WebClientVertxInit.init(vertx); + + // 2) 从分享链接自动识别网盘类型并创建解析器 + String shareUrl = "https://www.ilanzou.com/s/xxxx"; // 替换为实际分享链接 + IPanTool tool = ParserCreate.fromShareUrl(shareUrl) + // .setShareLinkInfoPwd("1234") // 如有提取码可设置 + .createTool(); + + // 3) 异步 -> 同步等待，获取文件列表 + List files = tool.parseFileList() + .toCompletionStage().toCompletableFuture().join(); + for (FileInfo f : files) { + System.out.printf("%s\t%s\t%s\n", + f.getFileName(), f.getSizeStr(), f.getParserUrl()); + } + + // 4) 原始解析输出（不同盘实现差异较大，仅供调试） + String raw = tool.parseSync(); + System.out.println("raw: " + (raw == null ? "null" : raw.substring(0, Math.min(raw.length(), 200)) + "...")); + + // 5) 生成 parser 短链 path（可用于上层路由聚合显示） + String path = ParserCreate.fromShareUrl(shareUrl).genPathSuffix(); + System.out.println("path suffix: /" + path); + + vertx.close(); + } +} +``` + +等价用法：已知网盘类型 + shareKey 构造 +```java +IPanTool tool = ParserCreate.fromType("lz") // 对应 PanDomainTemplate.LZ + .shareKey("abcd12") // 必填：分享 key + .setShareLinkInfoPwd("1234") // 可选：提取码 + .createTool(); +// 获取文件列表 +List files = tool.parseFileList().toCompletionStage().toCompletableFuture().join(); +``` + +要点： +- 必须先 WebClientVertxInit.init(Vertx)；若未显式初始化，内部将懒加载 Vertx.vertx()，建议显式注入以统一生命周期。 +- parseFileList 返回 Future>，可直接 join/await；parseSync 仅针对 parse() 的 String 结果。 +- 生成短链 path：ParserCreate.genPathSuffix()（用于页面/服务端聚合）。 + +--- + +## 1. 解析器约定 +- 输入：目标分享/目录页或接口的上下文（通常在实现类构造或初始化时已注入必要参数，如 shareKey、cookie、headers）。 +- 输出：Future>（文件/目录混合列表，必要时区分 file/folder）。 +- 错误：失败场景通过 Future 失败或返回空列表；日志由上层统一处理。 +- 并发：尽量使用 Vert.x Web Client 异步请求；注意限流与重试策略由实现类自定。 + +FileInfo 关键字段（节选）： +- fileId：唯一标识 +- fileName：展示名（建议带扩展名，如 basename） +- fileType：如 "file"/"folder" 或 mime（实现自定，保持一致即可） +- size（Long, 字节）/ sizeStr（原文字符串） +- createTime / updateTime：格式 yyyy-MM-dd HH:mm:ss（如源为时间戳或 yyyy-MM-dd 需转） +- parserUrl：非直连下载的中间链接或协议占位（如 BilPan://） +- filePath / previewUrl / extParameters：按需补充 + +工具类： +- FileSizeConverter：字符串容量转字节、字节转可读容量 + +--- + +## 2. 文件列表解析规范（按给定 JSON） +目标 JSON（摘要）： +- 列表路径：data.data[] +- 每项结构：item.data（含 attributes、id、type、relationships） +- type："file" 或 "folder" + +字段映射建议： +- 通用 + - fileId ← data.id + - createTime ← data.attributes.created_at（若格式不一致，上层再统一格式化） + - updateTime ← data.attributes.updated_at + - fileType： + - 对文件用 data.attributes.mimetype 或固定 "file" + - 对目录固定 "folder" +- 文件（type="file"） + - fileName ← 优先 attributes.basename（示例："GBT+28448-2019.pdf"），无则用 attributes.name + - sizeStr ← attributes.filesize（示例："18MB"） + - size ← 尝试用 FileSizeConverter.convertToBytes(sizeStr)，失败则置空 + - parserUrl ← attributes.file_url（示例：BilPan://downLoad?id=...） + - filePath/parentId ← relationships.parent.data.id（可放到 extParameters.parentId） + - previewUrl/thumbnail ← attributes.thumbnail（可选） +- 目录（type="folder"） + - fileName ← attributes.name + - size/sizeStr ← 置空 + - 统计字段（如 items/trashed_items）可入 extParameters + +边界与兼容： +- attributes.filesize 可能为空或为非标准字符串；转换失败时保留 sizeStr，忽略 size。 +- attributes.file_url 可能为占位协议（BilPan://），直链转换在下载阶段处理。 +- relationships.* 可能为空，读取前需判空。 + +伪代码（parseFileList 核心片段）： +```java +// 仅示意，按项目 Json 工具替换 +JsonObject root = ...; // 接口返回 +JsonArray arr = root.getJsonObject("data").getJsonArray("data"); +List list = new ArrayList<>(); +for (JsonObject wrap : arr) { + JsonObject d = wrap.getJsonObject("data"); + String type = d.getString("type"); + JsonObject attrs = d.getJsonObject("attributes"); + FileInfo fi = new FileInfo(); + fi.setFileId(d.getString("id")); + fi.setCreateTime(attrs.getString("created_at")); + fi.setUpdateTime(attrs.getString("updated_at")); + if ("file".equals(type)) { + String basename = attrs.getString("basename"); + fi.setFileName(basename != null ? basename : attrs.getString("name")); + fi.setFileType(attrs.getString("mimetype", "file")); + String sizeStr = attrs.getString("filesize"); + fi.setSizeStr(sizeStr); + try { if (sizeStr != null) fi.setSize(FileSizeConverter.convertToBytes(sizeStr)); } catch (Exception ignore) {} + fi.setParserUrl(attrs.getString("file_url")); + // parentId（可选） + JsonObject rel = d.getJsonObject("relationships"); + if (rel != null) { + JsonObject p = rel.getJsonObject("parent"); + if (p != null && p.getJsonObject("data") != null) { + String pid = p.getJsonObject("data").getString("id"); + Map ext = new HashMap<>(); + ext.put("parentId", pid); + fi.setExtParameters(ext); + } + } + } else { + fi.setFileName(attrs.getString("name")); + fi.setFileType("folder"); + } + list.add(fi); +} +return Future.succeededFuture(list); +``` + +--- + +## 3. curl 转 Java 11 HttpClient 示例 +以 GET 为例（来源：developer-oss.lanrar.com）： +```java +HttpClient client = HttpClient.newHttpClient(); +String q = "<替换为长查询串>"; +String url = "https://developer-oss.lanrar.com/file/?" + URLEncoder.encode(q, StandardCharsets.UTF_8); +HttpRequest req = HttpRequest.newBuilder(URI.create(url)) + .header("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7") + .header("accept-language", "zh-CN,zh;q=0.9") + .header("cache-control", "max-age=0") + .header("dnt", "1") + .header("priority", "u=0, i") + .header("referer", "https://developer-oss.lanrar.com/file/?" + q) + .header("sec-ch-ua", "\"Chromium\";v=\"140\", \"Not=A?Brand\";v=\"24\", \"Microsoft Edge\";v=\"140\"") + .header("sec-ch-ua-mobile", "?0") + .header("sec-ch-ua-platform", "\"macOS\"") + .header("sec-fetch-dest", "document") + .header("sec-fetch-mode", "navigate") + .header("sec-fetch-site", "same-origin") + .header("upgrade-insecure-requests", "1") + .header("user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36 Edg/140.0.0.0") + .header("Cookie", "acw_tc=; cdn_sec_tc=; acw_sc__v2=") + .GET() + .build(); +HttpResponse resp = client.send(req, HttpResponse.BodyHandlers.ofString()); +System.out.println(resp.statusCode()); +System.out.println(resp.body()); +``` + +POST 示例（来源：Weiyun Share BatchDownload，使用 JSON）： +```java +HttpClient client = HttpClient.newHttpClient(); +String url = "https://share.weiyun.com/webapp/json/weiyunShare/WeiyunShareBatchDownload?refer=chrome_mac&g_tk=1399845656&r=0.3925692266635241"; +String json = "{...与 curl/requests 等价 JSON 负载，使用占位参数...}"; +HttpRequest req = HttpRequest.newBuilder(URI.create(url)) + .header("accept", "application/json, text/plain, */*") + .header("content-type", "application/json;charset=UTF-8") + .header("origin", "https://share.weiyun.com") + .header("referer", "https://share.weiyun.com/") + .header("user-agent", "Mozilla/5.0 ...") + .header("Cookie", "uin=; skey=; p_skey=; ...") + .POST(HttpRequest.BodyPublishers.ofString(json, StandardCharsets.UTF_8)) + .build(); +HttpResponse resp = client.send(req, HttpResponse.BodyHandlers.ofString()); +``` +提示： +- Cookie/Token 使用占位并从外部注入，避免硬编码与泄露。 +- r/g_tk 等参数如需计算，请在实现类中封装。 + +--- + +## 4. IntelliJ IDEA `.http` 调试样例 +保存为 `requests.http`，可配合环境变量使用。 + +GET： +```http +### 开发者资源 GET 示例 +GET https://developer-oss.lanrar.com/file/?{{q}} +accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7 +accept-language: zh-CN,zh;q=0.9 +cache-control: max-age=0 +dnt: 1 +priority: u=0, i +referer: https://developer-oss.lanrar.com/file/?{{q}} +sec-ch-ua: "Chromium";v="140", "Not=A?Brand";v="24", "Microsoft Edge";v="140" +sec-ch-ua-mobile: ?0 +sec-ch-ua-platform: "macOS" +sec-fetch-dest: document +sec-fetch-mode: navigate +sec-fetch-site: same-origin +upgrade-insecure-requests: 1 +user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36 Edg/140.0.0.0 +Cookie: acw_tc={{acw_tc}}; cdn_sec_tc={{cdn_sec_tc}}; acw_sc__v2={{acw_sc_v2}} + +> {% client.log("status: " + response.status); %} + +### 环境变量（可在 HTTP Client Environment 中配置） +@q=替换为实际长查询串 +@acw_tc=your_acw_tc +@cdn_sec_tc=your_cdn_sec_tc +@acw_sc_v2=your_acw_sc__v2 +``` + +POST： +```http +### Weiyun 批量下载 POST 示例 +POST https://share.weiyun.com/webapp/json/weiyunShare/WeiyunShareBatchDownload?refer=chrome_mac&g_tk={{g_tk}}&r={{r}} +accept: application/json, text/plain, */* +content-type: application/json;charset=UTF-8 +origin: https://share.weiyun.com +referer: https://share.weiyun.com/{{share_key}} +user-agent: Mozilla/5.0 ... +Cookie: uin={{uin}}; skey={{skey}}; p_skey={{p_skey}}; p_uin={{p_uin}}; wyctoken={{wyctoken}} + +{ + "req_header": "{...}", + "req_body": "{...}" +} +``` + +--- + +## 5. 开发流程建议 +- 新增站点：在 impl 下新增 Tool，实现 IPanTool，复用 PanBase/模板类；补充单测。 +- 字段不全：尽量回填 sizeStr/createTime 等便于前端展示；不可用字段置空。 +- 单测：放置于 parser/src/test/java，尽量添加 1-2 个 happy path + 1 个边界用例。 + +## 6. 常见问题 +- 容量解析失败：保留 sizeStr，并忽略 size；避免抛出异常影响整体列表。 +- 协议占位下载链接：统一放至 parserUrl，直链转换由下载阶段处理。 +- 鉴权：Cookie/Token 过期问题由上层刷新或外部注入处理；解析器保持无状态最佳。 + +--- + +## 7. 参考 +- FileInfo：parser/src/main/java/cn/qaiu/entity/FileInfo.java +- IPanTool：parser/src/main/java/cn/qaiu/parser/IPanTool.java +- FileSizeConverter：parser/src/main/java/cn/qaiu/util/FileSizeConverter.java diff --git a/parser/pom.xml b/parser/pom.xml index f96bb21..0087d5b 100644 --- a/parser/pom.xml +++ b/parser/pom.xml @@ -3,64 +3,29 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 + - netdisk-fast-download cn.qaiu + netdisk-fast-download ${revision} - parser + cn.qaiu + parser + 10.1.9 jar + cn.qaiu:parser - NFD parser + NFD parser module https://qaiu.top - - UTF-8 - - - - - - ch.qos.logback - logback-classic - ${logback.version} - - - org.slf4j - slf4j-api - ${slf4j.version} - - - io.vertx - vertx-web-client - ${vertx.version} - - - org.apache.commons - commons-lang3 - ${commons-lang3.version} - - - - org.openjdk.nashorn - nashorn-core - 15.4 - - - - junit - junit - ${junit.version} - test - - MIT License https://opensource.org/license/mit + qaiu @@ -68,11 +33,13 @@ https://qaiu.top + - scm:git@github.com:qaiu/netdisk-fast-download.git - scm:git@github.com:qaiu/netdisk-fast-download.git - git@github.com:qaiu/netdisk-fast-download.git + scm:git:https://github.com/qaiu/netdisk-fast-download.git + scm:git:ssh://git@github.com:qaiu/netdisk-fast-download.git + https://github.com/qaiu/netdisk-fast-download + sonatype @@ -84,41 +51,135 @@ + + 0.1.8 + 17 + 17 + 17 + UTF-8 + + + 4.5.21 + 0.10.2 + 1.18.38 + 2.0.5 + 3.18.0 + 2.14.2 + 1.5.19 + 4.13.2 + + + + + + ch.qos.logback + logback-classic + ${logback.version} + runtime + + + org.slf4j + slf4j-api + ${slf4j.version} + + + + + io.vertx + vertx-web-client + ${vertx.version} + + + + + org.apache.commons + commons-lang3 + ${commons-lang3.version} + + + + + org.openjdk.nashorn + nashorn-core + 15.4 + compile + + + + + org.brotli + dec + 0.1.2 + + + + + junit + junit + ${junit.version} + test + + + + org.apache.maven.plugins maven-compiler-plugin - 3.8.1 + 3.13.0 - ${java.version} - ${java.version} + ${maven.compiler.source} + ${maven.compiler.target} + ${project.build.sourceEncoding} + + org.apache.maven.plugins maven-source-plugin - 3.3.1 - - true - + 3.3.0 attach-sources + + jar + + + + org.apache.maven.plugins - maven-site-plugin - 3.7.1 + maven-javadoc-plugin + 3.7.0 + + + attach-javadocs + + jar + + + + false + + -Xdoclint:none + + true + + + - + + org.apache.maven.plugins maven-gpg-plugin - 1.6 + 3.2.7 sign-artifacts @@ -126,19 +187,31 @@ sign + + + + --batch + --yes + --pinentry-mode + loopback + + + + org.sonatype.central central-publishing-maven-plugin 0.6.0 true - central + sonatype true + diff --git a/parser/src/main/java/cn/qaiu/parser/PanBase.java b/parser/src/main/java/cn/qaiu/parser/PanBase.java index fcff72f..778a07e 100644 --- a/parser/src/main/java/cn/qaiu/parser/PanBase.java +++ b/parser/src/main/java/cn/qaiu/parser/PanBase.java @@ -2,6 +2,7 @@ package cn.qaiu.parser; import cn.qaiu.WebClientVertxInit; import cn.qaiu.entity.ShareLinkInfo; +import cn.qaiu.util.HttpResponseHelper; import io.vertx.core.Future; import io.vertx.core.Handler; import io.vertx.core.Promise; @@ -17,10 +18,7 @@ import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.BufferedReader; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStreamReader; +import java.io.*; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Iterator; @@ -223,20 +221,7 @@ public abstract class PanBase implements IPanTool { * @return String */ protected String asText(HttpResponse res) { - // 检查响应头中的Content-Encoding是否为gzip - String contentEncoding = res.getHeader("Content-Encoding"); - try { - if ("gzip".equalsIgnoreCase(contentEncoding)) { - // 如果是gzip压缩的响应体，解压 - return decompressGzip((Buffer) res.body()); - } else { - return res.bodyAsString(); - } - } catch (Exception e) { - fail("解析失败: res格式异常"); - //throw new RuntimeException("解析失败: res格式异常"); - } - return null; + return HttpResponseHelper.asText(res); } protected void complete(String url) { @@ -279,22 +264,16 @@ public abstract class PanBase implements IPanTool { private String decompressGzip(Buffer compressedData) throws IOException { try (ByteArrayInputStream bais = new ByteArrayInputStream(compressedData.getBytes()); GZIPInputStream gzis = new GZIPInputStream(bais); - BufferedReader reader = new BufferedReader(new InputStreamReader(gzis, - StandardCharsets.UTF_8))) { + InputStreamReader isr = new InputStreamReader(gzis, StandardCharsets.UTF_8); + StringWriter writer = new StringWriter()) { - // 用于存储解压后的字符串 - StringBuilder decompressedData = new StringBuilder(); - - // 逐行读取解压后的数据 - String line; - while ((line = reader.readLine()) != null) { - decompressedData.append(line); + char[] buffer = new char[4096]; + int n; + while ((n = isr.read(buffer)) != -1) { + writer.write(buffer, 0, n); } - - // 此时decompressedData.toString()包含了解压后的字符串 - return decompressedData.toString(); + return writer.toString(); } - } protected String getDomainName(){ diff --git a/parser/src/main/java/cn/qaiu/parser/impl/CeTool.java b/parser/src/main/java/cn/qaiu/parser/impl/CeTool.java index 666b17e..968f6cc 100644 --- a/parser/src/main/java/cn/qaiu/parser/impl/CeTool.java +++ b/parser/src/main/java/cn/qaiu/parser/impl/CeTool.java @@ -2,16 +2,12 @@ package cn.qaiu.parser.impl; import cn.qaiu.entity.ShareLinkInfo; import cn.qaiu.parser.PanBase; -import cn.qaiu.parser.PanDomainTemplate; -import cn.qaiu.parser.ParserCreate; import io.vertx.core.Future; import io.vertx.core.buffer.Buffer; import io.vertx.core.json.JsonObject; import io.vertx.ext.web.client.HttpRequest; import java.net.URL; -import java.util.Arrays; -import java.util.Iterator; /** * Cloudreve自建网盘解析
diff --git a/parser/src/main/java/cn/qaiu/parser/impl/LzTool.java b/parser/src/main/java/cn/qaiu/parser/impl/LzTool.java index 75ae28d..8ef968f 100644 --- a/parser/src/main/java/cn/qaiu/parser/impl/LzTool.java +++ b/parser/src/main/java/cn/qaiu/parser/impl/LzTool.java @@ -29,6 +29,24 @@ public class LzTool extends PanBase { public static final String SHARE_URL_PREFIX = "https://wwww.lanzoum.com"; + MultiMap headers0 = HeaderUtils.parseHeaders(""" + Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7 + Accept-Encoding: gzip, deflate, br + Accept-Language: zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6 + Cache-Control: max-age=0 + Cookie: codelen=1; pc_ad1=1 + DNT: 1 + Priority: u=0, i + Sec-CH-UA: "Chromium";v="140", "Not=A?Brand";v="24", "Microsoft Edge";v="140" + Sec-CH-UA-Mobile: ?0 + Sec-CH-UA-Platform: "macOS" + Sec-Fetch-Dest: document + Sec-Fetch-Mode: navigate + Sec-Fetch-Site: cross-site + Sec-Fetch-User: ?1 + Upgrade-Insecure-Requests: 1 + User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36 Edg/140.0.0.0 + """); public LzTool(ShareLinkInfo shareLinkInfo) { super(shareLinkInfo); @@ -39,7 +57,7 @@ public class LzTool extends PanBase { String pwd = shareLinkInfo.getSharePassword(); WebClient client = clientNoRedirects; - client.getAbs(sUrl).send().onSuccess(res -> { + client.getAbs(sUrl).putHeaders(headers0).send().onSuccess(res -> { String html = res.bodyAsString(); // 匹配iframe Pattern compile = Pattern.compile("src=\"(/fn\\?[a-zA-Z\\d_+/=]{16,})\""); @@ -139,10 +157,13 @@ public class LzTool extends PanBase { client.postAbs(url).putHeaders(headers).sendForm(map).onSuccess(res2 -> { try { JsonObject urlJson = asJson(res2); + String name = urlJson.getString("inf"); if (urlJson.getInteger("zt") != 1) { - fail(urlJson.getString("inf")); + fail(name); return; } + // 文件名 + ((FileInfo)shareLinkInfo.getOtherParam().get("fileInfo")).setFileName(name); String downUrl = urlJson.getString("dom") + "/file/" + urlJson.getString("url"); headers.remove("Referer"); WebClientSession webClientSession = WebClientSession.create(client); diff --git a/parser/src/main/java/cn/qaiu/util/HttpResponseHelper.java b/parser/src/main/java/cn/qaiu/util/HttpResponseHelper.java new file mode 100644 index 0000000..145ea53 --- /dev/null +++ b/parser/src/main/java/cn/qaiu/util/HttpResponseHelper.java @@ -0,0 +1,128 @@ +package cn.qaiu.util; + +import io.vertx.core.buffer.Buffer; +import io.vertx.core.http.HttpHeaders; +import io.vertx.ext.web.client.HttpResponse; +import io.vertx.core.json.JsonObject; +//import org.brotli.dec.BrotliInputStream; +import org.brotli.dec.BrotliInputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.nio.charset.StandardCharsets; +import java.util.zip.GZIPInputStream; +import java.util.zip.Inflater; +import java.util.zip.InflaterInputStream; + +public class HttpResponseHelper { + static Logger LOGGER = LoggerFactory.getLogger(HttpResponseHelper.class); + + // -------------------- 公共方法 -------------------- + public static String asText(HttpResponse res) { + String encoding = res.getHeader(HttpHeaders.CONTENT_ENCODING.toString()); + try { + Buffer body = toBuffer(res); + if (encoding == null || "identity".equalsIgnoreCase(encoding)) { + return body.toString(StandardCharsets.UTF_8); + } + return decompress(body, encoding); + } catch (Exception e) { + LOGGER.error("asText: {}", e.getMessage(), e); + return null; + } + } + + public static JsonObject asJson(HttpResponse res) { + try { + String text = asText(res); + if (text != null) { + return new JsonObject(text); + } else { + LOGGER.error("asJson: asText响应数据为空"); + return JsonObject.of(); + } + } catch (Exception e) { + LOGGER.error("asJson: {}", e.getMessage(), e); + return JsonObject.of(); + } + } + + // -------------------- Buffer 转换 -------------------- + private static Buffer toBuffer(HttpResponse res) { + return res.body() instanceof Buffer ? (Buffer) res.body() : Buffer.buffer(res.bodyAsString()); + } + + // -------------------- 通用解压分发 -------------------- + private static String decompress(Buffer compressed, String encoding) throws IOException { + return switch (encoding.toLowerCase()) { + case "gzip" -> decompressGzip(compressed); + case "deflate" -> decompressDeflate(compressed); + case "br" -> decompressBrotli(compressed); + //case "zstd" -> decompressZstd(compressed); + default -> throw new UnsupportedOperationException("不支持的 Content-Encoding: " + encoding); + }; + } + + // -------------------- gzip -------------------- + private static String decompressGzip(Buffer compressed) throws IOException { + try (ByteArrayInputStream bais = new ByteArrayInputStream(compressed.getBytes()); + GZIPInputStream gzis = new GZIPInputStream(bais); + InputStreamReader isr = new InputStreamReader(gzis, StandardCharsets.UTF_8); + StringWriter writer = new StringWriter()) { + + char[] buffer = new char[4096]; + int n; + while ((n = isr.read(buffer)) != -1) { + writer.write(buffer, 0, n); + } + return writer.toString(); + } + } + + // -------------------- deflate -------------------- + private static String decompressDeflate(Buffer compressed) throws IOException { + byte[] bytes = compressed.getBytes(); + try { + return inflate(bytes, false); // zlib 包裹 + } catch (IOException e) { + return inflate(bytes, true); // 裸 deflate + } + } + + private static String inflate(byte[] data, boolean nowrap) throws IOException { + try (ByteArrayInputStream bais = new ByteArrayInputStream(data); + InflaterInputStream iis = new InflaterInputStream(bais, new Inflater(nowrap)); + InputStreamReader isr = new InputStreamReader(iis, StandardCharsets.UTF_8); + StringWriter writer = new StringWriter()) { + + char[] buffer = new char[4096]; + int n; + while ((n = isr.read(buffer)) != -1) { + writer.write(buffer, 0, n); + } + return writer.toString(); + } + } + + // -------------------- Brotli -------------------- + private static String decompressBrotli(Buffer compressed) throws IOException { + try (ByteArrayInputStream bais = new ByteArrayInputStream(compressed.getBytes()); + BrotliInputStream bis = new BrotliInputStream(bais); + InputStreamReader isr = new InputStreamReader(bis, StandardCharsets.UTF_8); + StringWriter writer = new StringWriter()) { + + char[] buffer = new char[4096]; + int n; + while ((n = isr.read(buffer)) != -1) { + writer.write(buffer, 0, n); + } + return writer.toString(); + } + } + + // -------------------- Zstandard -------------------- + private static String decompressZstd(Buffer compressed) { + throw new UnsupportedOperationException("Zstandard"); + } +} diff --git a/pom.xml b/pom.xml index d19a6f6..de327bd 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ 3.18.0 2.0.0 2.14.2 - 1.5.8 + 1.5.19 4.13.2 @@ -60,7 +60,7 @@ cn.qaiu parser - ${revision} + 10.1.9 diff --git a/web-service/src/main/java/cn/qaiu/lz/common/cache/CacheConfigLoader.java b/web-service/src/main/java/cn/qaiu/lz/common/cache/CacheConfigLoader.java index 258b348..f7b5cb4 100644 --- a/web-service/src/main/java/cn/qaiu/lz/common/cache/CacheConfigLoader.java +++ b/web-service/src/main/java/cn/qaiu/lz/common/cache/CacheConfigLoader.java @@ -1,6 +1,5 @@ package cn.qaiu.lz.common.cache; -import cn.qaiu.parser.PanDomainTemplate; import io.vertx.core.json.JsonObject; import java.util.HashMap; @@ -30,9 +29,6 @@ public class CacheConfigLoader { }); } - public static Integer getDuration(PanDomainTemplate pdt) { - return CONFIGS.get(pdt.name().toLowerCase()); - } public static Integer getDuration(String type) { String key = type.toLowerCase(); return CONFIGS.getOrDefault(key, -1);