mirror of
https://github.com/qaiu/netdisk-fast-download.git
synced 2025-12-16 20:33:03 +00:00
feat: 完善JavaScript解析器功能
- 优化JsScriptLoader,支持JAR包内和文件系统的自动资源文件发现 - 移除预定义文件列表,完全依赖自动检测 - 添加getNoRedirect方法支持重定向处理 - 添加sendMultipartForm方法支持文件上传 - 添加代理配置支持 - 修复JSON解析的压缩处理问题 - 添加默认请求头支持(Accept-Encoding、User-Agent、Accept-Language) - 更新文档,修正导出方式说明 - 优化README.md结构,删除不符合模块定位的内容 - 升级parser版本到10.2.1
This commit is contained in:
@@ -4,7 +4,7 @@
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<groupId>cn.qaiu</groupId>
|
||||
<artifactId>parser</artifactId>
|
||||
<version>10.1.17</version>
|
||||
<version>10.2.1</version>
|
||||
<name>cn.qaiu:parser</name>
|
||||
<description>NFD parser module</description>
|
||||
<url>https://qaiu.top</url>
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
#### 1. 新增类
|
||||
|
||||
##### CustomParserConfig.java
|
||||
- **位置:** `cn.qaiu.parser.CustomParserConfig`
|
||||
- **位置:** `cn.qaiu.parser.custom.CustomParserConfig`
|
||||
- **功能:** 自定义解析器配置类
|
||||
- **主要字段:**
|
||||
- `type`: 解析器类型标识(唯一,必填)
|
||||
@@ -30,7 +30,7 @@
|
||||
- 验证必填字段是否为空
|
||||
|
||||
##### CustomParserRegistry.java
|
||||
- **位置:** `cn.qaiu.parser.CustomParserRegistry`
|
||||
- **位置:** `cn.qaiu.parser.custom.CustomParserRegistry`
|
||||
- **功能:** 自定义解析器注册中心
|
||||
- **主要方法:**
|
||||
- `register(CustomParserConfig)`: 注册解析器
|
||||
@@ -71,7 +71,7 @@
|
||||
#### 3. 测试类
|
||||
|
||||
##### CustomParserTest.java
|
||||
- **位置:** `cn.qaiu.parser.CustomParserTest`
|
||||
- **位置:** `cn.qaiu.parser.custom.CustomParserTest`
|
||||
- **测试覆盖:**
|
||||
- ✅ 注册自定义解析器
|
||||
- ✅ 重复注册检测
|
||||
|
||||
@@ -185,8 +185,8 @@ WebClient 是基于 Vert.x 的异步 HTTP 客户端,其请求流程如下:
|
||||
在应用启动时注册你的解析器:
|
||||
|
||||
```java
|
||||
import cn.qaiu.parser.CustomParserConfig;
|
||||
import cn.qaiu.parser.CustomParserRegistry;
|
||||
import cn.qaiu.parser.custom.CustomParserConfig;
|
||||
import cn.qaiu.parser.custom.CustomParserRegistry;
|
||||
import com.example.parser.MyCustomPanTool;
|
||||
|
||||
public class Application {
|
||||
@@ -356,8 +356,8 @@ public MyCustomPanTool(ShareLinkInfo shareLinkInfo) {
|
||||
|
||||
```java
|
||||
import cn.qaiu.entity.ShareLinkInfo;
|
||||
import cn.qaiu.parser.CustomParserConfig;
|
||||
import cn.qaiu.parser.CustomParserRegistry;
|
||||
import cn.qaiu.parser.custom.CustomParserConfig;
|
||||
import cn.qaiu.parser.custom.CustomParserRegistry;
|
||||
import cn.qaiu.parser.IPanTool;
|
||||
import cn.qaiu.parser.ParserCreate;
|
||||
import cn.qaiu.parser.PanBase;
|
||||
|
||||
@@ -53,8 +53,8 @@ public class MyPanTool implements IPanTool {
|
||||
```java
|
||||
package com.example.myapp.config;
|
||||
|
||||
import cn.qaiu.parser.CustomParserConfig;
|
||||
import cn.qaiu.parser.CustomParserRegistry;
|
||||
import cn.qaiu.parser.custom.CustomParserConfig;
|
||||
import cn.qaiu.parser.custom.CustomParserRegistry;
|
||||
import com.example.myapp.parser.MyPanTool;
|
||||
|
||||
public class ParserRegistry {
|
||||
@@ -130,8 +130,8 @@ public class DownloadService {
|
||||
package com.example;
|
||||
|
||||
import cn.qaiu.entity.ShareLinkInfo;
|
||||
import cn.qaiu.parser.CustomParserConfig;
|
||||
import cn.qaiu.parser.CustomParserRegistry;
|
||||
import cn.qaiu.parser.custom.CustomParserConfig;
|
||||
import cn.qaiu.parser.custom.CustomParserRegistry;
|
||||
import cn.qaiu.parser.IPanTool;
|
||||
import cn.qaiu.parser.ParserCreate;
|
||||
import cn.qaiu.WebClientVertxInit;
|
||||
|
||||
@@ -4,6 +4,26 @@
|
||||
|
||||
本指南介绍如何使用JavaScript编写自定义网盘解析器,支持通过JavaScript代码实现网盘解析逻辑,无需编写Java代码。
|
||||
|
||||
## 目录
|
||||
|
||||
- [快速开始](#快速开始)
|
||||
- [API参考](#api参考)
|
||||
- [ShareLinkInfo对象](#sharelinkinfo对象)
|
||||
- [JsHttpClient对象](#jshttpclient对象)
|
||||
- [JsHttpResponse对象](#jshttpresponse对象)
|
||||
- [JsLogger对象](#jslogger对象)
|
||||
- [重定向处理](#重定向处理)
|
||||
- [代理支持](#代理支持)
|
||||
- [文件上传支持](#文件上传支持)
|
||||
- [实现方法](#实现方法)
|
||||
- [parse方法(必填)](#parse方法必填)
|
||||
- [parseFileList方法(可选)](#parsefilelist方法可选)
|
||||
- [parseById方法(可选)](#parsebyid方法可选)
|
||||
- [错误处理](#错误处理)
|
||||
- [调试技巧](#调试技巧)
|
||||
- [最佳实践](#最佳实践)
|
||||
- [示例解析器](#示例解析器)
|
||||
|
||||
## 快速开始
|
||||
|
||||
### 1. 创建JavaScript脚本
|
||||
@@ -40,9 +60,6 @@ function parse(shareLinkInfo, http, logger) {
|
||||
var response = http.get(url);
|
||||
return response.body();
|
||||
}
|
||||
|
||||
// 导出函数
|
||||
exports.parse = parse;
|
||||
```
|
||||
|
||||
### 2. 重启应用
|
||||
@@ -119,6 +136,16 @@ if (shareLinkInfo.hasOtherParam("customParam")) {
|
||||
// GET请求
|
||||
var response = http.get("https://api.example.com/data");
|
||||
|
||||
// GET请求并跟随重定向
|
||||
var redirectResponse = http.getWithRedirect("https://api.example.com/redirect");
|
||||
|
||||
// GET请求但不跟随重定向(用于获取Location头)
|
||||
var noRedirectResponse = http.getNoRedirect("https://api.example.com/redirect");
|
||||
if (noRedirectResponse.statusCode() >= 300 && noRedirectResponse.statusCode() < 400) {
|
||||
var location = noRedirectResponse.header("Location");
|
||||
console.log("重定向到: " + location);
|
||||
}
|
||||
|
||||
// POST请求
|
||||
var response = http.post("https://api.example.com/submit", {
|
||||
key: "value",
|
||||
@@ -129,12 +156,19 @@ var response = http.post("https://api.example.com/submit", {
|
||||
http.putHeader("User-Agent", "MyBot/1.0")
|
||||
.putHeader("Authorization", "Bearer token");
|
||||
|
||||
// 发送表单数据
|
||||
// 发送简单表单数据
|
||||
var formResponse = http.sendForm({
|
||||
username: "user",
|
||||
password: "pass"
|
||||
});
|
||||
|
||||
// 发送multipart表单数据(支持文件上传)
|
||||
var multipartResponse = http.sendMultipartForm("https://api.example.com/upload", {
|
||||
textField: "value",
|
||||
fileField: fileBuffer, // Buffer或byte[]类型
|
||||
binaryData: binaryArray // byte[]类型
|
||||
});
|
||||
|
||||
// 发送JSON数据
|
||||
var jsonResponse = http.sendJson({
|
||||
name: "test",
|
||||
@@ -190,6 +224,168 @@ if (logger.isDebugEnabled()) {
|
||||
}
|
||||
```
|
||||
|
||||
## 重定向处理
|
||||
|
||||
当网盘服务返回302重定向时,可以使用`getNoRedirect`方法获取真实的下载链接:
|
||||
|
||||
```javascript
|
||||
/**
|
||||
* 获取真实的下载链接(处理302重定向)
|
||||
* @param {string} downloadUrl - 原始下载链接
|
||||
* @param {JsHttpClient} http - HTTP客户端
|
||||
* @param {JsLogger} logger - 日志记录器
|
||||
* @returns {string} 真实的下载链接
|
||||
*/
|
||||
function getRealDownloadUrl(downloadUrl, http, logger) {
|
||||
try {
|
||||
logger.info("获取真实下载链接: " + downloadUrl);
|
||||
|
||||
// 使用不跟随重定向的方法获取Location头
|
||||
var headResponse = http.getNoRedirect(downloadUrl);
|
||||
|
||||
if (headResponse.statusCode() >= 300 && headResponse.statusCode() < 400) {
|
||||
// 处理重定向
|
||||
var location = headResponse.header("Location");
|
||||
if (location) {
|
||||
logger.info("获取到重定向链接: " + location);
|
||||
return location;
|
||||
}
|
||||
}
|
||||
|
||||
// 如果没有重定向或无法获取Location,返回原链接
|
||||
logger.debug("下载链接无需重定向或无法获取重定向信息");
|
||||
return downloadUrl;
|
||||
|
||||
} catch (e) {
|
||||
logger.error("获取真实下载链接失败: " + e.message);
|
||||
// 如果获取失败,返回原链接
|
||||
return downloadUrl;
|
||||
}
|
||||
}
|
||||
|
||||
// 在parse方法中使用
|
||||
function parse(shareLinkInfo, http, logger) {
|
||||
// ... 获取原始下载链接的代码 ...
|
||||
var originalUrl = "https://example.com/download?id=123";
|
||||
|
||||
// 获取真实的下载链接
|
||||
var realUrl = getRealDownloadUrl(originalUrl, http, logger);
|
||||
return realUrl;
|
||||
}
|
||||
```
|
||||
|
||||
## 代理支持
|
||||
|
||||
JavaScript解析器支持HTTP代理配置,代理信息通过`ShareLinkInfo`的`otherParam`传递:
|
||||
|
||||
```javascript
|
||||
function parse(shareLinkInfo, http, logger) {
|
||||
// 检查是否有代理配置
|
||||
var proxyConfig = shareLinkInfo.getOtherParam("proxy");
|
||||
if (proxyConfig) {
|
||||
logger.info("使用代理: " + proxyConfig.host + ":" + proxyConfig.port);
|
||||
}
|
||||
|
||||
// HTTP客户端会自动使用代理配置
|
||||
var response = http.get("https://api.example.com/data");
|
||||
return response.body();
|
||||
}
|
||||
```
|
||||
|
||||
代理配置格式:
|
||||
```json
|
||||
{
|
||||
"type": "HTTP", // 代理类型: HTTP, SOCKS4, SOCKS5
|
||||
"host": "proxy.example.com",
|
||||
"port": 8080,
|
||||
"username": "user", // 可选,代理认证用户名
|
||||
"password": "pass" // 可选,代理认证密码
|
||||
}
|
||||
```
|
||||
|
||||
## 文件上传支持
|
||||
|
||||
JavaScript解析器支持通过`sendMultipartForm`方法上传文件:
|
||||
|
||||
### 1. 简单文件上传
|
||||
|
||||
```javascript
|
||||
function uploadFile(shareLinkInfo, http, logger) {
|
||||
// 模拟文件数据(实际使用中可能是从其他地方获取)
|
||||
var fileData = new java.lang.String("Hello, World!").getBytes();
|
||||
|
||||
// 使用sendMultipartForm上传文件
|
||||
var response = http.sendMultipartForm("https://api.example.com/upload", {
|
||||
file: fileData,
|
||||
filename: "test.txt",
|
||||
description: "测试文件"
|
||||
});
|
||||
|
||||
return response.body();
|
||||
}
|
||||
```
|
||||
|
||||
### 2. 混合表单上传
|
||||
|
||||
```javascript
|
||||
function uploadMixedForm(shareLinkInfo, http, logger) {
|
||||
var fileData = getFileData();
|
||||
|
||||
// 同时上传文本字段和文件
|
||||
var response = http.sendMultipartForm("https://api.example.com/upload", {
|
||||
username: "user123",
|
||||
email: "user@example.com",
|
||||
file: fileData,
|
||||
description: "用户上传的文件"
|
||||
});
|
||||
|
||||
if (response.isSuccess()) {
|
||||
var result = response.json();
|
||||
return result.downloadUrl;
|
||||
} else {
|
||||
throw new Error("文件上传失败: " + response.statusCode());
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 3. 多文件上传
|
||||
|
||||
```javascript
|
||||
function uploadMultipleFiles(shareLinkInfo, http, logger) {
|
||||
var files = [
|
||||
{ name: "file1.txt", data: getFileData1() },
|
||||
{ name: "file2.jpg", data: getFileData2() }
|
||||
];
|
||||
|
||||
var uploadResults = [];
|
||||
|
||||
for (var i = 0; i < files.length; i++) {
|
||||
var file = files[i];
|
||||
var response = http.sendMultipartForm("https://api.example.com/upload", {
|
||||
file: file.data,
|
||||
filename: file.name,
|
||||
uploadIndex: i.toString()
|
||||
});
|
||||
|
||||
if (response.isSuccess()) {
|
||||
uploadResults.push({
|
||||
fileName: file.name,
|
||||
success: true,
|
||||
url: response.json().url
|
||||
});
|
||||
} else {
|
||||
uploadResults.push({
|
||||
fileName: file.name,
|
||||
success: false,
|
||||
error: response.statusCode()
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return uploadResults;
|
||||
}
|
||||
```
|
||||
|
||||
## 实现方法
|
||||
|
||||
JavaScript解析器支持三种方法,对应Java接口的三种同步方法:
|
||||
@@ -301,43 +497,49 @@ String fileUrl = tool.parseByIdSync(); // 调用 parseById() 函数
|
||||
- 如果JavaScript方法抛出异常,Java同步方法会抛出相应的异常
|
||||
- 建议在JavaScript方法中添加适当的错误处理和日志记录
|
||||
|
||||
## 导出方式
|
||||
## 函数定义方式
|
||||
|
||||
支持三种导出方式:
|
||||
|
||||
### 方式1:分别导出(推荐)
|
||||
JavaScript解析器使用全局函数定义,不需要`exports`对象:
|
||||
|
||||
```javascript
|
||||
function parse(shareLinkInfo, http, logger) { }
|
||||
function parseFileList(shareLinkInfo, http, logger) { }
|
||||
function parseById(shareLinkInfo, http, logger) { }
|
||||
/**
|
||||
* 解析单个文件下载链接(必填)
|
||||
* @param {ShareLinkInfo} shareLinkInfo - 分享链接信息
|
||||
* @param {JsHttpClient} http - HTTP客户端
|
||||
* @param {JsLogger} logger - 日志对象
|
||||
* @returns {string} 下载链接
|
||||
*/
|
||||
function parse(shareLinkInfo, http, logger) {
|
||||
// 实现解析逻辑
|
||||
return "https://example.com/download";
|
||||
}
|
||||
|
||||
exports.parse = parse;
|
||||
exports.parseFileList = parseFileList;
|
||||
exports.parseById = parseById;
|
||||
/**
|
||||
* 解析文件列表(可选)
|
||||
* @param {ShareLinkInfo} shareLinkInfo - 分享链接信息
|
||||
* @param {JsHttpClient} http - HTTP客户端
|
||||
* @param {JsLogger} logger - 日志对象
|
||||
* @returns {Array} 文件信息数组
|
||||
*/
|
||||
function parseFileList(shareLinkInfo, http, logger) {
|
||||
// 实现文件列表解析逻辑
|
||||
return [];
|
||||
}
|
||||
|
||||
/**
|
||||
* 根据文件ID获取下载链接(可选)
|
||||
* @param {ShareLinkInfo} shareLinkInfo - 分享链接信息
|
||||
* @param {JsHttpClient} http - HTTP客户端
|
||||
* @param {JsLogger} logger - 日志对象
|
||||
* @returns {string} 下载链接
|
||||
*/
|
||||
function parseById(shareLinkInfo, http, logger) {
|
||||
// 实现按ID解析逻辑
|
||||
return "https://example.com/download";
|
||||
}
|
||||
```
|
||||
|
||||
### 方式2:直接挂载
|
||||
|
||||
```javascript
|
||||
exports.parse = function(shareLinkInfo, http, logger) { };
|
||||
exports.parseFileList = function(shareLinkInfo, http, logger) { };
|
||||
exports.parseById = function(shareLinkInfo, http, logger) { };
|
||||
```
|
||||
|
||||
### 方式3:module.exports
|
||||
|
||||
```javascript
|
||||
function parse(shareLinkInfo, http, logger) { }
|
||||
function parseFileList(shareLinkInfo, http, logger) { }
|
||||
function parseById(shareLinkInfo, http, logger) { }
|
||||
|
||||
module.exports = {
|
||||
parse: parse,
|
||||
parseFileList: parseFileList,
|
||||
parseById: parseById
|
||||
};
|
||||
```
|
||||
**注意**:JavaScript解析器通过`engine.eval()`执行,函数必须定义为全局函数,不需要使用`exports`或`module.exports`。
|
||||
|
||||
## VSCode配置
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
- 语言/构建:Java 17 / Maven
|
||||
- 关键接口:cn.qaiu.parser.IPanTool(返回 Future<List<FileInfo>>),各站点位于 parser/src/main/java/cn/qaiu/parser/impl
|
||||
- 数据模型:cn.qaiu.entity.FileInfo(统一对外文件项)
|
||||
- JavaScript解析器:支持使用JavaScript编写自定义解析器,位于 parser/src/main/resources/custom-parsers/
|
||||
|
||||
---
|
||||
|
||||
@@ -75,6 +76,51 @@ List<FileInfo> files = tool.parseFileListSync();
|
||||
- 异步方法仍可用:parse()、parseFileList()、parseById() 返回 Future 对象
|
||||
- 生成短链 path:ParserCreate.genPathSuffix()(用于页面/服务端聚合)。
|
||||
|
||||
## JavaScript解析器快速开始
|
||||
|
||||
除了Java解析器,还支持使用JavaScript编写自定义解析器:
|
||||
|
||||
### 1. 创建JavaScript解析器
|
||||
|
||||
在 `parser/src/main/resources/custom-parsers/` 目录下创建 `.js` 文件:
|
||||
|
||||
```javascript
|
||||
// ==UserScript==
|
||||
// @name 我的解析器
|
||||
// @type my_parser
|
||||
// @displayName 我的网盘
|
||||
// @description 使用JavaScript实现的网盘解析器
|
||||
// @match https?://example\.com/s/(?<KEY>\w+)
|
||||
// @author yourname
|
||||
// @version 1.0.0
|
||||
// ==/UserScript==
|
||||
|
||||
/**
|
||||
* 解析单个文件下载链接
|
||||
* @param {ShareLinkInfo} shareLinkInfo - 分享链接信息
|
||||
* @param {JsHttpClient} http - HTTP客户端
|
||||
* @param {JsLogger} logger - 日志对象
|
||||
* @returns {string} 下载链接
|
||||
*/
|
||||
function parse(shareLinkInfo, http, logger) {
|
||||
var url = shareLinkInfo.getShareUrl();
|
||||
var response = http.get(url);
|
||||
return response.body();
|
||||
}
|
||||
```
|
||||
|
||||
### 2. JavaScript解析器特性
|
||||
|
||||
- **重定向处理**:支持`getNoRedirect()`方法获取302重定向的真实链接
|
||||
- **代理支持**:自动支持HTTP/SOCKS代理配置
|
||||
- **类型提示**:提供完整的JSDoc类型定义
|
||||
- **热加载**:修改后重启应用即可生效
|
||||
|
||||
### 3. 详细文档
|
||||
|
||||
- [JavaScript解析器开发指南](JAVASCRIPT_PARSER_GUIDE.md)
|
||||
- [自定义解析器开发指南](CUSTOM_PARSER_GUIDE.md)
|
||||
|
||||
---
|
||||
|
||||
## 1. 解析器约定
|
||||
@@ -97,196 +143,125 @@ FileInfo 关键字段(节选):
|
||||
|
||||
---
|
||||
|
||||
## 2. 文件列表解析规范(按给定 JSON)
|
||||
目标 JSON(摘要):
|
||||
- 列表路径:data.data[]
|
||||
- 每项结构:item.data(含 attributes、id、type、relationships)
|
||||
- type:"file" 或 "folder"
|
||||
## 2. 文件列表解析规范
|
||||
|
||||
字段映射建议:
|
||||
- 通用
|
||||
- fileId ← data.id
|
||||
- createTime ← data.attributes.created_at(若格式不一致,上层再统一格式化)
|
||||
- updateTime ← data.attributes.updated_at
|
||||
- fileType:
|
||||
- 对文件用 data.attributes.mimetype 或固定 "file"
|
||||
- 对目录固定 "folder"
|
||||
- 文件(type="file")
|
||||
- fileName ← 优先 attributes.basename(示例:"GBT+28448-2019.pdf"),无则用 attributes.name
|
||||
- sizeStr ← attributes.filesize(示例:"18MB")
|
||||
- size ← 尝试用 FileSizeConverter.convertToBytes(sizeStr),失败则置空
|
||||
- parserUrl ← attributes.file_url(示例:BilPan://downLoad?id=...)
|
||||
- filePath/parentId ← relationships.parent.data.id(可放到 extParameters.parentId)
|
||||
- previewUrl/thumbnail ← attributes.thumbnail(可选)
|
||||
- 目录(type="folder")
|
||||
- fileName ← attributes.name
|
||||
- size/sizeStr ← 置空
|
||||
- 统计字段(如 items/trashed_items)可入 extParameters
|
||||
### 通用解析原则
|
||||
|
||||
边界与兼容:
|
||||
- attributes.filesize 可能为空或为非标准字符串;转换失败时保留 sizeStr,忽略 size。
|
||||
- attributes.file_url 可能为占位协议(BilPan://),直链转换在下载阶段处理。
|
||||
- relationships.* 可能为空,读取前需判空。
|
||||
1. **数据结构识别**:根据网盘API响应结构确定文件列表的路径
|
||||
2. **字段映射**:将网盘特定字段映射到统一的`FileInfo`对象
|
||||
3. **类型区分**:正确识别文件和文件夹类型
|
||||
4. **数据转换**:处理时间格式、文件大小等数据格式转换
|
||||
|
||||
### FileInfo字段映射指南
|
||||
|
||||
| FileInfo字段 | 说明 | 映射建议 |
|
||||
|-------------|------|----------|
|
||||
| `fileName` | 文件名 | 优先使用文件名字段,无则使用标题字段 |
|
||||
| `fileId` | 文件ID | 使用网盘提供的唯一标识符 |
|
||||
| `fileType` | 文件类型 | "file"或"folder" |
|
||||
| `size` | 文件大小(字节) | 转换为字节数,文件夹可为0 |
|
||||
| `sizeStr` | 文件大小(可读) | 保持网盘原始格式或转换 |
|
||||
| `createTime` | 创建时间 | 统一时间格式 |
|
||||
| `updateTime` | 更新时间 | 统一时间格式 |
|
||||
| `parserUrl` | 下载链接 | 网盘提供的下载URL |
|
||||
| `previewUrl` | 预览链接 | 可选,网盘提供的预览URL |
|
||||
|
||||
### 常见数据转换
|
||||
|
||||
- **文件大小**:使用`FileSizeConverter`进行字符串与字节数转换
|
||||
- **时间格式**:统一转换为标准时间格式
|
||||
- **文件类型**:根据网盘API判断文件/文件夹类型
|
||||
|
||||
### 解析注意事项
|
||||
|
||||
- **数据验证**:检查必要字段是否存在,避免空指针异常
|
||||
- **格式兼容**:处理不同网盘的数据格式差异
|
||||
- **错误处理**:转换失败时提供合理的默认值
|
||||
- **扩展字段**:额外信息可存储在`extParameters`中
|
||||
|
||||
### 解析示例
|
||||
|
||||
伪代码(parseFileList 核心片段):
|
||||
```java
|
||||
// 仅示意,按项目 Json 工具替换
|
||||
JsonObject root = ...; // 接口返回
|
||||
JsonArray arr = root.getJsonObject("data").getJsonArray("data");
|
||||
List<FileInfo> list = new ArrayList<>();
|
||||
for (JsonObject wrap : arr) {
|
||||
JsonObject d = wrap.getJsonObject("data");
|
||||
String type = d.getString("type");
|
||||
JsonObject attrs = d.getJsonObject("attributes");
|
||||
FileInfo fi = new FileInfo();
|
||||
fi.setFileId(d.getString("id"));
|
||||
fi.setCreateTime(attrs.getString("created_at"));
|
||||
fi.setUpdateTime(attrs.getString("updated_at"));
|
||||
if ("file".equals(type)) {
|
||||
String basename = attrs.getString("basename");
|
||||
fi.setFileName(basename != null ? basename : attrs.getString("name"));
|
||||
fi.setFileType(attrs.getString("mimetype", "file"));
|
||||
String sizeStr = attrs.getString("filesize");
|
||||
fi.setSizeStr(sizeStr);
|
||||
try { if (sizeStr != null) fi.setSize(FileSizeConverter.convertToBytes(sizeStr)); } catch (Exception ignore) {}
|
||||
fi.setParserUrl(attrs.getString("file_url"));
|
||||
// parentId(可选)
|
||||
JsonObject rel = d.getJsonObject("relationships");
|
||||
if (rel != null) {
|
||||
JsonObject p = rel.getJsonObject("parent");
|
||||
if (p != null && p.getJsonObject("data") != null) {
|
||||
String pid = p.getJsonObject("data").getString("id");
|
||||
Map<String,Object> ext = new HashMap<>();
|
||||
ext.put("parentId", pid);
|
||||
fi.setExtParameters(ext);
|
||||
}
|
||||
// 通用解析模式示例
|
||||
JsonObject root = response.json(); // 获取API响应
|
||||
JsonArray fileList = root.getJsonArray("files"); // 根据实际API调整路径
|
||||
List<FileInfo> result = new ArrayList<>();
|
||||
|
||||
for (JsonObject item : fileList) {
|
||||
FileInfo fileInfo = new FileInfo();
|
||||
|
||||
// 基本字段映射
|
||||
fileInfo.setFileName(item.getString("name"));
|
||||
fileInfo.setFileId(item.getString("id"));
|
||||
fileInfo.setFileType(item.getString("type").equals("file") ? "file" : "folder");
|
||||
|
||||
// 文件大小处理
|
||||
String sizeStr = item.getString("size");
|
||||
if (sizeStr != null) {
|
||||
fileInfo.setSizeStr(sizeStr);
|
||||
try {
|
||||
fileInfo.setSize(FileSizeConverter.convertToBytes(sizeStr));
|
||||
} catch (Exception e) {
|
||||
// 转换失败时保持sizeStr,size为0
|
||||
}
|
||||
}
|
||||
} else {
|
||||
fi.setFileName(attrs.getString("name"));
|
||||
fi.setFileType("folder");
|
||||
}
|
||||
list.add(fi);
|
||||
|
||||
// 时间处理
|
||||
fileInfo.setCreateTime(formatTime(item.getString("createTime")));
|
||||
fileInfo.setUpdateTime(formatTime(item.getString("updateTime")));
|
||||
|
||||
// 下载链接
|
||||
fileInfo.setParserUrl(item.getString("downloadUrl"));
|
||||
|
||||
result.add(fileInfo);
|
||||
}
|
||||
return Future.succeededFuture(list);
|
||||
```
|
||||
|
||||
---
|
||||
### JavaScript解析器示例
|
||||
|
||||
## 3. curl 转 Java 11 HttpClient 示例
|
||||
以 GET 为例(来源:developer-oss.lanrar.com):
|
||||
```java
|
||||
HttpClient client = HttpClient.newHttpClient();
|
||||
String q = "<替换为长查询串>";
|
||||
String url = "https://developer-oss.lanrar.com/file/?" + URLEncoder.encode(q, StandardCharsets.UTF_8);
|
||||
HttpRequest req = HttpRequest.newBuilder(URI.create(url))
|
||||
.header("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7")
|
||||
.header("accept-language", "zh-CN,zh;q=0.9")
|
||||
.header("cache-control", "max-age=0")
|
||||
.header("dnt", "1")
|
||||
.header("priority", "u=0, i")
|
||||
.header("referer", "https://developer-oss.lanrar.com/file/?" + q)
|
||||
.header("sec-ch-ua", "\"Chromium\";v=\"140\", \"Not=A?Brand\";v=\"24\", \"Microsoft Edge\";v=\"140\"")
|
||||
.header("sec-ch-ua-mobile", "?0")
|
||||
.header("sec-ch-ua-platform", "\"macOS\"")
|
||||
.header("sec-fetch-dest", "document")
|
||||
.header("sec-fetch-mode", "navigate")
|
||||
.header("sec-fetch-site", "same-origin")
|
||||
.header("upgrade-insecure-requests", "1")
|
||||
.header("user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36 Edg/140.0.0.0")
|
||||
.header("Cookie", "acw_tc=<acw_tc>; cdn_sec_tc=<cdn_sec_tc>; acw_sc__v2=<acw_sc__v2>")
|
||||
.GET()
|
||||
.build();
|
||||
HttpResponse<String> resp = client.send(req, HttpResponse.BodyHandlers.ofString());
|
||||
System.out.println(resp.statusCode());
|
||||
System.out.println(resp.body());
|
||||
```
|
||||
|
||||
POST 示例(来源:Weiyun Share BatchDownload,使用 JSON):
|
||||
```java
|
||||
HttpClient client = HttpClient.newHttpClient();
|
||||
String url = "https://share.weiyun.com/webapp/json/weiyunShare/WeiyunShareBatchDownload?refer=chrome_mac&g_tk=1399845656&r=0.3925692266635241";
|
||||
String json = "{...与 curl/requests 等价 JSON 负载,使用占位参数...}";
|
||||
HttpRequest req = HttpRequest.newBuilder(URI.create(url))
|
||||
.header("accept", "application/json, text/plain, */*")
|
||||
.header("content-type", "application/json;charset=UTF-8")
|
||||
.header("origin", "https://share.weiyun.com")
|
||||
.header("referer", "https://share.weiyun.com/<shareKey>")
|
||||
.header("user-agent", "Mozilla/5.0 ...")
|
||||
.header("Cookie", "uin=<uin>; skey=<skey>; p_skey=<p_skey>; ...")
|
||||
.POST(HttpRequest.BodyPublishers.ofString(json, StandardCharsets.UTF_8))
|
||||
.build();
|
||||
HttpResponse<String> resp = client.send(req, HttpResponse.BodyHandlers.ofString());
|
||||
```
|
||||
提示:
|
||||
- Cookie/Token 使用占位并从外部注入,避免硬编码与泄露。
|
||||
- r/g_tk 等参数如需计算,请在实现类中封装。
|
||||
|
||||
---
|
||||
|
||||
## 4. IntelliJ IDEA `.http` 调试样例
|
||||
保存为 `requests.http`,可配合环境变量使用。
|
||||
|
||||
GET:
|
||||
```http
|
||||
### 开发者资源 GET 示例
|
||||
GET https://developer-oss.lanrar.com/file/?{{q}}
|
||||
accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7
|
||||
accept-language: zh-CN,zh;q=0.9
|
||||
cache-control: max-age=0
|
||||
dnt: 1
|
||||
priority: u=0, i
|
||||
referer: https://developer-oss.lanrar.com/file/?{{q}}
|
||||
sec-ch-ua: "Chromium";v="140", "Not=A?Brand";v="24", "Microsoft Edge";v="140"
|
||||
sec-ch-ua-mobile: ?0
|
||||
sec-ch-ua-platform: "macOS"
|
||||
sec-fetch-dest: document
|
||||
sec-fetch-mode: navigate
|
||||
sec-fetch-site: same-origin
|
||||
upgrade-insecure-requests: 1
|
||||
user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36 Edg/140.0.0.0
|
||||
Cookie: acw_tc={{acw_tc}}; cdn_sec_tc={{cdn_sec_tc}}; acw_sc__v2={{acw_sc_v2}}
|
||||
|
||||
> {% client.log("status: " + response.status); %}
|
||||
|
||||
### 环境变量(可在 HTTP Client Environment 中配置)
|
||||
@q=替换为实际长查询串
|
||||
@acw_tc=your_acw_tc
|
||||
@cdn_sec_tc=your_cdn_sec_tc
|
||||
@acw_sc_v2=your_acw_sc__v2
|
||||
```
|
||||
|
||||
POST:
|
||||
```http
|
||||
### Weiyun 批量下载 POST 示例
|
||||
POST https://share.weiyun.com/webapp/json/weiyunShare/WeiyunShareBatchDownload?refer=chrome_mac&g_tk={{g_tk}}&r={{r}}
|
||||
accept: application/json, text/plain, */*
|
||||
content-type: application/json;charset=UTF-8
|
||||
origin: https://share.weiyun.com
|
||||
referer: https://share.weiyun.com/{{share_key}}
|
||||
user-agent: Mozilla/5.0 ...
|
||||
Cookie: uin={{uin}}; skey={{skey}}; p_skey={{p_skey}}; p_uin={{p_uin}}; wyctoken={{wyctoken}}
|
||||
|
||||
{
|
||||
"req_header": "{...}",
|
||||
"req_body": "{...}"
|
||||
```javascript
|
||||
function parseFileList(shareLinkInfo, http, logger) {
|
||||
var response = http.get(shareLinkInfo.getShareUrl());
|
||||
var data = response.json();
|
||||
|
||||
var fileList = [];
|
||||
var files = data.files || data.data || data.items; // 根据实际API调整
|
||||
|
||||
for (var i = 0; i < files.length; i++) {
|
||||
var file = files[i];
|
||||
var fileInfo = {
|
||||
fileName: file.name || file.title,
|
||||
fileId: file.id,
|
||||
fileType: file.type === "file" ? "file" : "folder",
|
||||
size: file.size || 0,
|
||||
sizeStr: file.sizeStr || formatSize(file.size),
|
||||
createTime: file.createTime,
|
||||
updateTime: file.updateTime,
|
||||
parserUrl: file.downloadUrl || file.url
|
||||
};
|
||||
|
||||
fileList.push(fileInfo);
|
||||
}
|
||||
|
||||
return fileList;
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 5. 开发流程建议
|
||||
## 3. 开发流程建议
|
||||
- 新增站点:在 impl 下新增 Tool,实现 IPanTool,复用 PanBase/模板类;补充单测。
|
||||
- 字段不全:尽量回填 sizeStr/createTime 等便于前端展示;不可用字段置空。
|
||||
- 单测:放置于 parser/src/test/java,尽量添加 1-2 个 happy path + 1 个边界用例。
|
||||
|
||||
## 6. 常见问题
|
||||
## 4. 常见问题
|
||||
- 容量解析失败:保留 sizeStr,并忽略 size;避免抛出异常影响整体列表。
|
||||
- 协议占位下载链接:统一放至 parserUrl,直链转换由下载阶段处理。
|
||||
- 鉴权:Cookie/Token 过期问题由上层刷新或外部注入处理;解析器保持无状态最佳。
|
||||
|
||||
---
|
||||
|
||||
## 7. 参考
|
||||
## 5. 参考
|
||||
- FileInfo:parser/src/main/java/cn/qaiu/entity/FileInfo.java
|
||||
- IPanTool:parser/src/main/java/cn/qaiu/parser/IPanTool.java
|
||||
- FileSizeConverter:parser/src/main/java/cn/qaiu/util/FileSizeConverter.java
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
|
||||
<groupId>cn.qaiu</groupId>
|
||||
<artifactId>parser</artifactId>
|
||||
<version>10.1.17</version>
|
||||
<version>10.2.1</version>
|
||||
<packaging>jar</packaging>
|
||||
|
||||
<name>cn.qaiu:parser</name>
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
package cn.qaiu;
|
||||
|
||||
import cn.qaiu.parser.CustomParserRegistry;
|
||||
import io.vertx.core.Vertx;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import cn.qaiu.parser.custom.CustomParserRegistry;
|
||||
|
||||
public class WebClientVertxInit {
|
||||
private Vertx vertx = null;
|
||||
private static final WebClientVertxInit INSTANCE = new WebClientVertxInit();
|
||||
@@ -36,4 +37,4 @@ public class WebClientVertxInit {
|
||||
}
|
||||
return INSTANCE.vertx;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,10 @@
|
||||
package cn.qaiu.parser;
|
||||
|
||||
import cn.qaiu.entity.ShareLinkInfo;
|
||||
import cn.qaiu.parser.custom.CustomParserConfig;
|
||||
import cn.qaiu.parser.custom.CustomParserRegistry;
|
||||
import cn.qaiu.parser.customjs.JsParserExecutor;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
import java.net.URLEncoder;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package cn.qaiu.parser;
|
||||
package cn.qaiu.parser.custom;
|
||||
|
||||
import cn.qaiu.entity.ShareLinkInfo;
|
||||
import cn.qaiu.parser.IPanTool;
|
||||
|
||||
import java.util.Map;
|
||||
import java.util.regex.Pattern;
|
||||
@@ -1,8 +1,12 @@
|
||||
package cn.qaiu.parser;
|
||||
package cn.qaiu.parser.custom;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import cn.qaiu.parser.PanDomainTemplate;
|
||||
import cn.qaiu.parser.customjs.JsScriptLoader;
|
||||
import cn.qaiu.parser.customjs.JsScriptMetadataParser;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
@@ -1,4 +1,4 @@
|
||||
package cn.qaiu.parser;
|
||||
package cn.qaiu.parser.customjs;
|
||||
|
||||
import cn.qaiu.WebClientVertxInit;
|
||||
import cn.qaiu.util.HttpResponseHelper;
|
||||
@@ -6,11 +6,17 @@ import io.vertx.core.Future;
|
||||
import io.vertx.core.MultiMap;
|
||||
import io.vertx.core.Promise;
|
||||
import io.vertx.core.buffer.Buffer;
|
||||
import io.vertx.core.json.JsonArray;
|
||||
import io.vertx.core.json.JsonObject;
|
||||
import io.vertx.core.net.ProxyOptions;
|
||||
import io.vertx.core.net.ProxyType;
|
||||
import io.vertx.ext.web.client.HttpRequest;
|
||||
import io.vertx.ext.web.client.HttpResponse;
|
||||
import io.vertx.ext.web.client.WebClient;
|
||||
import io.vertx.ext.web.client.WebClientOptions;
|
||||
import io.vertx.ext.web.client.WebClientSession;
|
||||
import io.vertx.ext.web.multipart.MultipartForm;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
@@ -36,6 +42,48 @@ public class JsHttpClient {
|
||||
this.client = WebClient.create(WebClientVertxInit.get());
|
||||
this.clientSession = WebClientSession.create(client);
|
||||
this.headers = MultiMap.caseInsensitiveMultiMap();
|
||||
// 设置默认的Accept-Encoding头以支持压缩响应
|
||||
this.headers.set("Accept-Encoding", "gzip, deflate, br, zstd");
|
||||
// 设置默认的User-Agent头
|
||||
this.headers.set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36 Edg/140.0.0.0");
|
||||
// 设置默认的Accept-Language头
|
||||
this.headers.set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6");
|
||||
}
|
||||
|
||||
/**
|
||||
* 带代理配置的构造函数
|
||||
* @param proxyConfig 代理配置JsonObject,包含type、host、port、username、password
|
||||
*/
|
||||
public JsHttpClient(JsonObject proxyConfig) {
|
||||
if (proxyConfig != null && proxyConfig.containsKey("type")) {
|
||||
ProxyOptions proxyOptions = new ProxyOptions()
|
||||
.setType(ProxyType.valueOf(proxyConfig.getString("type").toUpperCase()))
|
||||
.setHost(proxyConfig.getString("host"))
|
||||
.setPort(proxyConfig.getInteger("port"));
|
||||
|
||||
if (StringUtils.isNotEmpty(proxyConfig.getString("username"))) {
|
||||
proxyOptions.setUsername(proxyConfig.getString("username"));
|
||||
}
|
||||
if (StringUtils.isNotEmpty(proxyConfig.getString("password"))) {
|
||||
proxyOptions.setPassword(proxyConfig.getString("password"));
|
||||
}
|
||||
|
||||
this.client = WebClient.create(WebClientVertxInit.get(),
|
||||
new WebClientOptions()
|
||||
.setUserAgentEnabled(false)
|
||||
.setProxyOptions(proxyOptions));
|
||||
this.clientSession = WebClientSession.create(client);
|
||||
} else {
|
||||
this.client = WebClient.create(WebClientVertxInit.get());
|
||||
this.clientSession = WebClientSession.create(client);
|
||||
}
|
||||
this.headers = MultiMap.caseInsensitiveMultiMap();
|
||||
// 设置默认的Accept-Encoding头以支持压缩响应
|
||||
this.headers.set("Accept-Encoding", "gzip, deflate, br, zstd");
|
||||
// 设置默认的User-Agent头
|
||||
this.headers.set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36 Edg/140.0.0.0");
|
||||
// 设置默认的Accept-Language头
|
||||
this.headers.set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6");
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -132,7 +180,7 @@ public class JsHttpClient {
|
||||
}
|
||||
|
||||
/**
|
||||
* 发送表单数据
|
||||
* 发送表单数据(简单键值对)
|
||||
* @param data 表单数据
|
||||
* @return HTTP响应
|
||||
*/
|
||||
@@ -152,6 +200,45 @@ public class JsHttpClient {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 发送multipart表单数据(支持文件上传)
|
||||
* @param url 请求URL
|
||||
* @param data 表单数据,支持:
|
||||
* - Map<String, String>: 文本字段
|
||||
* - Map<String, Object>: 混合字段,Object可以是String、byte[]或Buffer
|
||||
* @return HTTP响应
|
||||
*/
|
||||
public JsHttpResponse sendMultipartForm(String url, Map<String, Object> data) {
|
||||
return executeRequest(() -> {
|
||||
HttpRequest<Buffer> request = client.postAbs(url);
|
||||
if (!headers.isEmpty()) {
|
||||
request.putHeaders(headers);
|
||||
}
|
||||
|
||||
MultipartForm form = MultipartForm.create();
|
||||
|
||||
if (data != null) {
|
||||
for (Map.Entry<String, Object> entry : data.entrySet()) {
|
||||
String key = entry.getKey();
|
||||
Object value = entry.getValue();
|
||||
|
||||
if (value instanceof String) {
|
||||
form.attribute(key, (String) value);
|
||||
} else if (value instanceof byte[]) {
|
||||
form.binaryFileUpload(key, key, Buffer.buffer((byte[]) value), "application/octet-stream");
|
||||
} else if (value instanceof Buffer) {
|
||||
form.binaryFileUpload(key, key, (Buffer) value, "application/octet-stream");
|
||||
} else if (value != null) {
|
||||
// 其他类型转换为字符串
|
||||
form.attribute(key, value.toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return request.sendMultipartForm(form);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 发送JSON数据
|
||||
* @param data JSON数据
|
||||
@@ -224,26 +311,13 @@ public class JsHttpClient {
|
||||
*/
|
||||
public Object json() {
|
||||
try {
|
||||
String body = response.bodyAsString();
|
||||
if (body == null || body.trim().isEmpty()) {
|
||||
JsonObject jsonObject = HttpResponseHelper.asJson(response);
|
||||
if (jsonObject == null || jsonObject.isEmpty()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// 尝试解析为JSON对象
|
||||
try {
|
||||
JsonObject jsonObject = response.bodyAsJsonObject();
|
||||
// 将JsonObject转换为Map,这样JavaScript可以正确访问
|
||||
return jsonObject.getMap();
|
||||
} catch (Exception e) {
|
||||
// 如果解析为对象失败,尝试解析为数组
|
||||
try {
|
||||
return response.bodyAsJsonArray().getList();
|
||||
} catch (Exception e2) {
|
||||
// 如果都失败了,返回原始字符串
|
||||
log.warn("无法解析为JSON,返回原始字符串: {}", body);
|
||||
return body;
|
||||
}
|
||||
}
|
||||
// 将JsonObject转换为Map,这样JavaScript可以正确访问
|
||||
return jsonObject.getMap();
|
||||
} catch (Exception e) {
|
||||
log.error("解析JSON响应失败", e);
|
||||
throw new RuntimeException("解析JSON响应失败: " + e.getMessage(), e);
|
||||
@@ -1,4 +1,4 @@
|
||||
package cn.qaiu.parser;
|
||||
package cn.qaiu.parser.customjs;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
@@ -1,9 +1,12 @@
|
||||
package cn.qaiu.parser;
|
||||
package cn.qaiu.parser.customjs;
|
||||
|
||||
import cn.qaiu.entity.FileInfo;
|
||||
import cn.qaiu.entity.ShareLinkInfo;
|
||||
import cn.qaiu.parser.IPanTool;
|
||||
import cn.qaiu.parser.custom.CustomParserConfig;
|
||||
import io.vertx.core.Future;
|
||||
import io.vertx.core.Promise;
|
||||
import io.vertx.core.json.JsonObject;
|
||||
import org.openjdk.nashorn.api.scripting.ScriptObjectMirror;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
@@ -39,7 +42,14 @@ public class JsParserExecutor implements IPanTool {
|
||||
this.config = config;
|
||||
this.shareLinkInfo = shareLinkInfo;
|
||||
this.engine = initEngine();
|
||||
this.httpClient = new JsHttpClient();
|
||||
|
||||
// 检查是否有代理配置
|
||||
JsonObject proxyConfig = null;
|
||||
if (shareLinkInfo.getOtherParam().containsKey("proxy")) {
|
||||
proxyConfig = (JsonObject) shareLinkInfo.getOtherParam().get("proxy");
|
||||
}
|
||||
|
||||
this.httpClient = new JsHttpClient(proxyConfig);
|
||||
this.jsLogger = new JsLogger("JsParser-" + config.getType());
|
||||
this.shareLinkInfoWrapper = new JsShareLinkInfoWrapper(shareLinkInfo);
|
||||
}
|
||||
@@ -1,8 +1,10 @@
|
||||
package cn.qaiu.parser;
|
||||
package cn.qaiu.parser.customjs;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import cn.qaiu.parser.custom.CustomParserConfig;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
@@ -66,33 +68,28 @@ public class JsScriptLoader {
|
||||
List<CustomParserConfig> configs = new ArrayList<>();
|
||||
|
||||
try {
|
||||
// 获取资源目录的输入流
|
||||
InputStream resourceStream = JsScriptLoader.class.getClassLoader()
|
||||
.getResourceAsStream(RESOURCE_PATH);
|
||||
// 尝试使用反射方式获取JAR包内的资源文件列表
|
||||
List<String> resourceFiles = getResourceFileList();
|
||||
|
||||
if (resourceStream == null) {
|
||||
log.debug("资源目录 {} 不存在", RESOURCE_PATH);
|
||||
return configs;
|
||||
}
|
||||
// 按文件名排序,确保加载顺序一致
|
||||
resourceFiles.sort(String::compareTo);
|
||||
|
||||
// 读取资源目录下的所有文件
|
||||
String resourcePath = JsScriptLoader.class.getClassLoader()
|
||||
.getResource(RESOURCE_PATH).getPath();
|
||||
|
||||
try (Stream<Path> paths = Files.walk(Paths.get(resourcePath))) {
|
||||
paths.filter(Files::isRegularFile)
|
||||
.filter(path -> path.toString().endsWith(".js"))
|
||||
.filter(path -> !isExcludedFile(path.getFileName().toString()))
|
||||
.forEach(path -> {
|
||||
try {
|
||||
String jsCode = Files.readString(path, StandardCharsets.UTF_8);
|
||||
CustomParserConfig config = JsScriptMetadataParser.parseScript(jsCode);
|
||||
configs.add(config);
|
||||
log.debug("从资源目录加载脚本: {}", path.getFileName());
|
||||
} catch (Exception e) {
|
||||
log.warn("加载资源脚本失败: {}", path.getFileName(), e);
|
||||
}
|
||||
});
|
||||
for (String resourceFile : resourceFiles) {
|
||||
try {
|
||||
InputStream inputStream = JsScriptLoader.class.getClassLoader()
|
||||
.getResourceAsStream(resourceFile);
|
||||
|
||||
if (inputStream != null) {
|
||||
String jsCode = new String(inputStream.readAllBytes(), StandardCharsets.UTF_8);
|
||||
CustomParserConfig config = JsScriptMetadataParser.parseScript(jsCode);
|
||||
configs.add(config);
|
||||
|
||||
String fileName = resourceFile.substring(resourceFile.lastIndexOf('/') + 1);
|
||||
log.debug("从资源目录加载脚本: {}", fileName);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn("加载资源脚本失败: {}", resourceFile, e);
|
||||
}
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
@@ -102,6 +99,92 @@ public class JsScriptLoader {
|
||||
return configs;
|
||||
}
|
||||
|
||||
/**
|
||||
* 尝试使用反射方式获取JAR包内的资源文件列表
|
||||
*/
|
||||
private static List<String> getResourceFileList() {
|
||||
List<String> resourceFiles = new ArrayList<>();
|
||||
|
||||
try {
|
||||
// 尝试获取资源目录的URL
|
||||
java.net.URL resourceUrl = JsScriptLoader.class.getClassLoader()
|
||||
.getResource(RESOURCE_PATH);
|
||||
|
||||
if (resourceUrl != null) {
|
||||
String protocol = resourceUrl.getProtocol();
|
||||
|
||||
if ("jar".equals(protocol)) {
|
||||
// JAR包内的资源
|
||||
resourceFiles = getJarResourceFiles(resourceUrl);
|
||||
} else if ("file".equals(protocol)) {
|
||||
// 文件系统中的资源(开发环境)
|
||||
resourceFiles = getFileSystemResourceFiles(resourceUrl);
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.debug("使用反射方式获取资源文件列表失败,将使用预定义列表", e);
|
||||
}
|
||||
|
||||
return resourceFiles;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取JAR包内的资源文件列表
|
||||
*/
|
||||
private static List<String> getJarResourceFiles(java.net.URL jarUrl) {
|
||||
List<String> resourceFiles = new ArrayList<>();
|
||||
|
||||
try {
|
||||
String jarPath = jarUrl.getPath().substring(5, jarUrl.getPath().indexOf("!"));
|
||||
java.util.jar.JarFile jarFile = new java.util.jar.JarFile(jarPath);
|
||||
|
||||
java.util.Enumeration<java.util.jar.JarEntry> entries = jarFile.entries();
|
||||
while (entries.hasMoreElements()) {
|
||||
java.util.jar.JarEntry entry = entries.nextElement();
|
||||
String entryName = entry.getName();
|
||||
|
||||
if (entryName.startsWith(RESOURCE_PATH + "/") &&
|
||||
entryName.endsWith(".js") &&
|
||||
!isExcludedFile(entryName.substring(entryName.lastIndexOf('/') + 1))) {
|
||||
resourceFiles.add(entryName);
|
||||
}
|
||||
}
|
||||
|
||||
jarFile.close();
|
||||
} catch (Exception e) {
|
||||
log.debug("解析JAR包资源文件失败", e);
|
||||
}
|
||||
|
||||
return resourceFiles;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取文件系统中的资源文件列表
|
||||
*/
|
||||
private static List<String> getFileSystemResourceFiles(java.net.URL fileUrl) {
|
||||
List<String> resourceFiles = new ArrayList<>();
|
||||
|
||||
try {
|
||||
java.io.File resourceDir = new java.io.File(fileUrl.getPath());
|
||||
if (resourceDir.exists() && resourceDir.isDirectory()) {
|
||||
java.io.File[] files = resourceDir.listFiles();
|
||||
if (files != null) {
|
||||
for (java.io.File file : files) {
|
||||
if (file.isFile() && file.getName().endsWith(".js") &&
|
||||
!isExcludedFile(file.getName())) {
|
||||
resourceFiles.add(RESOURCE_PATH + "/" + file.getName());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.debug("解析文件系统资源文件失败", e);
|
||||
}
|
||||
|
||||
return resourceFiles;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 从外部目录加载JavaScript脚本
|
||||
*/
|
||||
@@ -1,9 +1,11 @@
|
||||
package cn.qaiu.parser;
|
||||
package cn.qaiu.parser.customjs;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import cn.qaiu.parser.custom.CustomParserConfig;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Matcher;
|
||||
@@ -1,4 +1,4 @@
|
||||
package cn.qaiu.parser;
|
||||
package cn.qaiu.parser.customjs;
|
||||
|
||||
import cn.qaiu.entity.ShareLinkInfo;
|
||||
import org.slf4j.Logger;
|
||||
@@ -59,7 +59,7 @@ public class HttpResponseHelper {
|
||||
case "gzip" -> decompressGzip(compressed);
|
||||
case "deflate" -> decompressDeflate(compressed);
|
||||
case "br" -> decompressBrotli(compressed);
|
||||
//case "zstd" -> decompressZstd(compressed);
|
||||
case "zstd" -> compressed.toString(StandardCharsets.UTF_8); // 暂时返回原始内容
|
||||
default -> throw new UnsupportedOperationException("不支持的 Content-Encoding: " + encoding);
|
||||
};
|
||||
}
|
||||
|
||||
@@ -29,9 +29,12 @@
|
||||
/**
|
||||
* @typedef {Object} JsHttpClient
|
||||
* @property {function(string): JsHttpResponse} get - 发起GET请求
|
||||
* @property {function(string): JsHttpResponse} getWithRedirect - 发起GET请求并跟随重定向
|
||||
* @property {function(string): JsHttpResponse} getNoRedirect - 发起GET请求但不跟随重定向(用于获取Location头)
|
||||
* @property {function(string, any=): JsHttpResponse} post - 发起POST请求
|
||||
* @property {function(string, string): JsHttpClient} putHeader - 设置请求头
|
||||
* @property {function(Object): JsHttpResponse} sendForm - 发送表单数据
|
||||
* @property {function(Object): JsHttpResponse} sendForm - 发送简单表单数据
|
||||
* @property {function(string, Object): JsHttpResponse} sendMultipartForm - 发送multipart表单数据(支持文件上传)
|
||||
* @property {function(any): JsHttpResponse} sendJson - 发送JSON数据
|
||||
*/
|
||||
|
||||
|
||||
@@ -2,8 +2,12 @@ package cn.qaiu.parser;
|
||||
|
||||
import cn.qaiu.entity.FileInfo;
|
||||
import cn.qaiu.entity.ShareLinkInfo;
|
||||
import cn.qaiu.parser.custom.CustomParserConfig;
|
||||
import cn.qaiu.parser.custom.CustomParserRegistry;
|
||||
import cn.qaiu.parser.customjs.JsParserExecutor;
|
||||
import cn.qaiu.WebClientVertxInit;
|
||||
import io.vertx.core.Vertx;
|
||||
import io.vertx.core.json.JsonObject;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.util.HashMap;
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package cn.qaiu.parser;
|
||||
|
||||
import cn.qaiu.entity.ShareLinkInfo;
|
||||
import cn.qaiu.parser.custom.CustomParserConfig;
|
||||
import cn.qaiu.parser.custom.CustomParserRegistry;
|
||||
import io.vertx.core.Future;
|
||||
import io.vertx.core.Promise;
|
||||
import org.junit.After;
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package cn.qaiu.parser;
|
||||
|
||||
import cn.qaiu.entity.ShareLinkInfo;
|
||||
import cn.qaiu.parser.custom.CustomParserConfig;
|
||||
import cn.qaiu.parser.custom.CustomParserRegistry;
|
||||
import io.vertx.core.Future;
|
||||
import io.vertx.core.Promise;
|
||||
|
||||
|
||||
@@ -2,6 +2,9 @@ package cn.qaiu.parser;
|
||||
|
||||
import cn.qaiu.entity.FileInfo;
|
||||
import cn.qaiu.entity.ShareLinkInfo;
|
||||
import cn.qaiu.parser.custom.CustomParserConfig;
|
||||
import cn.qaiu.parser.custom.CustomParserRegistry;
|
||||
import cn.qaiu.parser.customjs.JsParserExecutor;
|
||||
import cn.qaiu.WebClientVertxInit;
|
||||
import io.vertx.core.Vertx;
|
||||
import org.junit.Test;
|
||||
|
||||
@@ -2,6 +2,9 @@ package cn.qaiu.parser;
|
||||
|
||||
import org.junit.Test;
|
||||
|
||||
import cn.qaiu.parser.custom.CustomParserConfig;
|
||||
import cn.qaiu.parser.customjs.JsScriptLoader;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
|
||||
Reference in New Issue
Block a user