ahdoawhfo/netdisk-fast-download
1
0
Fork 0
mirror of https://github.com/qaiu/netdisk-fast-download.git synced 2025-12-16 04:13:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

1. 缓存优化

Browse Source
This commit is contained in:
qaiu
2024-09-18 13:24:33 +08:00
parent a0fe702c10
commit 59d2fb3010
24 changed files with 370 additions and 263 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
web-service/src/main/java/cn/qaiu/lz/common/cache/CacheConfigLoader.java vendored
View File

@@ -29,9 +29,9 @@ public class CacheConfigLoader {
}
public static Integer getDuration(PanDomainTemplate pdt) {
return CONFIGS.get(pdt.getShortName());
return CONFIGS.get(pdt.name().toLowerCase());
}
public static Integer getDuration(String shortName) {
return CONFIGS.get(shortName);
public static Integer getDuration(String type) {
return CONFIGS.get(type.toLowerCase());
}
}

81
web-service/src/main/java/cn/qaiu/lz/common/cache/CacheManager.java vendored
View File

@@ -6,6 +6,7 @@ import io.vertx.core.Future;
import io.vertx.core.Promise;
import io.vertx.core.json.JsonObject;
import io.vertx.jdbcclient.JDBCPool;
import io.vertx.sqlclient.Row;
import io.vertx.sqlclient.templates.SqlTemplate;
import java.util.HashMap;
@@ -16,7 +17,7 @@ public class CacheManager {
public Future<CacheLinkInfo> get(String cacheKey) {
String sql = "SELECT direct_link as directLink, expiration FROM cache_link_info WHERE share_key = #{share_key}";
String sql = "SELECT share_key as shareKey, direct_link as directLink, expiration FROM cache_link_info WHERE share_key = #{share_key}";
Map<String, Object> params = new HashMap<>();
params.put("share_key", cacheKey);
Promise<CacheLinkInfo> promise = Promise.promise();
@@ -29,7 +30,7 @@ public class CacheManager {
cacheHit = rows.iterator().next();
cacheHit.setCacheHit(true);
} else {
cacheHit = new CacheLinkInfo(JsonObject.of("cacheHit", false));
cacheHit = new CacheLinkInfo(JsonObject.of("cacheHit", false, "shareKey", cacheKey));
}
promise.complete(cacheHit);
}).onFailure(Throwable::printStackTrace);
@@ -49,4 +50,80 @@ public class CacheManager {
.execute(cacheLinkInfo)
.mapEmpty();
}
// 统计网盘厂商API解析次数
public Future<Integer> updateTotalByCached(String shareKey) {
Promise<Integer> promise = Promise.promise();
String sql = """
MERGE INTO `api_statistics_info` (`pan_type`, `share_key`, `cache_hit_total`, `update_ts`)
KEY (`share_key`)
VALUES (#{panType}, #{shareKey}, #{total}, #{ts})
""";
getShareKeyTotal(shareKey, "cache_hit_total").onSuccess(total -> {
Integer newTotal = (total == null ? 0 : total) + 1;
SqlTemplate.forUpdate(jdbcPool, sql)
.execute(new HashMap<>() {{
put("panType", getShareType(shareKey));
put("shareKey", shareKey);
put("total", newTotal);
put("ts", System.currentTimeMillis());
}})
.onSuccess(res -> promise.complete(res.rowCount()))
.onFailure(Throwable::printStackTrace);
});
return promise.future();
}
private String getShareType(String fullShareKey) {
// 将type和shareKey组合成一个字符串作为缓存key
return fullShareKey.split(":")[0];
}
// 统计网盘厂商API解析次数
public Future<Integer> updateTotalByParser(String shareKey) {
Promise<Integer> promise = Promise.promise();
String sql = """
MERGE INTO `api_statistics_info` (`pan_type`, `share_key`, `api_parser_total`, `update_ts`)
KEY (`share_key`)
VALUES (#{panType}, #{shareKey}, #{total}, #{ts})
""";
getShareKeyTotal(shareKey, "api_parser_total").onSuccess(total -> {
Integer newTotal = (total == null ? 0 : total) + 1;
SqlTemplate.forUpdate(jdbcPool, sql)
.execute(new HashMap<>() {{
put("panType", getShareType(shareKey));
put("shareKey", shareKey);
put("total", newTotal);
put("ts", System.currentTimeMillis());
}})
.onSuccess(res -> promise.complete(res.rowCount()))
.onFailure(Throwable::printStackTrace);
});
return promise.future();
}
public Future<Integer> getShareKeyTotal(String shareKey, String name) {
String sql = """
select `share_key`, sum({total_name}) sum_num
from `api_statistics_info`
group by `share_key` having `share_key` = #{shareKey};
""".replace("{total_name}", name);
Promise<Integer> promise = Promise.promise();
Map<String, Object> paramMap = new HashMap<>();
paramMap.put("shareKey", shareKey);
SqlTemplate.forQuery(jdbcPool, sql)
.mapTo(Row::toJson)
.execute(paramMap)
.onSuccess(res -> {
Integer total = res.iterator().hasNext() ?
res.iterator().next().getInteger("sum_num") : null;
promise.complete(total);
});
return promise.future();
}
}

2
web-service/src/main/java/cn/qaiu/lz/common/interceptorImpl/DefaultInterceptor.java
View File

@@ -20,7 +20,7 @@ public class DefaultInterceptor implements BeforeInterceptor {
@Override
public void handle(RoutingContext ctx) {
System.out.println("进入前置拦截器1->" + ctx.request().path());
// System.out.println("进入前置拦截器1->" + ctx.request().path());
doNext(ctx);
}

18
web-service/src/main/java/cn/qaiu/lz/common/interceptorImpl/LogStatistics.java
View File

@@ -4,7 +4,6 @@ import cn.qaiu.db.pool.JDBCPoolInit;
import cn.qaiu.lz.common.model.ParserLogInfo;
import cn.qaiu.vx.core.annotaions.HandleSortFilter;
import cn.qaiu.vx.core.interceptor.AfterInterceptor;
import cn.qaiu.vx.core.model.JsonResult;
import cn.qaiu.vx.core.util.CommonUtil;
import cn.qaiu.vx.core.util.SharedDataUtil;
import io.vertx.core.json.JsonArray;
@@ -36,21 +35,8 @@ public class LogStatistics implements AfterInterceptor {
ParserLogInfo parserLogInfo = new ParserLogInfo();
parserLogInfo.setPath(ctx.request().uri());
if (responseData == null) {
String location = ctx.response().headers().get("location");
if (location != null) {
parserLogInfo.setCode(200);
parserLogInfo.setData(location);
} else {
log.error("location不存在且responseData为空, path={}", ctx.request().path());
}
insert(parserLogInfo);
} else if (responseData.containsKey("code")) {
JsonResult<?> result = JsonResult.toJsonResult(responseData);
parserLogInfo.setCode(result.getCode());
parserLogInfo.setData(result.getCode() == 500 ? result.getMsg() : result.getData().toString());
insert(parserLogInfo);
if (responseData.containsKey("code") && responseData.getInteger("code") == 500) {
log.error("code 500: {} {}", ctx.request().path(), responseData.getString("msg"));
} else {
log.error("未知json日志: {}, path: {}", responseData.encode(), ctx.request().path());
}

2
web-service/src/main/java/cn/qaiu/lz/common/model/ParserLogInfo.java
View File

@@ -14,6 +14,8 @@ public class ParserLogInfo {
String id = SnowflakeIdWorker.getStringId();
@JsonFormat(pattern = "yyyy-MM-dd'T'HH:mm:ss.SSS", timezone = "GMT+8")
Date logTime = new Date();
@Length(varcharSize = 4096)
String path;
Integer code;

30
web-service/src/main/java/cn/qaiu/lz/web/http/ServerApi.java
View File

@@ -1,8 +1,8 @@
package cn.qaiu.lz.web.http;
import cn.qaiu.lz.common.util.URLParamUtil;
import cn.qaiu.lz.web.model.CacheLinkInfo;
import cn.qaiu.lz.web.service.CacheService;
import cn.qaiu.parser.PanDomainTemplate;
import cn.qaiu.vx.core.annotaions.RouteHandler;
import cn.qaiu.vx.core.annotaions.RouteMapping;
import cn.qaiu.vx.core.enums.RouteMethod;
@@ -31,8 +31,7 @@ public class ServerApi {
Promise<Void> promise = Promise.promise();
String url = URLParamUtil.parserParams(request);
PanDomainTemplate panDomainTemplate = PanDomainTemplate.fromShareUrl(url).setShareLinkInfoPwd(pwd);
cacheService.getAndSaveCachedShareLink(panDomainTemplate)
cacheService.getCachedByShareUrlAndPwd(url, pwd)
.onSuccess(res -> ResponseUtil.redirect(
response.putHeader("nfd-cache-hit", res.getCacheHit().toString())
.putHeader("nfd-cache-expires", res.getExpires()),
@@ -42,39 +41,32 @@ public class ServerApi {
}
@RouteMapping(value = "/json/parser", method = RouteMethod.GET, order = 3)
public Future<String> parseJson(HttpServerRequest request, String pwd) {
public Future<CacheLinkInfo> parseJson(HttpServerRequest request, String pwd) {
String url = URLParamUtil.parserParams(request);
return PanDomainTemplate.fromShareUrl(url).setShareLinkInfoPwd(pwd).createTool().parse();
return cacheService.getCachedByShareUrlAndPwd(url, pwd);
}
@RouteMapping(value = "/json/:type/:key", method = RouteMethod.GET, order = 2)
public Future<String> parseKeyJson(String type, String key) {
String code = "";
public Future<CacheLinkInfo> parseKeyJson(String type, String key) {
String pwd = "";
if (key.contains("@")) {
String[] keys = key.split("@");
key = keys[0];
code = keys[1];
pwd = keys[1];
}
return PanDomainTemplate.fromShortName(type)
.generateShareLink(key).setShareLinkInfoPwd(code).createTool().parse();
return cacheService.getCachedByShareKeyAndPwd(type, key, pwd);
}
@RouteMapping(value = "/:type/:key", method = RouteMethod.GET, order = 1)
public Future<Void> parseKey(HttpServerResponse response, String type, String key) {
Promise<Void> promise = Promise.promise();
String code = "";
String pwd = "";
if (key.contains("@")) {
String[] keys = key.split("@");
key = keys[0];
code = keys[1];
pwd = keys[1];
}
PanDomainTemplate panDomainTemplate = PanDomainTemplate
.fromShortName(type)
.generateShareLink(key)
.setShareLinkInfoPwd(code);
cacheService.getAndSaveCachedShareLink(panDomainTemplate)
cacheService.getCachedByShareKeyAndPwd(type, key, pwd)
.onSuccess(res -> ResponseUtil.redirect(
response.putHeader("nfd-cache-hit", res.getCacheHit().toString())
.putHeader("nfd-cache-expires", res.getExpires()),

68
web-service/src/main/java/cn/qaiu/lz/web/model/ApiStatisticsInfo.java Normal file
View File

@@ -0,0 +1,68 @@
package cn.qaiu.lz.web.model;
import cn.qaiu.db.ddl.Length;
import cn.qaiu.db.ddl.Table;
import cn.qaiu.lz.common.ToJson;
import io.vertx.codegen.annotations.DataObject;
import io.vertx.core.json.JsonObject;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* @author <a href="https://qaiu.top">QAIU</a>
* @date 2024/9/11 16:06
*/
@Table(value = "api_statistics_info", keyFields = "share_key")
@Data
@DataObject
@NoArgsConstructor
public class ApiStatisticsInfo implements ToJson {
/**
* pan type 单独拿出来便于统计.
*/
@Length(varcharSize = 4)
private String panType;
/**
* 分享key type:key
*/
@Length(varcharSize = 4096)
private String shareKey;
/**
* 命中缓存次数
*/
private Integer cacheHitTotal;
/**
* api解析次数
*/
private Integer apiParserTotal;
/**
* 更新时间戳
*/
private Long updateTs;
// 使用 JsonObject 构造
public ApiStatisticsInfo(JsonObject json) {
if (json.containsKey("panType")) {
this.setPanType(json.getString("panType"));
}
if (json.containsKey("shareKey")) {
this.setShareKey(json.getString("shareKey"));
}
if (json.containsKey("cacheHitTotal")) {
this.setCacheHitTotal(json.getInteger("cacheHitTotal"));
}
if (json.containsKey("apiParserTotal")) {
this.setApiParserTotal(json.getInteger("apiParserTotal"));
}
if (json.containsKey("updateTs")) {
this.setUpdateTs(json.getLong("updateTs"));
}
}
}

18
web-service/src/main/java/cn/qaiu/lz/web/model/CacheLinkInfo.java
View File

@@ -20,7 +20,7 @@ import lombok.NoArgsConstructor;
public class CacheLinkInfo implements ToJson {
/**
* 缓存key: type@ShareKey; e.g. lz@xxxx
* 缓存key: type:ShareKey; e.g. lz:xxxx
*/
@Length(varcharSize = 4096)
private String shareKey;
@@ -51,6 +51,20 @@ public class CacheLinkInfo implements ToJson {
// 使用 JsonObject 构造
public CacheLinkInfo(JsonObject json) {
CacheLinkInfoConverter.fromJson(json, this);
if (json.containsKey("shareKey")) {
this.setShareKey(json.getString("shareKey"));
}
if (json.containsKey("directLink")) {
this.setDirectLink(json.getString("directLink"));
}
if (json.containsKey("expires")) {
this.setExpires(json.getString("expires"));
}
if (json.containsKey("expiration")) {
this.setExpiration(json.getLong("expiration"));
}
this.setCacheHit(json.getBoolean("cacheHit", false));
}
}

23
web-service/src/main/java/cn/qaiu/lz/web/model/CacheLinkInfoConverter.java
View File

@@ -1,23 +0,0 @@
package cn.qaiu.lz.web.model;
import io.vertx.core.json.JsonObject;
// CacheLinkInfoConverter.java
public class CacheLinkInfoConverter {
public static void fromJson(JsonObject json, CacheLinkInfo obj) {
if (json.containsKey("shareKey")) {
obj.setShareKey(json.getString("shareKey"));
}
if (json.containsKey("directLink")) {
obj.setDirectLink(json.getString("directLink"));
}
if (json.containsKey("expires")) {
obj.setExpires(json.getString("expires"));
}
if (json.containsKey("expiration")) {
obj.setExpiration(json.getLong("expiration"));
}
obj.setCacheHit(json.getBoolean("cacheHit", false));
}
}

8
web-service/src/main/java/cn/qaiu/lz/web/model/StatisticsInfo.java
View File

@@ -10,14 +10,14 @@ import lombok.NoArgsConstructor;
@NoArgsConstructor
@DataObject
public class StatisticsInfo implements ToJson {
Integer fail;
Integer success;
Integer parserTotal;
Integer cacheTotal;
Integer total;
public StatisticsInfo(JsonObject jsonObject) {
this.fail = jsonObject.getInteger("fail");
this.success = jsonObject.getInteger("success");
this.parserTotal = jsonObject.getInteger("parserTotal");
this.cacheTotal = jsonObject.getInteger("cacheTotal");
this.total = jsonObject.getInteger("total");
}
}

5
web-service/src/main/java/cn/qaiu/lz/web/service/CacheService.java
View File

@@ -1,7 +1,6 @@
package cn.qaiu.lz.web.service;
import cn.qaiu.lz.web.model.CacheLinkInfo;
import cn.qaiu.parser.PanDomainTemplate;
import cn.qaiu.vx.core.base.BaseAsyncService;
import io.vertx.codegen.annotations.ProxyGen;
import io.vertx.core.Future;
@@ -13,5 +12,7 @@ import io.vertx.core.Future;
@ProxyGen
public interface CacheService extends BaseAsyncService {
Future<CacheLinkInfo> getAndSaveCachedShareLink(PanDomainTemplate shareLinkInfo);
Future<CacheLinkInfo> getCachedByShareKeyAndPwd(String type, String shareKey, String pwd);
Future<CacheLinkInfo> getCachedByShareUrlAndPwd(String shareUrl, String pwd);
}

29
web-service/src/main/java/cn/qaiu/lz/web/service/impl/CacheServiceImpl.java
View File

@@ -5,7 +5,7 @@ import cn.qaiu.lz.common.cache.CacheConfigLoader;
import cn.qaiu.lz.common.cache.CacheManager;
import cn.qaiu.lz.web.model.CacheLinkInfo;
import cn.qaiu.lz.web.service.CacheService;
import cn.qaiu.parser.PanDomainTemplate;
import cn.qaiu.parser.ParserCreate;
import cn.qaiu.vx.core.annotaions.Service;
import io.vertx.core.Future;
import io.vertx.core.Promise;
@@ -19,21 +19,22 @@ public class CacheServiceImpl implements CacheService {
private final CacheManager cacheManager = new CacheManager();
@Override
public Future<CacheLinkInfo> getAndSaveCachedShareLink(PanDomainTemplate template) {
private Future<CacheLinkInfo> getAndSaveCachedShareLink(ParserCreate parserCreate) {
Promise<CacheLinkInfo> promise = Promise.promise();
// 构建组合的缓存key
ShareLinkInfo shareLinkInfo = template.getShareLinkInfo();
ShareLinkInfo shareLinkInfo = parserCreate.getShareLinkInfo();
String cacheKey = generateCacheKey(shareLinkInfo.getType(), shareLinkInfo.getShareKey());
// 尝试从缓存中获取
cacheManager.get(cacheKey).onSuccess(result -> {
// 判断是否已过期
// 未命中或者过期
if (!result.getCacheHit() || result.getExpiration() < System.currentTimeMillis()) {
template.createTool().parse().onSuccess(redirectUrl -> {
// parse
result.setCacheHit(false);
result.setExpiration(0L);
parserCreate.createTool().parse().onSuccess(redirectUrl -> {
long expires = System.currentTimeMillis() +
CacheConfigLoader.getDuration(shareLinkInfo.getType()) * 60 * 1000;
CacheConfigLoader.getDuration(shareLinkInfo.getType()) * 60 * 1000L;
result.setDirectLink(redirectUrl);
// result.setExpires(generateDate(expires));
promise.complete(result);
@@ -45,10 +46,12 @@ public class CacheServiceImpl implements CacheService {
"shareKey", cacheKey
));
cacheManager.cacheShareLink(cacheLinkInfo).onFailure(Throwable::printStackTrace);
cacheManager.updateTotalByParser(cacheKey).onFailure(Throwable::printStackTrace);
}).onFailure(promise::fail);
} else {
result.setExpires(generateDate(result.getExpiration()));
promise.complete(result);
cacheManager.updateTotalByCached(cacheKey).onFailure(Throwable::printStackTrace);
}
}).onFailure(t -> promise.fail(t.fillInStackTrace()));
return promise.future();
@@ -62,4 +65,16 @@ public class CacheServiceImpl implements CacheService {
private String generateDate(Long ts) {
return DateFormatUtils.format(new Date(ts), "yyyy-MM-dd hh:mm:ss");
}
@Override
public Future<CacheLinkInfo> getCachedByShareKeyAndPwd(String type, String shareKey, String pwd) {
ParserCreate parserCreate = ParserCreate.fromType(type).shareKey(shareKey).setShareLinkInfoPwd(pwd);
return getAndSaveCachedShareLink(parserCreate);
}
@Override
public Future<CacheLinkInfo> getCachedByShareUrlAndPwd(String shareUrl, String pwd) {
ParserCreate parserCreate = ParserCreate.fromShareUrl(shareUrl).setShareLinkInfoPwd(pwd);
return getAndSaveCachedShareLink(parserCreate);
}
}

7
web-service/src/main/java/cn/qaiu/lz/web/service/impl/DbServiceImpl.java
View File

@@ -48,10 +48,9 @@ public class DbServiceImpl implements DbService {
JDBCPool client = JDBCPoolInit.instance().getPool();
Promise<StatisticsInfo> promise = Promise.promise();
String sql = """
select COUNT(CASE "code" WHEN 500 THEN "code" END ) "fail",
COUNT(CASE "code" WHEN 200 THEN "code" END ) "success",
count(1) "total"
from "t_parser_log_info"
select sum(api_parser_total) parserTotal,sum("cache_hit_total") cacheTotal,
sum(api_parser_total) + sum("cache_hit_total") total
from "api_statistics_info";
""";
SqlTemplate.forQuery(client, sql).mapTo(StatisticsInfo.class).execute(new HashMap<>()).onSuccess(row -> {
StatisticsInfo info;

2
web-service/src/main/resources/app-dev.yml
View File

@@ -54,7 +54,7 @@ cache:
iz:
le: 2879
lz:
qq:
qq: 999999
ws:
ye:

3
web-service/src/main/resources/conf/dictionaries.json
View File

@@ -1,3 +0,0 @@
{
}

8
web-service/src/main/resources/http-tools/test.http
View File

@@ -8,14 +8,14 @@ GET http://127.0.0.1:6400/json/parser?url=https://wwsd.lanzoue.com/iLany1e9bbbi
### 蓝奏云
# @no-redirect
GET http://127.0.0.1:6400/lz/i6SqHmp1yfc
GET http://127.0.0.1:6400/json/lz/i6SqHmp1yfc
### 蓝奏云https://acgtools.lanzoui.com/iUr7Qnu3sxc https://wwn.lanzouy.com/tp/ihLkw1gezutg https://wwn.lanzouy.com/ihLkw1gezutg
# @no-redirect
GET http://127.0.0.1:6400/lz/ihLkw1gezutg
### 蓝奏云
# @no-redirect
GET http://127.0.0.1:6400/lz/icBp6qqj82b@QAIU
GET http://127.0.0.1:6400/json/lz/icBp6qqj82b@QAIU
### 蓝奏云
GET http://127.0.0.1:6400/json/lz/ia2cntg
@@ -133,7 +133,7 @@ GET http://127.0.0.1:6400/le/2RkKbLP9BrppS9b43@ex2b
GET http://127.0.0.1:6400/json/le/2RkKbLP9BrppS9b43@ex2b
### PASS 文叔叔
GET http://127.0.0.1:6400/json/parser?url=https://f.ws59.cn/f/f25625rv6p6
GET http://127.0.0.1:6400/parser?url=https://f.ws59.cn/f/f25625rv6p6
###
https://f.wss.cc/f/f25625rv6p6
@@ -145,7 +145,7 @@ GET http://127.0.0.1:6400/parser?url=https://pan.huang1111.cn/s/g31PcQ&pwd=qaiu
### PASS QQ
# @no-redirect
GET http://127.0.0.1:6400/parser?url=https://iwx.mail.qq.com/ftn/download?func=3&key=c79c5732038ad41cf5ef1e3261663537f2cf4133636635371d4c484657050c0e5d561d070252021a0a0552024e5257530b4e0157540256525e5751565a053b374014540057560c070b511e53130d21f0361c829bf15b3e4a6355fbada6dd10dfdd11a33263663537386330&code=8c02cf57&k=c79c5732038ad41cf5ef1e3261663537f2cf4133636635371d4c484657050c0e5d561d070252021a0a0552024e5257530b4e0157540256525e5751565a053b374014540057560c070b511e53130d21f0361c829bf15b3e4a6355fbada6dd10dfdd11a33263663537386330
GET http://127.0.0.1:6400/json/parser?url=https://iwx.mail.qq.com/ftn/download?func=3&key=qweqe&code=8c02cf57&k=asdad