Skip to content

Commit

Permalink
feat: support chunk upload
Browse files Browse the repository at this point in the history
  • Loading branch information
AkagiYui committed Sep 13, 2023
1 parent fe04eff commit c7360b1
Show file tree
Hide file tree
Showing 15 changed files with 323 additions and 21 deletions.
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ API 文档:https://apifox.com/apidoc/project-2811497
- [x] [邮箱验证码(Redis)](src/main/java/com/akagiyui/drive/service/impl/MailServiceImpl.java)
- [ ] 图片验证码
- [x] [断点续传](src/main/java/com/akagiyui/drive/controller/FileController.java)
- [ ] 分片上传
- [x] [分片上传](src/main/java/com/akagiyui/drive/service/UploadService.java)
- [x] [分片下载](src/main/java/com/akagiyui/drive/controller/FileController.java)
- [ ] 文件秒传
- [x] 相同文件合并(在上传时会检测)
Expand Down Expand Up @@ -142,6 +142,7 @@ API 文档:https://apifox.com/apidoc/project-2811497
- [脚本之家: springboot切换使用undertow容器的方式](https://www.jb51.net/article/254623.htm)
- [知乎: SpringBoot开始定时任务的三种方式](https://zhuanlan.zhihu.com/p/622930121)
- [知乎: ObjectMapper,别再像个二货一样一直new了!](https://zhuanlan.zhihu.com/p/498705670)
- [哔哩哔哩: 【java工程师必知】SpringBoot Validation入参校验国际化](https://www.bilibili.com/video/av742302746/)
- [CSDN: 有关HikariPool-1 – Failed to validate connection com.mysql.cj.jdbc.ConnectionImp 错误的产生原因与解决方法](https://blog.csdn.net/qq_45886144/article/details/128984915)
- [CSDN: 数据库连接池选型 Druid vs HikariCP性能对比](https://blog.csdn.net/weixin_39098944/article/details/109228618)
- [CSDN: SpringBoot 使用 beforeBodyWrite 实现统一的接口返回类型](https://blog.csdn.net/qq_37170583/article/details/107470337)
Expand Down
8 changes: 8 additions & 0 deletions src/main/java/com/akagiyui/common/ResponseEnum.java
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,14 @@ public enum ResponseEnum {
* 任务已存在
*/
TASK_EXIST(10017, "Task exist"),
/**
* 校验失败
*/
VERIFY_FAILED(10018, "Verify failed"),
/**
* 任务不存在
*/
TASK_NOT_FOUND(10019, "Task not found"),
;

/**
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package com.akagiyui.common.limiter;
package com.akagiyui.drive.component.limiter;

import java.lang.annotation.*;
import java.util.concurrent.TimeUnit;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package com.akagiyui.common.limiter;
package com.akagiyui.drive.component.limiter;

import com.akagiyui.common.exception.TooManyRequestsException;
import com.google.common.collect.Maps;
Expand Down Expand Up @@ -29,7 +29,7 @@ public class LimitAspect {
*/
private final Map<String, RateLimiter> limitMap = Maps.newConcurrentMap();

@Around("@annotation(com.akagiyui.common.limiter.Limit)")
@Around("@annotation(com.akagiyui.drive.component.limiter.Limit)")
public Object around(@NotNull ProceedingJoinPoint joinPoint) throws Throwable {
// 获取方法签名
MethodSignature signature = (MethodSignature) joinPoint.getSignature();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import org.springframework.web.bind.annotation.ResponseStatus;
import org.springframework.web.bind.annotation.RestControllerAdvice;
import org.springframework.web.multipart.MaxUploadSizeExceededException;
import org.springframework.web.multipart.MultipartException;
import org.springframework.web.servlet.NoHandlerFoundException;

import java.io.IOException;
Expand Down Expand Up @@ -75,6 +76,7 @@ public ResponseEntity<Void> requestRejectedException(IOException ignored) {
MissingRequestHeaderException.class, // 缺少请求头
MaxUploadSizeExceededException.class, // 文件过大
MethodArgumentNotValidException.class, // 参数校验异常
MultipartException.class, // 文件上传异常
})
public ResponseResult<?> badRequestException(Exception e) {
// 目前可预见的是 JSON 解析错误
Expand Down
16 changes: 16 additions & 0 deletions src/main/java/com/akagiyui/drive/controller/FileController.java
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import com.akagiyui.drive.service.*;
import jakarta.annotation.Resource;
import jakarta.servlet.http.HttpServletResponse;
import jakarta.validation.constraints.Min;
import lombok.extern.slf4j.Slf4j;
import org.springframework.core.io.InputStreamResource;
import org.springframework.http.HttpHeaders;
Expand Down Expand Up @@ -89,6 +90,21 @@ public Boolean uploadRequest(@RequestBody @Validated PreUploadRequest request) {
return uploadService.requestUpload(request);
}

/**
* 上传文件分片
*/
@PostMapping("/upload/{hash}/chunk")
@RequirePermission(Permission.PERSONAL_UPLOAD)
public Boolean uploadChunk(
@PathVariable("hash") String fileHash,
@RequestParam("file") MultipartFile chunk,
@RequestParam("hash") String chunkHash,
@RequestParam("index") @Min(1) int chunkIndex
) {
uploadService.uploadChunk(fileHash, chunk, chunkHash, chunkIndex);
return true;
}

/**
* 获取文件列表
*
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package com.akagiyui.drive.controller;

import com.akagiyui.common.limiter.Limit;
import com.akagiyui.drive.component.limiter.Limit;
import com.akagiyui.drive.component.permission.RequirePermission;
import com.akagiyui.drive.entity.User;
import com.akagiyui.drive.model.Permission;
Expand Down
11 changes: 8 additions & 3 deletions src/main/java/com/akagiyui/drive/model/ChunkedUploadInfo.java
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
package com.akagiyui.drive.model;

import com.akagiyui.drive.model.request.PreUploadRequest;
import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.experimental.Accessors;

import java.io.Serializable;
Expand All @@ -16,6 +18,7 @@
*/
@Data
@Accessors(chain = true)
@NoArgsConstructor
public class ChunkedUploadInfo implements Serializable {
/**
* 整个文件hash
Expand Down Expand Up @@ -57,16 +60,17 @@ public class ChunkedUploadInfo implements Serializable {
*/
@Data
@AllArgsConstructor
static class Chunk implements Serializable {
@NoArgsConstructor
public static class Chunk implements Serializable {
/**
* 分片序号
*/
private final int index;
private int index;

/**
* 分片大小
*/
private final int size;
private int size;

/**
* 分片hash
Expand Down Expand Up @@ -96,6 +100,7 @@ public ChunkedUploadInfo(PreUploadRequest request) {
/**
* 是否上传完成
*/
@JsonIgnore
public boolean isUploadFinish() {
for (Chunk chunk : chunks) {
if (!chunk.isCheckSuccess()) {
Expand Down
33 changes: 33 additions & 0 deletions src/main/java/com/akagiyui/drive/model/StorageFile.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
package com.akagiyui.drive.model;

import lombok.Data;
import lombok.experimental.Accessors;

/**
* 存储中的文件
*
* @author AkagiYui
*/
@Data
@Accessors(chain = true)
public class StorageFile {
/**
* 文件名
*/
private String key;

/**
* 文件大小
*/
private long size;

/**
* 文件类型
*/
private String type;

/**
* 文件哈希
*/
private String hash;
}
5 changes: 5 additions & 0 deletions src/main/java/com/akagiyui/drive/service/FileInfoService.java
Original file line number Diff line number Diff line change
Expand Up @@ -52,4 +52,9 @@ public interface FileInfoService {
* 获取所有文件信息
*/
Stream<FileInfo> getAllFileInfo();

/**
* 添加文件信息
*/
void addFileInfo(FileInfo fileInfo);
}
19 changes: 18 additions & 1 deletion src/main/java/com/akagiyui/drive/service/StorageService.java
Original file line number Diff line number Diff line change
@@ -1,12 +1,17 @@
package com.akagiyui.drive.service;

import com.akagiyui.drive.model.StorageFile;
import org.springframework.core.io.InputStreamResource;
import org.springframework.scheduling.annotation.Async;

import java.io.OutputStream;

/**
* 存储服务接口
* 存储 服务接口
* <p>
* 存储服务是一个 Key-Value 存储服务,Key 为文件名,Value 为文件内容
* {key: String, value: byte[]/Stream}
* </p>
*
* @author AkagiYui
*/
Expand Down Expand Up @@ -49,4 +54,16 @@ public interface StorageService {
*/
@Async
void deleteFile(String key);

/**
* 存储分片
*/
void saveChunk(String userId, String fileHash, int chunkIndex, byte[] content);

/**
* 合并分片
*
* @return

Check warning on line 66 in src/main/java/com/akagiyui/drive/service/StorageService.java

View workflow job for this annotation

GitHub Actions / Qodana for JVM

Javadoc declaration problems

`@return` tag description is missing
*/
StorageFile mergeChunk(String userId, String fileHash, int chunkCount);
}
6 changes: 6 additions & 0 deletions src/main/java/com/akagiyui/drive/service/UploadService.java
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package com.akagiyui.drive.service;

import com.akagiyui.drive.model.request.PreUploadRequest;
import org.springframework.web.multipart.MultipartFile;

/**
* 上传 服务接口
Expand All @@ -12,4 +13,9 @@ public interface UploadService {
* 请求文件上传
*/
boolean requestUpload(PreUploadRequest preUploadRequest);

/**
* 上传分片
*/
void uploadChunk(String fileHash, MultipartFile chunk, String chunkHash, int chunkIndex);
}
Original file line number Diff line number Diff line change
Expand Up @@ -129,4 +129,9 @@ public void deleteFile(String id) {
public Stream<FileInfo> getAllFileInfo() {
return fileInfoRepository.findAllByOrderByUpdateTimeAsc();
}

@Override
public void addFileInfo(FileInfo fileInfo) {
fileInfoRepository.save(fileInfo);
}
}
Loading

0 comments on commit c7360b1

Please sign in to comment.