From 43d8d7aefd5f851e499eee5e8937e0ecc3fa96e5 Mon Sep 17 00:00:00 2001 From: "houcc@qq.com" Date: Sun, 29 Jul 2018 17:01:52 +0800 Subject: [PATCH 1/3] =?UTF-8?q?1.Gson=20=E6=94=B9=E4=B8=BA=20FASTJSON=202.?= =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=8E=BB=E4=B8=AD=20=E5=88=86=E5=B8=83?= =?UTF-8?q?=E5=BC=8F=E6=9C=8D=E5=8A=A1=E9=9B=86=E7=BE=A4=E3=80=82=203.?= =?UTF-8?q?=E6=96=B0=E5=A2=9Enetty=E4=B8=8A=E4=BC=A0=E6=96=B9=E5=BC=8F?= =?UTF-8?q?=EF=BC=8C=E5=B9=B6=E8=AE=BE=E7=BD=AE=E4=B8=BA=E9=BB=98=E8=AE=A4?= =?UTF-8?q?=E6=96=B9=E5=BC=8F=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- oss-server-sdk-java/pom.xml | 103 ++-- .../java/com/xiaominfo/oss/sdk/OSSClient.java | 386 ++------------ .../xiaominfo/oss/sdk/OSSClientProperty.java | 8 +- .../oss/sdk/client/FileBytesRequest.java | 17 +- .../oss/sdk/client/NettyFileRequest.java | 134 +++++ .../com/xiaominfo/oss/sdk/upload/IUpload.java | 21 + .../sdk/upload/handle/Base64UploadHandle.java | 142 +++++ .../sdk/upload/handle/NettyUploadHandle.java | 59 +++ .../sdk/upload/handle/netty/NettyClient.java | 74 +++ .../handle/netty/NettyClientHandler.java | 180 +++++++ .../handle/netty/NettyClientInitializer.java | 41 ++ .../sdk/upload/handle/netty/NettyMessage.java | 128 +++++ .../com/xiaominfo/oss/sdk/OSSClientTest.java | 45 ++ oss-server-spring-boot-starter/pom.xml | 17 +- oss-server/pom.xml | 499 +++++++++--------- .../xiaominfo/oss/OssServerApplication.java | 51 +- .../oss/api/MaterialApplication.java | 291 +++++----- .../xiaominfo/oss/api/ModelApplication.java | 37 +- .../com/xiaominfo/oss/api/NettyUpload.java | 4 + .../java/com/xiaominfo/oss/api/RootApis.java | 114 ++-- .../oss/common/SpringContextUtil.java | 70 +++ .../oss/config/JsonpMessageConverter.java | 13 +- .../oss/config/MaterialConfiguration.java | 14 +- .../oss/domain/FileBinaryRequest.java | 5 +- .../com/xiaominfo/oss/domain/FileStream.java | 15 +- .../oss/module/dao/OSSMaterialInfoMapper.java | 4 +- .../module/dao/OSSMaterialInfoSyncMapper.java | 27 + .../dao/mappings/OSSMaterialInfoMapper.xml | 42 +- .../oss/module/model/OSSMaterialInfoSync.java | 75 +++ .../oss/service/MaterialService.java | 33 +- .../oss/service/OSSMaterialInfoService.java | 11 +- .../service/OSSMaterialInfoSyncService.java | 28 + .../oss/service/impl/MaterialServiceImpl.java | 308 ++++++----- .../impl/OSSMaterialInfoServiceImpl.java | 30 +- .../impl/OSSMaterialInfoSyncServiceImpl.java | 184 +++++++ .../java/com/xiaominfo/oss/sync/Node.java | 32 ++ .../java/com/xiaominfo/oss/sync/Sync.java | 171 ++++++ .../com/xiaominfo/oss/sync/SyncMessage.java | 77 +++ .../oss/sync/netty/MasterServer.java | 61 +++ .../sync/netty/MasterSyncServerHandler.java | 185 +++++++ .../xiaominfo/oss/sync/netty/SalveServer.java | 71 +++ .../oss/sync/netty/SalveServerHandler.java | 103 ++++ .../sync/netty/SyncUploadServerHandler.java | 100 ++++ .../src/main/resources/application.properties | 22 +- .../oss/OssServerApplicationTests.java | 22 +- pom.xml | 113 ++++ 46 files changed, 3073 insertions(+), 1094 deletions(-) create mode 100644 oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/client/NettyFileRequest.java create mode 100644 oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/upload/IUpload.java create mode 100644 oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/upload/handle/Base64UploadHandle.java create mode 100644 oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/upload/handle/NettyUploadHandle.java create mode 100644 oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/upload/handle/netty/NettyClient.java create mode 100644 oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/upload/handle/netty/NettyClientHandler.java create mode 100644 oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/upload/handle/netty/NettyClientInitializer.java create mode 100644 oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/upload/handle/netty/NettyMessage.java create mode 100644 oss-server-sdk-java/src/test/java/com/xiaominfo/oss/sdk/OSSClientTest.java create mode 100644 oss-server/src/main/java/com/xiaominfo/oss/api/NettyUpload.java create mode 100644 oss-server/src/main/java/com/xiaominfo/oss/common/SpringContextUtil.java create mode 100644 oss-server/src/main/java/com/xiaominfo/oss/module/dao/OSSMaterialInfoSyncMapper.java create mode 100644 oss-server/src/main/java/com/xiaominfo/oss/module/model/OSSMaterialInfoSync.java create mode 100644 oss-server/src/main/java/com/xiaominfo/oss/service/OSSMaterialInfoSyncService.java create mode 100644 oss-server/src/main/java/com/xiaominfo/oss/service/impl/OSSMaterialInfoSyncServiceImpl.java create mode 100644 oss-server/src/main/java/com/xiaominfo/oss/sync/Node.java create mode 100644 oss-server/src/main/java/com/xiaominfo/oss/sync/Sync.java create mode 100644 oss-server/src/main/java/com/xiaominfo/oss/sync/SyncMessage.java create mode 100644 oss-server/src/main/java/com/xiaominfo/oss/sync/netty/MasterServer.java create mode 100644 oss-server/src/main/java/com/xiaominfo/oss/sync/netty/MasterSyncServerHandler.java create mode 100644 oss-server/src/main/java/com/xiaominfo/oss/sync/netty/SalveServer.java create mode 100644 oss-server/src/main/java/com/xiaominfo/oss/sync/netty/SalveServerHandler.java create mode 100644 oss-server/src/main/java/com/xiaominfo/oss/sync/netty/SyncUploadServerHandler.java create mode 100644 pom.xml diff --git a/oss-server-sdk-java/pom.xml b/oss-server-sdk-java/pom.xml index a82a3c7..ed18d16 100644 --- a/oss-server-sdk-java/pom.xml +++ b/oss-server-sdk-java/pom.xml @@ -2,11 +2,17 @@ + + com.xiaominfo.oss + oss-server-parent + 1.1 + + 4.0.0 com.xiaominfo.oss oss-server-sdk-java - 1.0 + 1.1 oss-server sdk java版本 UTF-8 @@ -17,40 +23,7 @@ true - - - xiaoym - 肖玉民 - lishi - xiaoymin@foxmail.com - - Java Development Enginner - - 2018-5-30 10:21:40 - - - - - nexus-lishiots-team-3rd - nexus-lishiots-team-3rd - http://192.168.0.242:9909/nexus/content/groups/public - - - - - nexus-lishiots-team-3rd - nexus-lishiots-team-3rd - http://192.168.0.242:9909/nexus/content/groups/public - - - - - - nexus-lishiots-team-3rd - http://192.168.0.242:9909/nexus/content/repositories/thirdparty/ - - @@ -58,11 +31,57 @@ commons-io 2.6 - - com.google.code.gson - gson - 2.8.5 + io.netty + netty-all + 4.1.25.Final + + + + net.logstash.logback + logstash-logback-encoder + 4.7 + + + org.slf4j + slf4j-api + ${org.slf4j.version} + + + org.slf4j + jcl-over-slf4j + ${org.slf4j.version} + runtime + + + ch.qos.logback + logback-classic + ${logback.version} + + + javax.mail + mail + + + javax.jms + jms + + + com.sun.jdmk + jmxtools + + + com.sun.jmx + jmxri + + + runtime + + + + com.alibaba + fastjson + 1.2.47 @@ -82,6 +101,12 @@ commons-codec 1.11 + + junit + junit + 4.12 + test + @@ -129,4 +154,4 @@ - \ No newline at end of file + diff --git a/oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/OSSClient.java b/oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/OSSClient.java index 099f950..7e3a4b8 100644 --- a/oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/OSSClient.java +++ b/oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/OSSClient.java @@ -7,380 +7,88 @@ package com.xiaominfo.oss.sdk; -import com.google.gson.*; -import com.google.gson.reflect.TypeToken; -import com.xiaominfo.oss.sdk.client.FileBytesRequest; import com.xiaominfo.oss.sdk.client.FileBytesResponse; import com.xiaominfo.oss.sdk.common.OSSClientMessage; -import org.apache.commons.codec.binary.Base64; -import org.apache.commons.io.FileUtils; -import org.apache.http.HttpStatus; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.methods.HttpUriRequest; -import org.apache.http.entity.ContentType; -import org.apache.http.entity.StringEntity; -import org.apache.http.entity.mime.MultipartEntityBuilder; -import org.apache.http.entity.mime.content.FileBody; -import org.apache.http.entity.mime.content.StringBody; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.util.EntityUtils; +import com.xiaominfo.oss.sdk.upload.IUpload; +import com.xiaominfo.oss.sdk.upload.handle.Base64UploadHandle; +import com.xiaominfo.oss.sdk.upload.handle.NettyUploadHandle; import java.io.File; -import java.io.IOException; -import java.lang.reflect.Type; -import java.util.ArrayList; -import java.util.List; /*** * * @since:oss-server 1.0 - * @author xiaoymin@foxmail.com + * @author xiaoymin@foxmail.com * 2018/05/30 17:00 */ public class OSSClient { - - /*** - * byte64 上传 - */ - private static final String v1_upload_file_binary_api="/oss/material/uploadByBinary"; - - /*** - * form表单上传 - */ - private static final String UPLOAD_FILE_FORM_API="/oss/material/uploadMaterialNonProUrl"; - - - + private static final String v1_upload_file_binary_api = "/oss/material/uploadByBinary"; private OSSClientProperty ossClientProperty; + private String mode = "NETTY";//支持base64 和 netty两种方式上传，默认netty - public OSSClient(OSSClientProperty ossClientProperty) { + public OSSClient(OSSClientProperty ossClientProperty, String mode) { this.ossClientProperty = ossClientProperty; + this.mode = mode; //初始化上传url地址 - if (ossClientProperty!=null&&ossClientProperty.getRemote()!=null&&!"".equalsIgnoreCase(ossClientProperty.getRemote())){ - StringBuffer endpoint=new StringBuffer(); + if (ossClientProperty != null && ossClientProperty.getRemote() != null && !"".equalsIgnoreCase(ossClientProperty.getRemote())) { + StringBuffer endpoint = new StringBuffer(); endpoint.append(ossClientProperty.getRemote()); - if (ossClientProperty.getRemote().endsWith("/")){ - endpoint.append(UPLOAD_FILE_FORM_API.substring(1)); - }else{ - endpoint.append(UPLOAD_FILE_FORM_API); + if (ossClientProperty.getRemote().endsWith("/")) { + endpoint.append(v1_upload_file_binary_api.substring(1)); + } else { + endpoint.append(v1_upload_file_binary_api); } - this.ossClientProperty.setRemote(endpoint.toString()); - } - } - - /*** - * 设置默认header - * @param request - */ - private void addDefaultHeader(HttpUriRequest request){ - request.addHeader("Content-Encoding","gzip"); - request.addHeader("Content-type", "application/json"); - } - - private void handleServerExceptionMessage(OSSClientMessage ossClientMessage,Exception e){ - ossClientMessage.setCode("8500"); - ossClientMessage.setMessage(e.getMessage()); - } - - /*** - * 创建数组 - * @param fileBytesRequests - * @return - */ - private JsonArray createFileArrs(List fileBytesRequests){ - JsonArray jsonArray=new JsonArray(); - for (FileBytesRequest fileBytesRequest:fileBytesRequests){ - JsonObject fileObj=new JsonObject(); - fileObj.addProperty("media_type",fileBytesRequest.getMediaType()); - fileObj.addProperty("file",fileBytesRequest.getFile()); - fileObj.addProperty("original_name",fileBytesRequest.getOriginalName()); - jsonArray.add(fileObj); - } - return jsonArray; - } - - /** - * 创建请求参数 - * @param fileBytesRequests - * @param project - * @return - */ - private JsonObject createRequestParams(List fileBytesRequests,String project){ - JsonObject param=new JsonObject(); - param.addProperty("project",project); - param.addProperty("appid",ossClientProperty.getAppid()); - param.addProperty("appsecret",ossClientProperty.getAppsecret()); - param.add("files",createFileArrs(fileBytesRequests)); - return param; - } - - private void addRequestParam(HttpPost request,List fileBytesRequests,String project){ - JsonObject param=createRequestParams(fileBytesRequests,ossClientProperty.getProject()); - request.setEntity(new StringEntity(param.toString(),"UTF-8")); - } - - - /*** - * 处理结果 - * @param closeableHttpResponse - * @param ossClientMessage - * @param type - * @throws IOException - */ - private void handleResult(CloseableHttpResponse closeableHttpResponse,OSSClientMessage ossClientMessage,Type type) throws IOException { - if (closeableHttpResponse.getStatusLine().getStatusCode()== HttpStatus.SC_OK) { - String content = EntityUtils.toString(closeableHttpResponse.getEntity(), "UTF-8"); - if (content != null && !"".equals(content)) { - JsonElement jsonE = new JsonParser().parse(content); - ossClientMessage.setCode(jsonE.getAsJsonObject().get("code").getAsString()); - ossClientMessage.setMessage(jsonE.getAsJsonObject().get("message").getAsString()); - JsonElement data=jsonE.getAsJsonObject().get("data"); - ossClientMessage.setData(new Gson().fromJson(data,type)); - } - } - } - - /*** - * 文件上传 - * @param file - * @return - */ - public OSSClientMessage uploadFile(File file){ - OSSClientMessage ossClientMessage=new OSSClientMessage<>(); - try{ - if (file.isDirectory()){ - //文件不能是目录 - throw new RuntimeException("file can't be directory "); - } - //获取文件原始名称 - String originalName=file.getName(); - String mediaType="unkown"; - int idx=originalName.lastIndexOf("."); - if (idx>0){ - mediaType=originalName.substring(idx+1); + if ("BASE64".equalsIgnoreCase(mode)){ + this.ossClientProperty.setRemote(endpoint.toString()); } - String filebyteString= Base64.encodeBase64String(FileUtils.readFileToByteArray(file)); - FileBytesRequest fileBytesRequest=new FileBytesRequest(); - fileBytesRequest.setFile(filebyteString); - fileBytesRequest.setMediaType(mediaType); - fileBytesRequest.setOriginalName(originalName); - ossClientMessage=uploadFileByte(fileBytesRequest); - }catch (Exception e){ - handleServerExceptionMessage(ossClientMessage,e); } - return ossClientMessage; } - /*** - * 批量上传文件 - * @param uploadFiles - * @return - */ - public OSSClientMessage> uploadFiles(List uploadFiles){ - OSSClientMessage> ossClientMessage=new OSSClientMessage<>(); - try{ - List fileBytesRequests=new ArrayList<>(); - for (File file:uploadFiles){ - if (file.isDirectory()){ - //文件不能是目录 - throw new RuntimeException("file {"+file.getName()+"} can't be directory "); - } - } - for (File file:uploadFiles){ - //获取文件原始名称 - String originalName=file.getName(); - String mediaType="unkown"; - int idx=originalName.lastIndexOf("."); - if (idx>0){ - mediaType=originalName.substring(idx+1); - } - String filebyteString= Base64.encodeBase64String(FileUtils.readFileToByteArray(file)); - FileBytesRequest fileBytesRequest=new FileBytesRequest(); - fileBytesRequest.setFile(filebyteString); - fileBytesRequest.setMediaType(mediaType); - fileBytesRequest.setOriginalName(originalName); - fileBytesRequests.add(fileBytesRequest); + public OSSClient(OSSClientProperty ossClientProperty) { + this.ossClientProperty = ossClientProperty; + this.mode = mode; + //初始化上传url地址 + if (ossClientProperty != null && ossClientProperty.getRemote() != null && !"".equalsIgnoreCase(ossClientProperty.getRemote())) { + StringBuffer endpoint = new StringBuffer(); + endpoint.append(ossClientProperty.getRemote()); + if (ossClientProperty.getRemote().endsWith("/")) { + endpoint.append(v1_upload_file_binary_api.substring(1)); + } else { + endpoint.append(v1_upload_file_binary_api); } - ossClientMessage=uploadFilesByte(fileBytesRequests); - }catch (Exception e){ - handleServerExceptionMessage(ossClientMessage,e); - } - return ossClientMessage; - } - /*** - * 字节字符串形式上传文件 - * @param fileBytesRequest - * @return - */ - public OSSClientMessage uploadFileByte(FileBytesRequest fileBytesRequest){ - OSSClientMessage ossClientMessage=new OSSClientMessage<>(); - CloseableHttpResponse closeableHttpResponse=null; - CloseableHttpClient httpClient=null; - try{ - HttpPost request=new HttpPost(ossClientProperty.getRemote()); - addDefaultHeader(request); - httpClient=HttpClients.createDefault(); - List fileBytesRequests=new ArrayList<>(); - fileBytesRequests.add(fileBytesRequest); - addRequestParam(request,fileBytesRequests,ossClientProperty.getProject()); - closeableHttpResponse=httpClient.execute(request); - if (closeableHttpResponse.getStatusLine().getStatusCode()== HttpStatus.SC_OK){ - String content= EntityUtils.toString(closeableHttpResponse.getEntity(),"UTF-8"); - if (content!=null&&!"".equals(content)){ - JsonElement jsonE=new JsonParser().parse(content); - ossClientMessage.setCode(jsonE.getAsJsonObject().get("code").getAsString()); - ossClientMessage.setMessage(jsonE.getAsJsonObject().get("message").getAsString()); - Type type=new TypeToken>(){}.getType(); - List fileBytesResponse=new Gson().fromJson(jsonE.getAsJsonObject().get("data"),type); - if (fileBytesResponse!=null&&fileBytesResponse.size()>0){ - ossClientMessage.setData(fileBytesResponse.get(0)); - } - } + if ("BASE64".equalsIgnoreCase(mode)){ + this.ossClientProperty.setRemote(endpoint.toString()); } - }catch (Exception e){ - handleServerExceptionMessage(ossClientMessage,e); } - return ossClientMessage; } - /*** - * form表单提交 + * 文件上传 * @param file * @return */ - public OSSClientMessage uploadFileByForm(File file){ - OSSClientMessage ossClientMessage=new OSSClientMessage<>(); - CloseableHttpResponse closeableHttpResponse=null; - CloseableHttpClient httpClient=null; + public OSSClientMessage uploadFile(File file) { + OSSClientMessage ossClientMessage = new OSSClientMessage<>(); try { - HttpPost request = new HttpPost(ossClientProperty.getRemote()); - httpClient=HttpClients.createDefault(); - MultipartEntityBuilder builder= MultipartEntityBuilder.create(); - //表单参数 - builder.addPart("appid",new StringBody(ossClientProperty.getAppid(), ContentType.MULTIPART_FORM_DATA)); - builder.addPart("appsecret",new StringBody(ossClientProperty.getAppsecret(), ContentType.MULTIPART_FORM_DATA)); - builder.addPart("project",new StringBody(ossClientProperty.getProject(), ContentType.MULTIPART_FORM_DATA)); - //添加文件 - builder.addPart("file",new FileBody(file)); - //参数赋值 - request.setEntity(builder.build()); - closeableHttpResponse=httpClient.execute(request); - if (closeableHttpResponse.getStatusLine().getStatusCode()== HttpStatus.SC_OK){ - String content= EntityUtils.toString(closeableHttpResponse.getEntity(),"UTF-8"); - if (content!=null&&!"".equals(content)){ - JsonElement jsonE=new JsonParser().parse(content); - ossClientMessage.setCode(jsonE.getAsJsonObject().get("code").getAsString()); - ossClientMessage.setMessage(jsonE.getAsJsonObject().get("message").getAsString()); - Type type=new TypeToken>(){}.getType(); - List fileBytesResponse=new Gson().fromJson(jsonE.getAsJsonObject().get("data"),type); - if (fileBytesResponse!=null&&fileBytesResponse.size()>0){ - ossClientMessage.setData(fileBytesResponse.get(0)); - } - } + if (file.isDirectory()) { + //文件不能是目录 + throw new RuntimeException("file can't be directory "); } - }catch (Exception e){ - handleServerExceptionMessage(ossClientMessage,e); - } - return ossClientMessage; - } - - - /** - * 批量上传文件 - * @param fileBytesRequests - * @return - */ - public OSSClientMessage> uploadFilesByte(List fileBytesRequests){ - OSSClientMessage> ossClientMessage=new OSSClientMessage<>(); - CloseableHttpResponse closeableHttpResponse=null; - CloseableHttpClient httpClient=null; - try{ - HttpPost request=new HttpPost(ossClientProperty.getRemote()); - addDefaultHeader(request); - httpClient=HttpClients.createDefault(); - addRequestParam(request,fileBytesRequests,ossClientProperty.getProject()); - closeableHttpResponse=httpClient.execute(request); - if (closeableHttpResponse.getStatusLine().getStatusCode()== HttpStatus.SC_OK){ - String content= EntityUtils.toString(closeableHttpResponse.getEntity(),"UTF-8"); - if (content!=null&&!"".equals(content)){ - JsonElement jsonE=new JsonParser().parse(content); - ossClientMessage.setCode(jsonE.getAsJsonObject().get("code").getAsString()); - ossClientMessage.setMessage(jsonE.getAsJsonObject().get("message").getAsString()); - Type type=new TypeToken>(){}.getType(); - List fileBytesResponse=new Gson().fromJson(jsonE.getAsJsonObject().get("data"),type); - if (fileBytesResponse!=null&&fileBytesResponse.size()>0){ - ossClientMessage.setData(fileBytesResponse); - } - } + IUpload handle = null; + switch (mode) { + case "BASE64": + handle = new Base64UploadHandle(ossClientProperty); + break; + default: + handle = new NettyUploadHandle(ossClientProperty); + break; } - }catch (Exception e){ - handleServerExceptionMessage(ossClientMessage,e); + ossClientMessage = handle.upload(file); + } catch (Exception e) { + ossClientMessage.setCode("8500"); + ossClientMessage.setMessage(e.getMessage()); } return ossClientMessage; - } - - - - - public static void main(String[] args) throws IOException { - //客户端上传 - String url="http://192.168.0.6:18000/oss/material/uploadByBinary"; - OSSClientProperty ossClientProperty=new OSSClientProperty(url,"province_IIII"); - OSSClient ossClient=new OSSClient(ossClientProperty); - File uploadFile=new File("C:\\Users\\xiaoymin\\Desktop\\aa.jpg"); - Gson gson=new Gson(); - OSSClientMessage ossClientMessage=ossClient.uploadFile(uploadFile); - System.out.println(gson.toJson(ossClientMessage)); - - /*System.out.println(uploadFile.getName()); - //获取文件原始名称 - String originalName=uploadFile.getName(); - String mediaType="unkown"; - int idx=originalName.lastIndexOf("."); - if (idx>0){ - mediaType=originalName.substring(idx+1); - } - System.out.println(mediaType); - String filebyteString= Base64.encodeBase64String(FileUtils.readFileToByteArray(uploadFile)); - CloseableHttpClient closeableHttpClient= HttpClients.createDefault(); - JsonObject fileObj=new JsonObject(); - fileObj.addProperty("media_type","png"); - fileObj.addProperty("file",filebyteString); - fileObj.addProperty("original_name","test.png"); - JsonArray jsonArray=new JsonArray(); - jsonArray.add(fileObj); - JsonObject param=new JsonObject(); - param.addProperty("project","province_III"); - param.add("files",jsonArray); - - request.addHeader("Content-Encoding","gzip"); - request.addHeader("Content-type", "application/json"); - request.setEntity(new StringEntity(param.toString(),"UTF-8")); - - CloseableHttpResponse response=closeableHttpClient.execute(request); - if (response.getStatusLine().getStatusCode()== HttpStatus.SC_OK){ - String content= EntityUtils.toString(response.getEntity(),"UTF-8"); - System.out.println(content); - }*/ - - - - - - - - - - - - - - - - } } diff --git a/oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/OSSClientProperty.java b/oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/OSSClientProperty.java index 2103192..9142b21 100644 --- a/oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/OSSClientProperty.java +++ b/oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/OSSClientProperty.java @@ -7,14 +7,16 @@ package com.xiaominfo.oss.sdk; +import java.io.Serializable; + /*** * client配置文件属性 * @since:oss-server 1.0 - * @author xiaoymin@foxmail.com + * @author xiaoymin@foxmail.com * 2018/05/30 19:22 */ -public class OSSClientProperty { - +public class OSSClientProperty implements Serializable { + private static final long serialVersionUID = 1L; /*** * 远程oss上传地址 */ diff --git a/oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/client/FileBytesRequest.java b/oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/client/FileBytesRequest.java index 929aaa8..e7cbb65 100644 --- a/oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/client/FileBytesRequest.java +++ b/oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/client/FileBytesRequest.java @@ -7,17 +7,15 @@ package com.xiaominfo.oss.sdk.client; -import com.google.gson.annotations.SerializedName; /*** * * @since:oss-server 1.0 - * @author xiaoymin@foxmail.com + * @author xiaoymin@foxmail.com * 2018/05/30 15:12 */ public class FileBytesRequest { - @SerializedName(value = "original_name") private String originalName; /*** @@ -27,8 +25,19 @@ public class FileBytesRequest { */ private String file; - @SerializedName(value = "media_type") private String mediaType; + /** + * 模块的名称 + */ + private String module; + + public String getModule() { + return module; + } + + public void setModule(String module) { + this.module = module; + } public String getOriginalName() { return originalName; diff --git a/oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/client/NettyFileRequest.java b/oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/client/NettyFileRequest.java new file mode 100644 index 0000000..67f70d4 --- /dev/null +++ b/oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/client/NettyFileRequest.java @@ -0,0 +1,134 @@ +/* + * Copyright (C) 2018 Zhejiang lishiots Technology CO.,LTD. + * All rights reserved. + * Official Web Site: http://www.lishiots.com. + * Developer Web Site: http://open.lishiots.com. + */ + +package com.xiaominfo.oss.sdk.client; + + +import com.xiaominfo.oss.sdk.OSSClientProperty; + +import java.io.File; +import java.io.Serializable; + +/*** + * + * @since:oss-server 1.0 + * @author xiaoymin@foxmail.com + * 2018/05/30 15:12 + */ +public class NettyFileRequest extends OSSClientProperty implements Serializable { + private static final long serialVersionUID = 1L; + + private String originalName; + + private File file; + + private String mediaType; + /** + * 模块的名称 + */ + private String module; + + private int starPos;// 开始位置 + private byte[] bytes;// 文件字节数组 + private int endPos;// 结尾位置 + + + private String projectPath;//项目的路径 + private String code;//通讯的code + private String message;//通讯的message + + private String uuid;//文件的id + + + public String getOriginalName() { + return originalName; + } + + public void setOriginalName(String originalName) { + this.originalName = originalName; + } + + public File getFile() { + return file; + } + + public void setFile(File file) { + this.file = file; + } + + public String getMediaType() { + return mediaType; + } + + public void setMediaType(String mediaType) { + this.mediaType = mediaType; + } + + public String getModule() { + return module; + } + + public void setModule(String module) { + this.module = module; + } + + public int getStarPos() { + return starPos; + } + + public void setStarPos(int starPos) { + this.starPos = starPos; + } + + public byte[] getBytes() { + return bytes; + } + + public void setBytes(byte[] bytes) { + this.bytes = bytes; + } + + public int getEndPos() { + return endPos; + } + + public void setEndPos(int endPos) { + this.endPos = endPos; + } + + public String getProjectPath() { + return projectPath; + } + + public void setProjectPath(String projectPath) { + this.projectPath = projectPath; + } + + public String getCode() { + return code; + } + + public void setCode(String code) { + this.code = code; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + public String getUuid() { + return uuid; + } + + public void setUuid(String uuid) { + this.uuid = uuid; + } +} diff --git a/oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/upload/IUpload.java b/oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/upload/IUpload.java new file mode 100644 index 0000000..f0aecd9 --- /dev/null +++ b/oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/upload/IUpload.java @@ -0,0 +1,21 @@ +package com.xiaominfo.oss.sdk.upload; + +import com.xiaominfo.oss.sdk.OSSClientProperty; +import com.xiaominfo.oss.sdk.client.FileBytesRequest; +import com.xiaominfo.oss.sdk.client.FileBytesResponse; +import com.xiaominfo.oss.sdk.common.OSSClientMessage; + +import java.io.File; + +public interface IUpload { + + + /** + * 文件上传 + * + * @return + */ + OSSClientMessage upload(File file); + + +} diff --git a/oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/upload/handle/Base64UploadHandle.java b/oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/upload/handle/Base64UploadHandle.java new file mode 100644 index 0000000..79025b4 --- /dev/null +++ b/oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/upload/handle/Base64UploadHandle.java @@ -0,0 +1,142 @@ +package com.xiaominfo.oss.sdk.upload.handle; + +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; +import com.xiaominfo.oss.sdk.OSSClientProperty; +import com.xiaominfo.oss.sdk.client.FileBytesRequest; +import com.xiaominfo.oss.sdk.client.FileBytesResponse; +import com.xiaominfo.oss.sdk.common.OSSClientMessage; +import com.xiaominfo.oss.sdk.upload.IUpload; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.io.FileUtils; +import org.apache.http.HttpStatus; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; + + +public class Base64UploadHandle implements IUpload { + public OSSClientProperty ossClientProperty; + + public Base64UploadHandle(OSSClientProperty ossClientProperty) { + this.ossClientProperty = ossClientProperty; + } + + /*** + * 字节字符串形式上传文件 + * @param file + * @return + */ + + @Override + public OSSClientMessage upload(File file) { + OSSClientMessage ossClientMessage = new OSSClientMessage<>(); + CloseableHttpResponse closeableHttpResponse = null; + CloseableHttpClient httpClient = null; + try { + //获取文件原始名称 + String originalName = file.getName(); + String mediaType = "unkown"; + int idx = originalName.lastIndexOf("."); + if (idx > 0) { + mediaType = originalName.substring(idx + 1); + } + String filebyteString = Base64.encodeBase64String(FileUtils.readFileToByteArray(file)); + FileBytesRequest fileBytesRequest = new FileBytesRequest(); + fileBytesRequest.setFile(filebyteString); + fileBytesRequest.setMediaType(mediaType); + fileBytesRequest.setOriginalName(originalName); + + + HttpPost request = new HttpPost(ossClientProperty.getRemote()); + addDefaultHeader(request); + httpClient = HttpClients.createDefault(); + List fileBytesRequests = new ArrayList<>(); + fileBytesRequests.add(fileBytesRequest); + addRequestParam(request, fileBytesRequests, ossClientProperty.getProject()); + closeableHttpResponse = httpClient.execute(request); + if (closeableHttpResponse.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { + String content = EntityUtils.toString(closeableHttpResponse.getEntity(), "UTF-8"); + if (content != null && !"".equals(content)) { + JSONObject jsonObject = JSONObject.parseObject(content); + String code = jsonObject.get("code").toString(); + String message = jsonObject.get("message").toString(); + String data = jsonObject.get("data") + ""; + ossClientMessage.setCode(code); + ossClientMessage.setMessage(message); + + List fileBytesResponse = JSONArray.parseArray(data, FileBytesResponse.class); + + if (fileBytesResponse != null && fileBytesResponse.size() > 0) { + ossClientMessage.setData(fileBytesResponse.get(0)); + } + } + } + } catch (Exception e) { + handleServerExceptionMessage(ossClientMessage, e); + } + return ossClientMessage; + } + + /*** + * 设置默认header + * @param request + */ + private void addDefaultHeader(HttpUriRequest request) { + request.addHeader("Content-Encoding", "gzip"); + request.addHeader("Content-type", "application/json"); + } + + private void handleServerExceptionMessage(OSSClientMessage ossClientMessage, Exception e) { + ossClientMessage.setCode("8500"); + ossClientMessage.setMessage(e.getMessage()); + } + + private void addRequestParam(HttpPost request, List fileBytesRequests, String project) { + JSONObject param = createRequestParams(fileBytesRequests, ossClientProperty.getProject()); + request.setEntity(new StringEntity(param.toString(), "UTF-8")); + } + + /** + * 创建请求参数 + * + * @param fileBytesRequests + * @param project + * @return + */ + private JSONObject createRequestParams(List fileBytesRequests, String project) { + JSONObject param = new JSONObject(); + param.put("project", project); + param.put("appid", ossClientProperty.getAppid()); + param.put("appsecret", ossClientProperty.getAppsecret()); + param.put("files", createFileArrs(fileBytesRequests)); + return param; + } + + + /*** + * 创建数组 + * @param fileBytesRequests + * @return + */ + private JSONArray createFileArrs(List fileBytesRequests) { + JSONArray jsonArray = new JSONArray(); + for (FileBytesRequest fileBytesRequest : fileBytesRequests) { + JSONObject fileObj = new JSONObject(); + fileObj.put("media_type", fileBytesRequest.getMediaType()); + fileObj.put("file", fileBytesRequest.getFile()); + fileObj.put("original_name", fileBytesRequest.getOriginalName()); + jsonArray.add(fileObj); + } + return jsonArray; + } + +} diff --git a/oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/upload/handle/NettyUploadHandle.java b/oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/upload/handle/NettyUploadHandle.java new file mode 100644 index 0000000..3f8d702 --- /dev/null +++ b/oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/upload/handle/NettyUploadHandle.java @@ -0,0 +1,59 @@ +package com.xiaominfo.oss.sdk.upload.handle; + + +import com.xiaominfo.oss.sdk.OSSClientProperty; +import com.xiaominfo.oss.sdk.client.FileBytesResponse; +import com.xiaominfo.oss.sdk.client.NettyFileRequest; +import com.xiaominfo.oss.sdk.common.OSSClientMessage; +import com.xiaominfo.oss.sdk.upload.IUpload; +import com.xiaominfo.oss.sdk.upload.handle.netty.NettyClient; + +import java.io.File; + +public class NettyUploadHandle implements IUpload { + public OSSClientProperty ossClientProperty; + + public NettyUploadHandle(OSSClientProperty ossClientProperty) { + this.ossClientProperty = ossClientProperty; + } + + @Override + public OSSClientMessage upload(File file) { + OSSClientMessage ossClientMessage = new OSSClientMessage<>(); + + try { + //获取文件原始名称 + NettyFileRequest nettyFileRequest = initNettyFileRequest(file); + String[] node = ossClientProperty.getRemote().split(":"); + + //调用 + NettyClient client = new NettyClient(node[0], Integer.parseInt(node[1])); + client.connect(nettyFileRequest); + ossClientMessage = client.getResponse(); + } catch (Exception e) { + ossClientMessage.setCode("8500"); + ossClientMessage.setMessage(e.getMessage()); + e.printStackTrace(); + } + return ossClientMessage; + } + + private NettyFileRequest initNettyFileRequest(File file) { + NettyFileRequest nettyFileRequest = new NettyFileRequest(); + //基础参数 + nettyFileRequest.setAppid(ossClientProperty.getAppid()); + nettyFileRequest.setAppsecret(ossClientProperty.getAppsecret()); + nettyFileRequest.setProject(ossClientProperty.getProject()); + //文件信息 + String originalName = file.getName(); + String mediaType = "unkown"; + int idx = originalName.lastIndexOf("."); + if (idx > 0) { + mediaType = originalName.substring(idx + 1); + } + nettyFileRequest.setOriginalName(originalName); + nettyFileRequest.setMediaType(mediaType); + nettyFileRequest.setFile(file); + return nettyFileRequest; + } +} diff --git a/oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/upload/handle/netty/NettyClient.java b/oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/upload/handle/netty/NettyClient.java new file mode 100644 index 0000000..aa9e7ca --- /dev/null +++ b/oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/upload/handle/netty/NettyClient.java @@ -0,0 +1,74 @@ +package com.xiaominfo.oss.sdk.upload.handle.netty; + +import com.xiaominfo.oss.sdk.client.FileBytesResponse; +import com.xiaominfo.oss.sdk.client.NettyFileRequest; +import com.xiaominfo.oss.sdk.common.OSSClientMessage; +import io.netty.bootstrap.Bootstrap; +import io.netty.channel.ChannelFuture; +import io.netty.channel.ChannelOption; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.nio.NioSocketChannel; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.CountDownLatch; + +public class NettyClient { + Logger logger = LoggerFactory.getLogger(NettyClient.class); + + private OSSClientMessage response; + private final String host; + private final int port; + + public NettyClient(String host, int port) { + this.host = host; + this.port = port; + } + + + public void connect(final NettyFileRequest syncMessage) throws Exception { + EventLoopGroup group = new NioEventLoopGroup(); + try { + final CountDownLatch lathc = new CountDownLatch(1); + NettyClientInitializer handler = new NettyClientInitializer(syncMessage, lathc); + Bootstrap b = new Bootstrap(); + b.group(group).channel(NioSocketChannel.class); + b.option(ChannelOption.TCP_NODELAY, true); + b.handler(handler); + ChannelFuture f = b.connect(host, port).sync(); + f.channel().closeFuture().sync(); + lathc.await(); + + NettyMessage message = handler.getMessage(); + logger.info("message ={}", message); + + OSSClientMessage clientMessage = new OSSClientMessage<>(); + FileBytesResponse response = new FileBytesResponse(message.getMaterialId(), message.getUrl(), message.getFilePath()); + response.setOriginalName(syncMessage.getOriginalName()); + response.setObjType(syncMessage.getMediaType()); + response.setByteLength(syncMessage.getFile().length()); + + if ("0000".equalsIgnoreCase(message.getCode())) { + clientMessage.setData(response); + } + clientMessage.setCode(message.getCode()); + clientMessage.setMessage(message.getMessage()); + this.response = clientMessage; + logger.info("FileUploadClient connect()结束"); + } finally { + group.shutdownGracefully(); + } + } + + public OSSClientMessage getResponse() { + return this.response; + } + + public void setResponse(OSSClientMessage response) { + this.response = response; + } +} + + + diff --git a/oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/upload/handle/netty/NettyClientHandler.java b/oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/upload/handle/netty/NettyClientHandler.java new file mode 100644 index 0000000..d213341 --- /dev/null +++ b/oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/upload/handle/netty/NettyClientHandler.java @@ -0,0 +1,180 @@ +package com.xiaominfo.oss.sdk.upload.handle.netty; + +import com.xiaominfo.oss.sdk.client.NettyFileRequest; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.util.concurrent.CountDownLatch; + +public class NettyClientHandler extends ChannelInboundHandlerAdapter { + private CountDownLatch lathc; + + private int byteRead; + private volatile int start = 0; + private volatile int lastLength = 0; + public RandomAccessFile randomAccessFile; + private NettyFileRequest syncMessage; + + private NettyMessage message; + private Logger log = LoggerFactory.getLogger(NettyClientHandler.class); + + public NettyClientHandler(NettyFileRequest ef, CountDownLatch lathc) { + if (ef.getFile().exists()) { + if (!ef.getFile().isFile()) { + log.info("Not a file :{}", ef.getFile()); + return; + } + } + this.syncMessage = ef; + this.lathc = lathc; + } + + @Override + public void channelInactive(ChannelHandlerContext ctx) throws Exception { + // TODO Auto-generated method stub + super.channelInactive(ctx); + log.info("客户端结束传递文件channelInactive()"); + } + + @Override + public void channelActive(ChannelHandlerContext ctx) { + try { + randomAccessFile = new RandomAccessFile(syncMessage.getFile(), + "r"); + randomAccessFile.seek(syncMessage.getStarPos()); + // lastLength = (int) randomAccessFile.length() / 10; + lastLength = 1024 * 10; + byte[] bytes = new byte[lastLength]; + if ((byteRead = randomAccessFile.read(bytes)) != -1) { + syncMessage.setEndPos(byteRead); + syncMessage.setBytes(bytes); + ctx.writeAndFlush(syncMessage); + } else { + } + //log.info("channelActive()文件已经读完 " + byteRead); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } catch (IOException i) { + i.printStackTrace(); + } + } + + @Override + public void channelRead(ChannelHandlerContext ctx, Object msg) + throws Exception { + if (msg instanceof Integer) { + start = (Integer) msg; + if (start != -1) { + randomAccessFile = new RandomAccessFile( + syncMessage.getFile(), "r"); + randomAccessFile.seek(start); + //log.info("长度：" + (randomAccessFile.length() - start)); + int a = (int) (randomAccessFile.length() - start); + int b = (int) (randomAccessFile.length() / 1024 * 2); + if (a < lastLength) { + lastLength = a; + } + //log.info("文件长度：" + (randomAccessFile.length()) + ",start:" + start + ",a:" + a + ",b:" + b + ",lastLength:" + lastLength); + byte[] bytes = new byte[lastLength]; + // log.info("-----------------------------" + bytes.length); + if ((byteRead = randomAccessFile.read(bytes)) != -1 + && (randomAccessFile.length() - start) > 0) { + // log.info("byte 长度：" + bytes.length); + syncMessage.setEndPos(byteRead); + syncMessage.setBytes(bytes); + try { + ctx.writeAndFlush(syncMessage); + } catch (Exception e) { + e.printStackTrace(); + } + } else { + randomAccessFile.close(); + ctx.close(); + log.info("文件已经读完channelRead()--------"); + lathc.countDown(); + } + } + } + + + if (msg instanceof NettyMessage) { + NettyMessage message = (NettyMessage) msg; + start = message.getStarPos(); + + if (!"0000".equalsIgnoreCase(message.getCode())) { + message.setMaterialId(""); + message.setUrl(""); + message.setFilePath(""); + ctx.close(); + log.info("上传文件出错:{}", message.getMessage()); + lathc.countDown(); + this.message = message; + } + + if (start != -1) { + randomAccessFile = new RandomAccessFile( + syncMessage.getFile(), "r"); + randomAccessFile.seek(start); + //log.info("长度：" + (randomAccessFile.length() - start)); + int a = (int) (randomAccessFile.length() - start); + int b = (int) (randomAccessFile.length() / 1024 * 2); + if (a < lastLength) { + lastLength = a; + } + message.setLength(randomAccessFile.length()); + //log.info("文件长度：" + (randomAccessFile.length()) + ",start:" + start + ",a:" + a + ",b:" + b + ",lastLength:" + lastLength); + byte[] bytes = new byte[lastLength]; + // log.info("-----------------------------" + bytes.length); + if ((byteRead = randomAccessFile.read(bytes)) != -1 + && (randomAccessFile.length() - start) > 0) { + // log.info("byte 长度：" + bytes.length); + syncMessage.setEndPos(byteRead); + syncMessage.setBytes(bytes); + syncMessage.setProjectPath(message.getFilePath()); + syncMessage.setUuid(message.getMaterialId()); + try { + ctx.writeAndFlush(syncMessage); + } catch (Exception e) { + e.printStackTrace(); + } + } else { + randomAccessFile.close(); + ctx.close(); + log.info("文件已经读完channelRead()--------" + byteRead); + lathc.countDown(); + this.message = message; + } + } else { + log.info("-11111111111111111111111111111111111111"); + } + } + + + } + + public void resetLatch(CountDownLatch initLathc) { + this.lathc = initLathc; + } + + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { + cause.printStackTrace(); + ctx.close(); + } + + + public NettyMessage getMessage() { + return message; + } + + public void setMessage(NettyMessage message) { + this.message = message; + } +} + diff --git a/oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/upload/handle/netty/NettyClientInitializer.java b/oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/upload/handle/netty/NettyClientInitializer.java new file mode 100644 index 0000000..5d2e3fc --- /dev/null +++ b/oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/upload/handle/netty/NettyClientInitializer.java @@ -0,0 +1,41 @@ +package com.xiaominfo.oss.sdk.upload.handle.netty; + +import com.xiaominfo.oss.sdk.client.NettyFileRequest; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.socket.SocketChannel; +import io.netty.handler.codec.serialization.ClassResolvers; +import io.netty.handler.codec.serialization.ObjectDecoder; +import io.netty.handler.codec.serialization.ObjectEncoder; + +import java.util.concurrent.CountDownLatch; + +public class NettyClientInitializer extends ChannelInitializer { + private CountDownLatch lathc; + private NettyClientHandler handler; + private NettyFileRequest syncMessage; + + public NettyClientInitializer(NettyFileRequest syncMessage, CountDownLatch lathc) { + this.syncMessage = syncMessage; + this.lathc = lathc; + } + + @Override + protected void initChannel(SocketChannel ch) throws Exception { + handler = new NettyClientHandler(syncMessage, lathc); + + ch.pipeline().addLast(new ObjectEncoder()); + ch.pipeline().addLast(new ObjectDecoder(ClassResolvers.weakCachingConcurrentResolver(null))); + //ch.pipeline().addLast(new NettyClientHandler(syncMessage, lathc)); + ch.pipeline().addLast(handler); + } + + public NettyMessage getMessage() { + return handler.getMessage(); + } + + //重置同步锁 + public void resetLathc(CountDownLatch initLathc) { + handler.resetLatch(initLathc); + } + +} diff --git a/oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/upload/handle/netty/NettyMessage.java b/oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/upload/handle/netty/NettyMessage.java new file mode 100644 index 0000000..a33d4a6 --- /dev/null +++ b/oss-server-sdk-java/src/main/java/com/xiaominfo/oss/sdk/upload/handle/netty/NettyMessage.java @@ -0,0 +1,128 @@ +package com.xiaominfo.oss.sdk.upload.handle.netty; + +import java.io.File; +import java.io.Serializable; +import java.util.Arrays; + +public class NettyMessage implements Serializable { + private static final long serialVersionUID = 1L; + private File file;// 文件 + private String filePath;// 文件路径 + private String materialId;//对象的id + private String file_md5;// 文件名 + private int starPos;// 开始位置 + private byte[] bytes;// 文件字节数组 + private int endPos;// 结尾位置 + private long length;// 文件总长度 + private String url; + + private String code;//通讯的code + private String message;//通讯的message + + public String getCode() { + return code; + } + + public void setCode(String code) { + this.code = code; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + public long getLength() { + return length; + } + + public void setLength(long length) { + this.length = length; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public String getMaterialId() { + return materialId; + } + + public void setMaterialId(String materialId) { + this.materialId = materialId; + } + + public String getFilePath() { + return filePath; + } + + public void setFilePath(String filePath) { + this.filePath = filePath; + } + + public File getFile() { + return file; + } + + public void setFile(File file) { + this.file = file; + } + + public String getFile_md5() { + return file_md5; + } + + public void setFile_md5(String file_md5) { + this.file_md5 = file_md5; + } + + public int getStarPos() { + return starPos; + } + + public void setStarPos(int starPos) { + this.starPos = starPos; + } + + public byte[] getBytes() { + return bytes; + } + + public void setBytes(byte[] bytes) { + this.bytes = bytes; + } + + public int getEndPos() { + return endPos; + } + + public void setEndPos(int endPos) { + this.endPos = endPos; + } + + public static long getSerialversionuid() { + return serialVersionUID; + } + + @Override + public String toString() { + return "NettyMessage{" + + "file=" + file + + ", filePath='" + filePath + '\'' + + ", materialId='" + materialId + '\'' + + ", file_md5='" + file_md5 + '\'' + + ", starPos=" + starPos + + ", bytes=" + Arrays.toString(bytes) + + ", endPos=" + endPos + + ", url='" + url + '\'' + + '}'; + } +} + diff --git a/oss-server-sdk-java/src/test/java/com/xiaominfo/oss/sdk/OSSClientTest.java b/oss-server-sdk-java/src/test/java/com/xiaominfo/oss/sdk/OSSClientTest.java new file mode 100644 index 0000000..69f9b2f --- /dev/null +++ b/oss-server-sdk-java/src/test/java/com/xiaominfo/oss/sdk/OSSClientTest.java @@ -0,0 +1,45 @@ +package com.xiaominfo.oss.sdk; + +import com.alibaba.fastjson.JSON; +import com.xiaominfo.oss.sdk.client.FileBytesResponse; +import com.xiaominfo.oss.sdk.common.OSSClientMessage; +import org.junit.Test; + +import java.io.File; + +public class OSSClientTest { + + @Test + public void uploadFile() { + //客户端上传 + String url = "http://127.0.0.1:18000"; + OSSClientProperty ossClientProperty = new OSSClientProperty(url, "province_IIII"); + ossClientProperty.setAppid("zh"); + ossClientProperty.setAppsecret("123123"); + OSSClient ossClient = new OSSClient(ossClientProperty, "BASE64"); + //File uploadFile=new File("C:\\Users\\xiaoymin\\Desktop\\aa.jpg"); + //File uploadFile = new File("D:\\source\\oss-server\\static\\wechat.jpg"); + File uploadFile = new File("F:\\得仕\\页面\\得仕宝前台页面改动效果图-20150430\\效果图 得仕宝产品列表页-修改后.png"); + + OSSClientMessage ossClientMessage = ossClient.uploadFile(uploadFile); + + System.out.println(JSON.toJSONString(ossClientMessage)); + } + + @Test + public void nettyUploadFile() { + //客户端上传 + String url = "127.0.0.1:18001"; + OSSClientProperty ossClientProperty = new OSSClientProperty(url, "province_IIII"); + ossClientProperty.setAppid("zh"); + ossClientProperty.setAppsecret("123123"); + OSSClient ossClient = new OSSClient(ossClientProperty); + //File uploadFile = new File("D:\\source\\oss-server\\static\\wechat.jpg"); + File uploadFile = new File("F:\\得仕\\页面\\得仕宝前台页面改动效果图-20150430\\效果图 得仕宝产品列表页-修改后.png"); + + OSSClientMessage ossClientMessage = ossClient.uploadFile(uploadFile); + + System.out.println("==========================================="); + System.out.println(JSON.toJSONString(ossClientMessage)); + } +} diff --git a/oss-server-spring-boot-starter/pom.xml b/oss-server-spring-boot-starter/pom.xml index 0ba6510..7567c7c 100644 --- a/oss-server-spring-boot-starter/pom.xml +++ b/oss-server-spring-boot-starter/pom.xml @@ -2,6 +2,12 @@ + + com.xiaominfo.oss + oss-server-parent + 1.1 + + 4.0.0 com.xiaominfo.oss oss-server-spring-boot-starter @@ -14,17 +20,6 @@ - - - xiaoymin - xiaoymin - xiaoymin@foxmail.com - 2018-6-25 10:30:55 - - Java Developer Enginner - - - UTF-8 1.7 diff --git a/oss-server/pom.xml b/oss-server/pom.xml index 5ee22e9..f4f0a8c 100644 --- a/oss-server/pom.xml +++ b/oss-server/pom.xml @@ -1,250 +1,259 @@ - 4.0.0 - - com.xiaominfo.oss - oss-server - 1.1 - jar - oss-server - oss-server上传组件,该组件为项目服务,可单独部署迁移 - - org.springframework.boot - spring-boot-starter-parent - 2.0.2.RELEASE - - - - - UTF-8 - UTF-8 - 1.8 - 1.7.25 - 1.2.3 - - true - - - - xiaoym - 肖玉民 - lishi - xiaoymin@foxmail.com - - Java Development Enginner - - 2018-5-30 10:21:40 - - - - - - org.springframework.boot - spring-boot-starter-web - - - org.springframework.boot - spring-boot-starter-aop - - - org.springframework.boot - spring-boot-starter-test - test - - - - org.springframework.boot - spring-boot-starter-freemarker - - - - - commons-io - commons-io - 2.6 - - - - com.google.code.gson - gson - 2.8.2 - - - - com.google.guava - guava - 24.0-jre - - - - commons-codec - commons-codec - 1.11 - - - - - - joda-time - joda-time - 2.9.9 - - - - net.logstash.logback - logstash-logback-encoder - 4.7 - - - org.slf4j - slf4j-api - ${org.slf4j.version} - - - org.slf4j - jcl-over-slf4j - ${org.slf4j.version} - runtime - - - ch.qos.logback - logback-classic - ${logback.version} - - - javax.mail - mail - - - javax.jms - jms - - - com.sun.jdmk - jmxtools - - - com.sun.jmx - jmxri - - - runtime - - - - cn.hutool - hutool-all - 4.0.7 - - - - com.alibaba - druid - 1.1.9 - - - - org.xerial - sqlite-jdbc - 3.21.0.1 - - - - com.baomidou - mybatisplus-spring-boot-starter - 1.0.5 - - - - - com.baomidou - mybatis-plus - 2.1.9 - - - - - org.apache.poi - poi - 3.17 - - - - org.apache.poi - poi-ooxml - 3.17 - - - - - - - - org.springframework.boot - spring-boot-maven-plugin - - - - org.apache.maven.plugins - maven-javadoc-plugin - 2.9.1 - - - - - org.apache.maven.plugins - maven-source-plugin - - - attach-sources - - jar - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - - 1.8 - 1.8 - UTF-8 - - - - org.apache.maven.plugins - maven-surefire-plugin - 2.18.1 - - ${skipTests} - - - - - - src/main/java - - **/*.properties - **/*.xml - - false - - - src/main/resources - - **/*.properties - **/*.xml - **/*.json - **/static/** - **/templates/** - - false - - - + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + + com.xiaominfo.oss + oss-server-parent + 1.1 + + + 4.0.0 + + com.xiaominfo.oss + oss-server + 1.1 + jar + oss-server + oss-server上传组件,该组件为项目服务,可单独部署迁移 + + + + UTF-8 + UTF-8 + 1.8 + 1.7.25 + 1.2.3 + + true + + + + xiaoym + 肖玉民 + lishi + xiaoymin@foxmail.com + + Java Development Enginner + + 2018-5-30 10:21:40 + + + + + + + com.xiaominfo.oss + oss-server-sdk-java + ${project.version} + + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-starter-aop + + + org.springframework.boot + spring-boot-starter-test + test + + + + org.springframework.boot + spring-boot-starter-freemarker + + + + + commons-io + commons-io + 2.6 + + + + + com.google.guava + guava + 24.0-jre + + + + commons-codec + commons-codec + 1.11 + + + + + + joda-time + joda-time + 2.9.9 + + + + net.logstash.logback + logstash-logback-encoder + 4.7 + + + org.slf4j + slf4j-api + ${org.slf4j.version} + + + org.slf4j + jcl-over-slf4j + ${org.slf4j.version} + runtime + + + ch.qos.logback + logback-classic + ${logback.version} + + + javax.mail + mail + + + javax.jms + jms + + + com.sun.jdmk + jmxtools + + + com.sun.jmx + jmxri + + + runtime + + + + cn.hutool + hutool-all + 4.0.7 + + + + com.alibaba + druid + 1.1.9 + + + + com.oracle + ojdbc6 + 10.2.0.1.0 + + + + com.baomidou + mybatisplus-spring-boot-starter + 1.0.5 + + + + + com.baomidou + mybatis-plus + 2.1.9 + + + + + org.apache.poi + poi + 3.17 + + + + org.apache.poi + poi-ooxml + 3.17 + + + + io.netty + netty-all + 4.1.25.Final + + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + org.apache.maven.plugins + maven-javadoc-plugin + 2.9.1 + + + + + org.apache.maven.plugins + maven-source-plugin + + + attach-sources + + jar + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.8 + 1.8 + UTF-8 + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.18.1 + + ${skipTests} + + + + + + src/main/java + + **/*.properties + **/*.xml + + false + + + src/main/resources + + **/*.properties + **/*.xml + **/*.json + **/static/** + **/templates/** + + false + + + diff --git a/oss-server/src/main/java/com/xiaominfo/oss/OssServerApplication.java b/oss-server/src/main/java/com/xiaominfo/oss/OssServerApplication.java index 89db9d1..739901c 100644 --- a/oss-server/src/main/java/com/xiaominfo/oss/OssServerApplication.java +++ b/oss-server/src/main/java/com/xiaominfo/oss/OssServerApplication.java @@ -2,9 +2,13 @@ package com.xiaominfo.oss; import cn.hutool.log.Log; import cn.hutool.log.LogFactory; +import com.xiaominfo.oss.common.SpringContextUtil; +import com.xiaominfo.oss.sync.Sync; +import com.xiaominfo.oss.sync.netty.MasterServer; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.ComponentScan; import org.springframework.core.env.Environment; import org.springframework.scheduling.annotation.EnableScheduling; @@ -13,20 +17,39 @@ import java.net.UnknownHostException; @SpringBootApplication @EnableScheduling +@ComponentScan(basePackages = {"com.xiaominfo.oss"}) public class OssServerApplication { - private final static Log log= LogFactory.get(); - - public static void main(String[] args) throws UnknownHostException { - ConfigurableApplicationContext application=SpringApplication.run(OssServerApplication.class, args); - Environment env = application.getEnvironment(); - log.info("\n----------------------------------------------------------\n\t" + - "Application '{}' is running! Access URLs:\n\t" + - "Local: \t\thttp://localhost:{}\n\t" + - "External: \thttp://{}:{}\n----------------------------------------------------------", - env.getProperty("spring.application.name"), - env.getProperty("server.port"), - InetAddress.getLocalHost().getHostAddress(), - env.getProperty("server.port")); - } + private final static Log log = LogFactory.get(); + + + public static void main(String[] args) throws UnknownHostException { + ConfigurableApplicationContext application = SpringApplication.run(OssServerApplication.class, args); + + //Sync.startMaster(); + Environment env = application.getEnvironment(); + log.info("\n--------------------------------------------------------------------------------------------------------------------\n\t" + + "Application '{}' is running! Access URLs:\n\t" + + "Local: \t\thttp://localhost:{}\n\t" + + "External: \thttp://{}:{}\n--------------------------------------------------------------------------------------------------------------------", + env.getProperty("spring.application.name"), + env.getProperty("server.port"), + InetAddress.getLocalHost().getHostAddress(), + env.getProperty("server.port")); + startMaster(); + } + + public static void startMaster() { + ((Runnable) () -> { + try { + String i = SpringContextUtil.getProperty("material.masterPort"); + int masterPort = Integer.parseInt(i); + new MasterServer().bind(masterPort); + } catch (Exception e) { + e.printStackTrace(); + } + }).run(); + + } + } diff --git a/oss-server/src/main/java/com/xiaominfo/oss/api/MaterialApplication.java b/oss-server/src/main/java/com/xiaominfo/oss/api/MaterialApplication.java index 3f92a67..820c9a5 100644 --- a/oss-server/src/main/java/com/xiaominfo/oss/api/MaterialApplication.java +++ b/oss-server/src/main/java/com/xiaominfo/oss/api/MaterialApplication.java @@ -10,17 +10,13 @@ package com.xiaominfo.oss.api; import cn.hutool.core.util.StrUtil; import cn.hutool.log.Log; import cn.hutool.log.LogFactory; -import com.google.gson.Gson; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import com.google.gson.reflect.TypeToken; -import com.google.gson.stream.JsonReader; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; import com.xiaominfo.oss.common.annotation.NotLogin; import com.xiaominfo.oss.common.pojo.Pagination; +import com.xiaominfo.oss.common.pojo.RestfulMessage; import com.xiaominfo.oss.domain.FileBinaryRequest; import com.xiaominfo.oss.domain.FileBinaryResponse; -import com.xiaominfo.oss.common.pojo.RestfulMessage; import com.xiaominfo.oss.exception.AssemblerException; import com.xiaominfo.oss.exception.ErrorCable; import com.xiaominfo.oss.exception.ErrorConstant; @@ -30,10 +26,10 @@ import com.xiaominfo.oss.module.model.OSSDeveloper; import com.xiaominfo.oss.module.model.OSSInformation; import com.xiaominfo.oss.module.model.OSSMaterialInfo; import com.xiaominfo.oss.service.*; +import io.netty.util.internal.StringUtil; import org.apache.commons.codec.net.URLCodec; import org.apache.poi.ss.usermodel.Workbook; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.http.HttpEntity; import org.springframework.web.bind.annotation.*; import org.springframework.web.multipart.MultipartFile; @@ -43,22 +39,20 @@ import javax.servlet.http.HttpServletResponse; import java.io.File; import java.io.IOException; import java.io.OutputStream; -import java.io.StringReader; -import java.lang.reflect.Type; import java.util.List; /*** * * @since:oss-server 1.0 - * @author xiaoymin@foxmail.com + * @author xiaoymin@foxmail.com * 2018/05/30 10:47 */ @RestController @RequestMapping("/oss/material") -public class MaterialApplication extends RootApis{ +public class MaterialApplication extends RootApis { - private Log log= LogFactory.get(); + private Log log = LogFactory.get(); @Autowired MaterialService materialService; @@ -80,46 +74,46 @@ public class MaterialApplication extends RootApis{ @GetMapping("/queryByPage") public Pagination queryByPage(OSSMaterialInfo ossMaterialInfo, - @RequestParam(value = "page",defaultValue = "1") Integer current_page, - @RequestParam(value = "rows",defaultValue = "10") Integer page_size){ - return ossMaterialInfoService.queryByPage(ossMaterialInfo,current_page,page_size); + @RequestParam(value = "page", defaultValue = "1") Integer current_page, + @RequestParam(value = "rows", defaultValue = "10") Integer page_size) { + return ossMaterialInfoService.queryByPage(ossMaterialInfo, current_page, page_size); } @GetMapping("/exportExcel") - public void exportExcel(HttpServletRequest request, HttpServletResponse response){ + public void exportExcel(HttpServletRequest request, HttpServletResponse response) { try { response.setContentType("application/vnd.ms-excel"); - String fileName=System.currentTimeMillis()+"-oss-server.xls"; - response.setHeader("Content-disposition", "attachment;filename="+fileName); - Workbook workbook=excelService.exportMaterials(); - OutputStream outputStream=response.getOutputStream(); + String fileName = System.currentTimeMillis() + "-oss-server.xls"; + response.setHeader("Content-disposition", "attachment;filename=" + fileName); + Workbook workbook = excelService.exportMaterials(); + OutputStream outputStream = response.getOutputStream(); workbook.write(outputStream); outputStream.flush(); outputStream.close(); - }catch (Exception e){ + } catch (Exception e) { log.error(e); } } @PostMapping(value = "/createDir") - public RestfulMessage createDirectory(@RequestParam(value = "dir",required = false) String dir, - @RequestParam(value = "pro") String pro){ - RestfulMessage restfulMessage=new RestfulMessage(); - try{ - String root=ossInformationService.queryOne().getRoot(); - StringBuffer dirStr=new StringBuffer(); + public RestfulMessage createDirectory(@RequestParam(value = "dir", required = false) String dir, + @RequestParam(value = "pro") String pro) { + RestfulMessage restfulMessage = new RestfulMessage(); + try { + String root = ossInformationService.queryOne().getRoot(); + StringBuffer dirStr = new StringBuffer(); dirStr.append(root).append(dir); validateProjectName(pro); dirStr.append("/").append(pro); - File file=new File(dirStr.toString()); + File file = new File(dirStr.toString()); createDirectoryQuietly(file); file.setExecutable(false); file.setWritable(false); file.setReadOnly(); successResultCode(restfulMessage); - }catch (Exception e){ - restfulMessage=wrapperException(e); + } catch (Exception e) { + restfulMessage = wrapperException(e); } return restfulMessage; } @@ -146,95 +140,90 @@ public class MaterialApplication extends RootApis{ * @return */ @NotLogin - @PostMapping(value = "/uploadByBinary",produces = "application/json;charset=UTF-8") - public RestfulMessage upload(HttpEntity entity){ - RestfulMessage restfulMessage=new RestfulMessage(); - try{ + @PostMapping(value = "/uploadByBinary", produces = "application/json;charset=UTF-8") + public RestfulMessage upload(HttpEntity entity) { + RestfulMessage restfulMessage = new RestfulMessage(); + try { log.info("/uploadByBinary..."); - String bodyStr=entity.getBody(); + String bodyStr = entity.getBody(); String decodeBodyStr = new String(URLCodec.decodeUrl(bodyStr.getBytes("UTF-8")), "UTF-8"); - StringReader strReader = new StringReader(decodeBodyStr); - JsonReader reader = new JsonReader(strReader); - reader.setLenient(true); - JsonElement je = new JsonParser().parse(reader); - if (je==null || je.isJsonNull() || !je.isJsonObject() ||("null".equals(je.toString())) || (je.toString() == null)) { - throw new AssemblerException(new ErrorCable(ErrorConstant.REQUEST_PARAMS_NOT_VALID,"not any data")); - } //获取project - JsonObject req=je.getAsJsonObject(); - //获取appid - JsonElement appIdEle=req.get("appid"); - JsonElement appSecretEle=req.get("appsecret"); - assertJsonNotEmpty(appIdEle,"appid can't be empty "); - assertJsonNotEmpty(appSecretEle,"appsecret can't be empty "); - //获取模块名称 - JsonElement moduleEle=req.get("module"); - String moduleStr=""; - if (moduleEle!=null&&!moduleEle.isJsonNull()&&moduleEle.isJsonPrimitive()){ - moduleStr=moduleEle.getAsString(); + JSONObject reqJson = JSONObject.parseObject(decodeBodyStr); + + if (reqJson == null || reqJson.isEmpty() || ("null".equals(reqJson.toString())) || (reqJson.toString() == null)) { + throw new AssemblerException(new ErrorCable(ErrorConstant.REQUEST_PARAMS_NOT_VALID, "not any data")); } - JsonElement projectEle=req.get("project"); - JsonElement files=req.get("files"); - assertJsonNotEmpty(projectEle,"project name can't be empty!!!"); - assertJsonNotEmpty(files,"files can't be empty!!!"); - if (!files.isJsonArray()){ - throw new AssemblerException(new ErrorCable(ErrorConstant.REQUEST_PARAMS_NOT_VALID,"files must be array")); + //获取appid "project" + String moduleStr = ""; + String appId = reqJson.getString("appid"); + String appSecret = reqJson.getString("appsecret"); + String project = reqJson.getString("project"); + JSONArray files = reqJson.getJSONArray("files"); + + assertArgumentNotEmpty(appId, "appid can't be empty"); + assertArgumentNotEmpty(appSecret, "appsecret can't be empty"); + assertArgumentNotEmpty(project, "project name can't be empty"); + assertArgumentNotEmpty(files, "files can't be empty"); + + if (!StringUtil.isNullOrEmpty(reqJson.getString("module"))) { + moduleStr = reqJson.getString("module"); } - OSSDeveloper ossDeveloper=ossDeveloperService.queryByAppid(appIdEle.getAsString(),appSecretEle.getAsString()); - assertArgumentNotEmpty(ossDeveloper,"appid or appsecret is invalid"); - String projectFilePathName=projectEle.getAsString(); + + OSSDeveloper ossDeveloper = ossDeveloperService.queryByAppid(appId, appSecret); + assertArgumentNotEmpty(ossDeveloper, "appid or appsecret is invalid"); + String projectFilePathName = project; validateProjectName(projectFilePathName); //判断文件夹code是否相等 - List ossAppInfos=ossAppInfoService.queryByDevIds(ossDeveloper.getId()); - if (ossAppInfos==null||ossAppInfos.size()==0){ - throw new AssemblerException(new ErrorCable(ErrorConstant.REQUEST_PARAMS_NOT_VALID,"You do not have permission to upload files")); + List ossAppInfos = ossAppInfoService.queryByDevIds(ossDeveloper.getId()); + if (ossAppInfos == null || ossAppInfos.size() == 0) { + throw new AssemblerException(new ErrorCable(ErrorConstant.REQUEST_PARAMS_NOT_VALID, "You do not have permission to upload files")); } - OSSAppInfo ossApp=null; - boolean flag=false; - for (OSSAppInfo ossAppInfo:ossAppInfos){ - if (StrUtil.equalsIgnoreCase(ossAppInfo.getCode(),projectFilePathName)){ - flag=true; - ossApp=ossAppInfo; + OSSAppInfo ossApp = null; + boolean flag = false; + for (OSSAppInfo ossAppInfo : ossAppInfos) { + if (StrUtil.equalsIgnoreCase(ossAppInfo.getCode(), projectFilePathName)) { + flag = true; + ossApp = ossAppInfo; break; } } - if (!flag){ - throw new AssemblerException(new ErrorCable(ErrorConstant.REQUEST_PARAMS_NOT_VALID,"You do not have permission to upload files")); + if (!flag) { + throw new AssemblerException(new ErrorCable(ErrorConstant.REQUEST_PARAMS_NOT_VALID, "You do not have permission to upload files")); } - OSSInformation ossInformation=ossInformationService.queryOne(); - String root=ossInformation.getRoot(); + OSSInformation ossInformation = ossInformationService.queryOne(); + String root = ossInformation.getRoot(); //验证文件夹规则,不能包含特殊字符 - File file=new File(root); + File file = new File(root); createDirectoryQuietly(file); - StringBuffer path=new StringBuffer(); + StringBuffer path = new StringBuffer(); path.append(file.getAbsolutePath()); path.append(File.separator); path.append(projectFilePathName); - if (StrUtil.isNotEmpty(moduleStr)){ - if (!moduleStr.startsWith("/")){ + if (StrUtil.isNotEmpty(moduleStr)) { + if (!moduleStr.startsWith("/")) { path.append("/"); } path.append(moduleStr); } - log.info("path:{}",path); - File projectFile=new File(path.toString()); - if (!projectFile.exists()){ - throw new AssemblerException(new ErrorCable(ErrorConstant.AUTHENTICATION_FAILED,"You do not have operating authority for this directory "+projectFilePathName+", or the directory was not created")); + log.info("path:{}", path); + File projectFile = new File(path.toString()); + if (!projectFile.exists()) { + throw new AssemblerException(new ErrorCable(ErrorConstant.AUTHENTICATION_FAILED, "You do not have operating authority for this directory " + projectFilePathName + ", or the directory was not created")); } createDirectoryQuietly(projectFile); - Type typeOf = new TypeToken>() {}.getType(); - List materials = new Gson().fromJson(files, typeOf); - for (FileBinaryRequest fileBinaryRequest:materials){ - assertArgumentNotEmpty(fileBinaryRequest.getMediaType(),"media_type is requried ..."); - assertArgumentNotEmpty(fileBinaryRequest.getFile(),"Not found file bytes, but it is requried ..."); + + List materials = JSONObject.parseArray(files.toJSONString(), FileBinaryRequest.class); + for (FileBinaryRequest fileBinaryRequest : materials) { + assertArgumentNotEmpty(fileBinaryRequest.getMediaType(), "media_type is requried ..."); + assertArgumentNotEmpty(fileBinaryRequest.getFile(), "Not found file bytes, but it is requried ..."); //fileBinaryRequest.setFileBytes(material.getFileBytes().replaceAll(" ", "+")); - fileBinaryRequest.setFile(fileBinaryRequest.getFile().replaceAll(" ","+")); + fileBinaryRequest.setFile(fileBinaryRequest.getFile().replaceAll(" ", "+")); } - List fileBinaryResponseList=materialService.saveAndStore(ossInformation,ossDeveloper,ossApp,projectFile,materials); + List fileBinaryResponseList = materialService.saveAndStore(ossInformation, ossDeveloper, ossApp, projectFile, materials); restfulMessage.setData(fileBinaryResponseList); successResultCode(restfulMessage); - }catch (Exception e){ - restfulMessage=wrapperException(e); + } catch (Exception e) { + restfulMessage = wrapperException(e); } return restfulMessage; } @@ -251,14 +240,14 @@ public class MaterialApplication extends RootApis{ @NotLogin @PostMapping("{project}/uploadMaterial") public RestfulMessage uploadMaterial(@PathVariable(value = "project") String project, - @RequestParam(value = "module",required = false) String module, - @RequestParam(value = "appid",required = false) String appid, - @RequestParam(value = "appsecret",required = false) String appsecret,@RequestParam(value="file") MultipartFile[] files){ - RestfulMessage restfulMessage=new RestfulMessage(); - try{ - restfulMessage=uploadFileByForm(project,appid,appsecret,module,files); - }catch (Exception e){ - restfulMessage=wrapperException(e); + @RequestParam(value = "module", required = false) String module, + @RequestParam(value = "appid", required = false) String appid, + @RequestParam(value = "appsecret", required = false) String appsecret, @RequestParam(value = "file") MultipartFile[] files) { + RestfulMessage restfulMessage = new RestfulMessage(); + try { + restfulMessage = uploadFileByForm(project, appid, appsecret, module, files); + } catch (Exception e) { + restfulMessage = wrapperException(e); } return restfulMessage; } @@ -270,67 +259,67 @@ public class MaterialApplication extends RootApis{ */ @NotLogin @PostMapping("/uploadMaterialNonProUrl") - public RestfulMessage uploadMaterialNonProUrl(@RequestParam(value = "project",required = false) String project, - @RequestParam(value = "module",required = false) String module, - @RequestParam(value = "appid",required = false) String appid, - @RequestParam(value = "appsecret",required = false) String appsecret,@RequestParam(value="file") MultipartFile[] files){ - RestfulMessage restfulMessage=new RestfulMessage(); - try{ - restfulMessage=uploadFileByForm(project,appid,appsecret,module,files); - }catch (Exception e){ - restfulMessage=wrapperException(e); + public RestfulMessage uploadMaterialNonProUrl(@RequestParam(value = "project", required = false) String project, + @RequestParam(value = "module", required = false) String module, + @RequestParam(value = "appid", required = false) String appid, + @RequestParam(value = "appsecret", required = false) String appsecret, @RequestParam(value = "file") MultipartFile[] files) { + RestfulMessage restfulMessage = new RestfulMessage(); + try { + restfulMessage = uploadFileByForm(project, appid, appsecret, module, files); + } catch (Exception e) { + restfulMessage = wrapperException(e); } return restfulMessage; } - private RestfulMessage uploadFileByForm(String project,String appid,String appsecret,String module,MultipartFile[] files) throws IOException { - RestfulMessage restfulMessage=new RestfulMessage(); - assertArgumentNotEmpty(project,"project can't be empty!!!"); - assertArgumentNotEmpty(appid,"appid can't be empty "); - assertArgumentNotEmpty(appsecret,"appsecret can't be empty "); - OSSDeveloper ossDeveloper=ossDeveloperService.queryByAppid(appid,appsecret); - assertArgumentNotEmpty(ossDeveloper,"appid or appsecret is invalid"); + private RestfulMessage uploadFileByForm(String project, String appid, String appsecret, String module, MultipartFile[] files) throws IOException { + RestfulMessage restfulMessage = new RestfulMessage(); + assertArgumentNotEmpty(project, "project can't be empty!!!"); + assertArgumentNotEmpty(appid, "appid can't be empty "); + assertArgumentNotEmpty(appsecret, "appsecret can't be empty "); + OSSDeveloper ossDeveloper = ossDeveloperService.queryByAppid(appid, appsecret); + assertArgumentNotEmpty(ossDeveloper, "appid or appsecret is invalid"); //判断文件夹code是否相等 - List ossAppInfos=ossAppInfoService.queryByDevIds(ossDeveloper.getId()); - if (ossAppInfos==null||ossAppInfos.size()==0){ - throw new AssemblerException(new ErrorCable(ErrorConstant.REQUEST_PARAMS_NOT_VALID,"You do not have permission to upload files")); + List ossAppInfos = ossAppInfoService.queryByDevIds(ossDeveloper.getId()); + if (ossAppInfos == null || ossAppInfos.size() == 0) { + throw new AssemblerException(new ErrorCable(ErrorConstant.REQUEST_PARAMS_NOT_VALID, "You do not have permission to upload files")); } - boolean flag=false; - OSSAppInfo ossApp=null; - for (OSSAppInfo ossAppInfo:ossAppInfos){ - if (StrUtil.equalsIgnoreCase(ossAppInfo.getCode(),project)){ - flag=true; - ossApp=ossAppInfo; + boolean flag = false; + OSSAppInfo ossApp = null; + for (OSSAppInfo ossAppInfo : ossAppInfos) { + if (StrUtil.equalsIgnoreCase(ossAppInfo.getCode(), project)) { + flag = true; + ossApp = ossAppInfo; break; } } - if (!flag){ - throw new AssemblerException(new ErrorCable(ErrorConstant.REQUEST_PARAMS_NOT_VALID,"You do not have permission to upload files")); + if (!flag) { + throw new AssemblerException(new ErrorCable(ErrorConstant.REQUEST_PARAMS_NOT_VALID, "You do not have permission to upload files")); } validateProjectName(project); - OSSInformation ossInformation=ossInformationService.queryOne(); - String root=ossInformation.getRoot(); + OSSInformation ossInformation = ossInformationService.queryOne(); + String root = ossInformation.getRoot(); //验证文件夹规则,不能包含特殊字符 - File file=new File(root); + File file = new File(root); createDirectoryQuietly(file); - StringBuffer path=new StringBuffer(); + StringBuffer path = new StringBuffer(); path.append(file.getAbsolutePath()); path.append(File.separator); path.append(project); - if (StrUtil.isNotEmpty(module)){ - if (!module.startsWith("/")){ + if (StrUtil.isNotEmpty(module)) { + if (!module.startsWith("/")) { path.append("/"); } path.append(module); } - log.info("path:{}",path); - File projectFile=new File(path.toString()); + log.info("path:{}", path); + File projectFile = new File(path.toString()); /*if (!projectFile.exists()){ throw new AssemblerException(new ErrorCable(ErrorConstant.AUTHENTICATION_FAILED,"You do not have operating authority for this directory "+project+", or the directory was not created")); }*/ createDirectoryQuietly(projectFile); - List fileBinaryResponseList=materialService.saveAndStore(ossInformation,ossDeveloper,ossApp,projectFile,files); + List fileBinaryResponseList = materialService.saveAndStore(ossInformation, ossDeveloper, ossApp, projectFile, files); restfulMessage.setData(fileBinaryResponseList); successResultCode(restfulMessage); return restfulMessage; @@ -338,30 +327,30 @@ public class MaterialApplication extends RootApis{ @PostMapping("/uploadBySys") - public RestfulMessage uploadSys(@RequestParam(value = "dir",required = false) String dir,@RequestParam(value="file") MultipartFile[] files){ - RestfulMessage restfulMessage=new RestfulMessage(); - try{ - OSSInformation ossInformation=ossInformationService.queryOne(); - String root=ossInformation.getRoot(); + public RestfulMessage uploadSys(@RequestParam(value = "dir", required = false) String dir, @RequestParam(value = "file") MultipartFile[] files) { + RestfulMessage restfulMessage = new RestfulMessage(); + try { + OSSInformation ossInformation = ossInformationService.queryOne(); + String root = ossInformation.getRoot(); //验证文件夹规则,不能包含特殊字符 - File file=new File(root); + File file = new File(root); createDirectoryQuietly(file); - StringBuffer path=new StringBuffer(); + StringBuffer path = new StringBuffer(); path.append(file.getAbsolutePath()); - if (StrUtil.isNotEmpty(dir)){ - if (!dir.startsWith("/")){ + if (StrUtil.isNotEmpty(dir)) { + if (!dir.startsWith("/")) { path.append(File.separator); } path.append(dir); } - log.info("path:{}",path); - File projectFile=new File(path.toString()); + log.info("path:{}", path); + File projectFile = new File(path.toString()); createDirectoryQuietly(projectFile); - List fileBinaryResponseList=materialService.saveAndStoreBySys(ossInformation,projectFile,files); + List fileBinaryResponseList = materialService.saveAndStoreBySys(ossInformation, projectFile, files); restfulMessage.setData(fileBinaryResponseList); successResultCode(restfulMessage); - }catch (Exception e){ - restfulMessage=wrapperException(e); + } catch (Exception e) { + restfulMessage = wrapperException(e); } return restfulMessage; } diff --git a/oss-server/src/main/java/com/xiaominfo/oss/api/ModelApplication.java b/oss-server/src/main/java/com/xiaominfo/oss/api/ModelApplication.java index cce0b71..05b800e 100644 --- a/oss-server/src/main/java/com/xiaominfo/oss/api/ModelApplication.java +++ b/oss-server/src/main/java/com/xiaominfo/oss/api/ModelApplication.java @@ -7,8 +7,8 @@ package com.xiaominfo.oss.api; +import com.alibaba.fastjson.JSON; import com.baomidou.mybatisplus.mapper.EntityWrapper; -import com.google.gson.Gson; import com.xiaominfo.oss.module.model.OSSDeveloper; import com.xiaominfo.oss.service.*; import org.springframework.beans.factory.annotation.Autowired; @@ -23,7 +23,7 @@ import java.util.List; /*** * 前端url跳转Application * @since:oss-server 1.0 - * @author xiaoymin@foxmail.com + * @author xiaoymin@foxmail.com * 2018/06/16 19:59 */ @Controller @@ -54,43 +54,44 @@ public class ModelApplication { /** * 主面板 + * * @return */ @GetMapping("/dashboard") - public String dashboard(Model model){ - List ossDevelopers=ossDeveloperService.queryAllDevs(); - model.addAttribute("apps",ossAppInfoService.selectCount(new EntityWrapper<>())); - model.addAttribute("devs",ossDeveloperService.selectCount(new EntityWrapper<>())); - model.addAttribute("files",ossMaterialInfoService.selectCount(new EntityWrapper<>())); - model.addAttribute("totalUseSpace",ossMaterialInfoService.queryTotalSpaceByteStr()); - String fileCounts=new Gson().toJson(ossStatisticDayService.queryCurrentWeekStaticsDay()); - String devCounts=new Gson().toJson(ossDevelopers); - model.addAttribute("fileCounts",fileCounts); - model.addAttribute("devCounts",devCounts); + public String dashboard(Model model) { + List ossDevelopers = ossDeveloperService.queryAllDevs(); + model.addAttribute("apps", ossAppInfoService.selectCount(new EntityWrapper<>())); + model.addAttribute("devs", ossDeveloperService.selectCount(new EntityWrapper<>())); + model.addAttribute("files", ossMaterialInfoService.selectCount(new EntityWrapper<>())); + model.addAttribute("totalUseSpace", ossMaterialInfoService.queryTotalSpaceByteStr()); + String fileCounts = JSON.toJSONString(ossStatisticDayService.queryCurrentWeekStaticsDay()); + String devCounts = JSON.toJSONString(ossDevelopers); + model.addAttribute("fileCounts", fileCounts); + model.addAttribute("devCounts", devCounts); return "dashboard"; } @GetMapping("/information") - public String information(Model model){ - model.addAttribute("ossInfo",ossInformationService.queryOne()); + public String information(Model model) { + model.addAttribute("ossInfo", ossInformationService.queryOne()); return "information"; } @GetMapping("/developer") - public String developer(){ + public String developer() { return "developer"; } @GetMapping("/appinfo") - public String appinfo(Model model){ - model.addAttribute("devList",ossDeveloperService.queryAllDevs()); + public String appinfo(Model model) { + model.addAttribute("devList", ossDeveloperService.queryAllDevs()); return "appinfo"; } @GetMapping("/materialinfo") - public String materialInfo(Model model){ + public String materialInfo(Model model) { return "material_info"; } diff --git a/oss-server/src/main/java/com/xiaominfo/oss/api/NettyUpload.java b/oss-server/src/main/java/com/xiaominfo/oss/api/NettyUpload.java new file mode 100644 index 0000000..a776141 --- /dev/null +++ b/oss-server/src/main/java/com/xiaominfo/oss/api/NettyUpload.java @@ -0,0 +1,4 @@ +package com.xiaominfo.oss.api; + +public class NettyUpload { +} diff --git a/oss-server/src/main/java/com/xiaominfo/oss/api/RootApis.java b/oss-server/src/main/java/com/xiaominfo/oss/api/RootApis.java index a61c002..c3b3ce9 100644 --- a/oss-server/src/main/java/com/xiaominfo/oss/api/RootApis.java +++ b/oss-server/src/main/java/com/xiaominfo/oss/api/RootApis.java @@ -11,10 +11,6 @@ import cn.hutool.core.util.ReUtil; import cn.hutool.core.util.StrUtil; import cn.hutool.log.Log; import cn.hutool.log.LogFactory; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; -import com.google.gson.stream.JsonReader; import com.xiaominfo.oss.common.pojo.RestfulMessage; import com.xiaominfo.oss.exception.AssemblerException; import com.xiaominfo.oss.exception.ErrorCable; @@ -25,12 +21,11 @@ import org.springframework.http.HttpHeaders; import org.springframework.http.MediaType; import java.io.File; -import java.io.StringReader; /*** * * @since:oss-server 1.0 - * @author xiaoymin@foxmail.com + * @author xiaoymin@foxmail.com * 2018/05/30 11:30 */ public class RootApis { @@ -38,22 +33,24 @@ public class RootApis { private static final Log log = LogFactory.get(); - protected void assertNotEmpty(String str,String msg){ - if (StrUtil.isEmpty(str)){ - throw new AssemblerException(new ErrorCable(ErrorConstant.REQUEST_PARAMS_NOT_VALID,"request params not valid...")); + protected void assertNotEmpty(String str, String msg) { + if (StrUtil.isEmpty(str)) { + throw new AssemblerException(new ErrorCable(ErrorConstant.REQUEST_PARAMS_NOT_VALID, "request params not valid...")); } } - protected void assertArgumentNotEmpty(String arguments,String msg){ - if (StrUtil.isEmpty(arguments)){ - throw new AssemblerException(new ErrorCable(ErrorConstant.REQUEST_PARAMS_NOT_VALID,msg)); + protected void assertArgumentNotEmpty(String arguments, String msg) { + if (StrUtil.isEmpty(arguments)) { + throw new AssemblerException(new ErrorCable(ErrorConstant.REQUEST_PARAMS_NOT_VALID, msg)); } } - protected void assertArgumentNotEmpty(Object arguments,String msg){ - if (arguments==null){ - throw new AssemblerException(new ErrorCable(ErrorConstant.REQUEST_PARAMS_NOT_VALID,msg)); + + public void assertArgumentNotEmpty(Object arguments, String msg) { + if (arguments == null) { + throw new AssemblerException(new ErrorCable(ErrorConstant.REQUEST_PARAMS_NOT_VALID, msg)); } } + protected void assertArgFalse(boolean aBoolean, String aMessage) { if (aBoolean) { throw new AssemblerException(new ErrorCable(ErrorConstant.REQUEST_PARAMS_NOT_VALID, aMessage)); @@ -65,30 +62,30 @@ public class RootApis { throw new AssemblerException(new ErrorCable(ErrorConstant.REQUEST_PARAMS_NOT_VALID, msg)); } } - - protected void assertJsonNotEmpty(JsonElement jsonElement, String msg){ - if (jsonElement==null||jsonElement.isJsonNull()){ - throw new AssemblerException(new ErrorCable(ErrorConstant.REQUEST_PARAMS_NOT_VALID,msg)); +/* + protected void assertJsonNotEmpty(JsonElement jsonElement, String msg) { + if (jsonElement == null || jsonElement.isJsonNull()) { + throw new AssemblerException(new ErrorCable(ErrorConstant.REQUEST_PARAMS_NOT_VALID, msg)); } - } + }*/ - protected RestfulMessage wrapperException(Exception e){ + protected RestfulMessage wrapperException(Exception e) { log.error(e); - RestfulMessage restfulMessage=new RestfulMessage(); - if (e!=null){ - String msg=e.getMessage(); - if (StrUtil.isEmpty(msg)){ + RestfulMessage restfulMessage = new RestfulMessage(); + if (e != null) { + String msg = e.getMessage(); + if (StrUtil.isEmpty(msg)) { restfulMessage.setCode(ErrorConstant.INTERNAL_SERVER_ERROR); restfulMessage.setMessage(msg); - }else if(msg.contains("|")){ - String[] em= StrUtil.split(msg,"|"); + } else if (msg.contains("|")) { + String[] em = StrUtil.split(msg, "|"); restfulMessage.setCode(Integer.parseInt(em[0])); restfulMessage.setMessage(em[1]); - }else { + } else { restfulMessage.setCode(ErrorConstant.INTERNAL_SERVER_ERROR); restfulMessage.setMessage(e.getMessage()); } - }else{ + } else { restfulMessage.setCode(ErrorConstant.INTERNAL_SERVER_ERROR); restfulMessage.setMessage("未知错误"); } @@ -99,19 +96,19 @@ public class RootApis { * 验证文件夹名称 * @param projectName */ - protected void validateProjectName(String projectName){ - if (StrUtil.isBlank(projectName)){ - throw new AssemblerException(new ErrorCable(ErrorConstant.REQUEST_PARAMS_NOT_VALID,"project name can't be empty!")); + public void validateProjectName(String projectName) { + if (StrUtil.isBlank(projectName)) { + throw new AssemblerException(new ErrorCable(ErrorConstant.REQUEST_PARAMS_NOT_VALID, "project name can't be empty!")); } - String regex="^.*?(\\\\|\\/|\\:|\\*|\\?|\\？|\\\"|\\“|\\”|\\>|\\<|\\|).*"; - if (ReUtil.isMatch(regex,projectName)){ - throw new AssemblerException(new ErrorCable(ErrorConstant.REQUEST_PARAMS_NOT_VALID,"The name of the file can not contain any of the following characters: / /: *?<>|")); + String regex = "^.*?(\\\\|\\/|\\:|\\*|\\?|\\？|\\\"|\\“|\\”|\\>|\\<|\\|).*"; + if (ReUtil.isMatch(regex, projectName)) { + throw new AssemblerException(new ErrorCable(ErrorConstant.REQUEST_PARAMS_NOT_VALID, "The name of the file can not contain any of the following characters: / /: *?<>|")); } //不能包含\s字符 //不能包含中文 - regex=".*?[\\u4e00-\\u9fa5\\s].*"; - if (ReUtil.isMatch(regex,projectName)){ - throw new AssemblerException(new ErrorCable(ErrorConstant.REQUEST_PARAMS_NOT_VALID,"The name of the file does not contain Chinese and space")); + regex = ".*?[\\u4e00-\\u9fa5\\s].*"; + if (ReUtil.isMatch(regex, projectName)) { + throw new AssemblerException(new ErrorCable(ErrorConstant.REQUEST_PARAMS_NOT_VALID, "The name of the file does not contain Chinese and space")); } @@ -122,50 +119,51 @@ public class RootApis { * @param e * @return */ - protected String traceException(Exception e){ - StackTraceElement[] stackTraceElements=e.getStackTrace(); - StringBuffer errs=new StringBuffer(); - for (StackTraceElement stackTraceElement:stackTraceElements){ + protected String traceException(Exception e) { + StackTraceElement[] stackTraceElements = e.getStackTrace(); + StringBuffer errs = new StringBuffer(); + for (StackTraceElement stackTraceElement : stackTraceElements) { errs.append(stackTraceElement.toString()); } return errs.toString(); } - protected void successResultCode(RestfulMessage restfulMessage){ + protected void successResultCode(RestfulMessage restfulMessage) { restfulMessage.setCode(ErrorConstant.SUCCESS); restfulMessage.setMessage("Success"); } - - protected JsonObject paramJson(HttpEntity requestEntity){ +/* + protected JsonObject paramJson(HttpEntity requestEntity) { String decodeBodyStr = acceptHttpRequest(requestEntity); StringReader strReader = new StringReader(decodeBodyStr); JsonReader reader = new JsonReader(strReader); reader.setLenient(true); return new JsonParser().parse(reader).getAsJsonObject(); } - protected JsonElement paramJsonElement(HttpEntity requestEntity){ + + protected JsonElement paramJsonElement(HttpEntity requestEntity) { String decodeBodyStr = acceptHttpRequest(requestEntity); StringReader strReader = new StringReader(decodeBodyStr); JsonReader reader = new JsonReader(strReader); reader.setLenient(true); return new JsonParser().parse(reader); - } + }*/ /*** * 创建文件夹 * @param file */ - protected void createDirectoryQuietly(File file){ - try{ - if (file!=null){ - if (!file.exists()){ - if (!file.mkdirs()){ - throw new RuntimeException(file.getName()+" is invalid,can't be create directory"); + public void createDirectoryQuietly(File file) { + try { + if (file != null) { + if (!file.exists()) { + if (!file.mkdirs()) { + throw new RuntimeException(file.getName() + " is invalid,can't be create directory"); } } } - }finally { - if (file!=null){ + } finally { + if (file != null) { file.setWritable(false); file.setExecutable(false); file.setReadOnly(); @@ -198,6 +196,7 @@ public class RootApis { } return decodeBodyStr; } + protected void postParamHandler(String bodyStr) { assertArgumentNotEmpty(bodyStr, "Request params is empty ..."); } @@ -205,11 +204,12 @@ public class RootApis { /** * 产生日志 + * * @param msg * @return */ - protected String log(String msg){ - return DateTime.now().toString("yyyy-MM-dd HH:mm:ss")+">>"+msg+"\r\n"; + protected String log(String msg) { + return DateTime.now().toString("yyyy-MM-dd HH:mm:ss") + ">>" + msg + "\r\n"; } } diff --git a/oss-server/src/main/java/com/xiaominfo/oss/common/SpringContextUtil.java b/oss-server/src/main/java/com/xiaominfo/oss/common/SpringContextUtil.java new file mode 100644 index 0000000..da32156 --- /dev/null +++ b/oss-server/src/main/java/com/xiaominfo/oss/common/SpringContextUtil.java @@ -0,0 +1,70 @@ +package com.xiaominfo.oss.common; + +import org.springframework.beans.BeansException; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.context.annotation.Lazy; +import org.springframework.core.env.Environment; +import org.springframework.stereotype.Component; + +/** + * 描述:获取spring上下文 + * + */ +@Lazy(value = false) +@Component +public class SpringContextUtil implements ApplicationContextAware { + // Spring应用上下文环境 + private static ApplicationContext applicationContext; + + /** + * 实现ApplicationContextAware接口的回调方法，设置上下文环境 + * + * @param applicationContext + */ + @Override + public void setApplicationContext(ApplicationContext applicationContext) { + SpringContextUtil.applicationContext = applicationContext; + } + + /** + * @return ApplicationContext + */ + public static ApplicationContext getApplicationContext() { + return applicationContext; + } + + /** + * 获取对象 + * 这里重写了bean方法，起主要作用 + * + * @param name + * @return Object 一个以所给名字注册的bean的实例 + * @throws BeansException + */ + public static Object getBean(String name) throws BeansException { + return applicationContext.getBean(name); + } + + /** + * 获取对象 + * + * @param var1 + * @param + * @return + */ + public static T getBean(Class var1) { + return applicationContext.getBean(var1); + } + + /** + * 直接回去配置文件中的参数 + * + * @param property + * @return + */ + public static String getProperty(String property) { + Environment environment = SpringContextUtil.getBean(Environment.class); + return environment.getProperty(property); + } +} diff --git a/oss-server/src/main/java/com/xiaominfo/oss/config/JsonpMessageConverter.java b/oss-server/src/main/java/com/xiaominfo/oss/config/JsonpMessageConverter.java index db9a6f4..9fd4c6e 100644 --- a/oss-server/src/main/java/com/xiaominfo/oss/config/JsonpMessageConverter.java +++ b/oss-server/src/main/java/com/xiaominfo/oss/config/JsonpMessageConverter.java @@ -3,13 +3,11 @@ * All rights reserved. * Official Web Site: http://www.xiaominfo.com. * Developer Web Site: http://open.xiaominfo.com. - */ + *//* + package com.xiaominfo.oss.config; import cn.hutool.core.util.StrUtil; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.JsonIOException; import org.springframework.http.converter.HttpMessageNotWritableException; import org.springframework.http.converter.json.GsonHttpMessageConverter; import org.springframework.lang.Nullable; @@ -21,11 +19,13 @@ import java.io.Reader; import java.io.Writer; import java.lang.reflect.Type; +*/ /*** * GsonJsonpHttpMessageConverter * @author xiaoymin@foxmail.com * 2018-5-31 12:48:07 - */ + *//* + public class JsonpMessageConverter extends GsonHttpMessageConverter { private String jsonPrefix; private String callbackParam="callback"; @@ -83,6 +83,7 @@ public class JsonpMessageConverter extends GsonHttpMessageConverter { this.callbackParam = callbackParam; } - + } +*/ diff --git a/oss-server/src/main/java/com/xiaominfo/oss/config/MaterialConfiguration.java b/oss-server/src/main/java/com/xiaominfo/oss/config/MaterialConfiguration.java index 304a61b..62d8a14 100644 --- a/oss-server/src/main/java/com/xiaominfo/oss/config/MaterialConfiguration.java +++ b/oss-server/src/main/java/com/xiaominfo/oss/config/MaterialConfiguration.java @@ -27,7 +27,7 @@ import java.util.List; /*** * * @since:oss-server 1.0 - * @author xiaoymin@foxmail.com + * @author xiaoymin@foxmail.com * 2018/05/30 11:06 */ @Configuration @@ -43,7 +43,7 @@ public class MaterialConfiguration implements WebMvcConfigurer { @Override public void configureMessageConverters(List> converters) { - converters.add(new JsonpMessageConverter()); + // converters.add(new JsonpMessageConverter()); } @Override @@ -52,8 +52,8 @@ public class MaterialConfiguration implements WebMvcConfigurer { } @Bean - public MaterialConfig materialConfig(){ - MaterialConfigImpl materialConfig=new MaterialConfigImpl(); + public MaterialConfig materialConfig() { + MaterialConfigImpl materialConfig = new MaterialConfigImpl(); materialConfig.setRootPath(root); materialConfig.setInvokingRoot(invokingRoot); materialConfig.setPathStyle(pathstyle); @@ -61,8 +61,8 @@ public class MaterialConfiguration implements WebMvcConfigurer { } @Bean - public FilterRegistrationBean corsFilter(){ - UrlBasedCorsConfigurationSource source =new UrlBasedCorsConfigurationSource(); + public FilterRegistrationBean corsFilter() { + UrlBasedCorsConfigurationSource source = new UrlBasedCorsConfigurationSource(); CorsConfiguration config = new CorsConfiguration(); config.setAllowCredentials(true); config.addAllowedOrigin("*"); @@ -75,7 +75,7 @@ public class MaterialConfiguration implements WebMvcConfigurer { } @Bean - public GlobalRequestMappingFilter globalRequestMappingFilter(){ + public GlobalRequestMappingFilter globalRequestMappingFilter() { return new GlobalRequestMappingFilter(); } diff --git a/oss-server/src/main/java/com/xiaominfo/oss/domain/FileBinaryRequest.java b/oss-server/src/main/java/com/xiaominfo/oss/domain/FileBinaryRequest.java index b94200f..0181ecb 100644 --- a/oss-server/src/main/java/com/xiaominfo/oss/domain/FileBinaryRequest.java +++ b/oss-server/src/main/java/com/xiaominfo/oss/domain/FileBinaryRequest.java @@ -7,22 +7,19 @@ package com.xiaominfo.oss.domain; -import com.google.gson.annotations.SerializedName; /*** * * @since:oss-server 1.0 - * @author xiaoymin@foxmail.com + * @author xiaoymin@foxmail.com * 2018/05/30 15:12 */ public class FileBinaryRequest { - @SerializedName(value = "original_name") private String originalName; private String file; - @SerializedName(value = "media_type") private String mediaType; public String getOriginalName() { diff --git a/oss-server/src/main/java/com/xiaominfo/oss/domain/FileStream.java b/oss-server/src/main/java/com/xiaominfo/oss/domain/FileStream.java index 9812d4e..9448ba9 100644 --- a/oss-server/src/main/java/com/xiaominfo/oss/domain/FileStream.java +++ b/oss-server/src/main/java/com/xiaominfo/oss/domain/FileStream.java @@ -7,33 +7,30 @@ package com.xiaominfo.oss.domain; -import com.google.gson.annotations.SerializedName; import java.util.Map; /*** * * @since:oss-server 1.0 - * @author xiaoymin@foxmail.com + * @author xiaoymin@foxmail.com * 2018/05/30 10:47 */ public class FileStream { - @SerializedName("file_stream_hex") + private String fileStreamHex; - @SerializedName("file_store_url") + private String fileStoreUrl; - @SerializedName("thumbs") + private Map thumbs; - public FileStream() - { + public FileStream() { } - public FileStream(String fileStreamHex, String fileStoreUrl) - { + public FileStream(String fileStreamHex, String fileStoreUrl) { this.fileStreamHex = fileStreamHex; this.fileStoreUrl = fileStoreUrl; } diff --git a/oss-server/src/main/java/com/xiaominfo/oss/module/dao/OSSMaterialInfoMapper.java b/oss-server/src/main/java/com/xiaominfo/oss/module/dao/OSSMaterialInfoMapper.java index e399a86..3e1cf6d 100644 --- a/oss-server/src/main/java/com/xiaominfo/oss/module/dao/OSSMaterialInfoMapper.java +++ b/oss-server/src/main/java/com/xiaominfo/oss/module/dao/OSSMaterialInfoMapper.java @@ -11,6 +11,7 @@ import com.baomidou.mybatisplus.mapper.BaseMapper; import com.xiaominfo.oss.module.entity.OSSMaterialInfoResult; import com.xiaominfo.oss.module.entity.OssMaterialCount; import com.xiaominfo.oss.module.model.OSSMaterialInfo; +import org.apache.ibatis.annotations.Param; import java.util.List; import java.util.Map; @@ -18,7 +19,7 @@ import java.util.Map; /*** * * @since:oss-server 1.0 - * @author xiaoymin@foxmail.com + * @author xiaoymin@foxmail.com * 2018/06/18 8:53 */ public interface OSSMaterialInfoMapper extends BaseMapper { @@ -43,4 +44,5 @@ public interface OSSMaterialInfoMapper extends BaseMapper { */ List queryCount(Map param); + List queryWaitSync(@Param("node") String node, @Param("thatNode") String thatNode); } diff --git a/oss-server/src/main/java/com/xiaominfo/oss/module/dao/OSSMaterialInfoSyncMapper.java b/oss-server/src/main/java/com/xiaominfo/oss/module/dao/OSSMaterialInfoSyncMapper.java new file mode 100644 index 0000000..3db4d07 --- /dev/null +++ b/oss-server/src/main/java/com/xiaominfo/oss/module/dao/OSSMaterialInfoSyncMapper.java @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2018 Zhejiang xiaominfo Technology CO.,LTD. + * All rights reserved. + * Official Web Site: http://www.xiaominfo.com. + * Developer Web Site: http://open.xiaominfo.com. + */ + +package com.xiaominfo.oss.module.dao; + +import com.baomidou.mybatisplus.mapper.BaseMapper; +import com.xiaominfo.oss.module.entity.OSSMaterialInfoResult; +import com.xiaominfo.oss.module.entity.OssMaterialCount; +import com.xiaominfo.oss.module.model.OSSMaterialInfo; +import com.xiaominfo.oss.module.model.OSSMaterialInfoSync; + +import java.util.List; +import java.util.Map; + +/*** + * + * @since:oss-server 1.0 + * @author xiaoymin@foxmail.com + * 2018/06/18 8:53 + */ +public interface OSSMaterialInfoSyncMapper extends BaseMapper { + +} diff --git a/oss-server/src/main/java/com/xiaominfo/oss/module/dao/mappings/OSSMaterialInfoMapper.xml b/oss-server/src/main/java/com/xiaominfo/oss/module/dao/mappings/OSSMaterialInfoMapper.xml index 7d2074b..c58a797 100644 --- a/oss-server/src/main/java/com/xiaominfo/oss/module/dao/mappings/OSSMaterialInfoMapper.xml +++ b/oss-server/src/main/java/com/xiaominfo/oss/module/dao/mappings/OSSMaterialInfoMapper.xml @@ -3,28 +3,29 @@ - - SELECT user_id userId, sum( len ) sumlen, count( * ) files, - date( create_time ) curDate + create_time curDate FROM oss_material_info t WHERE - date( t.create_time ) BETWEEN #{start} - AND #{end} + t.create_time >=#{start} + And t.create_time <= #{end} GROUP BY user_id, - date( create_time ) + create_time - SELECT m.id, m.original_name originalName, @@ -41,11 +42,30 @@ d.NAME, app.NAME AS appname FROM - oss_material_info AS m - LEFT JOIN oss_app_info AS app ON m.app_id = app.id - LEFT JOIN oss_developer AS d ON app.dev_id = d.id + oss_material_info m + LEFT JOIN oss_app_info app ON m.app_id = app.id + LEFT JOIN oss_developer d ON app.dev_id = d.id ORDER BY m.create_time DESC - limit #{start},#{page_size} + + + diff --git a/oss-server/src/main/java/com/xiaominfo/oss/module/model/OSSMaterialInfoSync.java b/oss-server/src/main/java/com/xiaominfo/oss/module/model/OSSMaterialInfoSync.java new file mode 100644 index 0000000..2762c34 --- /dev/null +++ b/oss-server/src/main/java/com/xiaominfo/oss/module/model/OSSMaterialInfoSync.java @@ -0,0 +1,75 @@ +/* + * Copyright (C) 2018 Zhejiang xiaominfo Technology CO.,LTD. + * All rights reserved. + * Official Web Site: http://www.xiaominfo.com. + * Developer Web Site: http://open.xiaominfo.com. + */ + +package com.xiaominfo.oss.module.model; + +import com.baomidou.mybatisplus.activerecord.Model; +import com.baomidou.mybatisplus.annotations.TableField; +import com.baomidou.mybatisplus.annotations.TableId; +import com.baomidou.mybatisplus.annotations.TableName; +import com.baomidou.mybatisplus.enums.IdType; + +import java.io.Serializable; + +/*** + * + * @since:oss-server 1.0 + * @author xiaoymin@foxmail.com + * 2018/06/18 8:28 + */ +@TableName("oss_material_info_sync") +public class OSSMaterialInfoSync extends Model { + @TableField(value = "material_id") + private String materialId; + + @TableField(value = "node") + private String node; + + @TableField(value = "type") + private String type; + + @TableField(value = "create_time") + private String createTime; + + + @Override + protected Serializable pkVal() { + return null; + } + + public String getNode() { + return node; + } + + public void setNode(String node) { + this.node = node; + } + + public String getMaterialId() { + return materialId; + } + + public void setMaterialId(String materialId) { + this.materialId = materialId; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public String getCreateTime() { + return createTime; + } + + public void setCreateTime(String createTime) { + this.createTime = createTime; + } +} diff --git a/oss-server/src/main/java/com/xiaominfo/oss/service/MaterialService.java b/oss-server/src/main/java/com/xiaominfo/oss/service/MaterialService.java index 77825c0..1934c48 100644 --- a/oss-server/src/main/java/com/xiaominfo/oss/service/MaterialService.java +++ b/oss-server/src/main/java/com/xiaominfo/oss/service/MaterialService.java @@ -13,6 +13,7 @@ import com.xiaominfo.oss.domain.RemoteResult; import com.xiaominfo.oss.module.model.OSSAppInfo; import com.xiaominfo.oss.module.model.OSSDeveloper; import com.xiaominfo.oss.module.model.OSSInformation; +import com.xiaominfo.oss.sdk.client.NettyFileRequest; import org.apache.tomcat.util.codec.DecoderException; import org.springframework.web.multipart.MultipartFile; @@ -23,28 +24,29 @@ import java.util.List; /*** * * @since:cloud-ims 1.0 - * @author xiaoymin@foxmail.com + * @author xiaoymin@foxmail.com * 2018/05/30 10:53 */ public interface MaterialService { - - /*** - * 保存文件 - * @param paramFileStream + * 存储binary二进制文件 + * @param fileBinaryRequests * @return - * @throws DecoderException - * @throws IOException */ - RemoteResult saveFile(FileStream paramFileStream) throws DecoderException, IOException; - + List saveAndStore(OSSInformation ossInformation, OSSDeveloper ossDeveloper, OSSAppInfo ossApp, File projectDirectory, List fileBinaryRequests) throws IOException; - /*** - * 存储binary二进制文件 - * @param fileBinaryRequests + /** + * 保存文件 + * @param ossInformation + * @param ossDeveloper + * @param ossApp + * @param projectDirectory + * @param file * @return + * @throws IOException */ - List saveAndStore(OSSInformation ossInformation,OSSDeveloper ossDeveloper, OSSAppInfo ossApp, File projectDirectory, List fileBinaryRequests) throws IOException; + FileBinaryResponse saveAndStore(OSSInformation ossInformation, OSSDeveloper ossDeveloper, OSSAppInfo ossApp, + NettyFileRequest ef, File file) throws IOException; /*** @@ -53,9 +55,10 @@ public interface MaterialService { * @param multipartFile * @return */ - List saveAndStore(OSSInformation ossInformation,OSSDeveloper ossDeveloper, OSSAppInfo ossApp, File projectDirectory, MultipartFile[] multipartFile) throws IOException; + List saveAndStore(OSSInformation ossInformation, OSSDeveloper ossDeveloper, OSSAppInfo ossApp, File projectDirectory, MultipartFile[] multipartFile) throws IOException; + + List saveAndStoreBySys(OSSInformation ossInformation, File projectDirectory, MultipartFile[] multipartFile) throws IOException; - List saveAndStoreBySys(OSSInformation ossInformation,File projectDirectory, MultipartFile[] multipartFile) throws IOException; } diff --git a/oss-server/src/main/java/com/xiaominfo/oss/service/OSSMaterialInfoService.java b/oss-server/src/main/java/com/xiaominfo/oss/service/OSSMaterialInfoService.java index f4611d8..c6a4ff0 100644 --- a/oss-server/src/main/java/com/xiaominfo/oss/service/OSSMaterialInfoService.java +++ b/oss-server/src/main/java/com/xiaominfo/oss/service/OSSMaterialInfoService.java @@ -9,13 +9,14 @@ package com.xiaominfo.oss.service; import com.baomidou.mybatisplus.service.IService; import com.xiaominfo.oss.common.pojo.Pagination; import com.xiaominfo.oss.module.entity.OSSMaterialInfoResult; -import com.xiaominfo.oss.module.model.OSSDeveloper; import com.xiaominfo.oss.module.model.OSSMaterialInfo; +import java.util.List; + /*** * * @since:cloud-ims 1.0 - * @author xiaoymin@foxmail.com + * @author xiaoymin@foxmail.com * 2018/06/18 8:54 */ public interface OSSMaterialInfoService extends IService { @@ -30,4 +31,10 @@ public interface OSSMaterialInfoService extends IService { Pagination queryByPage(OSSMaterialInfo ossMaterialInfo, Integer current_page, Integer page_size); + /** + * 等待同步的文件 + * + * @param node + */ + List waitSync(String node, String thatNode); } diff --git a/oss-server/src/main/java/com/xiaominfo/oss/service/OSSMaterialInfoSyncService.java b/oss-server/src/main/java/com/xiaominfo/oss/service/OSSMaterialInfoSyncService.java new file mode 100644 index 0000000..bc5b7a3 --- /dev/null +++ b/oss-server/src/main/java/com/xiaominfo/oss/service/OSSMaterialInfoSyncService.java @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2018 Zhejiang xiaominfo Technology CO.,LTD. + * All rights reserved. + * Official Web Site: http://www.xiaominfo.com. + * Developer Web Site: http://open.xiaominfo.com. + */ +package com.xiaominfo.oss.service; + +import com.baomidou.mybatisplus.service.IService; +import com.xiaominfo.oss.module.model.*; +import com.xiaominfo.oss.sdk.client.NettyFileRequest; + +/*** + * + * @since:cloud-ims 1.0 + * @author xiaoymin@foxmail.com + * 2018/06/18 8:54 + */ +public interface OSSMaterialInfoSyncService extends IService { + + + OSSMaterialInfoSync createSync(String materialId, String Type); + + + + + String buildPath(NettyFileRequest ef, OSSInformation ossInformation, OSSDeveloper ossDeveloper, OSSAppInfo ossApp); +} diff --git a/oss-server/src/main/java/com/xiaominfo/oss/service/impl/MaterialServiceImpl.java b/oss-server/src/main/java/com/xiaominfo/oss/service/impl/MaterialServiceImpl.java index 7bc1b4b..cdf0813 100644 --- a/oss-server/src/main/java/com/xiaominfo/oss/service/impl/MaterialServiceImpl.java +++ b/oss-server/src/main/java/com/xiaominfo/oss/service/impl/MaterialServiceImpl.java @@ -15,8 +15,6 @@ import com.xiaominfo.oss.api.RootApis; import com.xiaominfo.oss.common.pojo.ThreadLocalHolder; import com.xiaominfo.oss.domain.FileBinaryRequest; import com.xiaominfo.oss.domain.FileBinaryResponse; -import com.xiaominfo.oss.domain.FileStream; -import com.xiaominfo.oss.domain.RemoteResult; import com.xiaominfo.oss.exception.AssemblerException; import com.xiaominfo.oss.exception.ErrorCable; import com.xiaominfo.oss.exception.ErrorConstant; @@ -24,7 +22,9 @@ import com.xiaominfo.oss.module.model.OSSAppInfo; import com.xiaominfo.oss.module.model.OSSDeveloper; import com.xiaominfo.oss.module.model.OSSInformation; import com.xiaominfo.oss.module.model.OSSMaterialInfo; +import com.xiaominfo.oss.sdk.client.NettyFileRequest; import com.xiaominfo.oss.service.*; +import com.xiaominfo.oss.sync.Sync; import com.xiaominfo.oss.utils.CommonUtils; import com.xiaominfo.oss.utils.IdGen; import org.apache.commons.codec.binary.Base64; @@ -36,14 +36,17 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; -import java.io.*; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; import java.math.BigDecimal; import java.util.List; /*** * * @since:oss-server 1.0 - * @author xiaoymin@foxmail.com + * @author xiaoymin@foxmail.com * 2018/05/30 11:00 */ @Service @@ -52,6 +55,9 @@ public class MaterialServiceImpl extends RootApis implements MaterialService { @Autowired OSSMaterialInfoService ossMaterialInfoService; + @Autowired + OSSMaterialInfoSyncService ossMaterialInfoSyncService; + @Autowired OSSDeveloperService ossDeveloperService; @@ -64,68 +70,61 @@ public class MaterialServiceImpl extends RootApis implements MaterialService { @Value(value = "${material.excludeFileTypes}") String excludeFileTypes; - private Log log= LogFactory.get(); - + private Log log = LogFactory.get(); @Override - public RemoteResult saveFile(FileStream paramFileStream) throws IOException { - - return null; - } - - @Override - public List saveAndStore(OSSInformation ossInformation,OSSDeveloper ossDeveloper, OSSAppInfo ossApp, File projectDirectory, List fileBinaryRequests) throws IOException { - List excludeFileTypeLists=Lists.newArrayList(StrUtil.split(excludeFileTypes,",")); - String ip= CommonUtils.getIpAddr(ThreadLocalHolder.getRequest()); + public List saveAndStore(OSSInformation ossInformation, OSSDeveloper ossDeveloper, OSSAppInfo ossApp, File projectDirectory, List fileBinaryRequests) throws IOException { + List excludeFileTypeLists = Lists.newArrayList(StrUtil.split(excludeFileTypes, ",")); + String ip = CommonUtils.getIpAddr(ThreadLocalHolder.getRequest()); //存ossMaterialInfo对象 - List ossMaterialInfos=Lists.newArrayList(); - String root=ossInformation.getRoot(); - File rootFile=new File(root); - int start=rootFile.getAbsolutePath().length(); - List fileBinaryResponses= Lists.newArrayList(); - String currentTimeString= DateTime.now().toString("yyyyMMddHHmmss"); + List ossMaterialInfos = Lists.newArrayList(); + String root = ossInformation.getRoot(); + File rootFile = new File(root); + int start = rootFile.getAbsolutePath().length(); + List fileBinaryResponses = Lists.newArrayList(); + String currentTimeString = DateTime.now().toString("yyyyMMddHHmmss"); //按月目录 final String monthPath = currentTimeString.substring(0, 6); //按日目录,如W020151111 final String dayPath = currentTimeString.substring(6, 8); - final String fileDirectory=projectDirectory.getAbsolutePath()+"/"+monthPath+"/"+dayPath; - log.info("directory:{}",fileDirectory); - File saveFileDirectory=new File(fileDirectory); + final String fileDirectory = projectDirectory.getAbsolutePath() + "/" + monthPath + "/" + dayPath; + log.info("directory:{}", fileDirectory); + File saveFileDirectory = new File(fileDirectory); createDirectoryQuietly(saveFileDirectory); - BigDecimal totalUseSpaceByDev=new BigDecimal(ossDeveloper.getUseSpace()); - BigDecimal appTotalUseSpace=new BigDecimal(ossApp.getUseSpace()); - for (FileBinaryRequest fileBinaryRequest:fileBinaryRequests){ - if (excludeFileTypeLists.contains(fileBinaryRequest.getMediaType())){ - throw new AssemblerException(new ErrorCable(ErrorConstant.REQUEST_PARAMS_NOT_VALID,"Your uploaded file may harm the server")); + BigDecimal totalUseSpaceByDev = new BigDecimal(ossDeveloper.getUseSpace()); + BigDecimal appTotalUseSpace = new BigDecimal(ossApp.getUseSpace()); + for (FileBinaryRequest fileBinaryRequest : fileBinaryRequests) { + if (excludeFileTypeLists.contains(fileBinaryRequest.getMediaType())) { + throw new AssemblerException(new ErrorCable(ErrorConstant.REQUEST_PARAMS_NOT_VALID, "Your uploaded file may harm the server")); } - String originalName=fileBinaryRequest.getOriginalName(); + String originalName = fileBinaryRequest.getOriginalName(); byte[] bs = Base64.decodeBase64(fileBinaryRequest.getFile()); //文件id - String uuid=IdGen.getUUID(); + String uuid = IdGen.getUUID(); //生成uuid文件名称 - String fileName= uuid+"."+fileBinaryRequest.getMediaType(); + String fileName = uuid + "." + fileBinaryRequest.getMediaType(); //静态资源存储路径 - StringBuffer storePathBuffer=new StringBuffer(); - String pre=projectDirectory.getAbsolutePath().substring(start); - pre= com.xiaominfo.oss.utils.FileUtils.transforSysSpec(pre); - if (!pre.startsWith("/")){ + StringBuffer storePathBuffer = new StringBuffer(); + String pre = projectDirectory.getAbsolutePath().substring(start); + pre = com.xiaominfo.oss.utils.FileUtils.transforSysSpec(pre); + if (!pre.startsWith("/")) { storePathBuffer.append("/"); } storePathBuffer.append(pre).append("/") .append(monthPath).append("/").append(dayPath).append("/").append(fileName); //判断最后一位是否是/ - String invokePath=ossInformation.getInvokingRoot(); - if (ossInformation.getInvokingRoot().endsWith("/")){ - invokePath= ossInformation.getInvokingRoot().substring(0,invokePath.lastIndexOf("/")); + String invokePath = ossInformation.getInvokingRoot(); + if (ossInformation.getInvokingRoot().endsWith("/")) { + invokePath = ossInformation.getInvokingRoot().substring(0, invokePath.lastIndexOf("/")); } - String url=invokePath+storePathBuffer.toString(); - File targetFile=new File(saveFileDirectory.getAbsolutePath()+"/"+fileName); + String url = invokePath + storePathBuffer.toString(); + File targetFile = new File(saveFileDirectory.getAbsolutePath() + "/" + fileName); //输出文件 - FileUtils.writeByteArrayToFile(targetFile,bs); + FileUtils.writeByteArrayToFile(targetFile, bs); //不可执行,防止恶意脚本攻击system com.xiaominfo.oss.utils.FileUtils.fileUnModified(targetFile); - FileBinaryResponse fileBinaryResponse=new FileBinaryResponse(uuid,url,storePathBuffer.toString()); + FileBinaryResponse fileBinaryResponse = new FileBinaryResponse(uuid, url, storePathBuffer.toString()); //赋值其他属性 fileBinaryResponse.setByteLength(targetFile.length()); fileBinaryResponse.setByteToStr(com.xiaominfo.oss.utils.FileUtils.byteToString(targetFile.length())); @@ -134,82 +133,142 @@ public class MaterialServiceImpl extends RootApis implements MaterialService { fileBinaryResponses.add(fileBinaryResponse); //添加ossMaterial对象 //追加 - totalUseSpaceByDev=totalUseSpaceByDev.add(new BigDecimal(targetFile.length())); + totalUseSpaceByDev = totalUseSpaceByDev.add(new BigDecimal(targetFile.length())); //项目 - appTotalUseSpace=appTotalUseSpace.add(new BigDecimal(targetFile.length())); - ossMaterialInfos.add(createTargetMaterial(ip,originalName,fileBinaryRequest.getMediaType(),ossApp,targetFile,fileBinaryResponse)); + appTotalUseSpace = appTotalUseSpace.add(new BigDecimal(targetFile.length())); + ossMaterialInfos.add(createTargetMaterial(ip, originalName, fileBinaryRequest.getMediaType(), ossApp, targetFile, fileBinaryResponse)); + //集群间同步 + Sync.syncSalve(root, targetFile, fileBinaryResponse.getId()); + //更新数据库 + ossMaterialInfoSyncService.createSync(fileBinaryResponse.getId(), "Master"); } ossMaterialInfoService.insertBatch(ossMaterialInfos); //更新ossdev - ossDeveloperService.updateById(new OSSDeveloper(ossDeveloper.getId(),totalUseSpaceByDev.longValue(), com.xiaominfo.oss.utils.FileUtils.byteToString(totalUseSpaceByDev.longValue()))); + ossDeveloperService.updateById(new OSSDeveloper(ossDeveloper.getId(), totalUseSpaceByDev.longValue(), com.xiaominfo.oss.utils.FileUtils.byteToString(totalUseSpaceByDev.longValue()))); //更新项目占用空间 - ossAppInfoService.updateById(new OSSAppInfo(ossApp.getId(),appTotalUseSpace.longValue(), com.xiaominfo.oss.utils.FileUtils.byteToString(appTotalUseSpace.longValue()))); + ossAppInfoService.updateById(new OSSAppInfo(ossApp.getId(), appTotalUseSpace.longValue(), com.xiaominfo.oss.utils.FileUtils.byteToString(appTotalUseSpace.longValue()))); return fileBinaryResponses; } @Override - public List saveAndStore(OSSInformation ossInformation,OSSDeveloper ossDeveloper, OSSAppInfo ossApp,File projectDirectory, MultipartFile[] multipartFiles) throws IOException { - List excludeFileTypeLists=Lists.newArrayList(StrUtil.split(excludeFileTypes,",")); - String ip= CommonUtils.getIpAddr(ThreadLocalHolder.getRequest()); + public FileBinaryResponse saveAndStore(OSSInformation ossInformation, OSSDeveloper ossDeveloper, OSSAppInfo ossApp, NettyFileRequest ef, File file) throws IOException { + //存ossMaterialInfo对象 + List ossMaterialInfos = Lists.newArrayList(); + String root = ossInformation.getRoot(); + File rootFile = new File(root); + int start = rootFile.getAbsolutePath().length(); + List fileBinaryResponses = Lists.newArrayList(); + + BigDecimal totalUseSpaceByDev = new BigDecimal(ossDeveloper.getUseSpace()); + BigDecimal appTotalUseSpace = new BigDecimal(ossApp.getUseSpace()); + + + String originalName = ef.getOriginalName(); + + //文件id + String uuid = ef.getUuid(); + //静态资源存储路径 + String pre = ef.getProjectPath().substring(start); + pre = com.xiaominfo.oss.utils.FileUtils.transforSysSpec(pre); + //判断最后一位是否是/ + String invokePath = ossInformation.getInvokingRoot(); + if (ossInformation.getInvokingRoot().endsWith("/")) { + invokePath = ossInformation.getInvokingRoot().substring(0, invokePath.lastIndexOf("/")); + } + String url = invokePath + pre; + //输出文件 + File targetFile = new File(ef.getProjectPath()); + //不可执行,防止恶意脚本攻击system + FileBinaryResponse fileBinaryResponse = new FileBinaryResponse(uuid, url, pre); + //赋值其他属性 + fileBinaryResponse.setByteLength(targetFile.length()); + fileBinaryResponse.setByteToStr(com.xiaominfo.oss.utils.FileUtils.byteToString(targetFile.length())); + fileBinaryResponse.setOriginalName(originalName); + fileBinaryResponse.setObjType(ef.getMediaType()); + fileBinaryResponses.add(fileBinaryResponse); + //添加ossMaterial对象 + //追加 + totalUseSpaceByDev = totalUseSpaceByDev.add(new BigDecimal(targetFile.length())); + //项目 + appTotalUseSpace = appTotalUseSpace.add(new BigDecimal(targetFile.length())); + ossMaterialInfos.add(createTargetMaterial("", originalName, ef.getMediaType(), ossApp, targetFile, fileBinaryResponse)); + //集群间同步 + log.info("开启同步 start"); + Sync.syncSalve(root, targetFile, fileBinaryResponse.getId()); + log.info("开启同步 end"); + //更新数据库 + ossMaterialInfoSyncService.createSync(fileBinaryResponse.getId(), "Master"); + ossMaterialInfoService.insertBatch(ossMaterialInfos); + //更新ossdev + ossDeveloperService.updateById(new OSSDeveloper(ossDeveloper.getId(), totalUseSpaceByDev.longValue(), com.xiaominfo.oss.utils.FileUtils.byteToString(totalUseSpaceByDev.longValue()))); + //更新项目占用空间 + ossAppInfoService.updateById(new OSSAppInfo(ossApp.getId(), appTotalUseSpace.longValue(), com.xiaominfo.oss.utils.FileUtils.byteToString(appTotalUseSpace.longValue()))); + return fileBinaryResponse; + } + + @Override + public List saveAndStore(OSSInformation ossInformation, OSSDeveloper ossDeveloper, OSSAppInfo ossApp, File projectDirectory, MultipartFile[] multipartFiles) throws IOException { + List excludeFileTypeLists = Lists.newArrayList(StrUtil.split(excludeFileTypes, ",")); + String ip = CommonUtils.getIpAddr(ThreadLocalHolder.getRequest()); //存ossMaterialInfo对象 - List ossMaterialInfos=Lists.newArrayList(); - String root=ossInformation.getRoot(); - File rootFile=new File(root); - int start=rootFile.getAbsolutePath().length(); - List fileBinaryResponses= Lists.newArrayList(); - String currentTimeString= DateTime.now().toString("yyyyMMddHHmmss"); + List ossMaterialInfos = Lists.newArrayList(); + String root = ossInformation.getRoot(); + File rootFile = new File(root); + int start = rootFile.getAbsolutePath().length(); + List fileBinaryResponses = Lists.newArrayList(); + String currentTimeString = DateTime.now().toString("yyyyMMddHHmmss"); //按月目录 final String monthPath = currentTimeString.substring(0, 6); //按日目录,如W020151111 final String dayPath = currentTimeString.substring(6, 8); - final String fileDirectory=projectDirectory.getAbsolutePath()+"/"+monthPath+"/"+dayPath; - log.info("directory:{}",fileDirectory); - File saveFileDirectory=new File(fileDirectory); + final String fileDirectory = projectDirectory.getAbsolutePath() + "/" + monthPath + "/" + dayPath; + log.info("directory:{}", fileDirectory); + File saveFileDirectory = new File(fileDirectory); createDirectoryQuietly(saveFileDirectory); - BigDecimal totalUseSpaceByDev=new BigDecimal(ossDeveloper.getUseSpace()); - BigDecimal appTotalUseSpace=new BigDecimal(ossApp.getUseSpace()); - for (MultipartFile multpFile:multipartFiles){ - String originalName=multpFile.getOriginalFilename(); + BigDecimal totalUseSpaceByDev = new BigDecimal(ossDeveloper.getUseSpace()); + BigDecimal appTotalUseSpace = new BigDecimal(ossApp.getUseSpace()); + for (MultipartFile multpFile : multipartFiles) { + String originalName = multpFile.getOriginalFilename(); //文件id - String uuid=IdGen.getUUID(); - String fileName=uuid; + String uuid = IdGen.getUUID(); + String fileName = uuid; String mediaType = ""; - if (originalName.lastIndexOf(".")>0){ - mediaType=multpFile.getOriginalFilename().substring(multpFile.getOriginalFilename().lastIndexOf(".")+1); - fileName=fileName+"."+mediaType; + if (originalName.lastIndexOf(".") > 0) { + mediaType = multpFile.getOriginalFilename().substring(multpFile.getOriginalFilename().lastIndexOf(".") + 1); + fileName = fileName + "." + mediaType; } - if (excludeFileTypeLists.contains(mediaType)){ - throw new AssemblerException(new ErrorCable(ErrorConstant.REQUEST_PARAMS_NOT_VALID,"Your uploaded file may harm the server ")); + if (excludeFileTypeLists.contains(mediaType)) { + throw new AssemblerException(new ErrorCable(ErrorConstant.REQUEST_PARAMS_NOT_VALID, "Your uploaded file may harm the server ")); } //静态资源存储路径 - StringBuffer storePathBuffer=new StringBuffer(); - String pre=projectDirectory.getAbsolutePath().substring(start); - pre= com.xiaominfo.oss.utils.FileUtils.transforSysSpec(pre); - if (!pre.startsWith("/")){ + StringBuffer storePathBuffer = new StringBuffer(); + String pre = projectDirectory.getAbsolutePath().substring(start); + pre = com.xiaominfo.oss.utils.FileUtils.transforSysSpec(pre); + if (!pre.startsWith("/")) { storePathBuffer.append("/"); } storePathBuffer.append(pre).append("/") .append(monthPath).append("/").append(dayPath).append("/").append(fileName); //判断最后一位是否是/ - String invokePath=ossInformation.getInvokingRoot(); - if (ossInformation.getInvokingRoot().endsWith("/")){ - invokePath= ossInformation.getInvokingRoot().substring(0,invokePath.lastIndexOf("/")); + String invokePath = ossInformation.getInvokingRoot(); + if (ossInformation.getInvokingRoot().endsWith("/")) { + invokePath = ossInformation.getInvokingRoot().substring(0, invokePath.lastIndexOf("/")); } - String url=invokePath+storePathBuffer.toString(); - File targetFile=new File(saveFileDirectory.getAbsolutePath()+"/"+fileName); - FileOutputStream fos=new FileOutputStream(targetFile); - InputStream ins=multpFile.getInputStream(); - byte[] byts=new byte[1024*1024]; - int len=-1; - while ((len=ins.read(byts))!=-1){ - fos.write(byts,0,len); + String url = invokePath + storePathBuffer.toString(); + File targetFile = new File(saveFileDirectory.getAbsolutePath() + "/" + fileName); + FileOutputStream fos = new FileOutputStream(targetFile); + InputStream ins = multpFile.getInputStream(); + byte[] byts = new byte[1024 * 1024]; + int len = -1; + while ((len = ins.read(byts)) != -1) { + fos.write(byts, 0, len); } //关闭输入输出流 IOUtils.closeQuietly(fos); IOUtils.closeQuietly(ins); //不可执行,防止恶意脚本攻击system com.xiaominfo.oss.utils.FileUtils.fileUnModified(targetFile); - FileBinaryResponse fileBinaryResponse=new FileBinaryResponse(uuid,url,storePathBuffer.toString()); + FileBinaryResponse fileBinaryResponse = new FileBinaryResponse(uuid, url, storePathBuffer.toString()); //赋值其他属性 fileBinaryResponse.setByteLength(targetFile.length()); fileBinaryResponse.setByteToStr(com.xiaominfo.oss.utils.FileUtils.byteToString(targetFile.length())); @@ -218,23 +277,23 @@ public class MaterialServiceImpl extends RootApis implements MaterialService { fileBinaryResponses.add(fileBinaryResponse); //添加ossMaterial对象 //追加 - totalUseSpaceByDev=totalUseSpaceByDev.add(new BigDecimal(targetFile.length())); + totalUseSpaceByDev = totalUseSpaceByDev.add(new BigDecimal(targetFile.length())); //项目 - appTotalUseSpace=appTotalUseSpace.add(new BigDecimal(targetFile.length())); - ossMaterialInfos.add(createTargetMaterial(ip,originalName,mediaType,ossApp,targetFile,fileBinaryResponse)); + appTotalUseSpace = appTotalUseSpace.add(new BigDecimal(targetFile.length())); + ossMaterialInfos.add(createTargetMaterial(ip, originalName, mediaType, ossApp, targetFile, fileBinaryResponse)); } ossMaterialInfoService.insertBatch(ossMaterialInfos); //更新ossdev - ossDeveloperService.updateById(new OSSDeveloper(ossDeveloper.getId(),totalUseSpaceByDev.longValue(), com.xiaominfo.oss.utils.FileUtils.byteToString(totalUseSpaceByDev.longValue()))); + ossDeveloperService.updateById(new OSSDeveloper(ossDeveloper.getId(), totalUseSpaceByDev.longValue(), com.xiaominfo.oss.utils.FileUtils.byteToString(totalUseSpaceByDev.longValue()))); //更新项目占用空间 - ossAppInfoService.updateById(new OSSAppInfo(ossApp.getId(),appTotalUseSpace.longValue(), com.xiaominfo.oss.utils.FileUtils.byteToString(appTotalUseSpace.longValue()))); + ossAppInfoService.updateById(new OSSAppInfo(ossApp.getId(), appTotalUseSpace.longValue(), com.xiaominfo.oss.utils.FileUtils.byteToString(appTotalUseSpace.longValue()))); return fileBinaryResponses; } - private OSSMaterialInfo createTargetMaterial(String ip,String originalName,String mediaType,OSSAppInfo ossAppInfo,File targetFile,FileBinaryResponse fileBinaryResponse){ + private OSSMaterialInfo createTargetMaterial(String ip, String originalName, String mediaType, OSSAppInfo ossAppInfo, File targetFile, FileBinaryResponse fileBinaryResponse) { //添加ossMaterial对象 - OSSMaterialInfo ossMaterialInfo=new OSSMaterialInfo(); + OSSMaterialInfo ossMaterialInfo = new OSSMaterialInfo(); ossMaterialInfo.setOriginalName(originalName); ossMaterialInfo.setFromIp(ip); ossMaterialInfo.setCreateTime(DateTime.now().toString("yyyy-MM-dd HH:mm:ss")); @@ -251,53 +310,52 @@ public class MaterialServiceImpl extends RootApis implements MaterialService { } @Override - public List saveAndStoreBySys(OSSInformation ossInformation,File projectDirectory, MultipartFile[] multipartFiles) throws IOException { - String root=ossInformation.getRoot(); - File rootFile=new File(root); - int start=rootFile.getAbsolutePath().length(); - List fileBinaryResponses= Lists.newArrayList(); + public List saveAndStoreBySys(OSSInformation ossInformation, File projectDirectory, MultipartFile[] multipartFiles) throws IOException { + String root = ossInformation.getRoot(); + File rootFile = new File(root); + int start = rootFile.getAbsolutePath().length(); + List fileBinaryResponses = Lists.newArrayList(); //按日目录,如W020151111 - final String fileDirectory=projectDirectory.getAbsolutePath(); - log.info("directory:{}",fileDirectory); - File saveFileDirectory=new File(fileDirectory); + final String fileDirectory = projectDirectory.getAbsolutePath(); + log.info("directory:{}", fileDirectory); + File saveFileDirectory = new File(fileDirectory); createDirectoryQuietly(saveFileDirectory); - for (MultipartFile multpFile:multipartFiles){ + for (MultipartFile multpFile : multipartFiles) { //文件id - String uuid=IdGen.getUUID(); - String fileName=uuid; + String uuid = IdGen.getUUID(); + String fileName = uuid; String mediaType = ""; - if (multpFile.getOriginalFilename().lastIndexOf(".")>0){ - mediaType=multpFile.getOriginalFilename().substring(multpFile.getOriginalFilename().lastIndexOf(".")+1); - fileName=fileName+"."+mediaType; + if (multpFile.getOriginalFilename().lastIndexOf(".") > 0) { + mediaType = multpFile.getOriginalFilename().substring(multpFile.getOriginalFilename().lastIndexOf(".") + 1); + fileName = fileName + "." + mediaType; } //静态资源存储路径 - StringBuffer storePathBuffer=new StringBuffer(); - String pre=projectDirectory.getAbsolutePath().substring(start); - pre= com.xiaominfo.oss.utils.FileUtils.transforSysSpec(pre); - if (!pre.startsWith("/")){ + StringBuffer storePathBuffer = new StringBuffer(); + String pre = projectDirectory.getAbsolutePath().substring(start); + pre = com.xiaominfo.oss.utils.FileUtils.transforSysSpec(pre); + if (!pre.startsWith("/")) { storePathBuffer.append("/"); } - storePathBuffer.append(pre).append("/").append(fileName); //判断最后一位是否是/ - String invokePath=ossInformation.getInvokingRoot(); - if (ossInformation.getInvokingRoot().endsWith("/")){ - invokePath= ossInformation.getInvokingRoot().substring(0,invokePath.lastIndexOf("/")); + String invokePath = ossInformation.getInvokingRoot(); + if (ossInformation.getInvokingRoot().endsWith("/")) { + invokePath = ossInformation.getInvokingRoot().substring(0, invokePath.lastIndexOf("/")); } - String url=invokePath+storePathBuffer.toString(); - File targetFile=new File(saveFileDirectory.getAbsolutePath()+"/"+fileName); - FileOutputStream fos=new FileOutputStream(targetFile); - InputStream ins=multpFile.getInputStream(); - byte[] byts=new byte[1024*1024]; - int len=-1; - while ((len=ins.read(byts))!=-1){ - fos.write(byts,0,len); + String url = invokePath + pre; + File targetFile = new File(saveFileDirectory.getAbsolutePath() + "/" + fileName); + FileOutputStream fos = new FileOutputStream(targetFile); + InputStream ins = multpFile.getInputStream(); + byte[] byts = new byte[1024 * 1024]; + int len = -1; + while ((len = ins.read(byts)) != -1) { + fos.write(byts, 0, len); } //关闭输入输出流 IOUtils.closeQuietly(fos); IOUtils.closeQuietly(ins); //不可执行,防止恶意脚本攻击system targetFile.setExecutable(false); - FileBinaryResponse fileBinaryResponse=new FileBinaryResponse(uuid,url,storePathBuffer.toString()); + FileBinaryResponse fileBinaryResponse = new FileBinaryResponse(uuid, url, storePathBuffer.toString()); fileBinaryResponses.add(fileBinaryResponse); } return fileBinaryResponses; diff --git a/oss-server/src/main/java/com/xiaominfo/oss/service/impl/OSSMaterialInfoServiceImpl.java b/oss-server/src/main/java/com/xiaominfo/oss/service/impl/OSSMaterialInfoServiceImpl.java index cb3a50d..ee6433b 100644 --- a/oss-server/src/main/java/com/xiaominfo/oss/service/impl/OSSMaterialInfoServiceImpl.java +++ b/oss-server/src/main/java/com/xiaominfo/oss/service/impl/OSSMaterialInfoServiceImpl.java @@ -14,7 +14,6 @@ import com.google.common.collect.Maps; import com.xiaominfo.oss.common.pojo.Pagination; import com.xiaominfo.oss.module.dao.OSSMaterialInfoMapper; import com.xiaominfo.oss.module.entity.OSSMaterialInfoResult; -import com.xiaominfo.oss.module.model.OSSDeveloper; import com.xiaominfo.oss.module.model.OSSMaterialInfo; import com.xiaominfo.oss.service.OSSMaterialInfoService; import com.xiaominfo.oss.utils.FileUtils; @@ -30,19 +29,19 @@ import java.util.Map; /*** * * @since:oss-server 1.0 - * @author xiaoymin@foxmail.com + * @author xiaoymin@foxmail.com * 2018/06/18 8:54 */ @Service -public class OSSMaterialInfoServiceImpl extends ServiceImpl implements OSSMaterialInfoService { +public class OSSMaterialInfoServiceImpl extends ServiceImpl implements OSSMaterialInfoService { @Autowired OSSMaterialInfoMapper ossMaterialInfoMapper; @Override public String queryTotalSpaceByteStr() { - Integer len=ossMaterialInfoMapper.selectMaterialUseSpace(); - if (len==null){ + Integer len = ossMaterialInfoMapper.selectMaterialUseSpace(); + if (len == null) { return "0kb"; } return FileUtils.byteToString(new BigDecimal(len).longValue()); @@ -55,15 +54,20 @@ public class OSSMaterialInfoServiceImpl extends ServiceImpl queryByPage(OSSMaterialInfo ossMaterialInfo, Integer current_page, Integer page_size) { - Wrapper wrapper=new EntityWrapper<>(); - Integer count=ossMaterialInfoMapper.selectCount(wrapper); - Map params= Maps.newHashMap(); - RowBounds rowBounds= PaginationUtils.getBounds(current_page,page_size); + Wrapper wrapper = new EntityWrapper<>(); + Integer count = ossMaterialInfoMapper.selectCount(wrapper); + Map params = Maps.newHashMap(); + RowBounds rowBounds = PaginationUtils.getBounds(current_page, page_size); //分页 - params.put("start",rowBounds.getOffset()); - params.put("page_size",page_size); - List list=ossMaterialInfoMapper.queryByPage(params); - Pagination pagination=PaginationUtils.transfor(list,count,current_page,page_size); + params.put("start", rowBounds.getOffset()); + params.put("page_size", page_size); + List list = ossMaterialInfoMapper.queryByPage(params); + Pagination pagination = PaginationUtils.transfor(list, count, current_page, page_size); return pagination; } + + @Override + public List waitSync(String node,String thatNode) { + return ossMaterialInfoMapper.queryWaitSync(node,thatNode); + } } diff --git a/oss-server/src/main/java/com/xiaominfo/oss/service/impl/OSSMaterialInfoSyncServiceImpl.java b/oss-server/src/main/java/com/xiaominfo/oss/service/impl/OSSMaterialInfoSyncServiceImpl.java new file mode 100644 index 0000000..87378e2 --- /dev/null +++ b/oss-server/src/main/java/com/xiaominfo/oss/service/impl/OSSMaterialInfoSyncServiceImpl.java @@ -0,0 +1,184 @@ +/* + * Copyright (C) 2018 Zhejiang xiaominfo Technology CO.,LTD. + * All rights reserved. + * Official Web Site: http://www.xiaominfo.com. + * Developer Web Site: http://open.xiaominfo.com. + */ + +package com.xiaominfo.oss.service.impl; + +import cn.hutool.core.bean.BeanUtil; +import cn.hutool.core.util.StrUtil; +import com.baomidou.mybatisplus.service.impl.ServiceImpl; +import com.google.common.collect.Lists; +import com.xiaominfo.oss.api.RootApis; +import com.xiaominfo.oss.common.SpringContextUtil; +import com.xiaominfo.oss.exception.AssemblerException; +import com.xiaominfo.oss.exception.ErrorCable; +import com.xiaominfo.oss.exception.ErrorConstant; +import com.xiaominfo.oss.module.dao.OSSMaterialInfoSyncMapper; +import com.xiaominfo.oss.module.model.OSSAppInfo; +import com.xiaominfo.oss.module.model.OSSDeveloper; +import com.xiaominfo.oss.module.model.OSSInformation; +import com.xiaominfo.oss.module.model.OSSMaterialInfoSync; +import com.xiaominfo.oss.sdk.client.NettyFileRequest; +import com.xiaominfo.oss.service.OSSAppInfoService; +import com.xiaominfo.oss.service.OSSDeveloperService; +import com.xiaominfo.oss.service.OSSInformationService; +import com.xiaominfo.oss.service.OSSMaterialInfoSyncService; +import com.xiaominfo.oss.sync.Node; +import com.xiaominfo.oss.utils.IdGen; +import io.netty.util.internal.StringUtil; +import org.joda.time.DateTime; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.io.File; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.List; + +/*** + * + * @since:oss-server 1.0 + * @author xiaoymin@foxmail.com + * 2018/06/18 8:54 + */ +@Service +public class OSSMaterialInfoSyncServiceImpl extends ServiceImpl implements OSSMaterialInfoSyncService { + + Logger log = LoggerFactory.getLogger(OSSMaterialInfoSyncServiceImpl.class); + @Autowired + OSSMaterialInfoSyncMapper ossMaterialInfoSyncMapper; + @Autowired + OSSDeveloperService ossDeveloperService; + @Autowired + OSSAppInfoService ossAppInfoService; + @Autowired + OSSInformationService ossInformationService; + + @Override + public OSSMaterialInfoSync createSync(String materialId, String type) { + String localIp = ""; + try { + localIp = InetAddress.getLocalHost().getHostAddress().toString(); + } catch (UnknownHostException e) { + log.info("未找到本机ip：", e); + } + int masterPort = Integer.parseInt(SpringContextUtil.getProperty("material.masterPort")); + OSSMaterialInfoSync sync = new OSSMaterialInfoSync(); + //sync.setType("Master"); + sync.setType(type); + sync.setNode(new Node(masterPort, localIp).toString()); + sync.setMaterialId(materialId); + sync.setCreateTime(DateTime.now().toString("yyyy-MM-dd HH:mm:ss")); + this.insert(sync); + return sync; + } + + + @Override + public String buildPath(NettyFileRequest ef, OSSInformation ossInformationRef, OSSDeveloper ossDeveloperRef, OSSAppInfo ossAppRef) { + //第一次的时候才生成路径和做检查 + if (!StringUtil.isNullOrEmpty(ef.getProjectPath())) { + return ef.getProjectPath(); + } + + List excludeFileTypeLists = Lists.newArrayList(StrUtil.split(SpringContextUtil.getProperty("material.excludeFileTypes"), ",")); + if (excludeFileTypeLists.contains(ef.getMediaType())) { + throw new AssemblerException(new ErrorCable(ErrorConstant.REQUEST_PARAMS_NOT_VALID, "Your uploaded file may harm the server")); + } + + RootApis apis = new RootApis(); + String moduleStr = ""; + apis.assertArgumentNotEmpty(ef.getAppid(), "appid can't be empty"); + apis.assertArgumentNotEmpty(ef.getAppsecret(), "appsecret can't be empty"); + apis.assertArgumentNotEmpty(ef.getProject(), "project name can't be empty"); + apis.assertArgumentNotEmpty(ef.getFile(), "files can't be empty"); + if (!StringUtil.isNullOrEmpty(ef.getModule())) { + moduleStr = ef.getModule(); + } + + OSSDeveloper ossDeveloper = ossDeveloperService.queryByAppid(ef.getAppid(), ef.getAppsecret()); + apis.assertArgumentNotEmpty(ossDeveloper, "appid or appsecret is invalid"); + String projectFilePathName = ef.getProject(); + apis.validateProjectName(projectFilePathName); + //判断文件夹code是否相等 + List ossAppInfos = ossAppInfoService.queryByDevIds(ossDeveloper.getId()); + if (ossAppInfos == null || ossAppInfos.size() == 0) { + throw new AssemblerException(new ErrorCable(ErrorConstant.REQUEST_PARAMS_NOT_VALID, "You do not have permission to upload files")); + } + OSSAppInfo ossApp = null; + + for (OSSAppInfo ossAppInfo : ossAppInfos) { + if (StrUtil.equalsIgnoreCase(ossAppInfo.getCode(), projectFilePathName)) { + ossApp = ossAppInfo; + break; + } + } + if (null==ossApp) { + throw new AssemblerException(new ErrorCable(ErrorConstant.REQUEST_PARAMS_NOT_VALID, "You do not have permission to upload files")); + } + + OSSInformation ossInformation = ossInformationService.queryOne(); + String root = ossInformation.getRoot(); + //验证文件夹规则,不能包含特殊字符 + File rootFile = new File(root); + //createDirectoryQuietly(file); + apis.createDirectoryQuietly(rootFile); + + StringBuffer path = new StringBuffer(); + path.append(rootFile.getAbsolutePath()); + path.append(File.separator); + path.append(projectFilePathName); + if (StrUtil.isNotEmpty(moduleStr)) { + if (!moduleStr.startsWith("/")) { + path.append("/"); + } + path.append(moduleStr); + } + log.info("path:{}", path); + File projectFile = new File(path.toString()); + if (!projectFile.exists()) { + throw new AssemblerException(new ErrorCable(ErrorConstant.AUTHENTICATION_FAILED, "You do not have operating authority for this directory " + projectFilePathName + ", or the directory was not created")); + } + + + int start = rootFile.getAbsolutePath().length(); + String currentTimeString = DateTime.now().toString("yyyyMMddHHmmss"); + //按月目录 + final String monthPath = currentTimeString.substring(0, 6); + //按日目录,如W020151111 + final String dayPath = currentTimeString.substring(6, 8); + final String fileDirectory = projectFile.getAbsolutePath() + "/" + monthPath + "/" + dayPath; + log.info("directory:{}", fileDirectory); + File saveFileDirectory = new File(fileDirectory); + apis.createDirectoryQuietly(saveFileDirectory); + + + //文件id + String uuid = IdGen.getUUID(); + //生成uuid文件名称 + String fileName = uuid + "." + ef.getMediaType(); + //静态资源存储路径 + StringBuffer storePathBuffer = new StringBuffer(); + String pre = projectFile.getAbsolutePath().substring(start); + pre = com.xiaominfo.oss.utils.FileUtils.transforSysSpec(pre); + if (!pre.startsWith("/")) { + storePathBuffer.append("/"); + } + storePathBuffer.append(pre).append("/").append(monthPath).append("/").append(dayPath).append("/").append(fileName); + ef.setUuid(uuid); + //输出文件 + String saveFilePath = saveFileDirectory.getAbsolutePath() + "/" + fileName; + BeanUtil.copyProperties(ossInformation, ossInformationRef); + BeanUtil.copyProperties(ossDeveloper, ossDeveloperRef); + BeanUtil.copyProperties(ossApp, ossAppRef); + log.info("saveFilePath = [{}]", saveFilePath); + return saveFilePath; + } + + +} diff --git a/oss-server/src/main/java/com/xiaominfo/oss/sync/Node.java b/oss-server/src/main/java/com/xiaominfo/oss/sync/Node.java new file mode 100644 index 0000000..511df52 --- /dev/null +++ b/oss-server/src/main/java/com/xiaominfo/oss/sync/Node.java @@ -0,0 +1,32 @@ +package com.xiaominfo.oss.sync; + +public class Node { + private int port; + private String ip; + + public Node(int port, String ip) { + this.port = port; + this.ip = ip; + } + + public int getPort() { + return port; + } + + public void setPort(int port) { + this.port = port; + } + + public String getIp() { + return ip; + } + + public void setIp(String ip) { + this.ip = ip; + } + + @Override + public String toString() { + return ip + ":" + port; + } +} diff --git a/oss-server/src/main/java/com/xiaominfo/oss/sync/Sync.java b/oss-server/src/main/java/com/xiaominfo/oss/sync/Sync.java new file mode 100644 index 0000000..496f982 --- /dev/null +++ b/oss-server/src/main/java/com/xiaominfo/oss/sync/Sync.java @@ -0,0 +1,171 @@ +package com.xiaominfo.oss.sync; + +import com.xiaominfo.oss.common.SpringContextUtil; +import com.xiaominfo.oss.module.model.OSSMaterialInfo; +import com.xiaominfo.oss.service.OSSInformationService; +import com.xiaominfo.oss.service.OSSMaterialInfoService; +import com.xiaominfo.oss.sync.netty.SalveServer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * 集群分发 + */ +public class Sync { + private static Logger logger = LoggerFactory.getLogger(Sync.class); + private static List nodeList = new ArrayList<>(); + + + static { + String[] node = SpringContextUtil.getProperty("material.nodes").split(","); + for (String str : node) { + int port = Integer.parseInt(str.split(":")[1]); + String ip = str.split(":")[0]; + Node n = new Node(port, ip); + nodeList.add(n); + } + startMaster(); + } + + /** + * 同步到其他节点 + */ + public static void syncSalve(String root, File file, String materialId) { + for (Node node : nodeList) { + try { + Sync.startSalve(root, node, file, materialId); + //syncMessage(root, node, file, materialId); + } catch (Exception e) { + logger.error("node=[{}]同步失败,fileName=[{}]", node, file.getName()); + } + } + } + + private static void startSalve(String root, Node node, File file, String materialId) { + ExecutorService executorService = Executors.newFixedThreadPool(10); + Runnable runnable = () -> Sync.syncMessage(root, node, file, materialId); + executorService.submit(runnable); + } + + /** + * 发送文件同步 + * + * @param root + * @param node + * @param file + * @param materialId + */ + protected static void syncMessage(String root, Node node, File file, String materialId) { + try { + SyncMessage uploadFile = new SyncMessage(); + String fileMd5 = file.getName();// 文件名 + uploadFile.setFile(file); + uploadFile.setFile_md5(fileMd5); + uploadFile.setFilePath(file.getParent().replaceFirst(root.replaceAll("/", "\\\\").replaceAll("\\\\", "\\\\\\\\"), "") + File.separator); + uploadFile.setStarPos(0);// 文件开始位置 + uploadFile.setMaterialId(materialId); + logger.info("开始 node = [{}] 节点同步 file = [{}]...", node, file.getName()); + if (file.exists()) { + new SalveServer().connect(node.getPort(), node.getIp(), uploadFile); + } else { + logger.info("file = [{}] 不存在,不同步", file.getName()); + } + + } catch (Exception e) { + e.printStackTrace(); + } + } + + public static void startMaster() { + String localIp = ""; + try { + localIp = InetAddress.getLocalHost().getHostAddress().toString(); + } catch (UnknownHostException e) { + logger.info("未找到本机ip：", e); + } + int masterPort = Integer.parseInt(SpringContextUtil.getProperty("material.masterPort")); + Node node = new Node(masterPort, localIp); + + + OSSInformationService ossInformationService = SpringContextUtil.getBean(OSSInformationService.class); + String root = ossInformationService.queryOne().getRoot(); + + ExecutorService executorService = Executors.newFixedThreadPool(nodeList.size()); + + for (Node thatNode : nodeList) { + try { + logger.info("开启 node[{}] 文件同步线程", thatNode); + SyncRunnable syncRunnable = new SyncRunnable(); + syncRunnable.setNode(node); + syncRunnable.setThatNode(thatNode); + syncRunnable.setRoot(root); + executorService.submit(syncRunnable); + } catch (Exception e) { + logger.error("node=[{}]同步失败", e); + } + } + } + +} + +class SyncRunnable implements Runnable { + private static Logger logger = LoggerFactory.getLogger(Sync.class); + private String root; + private Node node; + private Node thatNode; + + @Override + public void run() { + while (true) { + OSSMaterialInfoService syncService = SpringContextUtil.getBean(OSSMaterialInfoService.class); + List materialInfos = syncService.waitSync(node.toString(), thatNode.toString()); + + for (OSSMaterialInfo info : materialInfos) { + try { + File file = new File(root + File.separator + info.getStorePath()); + //Sync.syncSalve(root, file, info.getId()); + Sync.syncMessage(root, thatNode, file, info.getId()); + } catch (Exception e) { + logger.error("当前文件同步失败 material id =[{}]", info.getId(), e); + } + } + try { + Thread.sleep(1000 * 60 * 10);//10分钟 + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + + public String getRoot() { + return root; + } + + public void setRoot(String root) { + this.root = root; + } + + public Node getThatNode() { + return thatNode; + } + + public void setThatNode(Node thatNode) { + this.thatNode = thatNode; + } + + public Node getNode() { + return node; + } + + public void setNode(Node node) { + this.node = node; + } +} diff --git a/oss-server/src/main/java/com/xiaominfo/oss/sync/SyncMessage.java b/oss-server/src/main/java/com/xiaominfo/oss/sync/SyncMessage.java new file mode 100644 index 0000000..8f5865f --- /dev/null +++ b/oss-server/src/main/java/com/xiaominfo/oss/sync/SyncMessage.java @@ -0,0 +1,77 @@ +package com.xiaominfo.oss.sync; + +import java.io.File; +import java.io.Serializable; + +public class SyncMessage implements Serializable { + private static final long serialVersionUID = 1L; + private File file;// 文件 + private String filePath;// 文件路径 + private String materialId;//对象的id + private String file_md5;// 文件名 + private int starPos;// 开始位置 + private byte[] bytes;// 文件字节数组 + private int endPos;// 结尾位置 + + public String getMaterialId() { + return materialId; + } + + public void setMaterialId(String materialId) { + this.materialId = materialId; + } + + public String getFilePath() { + return filePath; + } + + public void setFilePath(String filePath) { + this.filePath = filePath; + } + + public File getFile() { + return file; + } + + public void setFile(File file) { + this.file = file; + } + + public String getFile_md5() { + return file_md5; + } + + public void setFile_md5(String file_md5) { + this.file_md5 = file_md5; + } + + public int getStarPos() { + return starPos; + } + + public void setStarPos(int starPos) { + this.starPos = starPos; + } + + public byte[] getBytes() { + return bytes; + } + + public void setBytes(byte[] bytes) { + this.bytes = bytes; + } + + public int getEndPos() { + return endPos; + } + + public void setEndPos(int endPos) { + this.endPos = endPos; + } + + public static long getSerialversionuid() { + return serialVersionUID; + } + +} + diff --git a/oss-server/src/main/java/com/xiaominfo/oss/sync/netty/MasterServer.java b/oss-server/src/main/java/com/xiaominfo/oss/sync/netty/MasterServer.java new file mode 100644 index 0000000..a664b9a --- /dev/null +++ b/oss-server/src/main/java/com/xiaominfo/oss/sync/netty/MasterServer.java @@ -0,0 +1,61 @@ +package com.xiaominfo.oss.sync.netty; + +import com.xiaominfo.oss.sync.Sync; +import io.netty.bootstrap.ServerBootstrap; +import io.netty.channel.*; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.nio.NioServerSocketChannel; +import io.netty.handler.codec.serialization.ClassResolvers; +import io.netty.handler.codec.serialization.ObjectDecoder; +import io.netty.handler.codec.serialization.ObjectEncoder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MasterServer { + public static final int FILE_PORT = 18002; + private Logger logger = LoggerFactory.getLogger(Sync.class); + + public void bind(int port) throws Exception { + EventLoopGroup bossGroup = new NioEventLoopGroup(); + EventLoopGroup workerGroup = new NioEventLoopGroup(); + try { + ServerBootstrap b = new ServerBootstrap(); + b.group(bossGroup, workerGroup).channel(NioServerSocketChannel.class).option(ChannelOption.SO_BACKLOG, 1024).childHandler(new ChannelInitializer() { + + @Override + protected void initChannel(Channel ch) throws Exception { + System.out.println("有客户端连接上来:" + ch.localAddress().toString()); + ch.pipeline().addLast(new ObjectEncoder()); + ch.pipeline().addLast(new ObjectDecoder(Integer.MAX_VALUE, ClassResolvers.weakCachingConcurrentResolver(null))); // 最大长度 + ch.pipeline().addLast(new MasterSyncServerHandler()); + } + }); + ChannelFuture f = b.bind(port).sync(); + logger.info("集群同步端口已经启动. port = [{}]", port); + f.channel().closeFuture().sync(); + } finally { + bossGroup.shutdownGracefully(); + workerGroup.shutdownGracefully(); + } + } + + public static void main(String[] args) { + int port = FILE_PORT; + if (args != null && args.length > 0) { + try { + port = Integer.valueOf(args[0]); + } catch (NumberFormatException e) { + e.printStackTrace(); + } + } + try { + new MasterServer().bind(port); + } catch (Exception e) { + e.printStackTrace(); + } + } + + +} + + diff --git a/oss-server/src/main/java/com/xiaominfo/oss/sync/netty/MasterSyncServerHandler.java b/oss-server/src/main/java/com/xiaominfo/oss/sync/netty/MasterSyncServerHandler.java new file mode 100644 index 0000000..060a4c2 --- /dev/null +++ b/oss-server/src/main/java/com/xiaominfo/oss/sync/netty/MasterSyncServerHandler.java @@ -0,0 +1,185 @@ +package com.xiaominfo.oss.sync.netty; + +import com.xiaominfo.oss.common.SpringContextUtil; +import com.xiaominfo.oss.domain.FileBinaryResponse; +import com.xiaominfo.oss.exception.AssemblerException; +import com.xiaominfo.oss.exception.ErrorConstant; +import com.xiaominfo.oss.module.model.OSSAppInfo; +import com.xiaominfo.oss.module.model.OSSDeveloper; +import com.xiaominfo.oss.module.model.OSSInformation; +import com.xiaominfo.oss.sdk.client.NettyFileRequest; +import com.xiaominfo.oss.sdk.upload.handle.netty.NettyMessage; +import com.xiaominfo.oss.service.MaterialService; +import com.xiaominfo.oss.service.OSSInformationService; +import com.xiaominfo.oss.service.OSSMaterialInfoSyncService; +import com.xiaominfo.oss.sync.SyncMessage; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.RandomAccessFile; + +public class MasterSyncServerHandler extends ChannelInboundHandlerAdapter { + private int byteRead; + private volatile int start = 0; + private String file_dir = "F:"; + private Logger log = LoggerFactory.getLogger(SalveServerHandler.class); + private OSSMaterialInfoSyncService ossMaterialInfoSyncService = SpringContextUtil.getBean(OSSMaterialInfoSyncService.class); + private OSSInformationService ossInformationService = SpringContextUtil.getBean(OSSInformationService.class); + private MaterialService materialService = SpringContextUtil.getBean(MaterialService.class); + + + private OSSInformation ossInformation = new OSSInformation(); + private OSSDeveloper ossDeveloper = new OSSDeveloper(); + private OSSAppInfo ossApp = new OSSAppInfo(); + + @Override + public void channelActive(ChannelHandlerContext ctx) throws Exception { + super.channelActive(ctx); + log.info("服务端1：channelActive()"); + file_dir = ossInformationService.queryOne().getRoot(); + } + + @Override + public void channelInactive(ChannelHandlerContext ctx) throws Exception { + super.channelInactive(ctx); + log.info("服务端1：channelInactive()"); + ctx.flush(); + ctx.close(); + + } + + + @Override + public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { + if (msg instanceof SyncMessage) { + readSyncMessage(ctx, (SyncMessage) msg); + } else if (msg instanceof NettyFileRequest) { + uploadByBinary(ctx, (NettyFileRequest) msg); + + } + } + + public void uploadByBinary(ChannelHandlerContext ctx, NettyFileRequest ef) throws Exception { + try { + String path = ossMaterialInfoSyncService.buildPath(ef, ossInformation, ossDeveloper, ossApp); + ef.setProjectPath(path); + ef.setCode("0000"); + //保存文件 + readNettyFile(ctx, ef); + } catch (AssemblerException e) { + log.error("生成路径失败:", e); + writeErrorLogAndFlush(ctx, ErrorConstant.INTERNAL_SERVER_ERROR + "", e.getMessage()); + } catch (Exception e) { + log.error("生成路径失败:", e); + writeErrorLogAndFlush(ctx, ErrorConstant.INTERNAL_SERVER_ERROR + "", "系统异常，请联系管理员"); + } + } + + private void writeErrorLogAndFlush(ChannelHandlerContext ctx, String code, String message) throws Exception { + NettyMessage nettyMessage = new NettyMessage(); + nettyMessage.setCode(code); + nettyMessage.setMessage(message); + ctx.writeAndFlush(nettyMessage); + channelInactive(ctx); + } + + public void readNettyFile(ChannelHandlerContext ctx, NettyFileRequest ef) throws Exception { + byte[] bytes = ef.getBytes(); + byteRead = ef.getEndPos(); + + File file = new File(ef.getProjectPath()); + createDirectoryQuietly(file); + RandomAccessFile randomAccessFile = new RandomAccessFile(file, "rw"); + randomAccessFile.seek(start); + randomAccessFile.write(bytes); + start = start + byteRead; + //log.debug("path:" + ef.getProjectPath() + "," + byteRead); + if (byteRead > 0) { + NettyMessage message = new NettyMessage(); + message.setStarPos(start); + message.setMaterialId(ef.getUuid()); + message.setFilePath(ef.getProjectPath()); + message.setCode("0000"); + + if (byteRead != 1024 * 10) { + //最后一次上传，保存数据 + FileBinaryResponse response = materialService.saveAndStore(ossInformation, ossDeveloper, ossApp, ef, file); + message.setMaterialId(response.getId()); + message.setUrl(response.getUrl()); + message.setFilePath(response.getStore()); + } + ctx.writeAndFlush(message); + randomAccessFile.close(); + + if (byteRead != 1024 * 10) { + channelInactive(ctx); + } + } else { + ctx.close(); + } + } + + public void readSyncMessage(ChannelHandlerContext ctx, SyncMessage ef) throws Exception { + byte[] bytes = ef.getBytes(); + byteRead = ef.getEndPos(); + String md5 = ef.getFile_md5();//文件名 + String path = file_dir + File.separator + ef.getFilePath() + ef.getFile().getName(); + File file = new File(path); + createDirectoryQuietly(file); + RandomAccessFile randomAccessFile = new RandomAccessFile(file, "rw"); + randomAccessFile.seek(start); + randomAccessFile.write(bytes); + start = start + byteRead; + // log.debug("path:" + path + "," + byteRead); + if (byteRead > 0) { + ctx.writeAndFlush(start); + randomAccessFile.close(); + if (byteRead != 1024 * 10) { + //更新数据库 + ossMaterialInfoSyncService.createSync(ef.getMaterialId(), "Salve"); + channelInactive(ctx); + } + } else { + //ctx.flush(); + ctx.close(); + } + log.info("SyncMessage:{}" + ef); + } + + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { + cause.printStackTrace(); + ctx.close(); + log.info("FileUploadServerHandler--exceptionCaught()"); + } + + /*** + * 创建文件夹 + * @param files + */ + protected void createDirectoryQuietly(File files) { + File file = new File(files.getParent()); + try { + if (file != null) { + if (!file.exists()) { + if (!file.mkdirs()) { + throw new RuntimeException(file.getName() + " is invalid,can't be create directory"); + } + } + } + } finally { + if (file != null) { + file.setWritable(false); + file.setExecutable(false); + file.setReadOnly(); + } + + } + + } +} + diff --git a/oss-server/src/main/java/com/xiaominfo/oss/sync/netty/SalveServer.java b/oss-server/src/main/java/com/xiaominfo/oss/sync/netty/SalveServer.java new file mode 100644 index 0000000..d405872 --- /dev/null +++ b/oss-server/src/main/java/com/xiaominfo/oss/sync/netty/SalveServer.java @@ -0,0 +1,71 @@ +package com.xiaominfo.oss.sync.netty; + +import com.xiaominfo.oss.sync.SyncMessage; +import io.netty.bootstrap.Bootstrap; +import io.netty.channel.*; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.nio.NioSocketChannel; +import io.netty.handler.codec.serialization.ClassResolvers; +import io.netty.handler.codec.serialization.ObjectDecoder; +import io.netty.handler.codec.serialization.ObjectEncoder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; + +public class SalveServer { + public static final int FILE_PORT = 18001; + private static Logger logger = LoggerFactory.getLogger(SalveServer.class); + + public void connect(int port, String host, + final SyncMessage syncMessage) throws Exception { + EventLoopGroup group = new NioEventLoopGroup(); + try { + Bootstrap b = new Bootstrap(); + b.group(group).channel(NioSocketChannel.class) + .option(ChannelOption.TCP_NODELAY, true) + .handler(new ChannelInitializer() { + + @Override + protected void initChannel(Channel ch) throws Exception { + ch.pipeline().addLast(new ObjectEncoder()); + ch.pipeline().addLast(new ObjectDecoder(ClassResolvers.weakCachingConcurrentResolver(null))); + ch.pipeline().addLast(new SalveServerHandler(syncMessage)); + } + }); + ChannelFuture f = b.connect(host, port).sync(); + f.channel().closeFuture().sync(); + logger.info("file = [{}] 同步完成", syncMessage.getFile().getName()); + } finally { + group.shutdownGracefully(); + } + } + + public static void main(String[] args) { + int port = FILE_PORT; + if (args != null && args.length > 0) { + try { + port = Integer.valueOf(args[0]); + } catch (NumberFormatException e) { + e.printStackTrace(); + } + } + try { + SyncMessage uploadFile = new SyncMessage(); + File file = new File("D:\\迅雷下载\\idman630build7.exe");// d:/source.rar,D:/2014work/apache-maven-3.5.0-bin.tar.gz + String fileMd5 = file.getName();// 文件名 + uploadFile.setFile(file); + uploadFile.setFile_md5(fileMd5); + uploadFile.setFilePath("\\sync\\123\\"); + uploadFile.setStarPos(0);// 文件开始位置 + new SalveServer().connect(port, "127.0.0.1", uploadFile); + } catch (Exception e) { + e.printStackTrace(); + } + } + + +} + + + diff --git a/oss-server/src/main/java/com/xiaominfo/oss/sync/netty/SalveServerHandler.java b/oss-server/src/main/java/com/xiaominfo/oss/sync/netty/SalveServerHandler.java new file mode 100644 index 0000000..b27683a --- /dev/null +++ b/oss-server/src/main/java/com/xiaominfo/oss/sync/netty/SalveServerHandler.java @@ -0,0 +1,103 @@ +package com.xiaominfo.oss.sync.netty; + +import com.xiaominfo.oss.sync.SyncMessage; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.RandomAccessFile; + +public class SalveServerHandler extends ChannelInboundHandlerAdapter { + private int byteRead; + private volatile int start = 0; + private volatile int lastLength = 0; + public RandomAccessFile randomAccessFile; + private SyncMessage syncMessage; + private Logger log = LoggerFactory.getLogger(SalveServerHandler.class); + + public SalveServerHandler(SyncMessage ef) { + if (ef.getFile().exists()) { + if (!ef.getFile().isFile()) { + log.info("Not a file :{}", ef.getFile()); + return; + } + } + this.syncMessage = ef; + } + + @Override + public void channelInactive(ChannelHandlerContext ctx) throws Exception { + // TODO Auto-generated method stub + super.channelInactive(ctx); + log.info("客户端结束传递文件channelInactive()"); + } + + @Override + public void channelActive(ChannelHandlerContext ctx) { + try { + randomAccessFile = new RandomAccessFile(syncMessage.getFile(), + "r"); + randomAccessFile.seek(syncMessage.getStarPos()); + // lastLength = (int) randomAccessFile.length() / 10; + lastLength = 1024 * 10; + byte[] bytes = new byte[lastLength]; + if ((byteRead = randomAccessFile.read(bytes)) != -1) { + syncMessage.setEndPos(byteRead); + syncMessage.setBytes(bytes); + ctx.writeAndFlush(syncMessage); + } else { + } + // log.info("channelActive()文件已经读完 " + byteRead); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } catch (IOException i) { + i.printStackTrace(); + } + } + + @Override + public void channelRead(ChannelHandlerContext ctx, Object msg) + throws Exception { + if (msg instanceof Integer) { + start = (Integer) msg; + if (start != -1) { + randomAccessFile = new RandomAccessFile( + syncMessage.getFile(), "r"); + randomAccessFile.seek(start); + //log.debug("长度：" + (randomAccessFile.length() - start)); + int a = (int) (randomAccessFile.length() - start); + int b = (int) (randomAccessFile.length() / 1024 * 2); + if (a < lastLength) { + lastLength = a; + } + // log.debug("文件长度：" + (randomAccessFile.length()) + ",start:" + start + ",a:" + a + ",b:" + b + ",lastLength:" + lastLength); + byte[] bytes = new byte[lastLength]; + if ((byteRead = randomAccessFile.read(bytes)) != -1 + && (randomAccessFile.length() - start) > 0) { + syncMessage.setEndPos(byteRead); + syncMessage.setBytes(bytes); + try { + ctx.writeAndFlush(syncMessage); + } catch (Exception e) { + log.error("写入文件异常", e); + } + } else { + randomAccessFile.close(); + ctx.close(); + log.info("文件已经读完channelRead()--------" + byteRead); + } + } + } + } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { + cause.printStackTrace(); + ctx.close(); + } + +} + diff --git a/oss-server/src/main/java/com/xiaominfo/oss/sync/netty/SyncUploadServerHandler.java b/oss-server/src/main/java/com/xiaominfo/oss/sync/netty/SyncUploadServerHandler.java new file mode 100644 index 0000000..3794d2c --- /dev/null +++ b/oss-server/src/main/java/com/xiaominfo/oss/sync/netty/SyncUploadServerHandler.java @@ -0,0 +1,100 @@ +package com.xiaominfo.oss.sync.netty; + +import com.xiaominfo.oss.common.SpringContextUtil; +import com.xiaominfo.oss.sdk.client.NettyFileRequest; +import com.xiaominfo.oss.service.OSSInformationService; +import com.xiaominfo.oss.service.OSSMaterialInfoSyncService; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInboundHandlerAdapter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.RandomAccessFile; + +public class SyncUploadServerHandler extends ChannelInboundHandlerAdapter { + private int byteRead; + private volatile int start = 0; + private String file_dir = "F:"; + private Logger log = LoggerFactory.getLogger(SalveServerHandler.class); + private OSSMaterialInfoSyncService ossMaterialInfoSyncService = SpringContextUtil.getBean(OSSMaterialInfoSyncService.class); + private OSSInformationService ossInformationService = SpringContextUtil.getBean(OSSInformationService.class); + + @Override + public void channelActive(ChannelHandlerContext ctx) throws Exception { + super.channelActive(ctx); + log.info("服务端2：channelActive()"); + file_dir = ossInformationService.queryOne().getRoot(); + } + + @Override + public void channelInactive(ChannelHandlerContext ctx) throws Exception { + super.channelInactive(ctx); + log.info("服务端2：channelInactive()"); + ctx.flush(); + ctx.close(); + + } + + + @Override + public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { + if (msg instanceof NettyFileRequest) { + NettyFileRequest ef = (NettyFileRequest) msg; + byte[] bytes = ef.getBytes(); + byteRead = ef.getEndPos(); + + String path = file_dir + File.separator + ef.getFile().getName(); + File file = new File(path); + createDirectoryQuietly(file); + RandomAccessFile randomAccessFile = new RandomAccessFile(file, "rw"); + randomAccessFile.seek(start); + randomAccessFile.write(bytes); + start = start + byteRead; + log.debug("path:" + path + "," + byteRead); + if (byteRead > 0) { + ctx.writeAndFlush(start); + randomAccessFile.close(); + if (byteRead != 1024 * 10) { + channelInactive(ctx); + } + } else { + ctx.close(); + } + log.info("SyncMessage:{}" + ef); + } + } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { + cause.printStackTrace(); + ctx.close(); + log.info("FileUploadServerHandler--exceptionCaught()"); + } + + /*** + * 创建文件夹 + * @param files + */ + protected void createDirectoryQuietly(File files) { + File file = new File(files.getParent()); + try { + if (file != null) { + if (!file.exists()) { + if (!file.mkdirs()) { + throw new RuntimeException(file.getName() + " is invalid,can't be create directory"); + } + } + } + } finally { + if (file != null) { + file.setWritable(false); + file.setExecutable(false); + file.setReadOnly(); + } + + } + + } +} + diff --git a/oss-server/src/main/resources/application.properties b/oss-server/src/main/resources/application.properties index 6c9c152..92982d9 100644 --- a/oss-server/src/main/resources/application.properties +++ b/oss-server/src/main/resources/application.properties @@ -3,6 +3,10 @@ # ----------------------------------------------------------------------------------------------------------------# server.port=18000 spring.application.name=OSS Server Application v1.0 +#ļ˿ +material.masterPort=18001 +#ļȺ +material.nodes=172.30.45.160:18001 #زĴ洢· -- /β material.root=/home/material/ #ز·Ŀ¼ @@ -40,31 +44,29 @@ spring.freemarker.request-context-attribute=request #֤û oss.security.userName=admin oss.security.password=adminA123 - spring.datasource.type=com.alibaba.druid.pool.DruidDataSource -spring.datasource.driver-class-name=org.sqlite.JDBC -spring.datasource.url=jdbc:sqlite:D:\\sqlitedb\\oss.db +spring.datasource.driver-class-name=oracle.jdbc.driver.OracleDriver +spring.datasource.url=jdbc:oracle:thin:@172.30.45.182:1521:z45db2 #spring.datasource.url=jdbc:sqlite:classpath:resource:\\db\\datashare.db -#spring.datasource.username=root -#spring.datasource.password=Lskj1234 -spring.datasource.platform=sqlite +spring.datasource.username=resetsalebatch +spring.datasource.password=resetsalebatch +#spring.datasource.platform=sqlite spring.datasource.min-idle=3 spring.datasource.initial-size=3 spring.datasource.max-active=20 spring.datasource.max-wait=60000 spring.datasource.time-between-eviction-runs-millis=600000 spring.datasource.min-evictable-idle-time-millis=30000 -spring.datasource.validation-query=select 'x' +spring.datasource.validation-query=select 1 from dual spring.datasource.test-while-idle=true spring.datasource.test-on-borrow=false spring.datasource.test-on-return=false spring.datasource.pool-prepared-statements=true spring.datasource.max-pool-prepared-statement-per-connection-size=20 -spring.datasource.connection-properties=druid.stat.mergeSql=true;druid.stat.slowSqlMillis=5000 +#spring.datasource.connection-properties=druid.stat.mergeSql=true;druid.stat.slowSqlMillis=5000 spring.datasource.filters=stat,wall,slf4j - ## mybatis-plus ļ mybatis-plus.mapper-locations=classpath:/com/xiaominfo/oss/module/dao/mappings/*Mapper.xml mybatis-plus.type-aliases-package=com.xiaominfo.oss.module.model ## ־ -mybatis-plus.configuration.log-impl=com.xiaominfo.oss.extend.mybatis.MybatisStdOutImpl \ No newline at end of file +mybatis-plus.configuration.log-impl=com.xiaominfo.oss.extend.mybatis.MybatisStdOutImpl diff --git a/oss-server/src/test/java/com/xiaominfo/oss/OssServerApplicationTests.java b/oss-server/src/test/java/com/xiaominfo/oss/OssServerApplicationTests.java index 2554c7f..5b0b86d 100644 --- a/oss-server/src/test/java/com/xiaominfo/oss/OssServerApplicationTests.java +++ b/oss-server/src/test/java/com/xiaominfo/oss/OssServerApplicationTests.java @@ -1,6 +1,7 @@ package com.xiaominfo.oss; -import com.google.gson.Gson; + +import com.alibaba.fastjson.JSON; import com.xiaominfo.oss.service.OSSStatisticDayService; import org.junit.Test; import org.junit.runner.RunWith; @@ -12,17 +13,18 @@ import org.springframework.test.context.junit4.SpringRunner; @SpringBootTest public class OssServerApplicationTests { - @Autowired - OSSStatisticDayService ossStatisticDayService; + @Autowired + OSSStatisticDayService ossStatisticDayService; - @Test - public void contextLoads() { - } + @Test + public void contextLoads() { + } - @Test - public void test(){ - System.out.println(new Gson().toJson(ossStatisticDayService.queryCurrentWeekStaticsDay()));; - } + @Test + public void test() { + System.out.println(JSON.toJSONString(ossStatisticDayService.queryCurrentWeekStaticsDay())); + ; + } } diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..5677bc5 --- /dev/null +++ b/pom.xml @@ -0,0 +1,113 @@ + + + 4.0.0 + + com.xiaominfo.oss + oss-server-parent + 1.1 + pom + oss-server + oss-server上传组件,该组件为项目服务,可单独部署迁移 + + + + + + org.springframework.boot + spring-boot-dependencies + 2.0.2.RELEASE + pom + import + + + + + + + oss-server + oss-server-sdk-java + + + + + UTF-8 + UTF-8 + 1.8 + 1.7.25 + 1.2.3 + + true + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + org.apache.maven.plugins + maven-javadoc-plugin + 2.9.1 + + + + + org.apache.maven.plugins + maven-source-plugin + + + attach-sources + + jar + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.8 + 1.8 + UTF-8 + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.18.1 + + ${skipTests} + + + + + + src/main/java + + **/*.properties + **/*.xml + + false + + + src/main/resources + + **/*.properties + **/*.xml + **/*.json + **/static/** + **/templates/** + + false + + + + + + -- Gitee From e919aa5f166d084a31d6b0069b9d4a226b9dc6e8 Mon Sep 17 00:00:00 2001 From: "houcc@qq.com" Date: Sun, 29 Jul 2018 17:06:20 +0800 Subject: [PATCH 2/3] =?UTF-8?q?1.Gson=20=E6=94=B9=E4=B8=BA=20FASTJSON=202.?= =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=8E=BB=E4=B8=AD=20=E5=88=86=E5=B8=83?= =?UTF-8?q?=E5=BC=8F=E6=9C=8D=E5=8A=A1=E9=9B=86=E7=BE=A4=E3=80=82=203.?= =?UTF-8?q?=E6=96=B0=E5=A2=9Enetty=E4=B8=8A=E4=BC=A0=E6=96=B9=E5=BC=8F?= =?UTF-8?q?=EF=BC=8C=E5=B9=B6=E8=AE=BE=E7=BD=AE=E4=B8=BA=E9=BB=98=E8=AE=A4?= =?UTF-8?q?=E6=96=B9=E5=BC=8F=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 17 ++++++++++++++++- .../src/main/resources/application.properties | 8 ++++---- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 2257131..edd78d3 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,21 @@ **oss-server项目交流群：608374991** ## 项目介绍 +################################################################################ +在原来的oss-server的基础上做了如下修改 +> 1.Gson 改为 FASTJSON +> +> 2.新增去中 分布式服务集群。 +> +> 3.新增netty上传方式，并设置为默认方式。 +> +> 4.数据库改为Oracle。 +> +> 5.其他保持不变 +################################################################################ + + + oss-server是针对项目开发时提供的小型对象存储系统,开发者在针对文件上传时业务剥离,同时方便文件迁移，为满足单个项目，多个系统的情况下，提供统一的oss服务 @@ -208,4 +223,4 @@ material.invokingRoot=http://192.168.0.7/ ## 交流 -![](static/wechat.jpg) \ No newline at end of file +![](static/wechat.jpg) diff --git a/oss-server/src/main/resources/application.properties b/oss-server/src/main/resources/application.properties index 92982d9..1603cee 100644 --- a/oss-server/src/main/resources/application.properties +++ b/oss-server/src/main/resources/application.properties @@ -6,7 +6,7 @@ spring.application.name=OSS Server Application v1.0 #ļ˿ material.masterPort=18001 #ļȺ -material.nodes=172.30.45.160:18001 +material.nodes=172.30.45.2:18001 #زĴ洢· -- /β material.root=/home/material/ #ز·Ŀ¼ @@ -46,10 +46,10 @@ oss.security.userName=admin oss.security.password=adminA123 spring.datasource.type=com.alibaba.druid.pool.DruidDataSource spring.datasource.driver-class-name=oracle.jdbc.driver.OracleDriver -spring.datasource.url=jdbc:oracle:thin:@172.30.45.182:1521:z45db2 +spring.datasource.url=jdbc:oracle:thin:@172.30.45.2:1521:z45db2 #spring.datasource.url=jdbc:sqlite:classpath:resource:\\db\\datashare.db -spring.datasource.username=resetsalebatch -spring.datasource.password=resetsalebatch +spring.datasource.username=111 +spring.datasource.password=111 #spring.datasource.platform=sqlite spring.datasource.min-idle=3 spring.datasource.initial-size=3 -- Gitee From d686d2989eab81a0e2e47948233c60f3f79cba06 Mon Sep 17 00:00:00 2001 From: "houcc@qq.com" Date: Sun, 29 Jul 2018 17:07:29 +0800 Subject: [PATCH 3/3] =?UTF-8?q?1.Gson=20=E6=94=B9=E4=B8=BA=20FASTJSON=202.?= =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=8E=BB=E4=B8=AD=20=E5=88=86=E5=B8=83?= =?UTF-8?q?=E5=BC=8F=E6=9C=8D=E5=8A=A1=E9=9B=86=E7=BE=A4=E3=80=82=203.?= =?UTF-8?q?=E6=96=B0=E5=A2=9Enetty=E4=B8=8A=E4=BC=A0=E6=96=B9=E5=BC=8F?= =?UTF-8?q?=EF=BC=8C=E5=B9=B6=E8=AE=BE=E7=BD=AE=E4=B8=BA=E9=BB=98=E8=AE=A4?= =?UTF-8?q?=E6=96=B9=E5=BC=8F=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index edd78d3..c70ee3f 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ **oss-server项目交流群：608374991** ## 项目介绍 -################################################################################ +>################################################################################ 在原来的oss-server的基础上做了如下修改 > 1.Gson 改为 FASTJSON > @@ -16,7 +16,7 @@ > 4.数据库改为Oracle。 > > 5.其他保持不变 -################################################################################ +>################################################################################ -- Gitee