优化代码

src/main/java/com/jkcredit/traffic/record/action/TrafficRecordsAction.java

@@ -35,37 +35,4 @@ public class TrafficRecordsAction {
     public CommonResponseObject pushErrorRecords(@RequestParam String path) {
         return service.pushErrorRecords(path);
     }
-
-    @PostMapping("/encrypt")
-    public String vehicleIdEncrypt(@RequestParam String vehicleId, @RequestParam String algorithm,
-                                    @RequestParam String keyId, @RequestParam String metadata, @RequestParam String ivStr) {
-        try {
-            byte[] plainData = vehicleId.getBytes("UTF-8");
-            byte[] iv = ivStr.getBytes("UTF-8");
-            Long startTime = System.currentTimeMillis();
-            // 加密
-            byte[] cipherData = CipherSuiteUtils.encrypt(plainData, algorithm, keyId, metadata, iv);
-            return DatatypeConverter.printHexBinary(cipherData);
-        } catch (UnsupportedEncodingException ue) {
-            ue.printStackTrace();
-            log.error("UnsupportedEncodingException:", ue);
-            return "";
-        } catch (CipherSuiteException cse) {
-            cse.printStackTrace();
-            log.error("CipherSuiteException:", cse);
-            return "";
-        } catch (CipherSuiteMacException csme) {
-            csme.printStackTrace();
-            log.error("CipherSuiteMacException:", csme);
-            return "";
-        } catch (Exception e) {
-            e.printStackTrace();
-            log.error("Exception:", e);
-            return "";
-        } catch (Error error) {
-            error.printStackTrace();
-            log.error("Error:", error);
-            return "";
-        }
-    }
 }

src/main/java/com/jkcredit/traffic/record/config/ThreadPoolConfig.java

@@ -12,8 +12,8 @@ import java.util.concurrent.*;
  * @author xusonglin
  * @version V1.0
  **/
-@Configuration
-@EnableAsync
+//@Configuration
+//@EnableAsync
 public class ThreadPoolConfig {
     @Value("${thread.pool.coreSize}")
     private Integer coreSize;

src/main/java/com/jkcredit/traffic/record/service/TrafficRecordsAnalyseServiceImpl.java

@@ -8,7 +8,6 @@ import com.ciphergateway.ciphersuite.CipherSuiteUtils;
 import com.jkcredit.traffic.record.constant.CommonConstant;
 import com.jkcredit.traffic.record.model.CommonResponseObject;
 import com.jkcredit.traffic.record.model.MonthResult;
-import com.jkcredit.traffic.record.test.AsyncPushTask;
 import com.jkcredit.traffic.record.util.CsvUtil;
 import com.jkcredit.traffic.record.util.DivideDataUtil;
 import com.jkcredit.traffic.record.util.FileUtil;
@@ -20,7 +19,6 @@ import org.apache.spark.sql.Encoders;
 import org.apache.spark.sql.SparkSession;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
-import org.springframework.scheduling.annotation.AsyncResult;
 import org.springframework.stereotype.Service;
 
 import javax.xml.bind.DatatypeConverter;
@@ -29,7 +27,6 @@ import java.io.UnsupportedEncodingException;
 import java.net.SocketTimeoutException;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 
 /**
@@ -41,8 +38,6 @@ import java.util.concurrent.TimeUnit;
 public class TrafficRecordsAnalyseServiceImpl implements TrafficRecordsAnalyseService {
     @Autowired
     SparkSession sparkSession;
-    @Autowired
-    AsyncPushTask asyncPushTask;
     @Value("${recordsPush.url}")
     private String recordsPushUrl;
     @Value("${recordsEncrypt.keyId}")
@@ -62,11 +57,12 @@ public class TrafficRecordsAnalyseServiceImpl implements TrafficRecordsAnalyseSe
 
     @Override
     public CommonResponseObject pushTrafficRecords(String filePath) {
-        log.info("开始处理");
+        log.info("pushTrafficRecordBegin");
+        long startTime = System.currentTimeMillis();
         List<File> files = new ArrayList<>();
         File folder = new File(filePath);
         File[] tempList = folder.listFiles();
-        if (tempList.length == 0) {
+        if (tempList == null || tempList.length == 0) {
             return new CommonResponseObject(CommonConstant.ERROR_CODE_FILE_NOT_EXIST,
                     CommonConstant.ERROR_MESSAGE_FILE_NOT_EXIST);
         }
@@ -93,19 +89,19 @@ public class TrafficRecordsAnalyseServiceImpl implements TrafficRecordsAnalyseSe
                     // 数据脱敏
                     List<MonthResult> maskResults = desensitize(j + "", tempFolder + "/temp_" + j + ".csv");
 
-                    // 数据分割，异步推送
+                    // 数据分割，推送数据
                     push(maskResults);
                     boolean deleteResult = FileUtil.INSTANCE.deleteFile(tempFolder + "/temp_" + j + ".csv");
-                    log.info("删除临时文件:{}, 删除结果:{}", tempFolder + "/temp_" + j + ".csv", deleteResult);
+                    log.info("deleteTempFile:{}, deleteResult:{}", tempFolder + "/temp_" + j + ".csv", deleteResult);
                     maskResults = null;
                 }
             } catch (Exception e) {
                 e.printStackTrace();
             }
             boolean deleteResult = FileUtil.INSTANCE.deleteDirectory(tempFolder);
-            log.info("删除临时目录:{}, 删除结果:{}", tempFolder, deleteResult);
+            log.info("deleteTempDir:{}, deleteResult:{}", tempFolder, deleteResult);
         }
-        log.info("结束处理");
+        log.info("pushTrafficRecordEnd, costTime:{}", System.currentTimeMillis() - startTime);
         return new CommonResponseObject().success();
     }
 
@@ -114,19 +110,19 @@ public class TrafficRecordsAnalyseServiceImpl implements TrafficRecordsAnalyseSe
         List<File> files = new ArrayList<>();
         File folder = new File(filePath);
         File[] tempList = folder.listFiles();
-        if (tempList.length == 0) {
+        if (tempList == null || tempList.length == 0) {
             return new CommonResponseObject(CommonConstant.ERROR_CODE_FILE_NOT_EXIST,
                     CommonConstant.ERROR_MESSAGE_FILE_NOT_EXIST);
         }
 
-        for (int i = 0; i < tempList.length; i++) {
-            if (tempList[i].isFile() && tempList[i].getName().split("\\.")[1].equals("csv")) {
-                files.add(tempList[i]);
+        for (File file : tempList) {
+            if (file.isFile() && file.getName().split("\\.")[1].equals("csv")) {
+                files.add(file);
             }
         }
 
         for (int i = 0; i < files.size(); i++) {
-            List<MonthResult> errorRecords = getMonthResult("ErrorRecord" +i, files.get(i).getAbsolutePath());
+            List<MonthResult> errorRecords = getMonthResult("ErrorRecord" + i, files.get(i).getAbsolutePath());
             push(errorRecords);
         }
         return new CommonResponseObject().success();
@@ -138,14 +134,14 @@ public class TrafficRecordsAnalyseServiceImpl implements TrafficRecordsAnalyseSe
                 .option("header", false)
                 .csv(filePath)
                 .toDF("month", "vehicleid", "max_exTime", "sum_travel_time", "max_travel_time", "sum_feemileage",
-                        "sum_fee", "sum_weight_mileage","exTimes_count", "axlecount", "vehicletype",
+                        "sum_fee", "sum_weight_mileage", "exTimes_count", "axlecount", "vehicletype",
                         "travel_provinces_count", "transtime_count")
                 .as(Encoders.bean(MonthResult.class))
                 .coalesce(36);
         String tempViewName = "monthDataset" + i;
         monthResultDataset.createOrReplaceTempView(tempViewName);
-        Dataset<MonthResult> rows = sparkSession.sql("select * from " + tempViewName + " where sum_travel_time >= 0 "+
-                "and max_travel_time >=0" )
+        Dataset<MonthResult> rows = sparkSession.sql("select * from " + tempViewName + " where sum_travel_time >= 0 " +
+                "and max_travel_time >=0")
                 .as(Encoders.bean(MonthResult.class));
 //        rows.write().csv("/Users/jkxy/Desktop/outPut/test" + System.currentTimeMillis());
         List<MonthResult> results = new ArrayList<>(3000000);
@@ -156,26 +152,27 @@ public class TrafficRecordsAnalyseServiceImpl implements TrafficRecordsAnalyseSe
 
     private List<MonthResult> desensitize(String value, String filePath) {
         List<MonthResult> results = getMonthResult(value, filePath);
-        log.info("待推送数据数量::{}", results.size());
+        log.info("desensitizeCount:{}", results.size());
         // 数据加密分区间
         List<MonthResult> maskResults = new ArrayList<>();
         for (int i = 0; i < results.size(); i++) {
             // 车牌号加密；
             String encryptPlateNumber = vehicleIdEncrypt(results.get(i).getVehicleid());
             if (StringUtils.isBlank(encryptPlateNumber)) {
-                log.error("加密失败:{}", results.get(i).toString());
+                log.error("encryptPlateNumberError:{}", results.get(i).toString());
                 continue;
             }
             results.get(i).setVehicleid(encryptPlateNumber);
             maskResults.add(divideData(results.get(i)));
         }
-        log.info("数据脱敏完成数量:{}", results.size());
+        log.info("desensitizeSuccessCount:{}", results.size());
         return maskResults;
     }
 
     public void push(List<MonthResult> maskResults) {
+        // 将数据每10000条分为一个数据包，通过http请求推送
         int times;
-        if (maskResults.size() % 10000 != 0 ) {
+        if (maskResults.size() % 10000 != 0) {
             times = (maskResults.size() / 10000) + 1;
         } else {
             times = maskResults.size() / 10000;
@@ -183,24 +180,23 @@ public class TrafficRecordsAnalyseServiceImpl implements TrafficRecordsAnalyseSe
         List<List<MonthResult>> partResultsList = new ArrayList<>();
         for (int i = 0; i < times; i++) {
             List<MonthResult> partResults;
-            if (i == times-1) {
-                partResults = maskResults.subList(i*10000, maskResults.size());
+            if (i == times - 1) {
+                partResults = maskResults.subList(i * 10000, maskResults.size());
             } else {
-                partResults = maskResults.subList(i*10000, (i+1)*10000);
+                partResults = maskResults.subList(i * 10000, (i + 1) * 10000);
             }
             partResultsList.add(partResults);
         }
+
+        // 推送数据
         for (List<MonthResult> partResults : partResultsList) {
-            try {
-                pushRecords(partResults);
-            } catch (Exception e) {
-                e.printStackTrace();
-            }
+            pushRecords(partResults);
         }
     }
 
     /**
      * 对vehicle_id字段进行加密
+     *
      * @param vehicleId 待加密vehicleId
      * @return 加密后vehicleId
      */
@@ -208,7 +204,6 @@ public class TrafficRecordsAnalyseServiceImpl implements TrafficRecordsAnalyseSe
         try {
             byte[] plainData = vehicleId.getBytes("UTF-8");
             byte[] iv = ivStr.getBytes("UTF-8");
-            Long startTime = System.currentTimeMillis();
             // 加密
             byte[] cipherData = CipherSuiteUtils.encrypt(plainData, algorithm, keyId, metadata, iv);
             return DatatypeConverter.printHexBinary(cipherData);
@@ -237,6 +232,7 @@ public class TrafficRecordsAnalyseServiceImpl implements TrafficRecordsAnalyseSe
 
     /**
      * 对数据脱敏，分区间
+     *
      * @param monthResult 待脱敏月结果数据
      * @return 脱敏后月结果数据
      */
@@ -250,7 +246,7 @@ public class TrafficRecordsAnalyseServiceImpl implements TrafficRecordsAnalyseSe
         return monthResult;
     }
 
-    public Future<String> pushRecords(List<MonthResult> resultList) throws InterruptedException {
+    public void pushRecords(List<MonthResult> resultList) {
         MediaType mediaType = MediaType.parse(MEDIA_TYPE);
         RequestBody requestBody = RequestBody.create(mediaType, JSON.toJSONString(resultList));
 
@@ -259,29 +255,29 @@ public class TrafficRecordsAnalyseServiceImpl implements TrafficRecordsAnalyseSe
                 .url(recordsPushUrl)
                 .build();
         OkHttpClient client = okHttpClient.newBuilder()
-                .connectTimeout(5000, TimeUnit.MILLISECONDS)
-                .readTimeout(30000, TimeUnit.MILLISECONDS)
-                .writeTimeout(30000, TimeUnit.MILLISECONDS)
+                .connectTimeout(1000, TimeUnit.MILLISECONDS)
+                .readTimeout(10000, TimeUnit.MILLISECONDS)
+                .writeTimeout(10000, TimeUnit.MILLISECONDS)
                 .build();
         String responseContext = "";
         List<MonthResult> errorList = new ArrayList<>();
         try {
             Long startTime = System.currentTimeMillis();
             Response response = client.newCall(okRequest).execute();
-            log.info("pushRecords -- costTime:{}", System.currentTimeMillis()-startTime);
             if (response.body() != null) {
                 responseContext = response.body().string();
             }
-            log.info("pushRecords -- responseContext:{}", responseContext);
+            log.info("pushRecords -- responseContext:{}, costTime:{}", responseContext, System.currentTimeMillis() - startTime);
             response.close();
         } catch (SocketTimeoutException ste) {
-            ste.printStackTrace();
+            log.error("pushRecordsException:", ste);
             errorList = resultList;
         } catch (Exception e) {
-            e.printStackTrace();
+            log.error("pushRecordsException:", e);
+            errorList = resultList;
         }
         CommonResponseObject responseObject = JSON.toJavaObject(JSON.parseObject(responseContext), CommonResponseObject.class);
-        if (!responseObject.getCode().equals(CommonConstant.SUCCESS_CODE)) {
+        if (responseObject != null && !responseObject.getCode().equals(CommonConstant.SUCCESS_CODE)) {
             JSONArray responseErrorArray = JSON.parseArray(responseObject.getMessage());
             errorList = responseErrorArray.toJavaList(MonthResult.class);
         }
@@ -292,8 +288,7 @@ public class TrafficRecordsAnalyseServiceImpl implements TrafficRecordsAnalyseSe
         if (errorData.size() != 0) {
             // 异常数据写入本地文件
             FileUtil.INSTANCE.addAll(errorData);
-            FileUtil.INSTANCE.write(errorRecordsPath+System.currentTimeMillis()+".csv");
+            FileUtil.INSTANCE.write(errorRecordsPath + System.currentTimeMillis() + ".csv");
         }
-        return new AsyncResult<>("推送完成");
     }
 }

src/main/java/com/jkcredit/traffic/record/test/AsyncPushTask.java

@@ -1,235 +0,0 @@
-package com.jkcredit.traffic.record.test;
-
-import com.alibaba.fastjson.JSON;
-import com.alibaba.fastjson.JSONArray;
-import com.alibaba.fastjson.JSONObject;
-import com.ciphergateway.ciphersuite.CipherSuiteException;
-import com.ciphergateway.ciphersuite.CipherSuiteMacException;
-import com.ciphergateway.ciphersuite.CipherSuiteUtils;
-import com.jkcredit.traffic.record.constant.CommonConstant;
-import com.jkcredit.traffic.record.model.CommonResponseObject;
-import com.jkcredit.traffic.record.model.MonthResult;
-import com.jkcredit.traffic.record.service.TrafficRecordsAnalyseService;
-import com.jkcredit.traffic.record.util.DivideDataUtil;
-import com.jkcredit.traffic.record.util.FileUtil;
-import lombok.extern.slf4j.Slf4j;
-import okhttp3.*;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.spark.sql.Dataset;
-import org.apache.spark.sql.Encoders;
-import org.apache.spark.sql.SparkSession;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.scheduling.annotation.Async;
-import org.springframework.scheduling.annotation.AsyncResult;
-import org.springframework.stereotype.Component;
-import org.springframework.stereotype.Service;
-
-import javax.xml.bind.DatatypeConverter;
-import java.io.File;
-import java.io.UnsupportedEncodingException;
-import java.net.SocketTimeoutException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-/**
- * @author xusonglin
- * @version V1.0
- **/
-@Service
-@Slf4j
-public class AsyncPushTask {
-    @Autowired
-    SparkSession sparkSession;
-
-    @Value("${recordsPush.url}")
-    private String recordsPushUrl;
-    @Value("${recordsEncrypt.keyId}")
-    private String keyId;
-    @Value("${recordsEncrypt.metadata}")
-    private String metadata;
-    @Value("${recordsEncrypt.ivStr}")
-    private String ivStr;
-    @Value("${recordsEncrypt.algorithm}")
-    private String algorithm;
-    @Value("${errorRecords.path}")
-    private String errorRecordsPath;
-
-    private static final String MEDIA_TYPE = "application/json;charset=UTF-8";
-    private OkHttpClient okHttpClient = new OkHttpClient.Builder().build();
-
-    @Async("executor")
-    public Future<String> pushRecords(List<MonthResult> resultList) throws InterruptedException {
-        log.info("&&&&&&&&&&&&&&&&&&&&&&&&&&开启线程&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&");
-        MediaType mediaType = MediaType.parse(MEDIA_TYPE);
-        RequestBody requestBody = RequestBody.create(mediaType, JSON.toJSONString(resultList));
-
-        Request okRequest = new Request.Builder()
-                .post(requestBody)
-                .url(recordsPushUrl)
-                .build();
-        OkHttpClient client = okHttpClient.newBuilder()
-                .connectTimeout(5000, TimeUnit.MILLISECONDS)
-                .readTimeout(30000, TimeUnit.MILLISECONDS)
-                .writeTimeout(30000, TimeUnit.MILLISECONDS)
-                .build();
-        String responseContext = "";
-        List<MonthResult> errorList = new ArrayList<>();
-        try {
-            Long startTime = System.currentTimeMillis();
-            Response response = client.newCall(okRequest).execute();
-            log.info("pushRecords -- costTime:{}", System.currentTimeMillis()-startTime);
-            if (response.body() != null) {
-                responseContext = response.body().string();
-            }
-            log.info("pushRecords -- responseContext:{}", responseContext);
-            response.close();
-        } catch (SocketTimeoutException ste) {
-            ste.printStackTrace();
-            errorList = resultList;
-        } catch (Exception e) {
-            e.printStackTrace();
-        }
-        if (StringUtils.isNotBlank(responseContext)) {
-            CommonResponseObject responseObject = JSON.toJavaObject(JSON.parseObject(responseContext), CommonResponseObject.class);
-            if (!responseObject.getCode().equals(CommonConstant.SUCCESS_CODE)) {
-                JSONArray responseErrorArray = JSON.parseArray(responseObject.getMessage());
-                errorList = responseErrorArray.toJavaList(MonthResult.class);
-            }
-        }
-
-        List<String> errorData = new ArrayList<>();
-        for (MonthResult monthResult : errorList) {
-            errorData.add(monthResult.toString());
-        }
-        if (errorData.size() != 0) {
-            // 异常数据写入本地文件
-            FileUtil.INSTANCE.addAll(errorData);
-            FileUtil.INSTANCE.write(errorRecordsPath+System.currentTimeMillis()+".csv");
-        }
-        return new AsyncResult<>("推送完成");
-    }
-
-//    @Async
-//    public void push(String value, String filePath) {
-//        List<MonthResult> results = getMonthResult(value, filePath);
-//        log.info("************************{}##########################", results.size());
-//        // 数据加密分区间
-//        List<MonthResult> maskResults = new ArrayList<>();
-//        for (MonthResult monthResult : results) {
-//            // 车牌号加密；
-//            String encryptPlateNumber = vehicleIdEncrypt(monthResult.getVehicleid());
-//            if (StringUtils.isBlank(encryptPlateNumber)) {
-//                // todo
-//                log.info(JSON.toJSONString(monthResult));
-//                continue;
-//            }
-//            maskResults.add(divideData(monthResult));
-//        }
-//        results = null;
-//
-//        int times;
-//        if (maskResults.size() % 10000 != 0 ) {
-//            times = (maskResults.size() / 10000) + 1;
-//        } else {
-//            times = maskResults.size() / 10000;
-//        }
-//        List<List<MonthResult>> partResultsList = new ArrayList<>();
-//        for (int i = 0; i < times; i++) {
-//            List<MonthResult> partResults;
-//            if (i == times-1) {
-//                partResults = maskResults.subList(i*10000, maskResults.size());
-//            } else {
-//                partResults = maskResults.subList(i*10000, (i+1)*10000);
-//            }
-//            partResultsList.add(partResults);
-//        }
-//        // todo 此处需要考虑异步
-//        for (List<MonthResult> partResults : partResultsList) {
-//            try {
-//                pushRecords(partResults);
-//            } catch (Exception e) {
-//                e.printStackTrace();
-//            }
-//        }
-//    }
-//
-//    private List<MonthResult> getMonthResult(String i, String filePath) {
-//        Dataset<MonthResult> monthResultDataset = sparkSession
-//                .read()
-//                .option("header", true)
-//                .csv(filePath)
-//                .toDF("month", "vehicleid", "max_exTime", "sum_travel_time", "max_travel_time", "sum_feemileage",
-//                        "sum_fee", "sum_weight_mileage","exTimes_count", "axlecount", "vehicletype",
-//                        "travel_provinces_count", "transtime_count")
-//                .as(Encoders.bean(MonthResult.class))
-//                .coalesce(36);
-//        String tempViewName = "monthDataset" + i;
-//        monthResultDataset.createOrReplaceTempView(tempViewName);
-//
-//        Dataset<MonthResult> rows = sparkSession.sql("select * from " + tempViewName + " where sum_travel_time < 2678400 "+
-//                "and max_travel_time < 604800 and sum_feemileage < 89280000 and sum_weight_mileage < 89280000 and exTimes_count < 1200" )
-//                .as(Encoders.bean(MonthResult.class));
-////        rows.write().csv("/Users/jkxy/Desktop/outPut/test" + System.currentTimeMillis());
-//        List<MonthResult> results = rows.javaRDD().collect();
-//        sparkSession.sqlContext().dropTempTable(tempViewName);
-//        return results;
-//    }
-//
-//    /**
-//     * 对vehicle_id字段进行加密
-//     * @param vehicleId 待加密vehicleId
-//     * @return 加密后vehicleId
-//     */
-//    private String vehicleIdEncrypt(String vehicleId) {
-//        try {
-//            byte[] plainData = vehicleId.getBytes("UTF-8");
-//            byte[] iv = ivStr.getBytes("UTF-8");
-//            Long startTime = System.currentTimeMillis();
-//            // 加密
-//            byte[] cipherData = CipherSuiteUtils.encrypt(plainData, algorithm, keyId, metadata, iv);
-//            log.info("encrypt result: " + DatatypeConverter.printHexBinary(cipherData));
-//            log.info("encrypt success. time:" + (System.currentTimeMillis() - startTime) + "ms.");
-//            return DatatypeConverter.printHexBinary(cipherData);
-//        } catch (UnsupportedEncodingException ue) {
-//            ue.printStackTrace();
-//            log.error("UnsupportedEncodingException:", ue);
-//            return "";
-//        } catch (CipherSuiteException cse) {
-//            cse.printStackTrace();
-//            log.error("CipherSuiteException:", cse);
-//            return "";
-//        } catch (CipherSuiteMacException csme) {
-//            csme.printStackTrace();
-//            log.error("CipherSuiteMacException:", csme);
-//            return "";
-//        } catch (Exception e) {
-//            e.printStackTrace();
-//            log.error("Exception:", e);
-//            return "";
-//        } catch (Error error) {
-//            error.printStackTrace();
-//            log.error("Error:", error);
-//            return "";
-//        }
-//    }
-//
-//    /**
-//     * 对数据脱敏，分区间
-//     * @param monthResult 待脱敏月结果数据
-//     * @return 脱敏后月结果数据
-//     */
-//    private MonthResult divideData(MonthResult monthResult) {
-//        monthResult.setExTimes_count(DivideDataUtil.divideExTimesCount(monthResult.getExTimes_count()));
-//        monthResult.setSum_fee(DivideDataUtil.divideSumFee(monthResult.getSum_fee()));
-//        monthResult.setSum_travel_time(DivideDataUtil.divideSumTravelTime(monthResult.getSum_travel_time()));
-//        monthResult.setMax_travel_time(DivideDataUtil.divideMaxTravelTime(monthResult.getMax_travel_time()));
-//        monthResult.setSum_feemileage(DivideDataUtil.divideSumFeeMileage(monthResult.getSum_feemileage()));
-//        monthResult.setSum_weight_mileage(DivideDataUtil.divideSumWeightMileage(monthResult.getSum_weight_mileage()));
-//        return monthResult;
-//    }
-}

src/main/java/com/jkcredit/traffic/record/test/CipherSuiteUtilsTest.java

@@ -1,125 +0,0 @@
-package com.jkcredit.traffic.record.test;
-
-import com.ciphergateway.ciphersuite.*;
-
-import javax.xml.bind.DatatypeConverter;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Scanner;
-
-/**
- * @ClassName : CipherSuiteUtilsTest
- * @Description : 密码套件工具测试
- * @Author : bcx
- * @Date: 2020-05-13 14:03
- */
-public class CipherSuiteUtilsTest {
-
-
-    public static void main(String[] args) throws Exception {
-        Scanner scanner = new Scanner(System.in);
-        List<String> algorithmList = new ArrayList<String>();
-        for (AlgorithmEnum algorithmEnum : AlgorithmEnum.values()) {
-            algorithmList.add(algorithmEnum.name());
-        }
-        System.out.println("请选择算法：");
-        int index = 1;
-        for (String algorithm : algorithmList) {
-            System.out.println("【" + (index++) + "】 " + algorithm);
-        }
-
-
-        int algorithmIndex = scanner.nextInt();
-        if (algorithmIndex > algorithmList.size() || algorithmIndex < 1) {
-            System.out.println("算法序号不合法");
-            return;
-        }
-        String algorithm = algorithmList.get(algorithmIndex - 1);
-
-        System.out.println("请输入密钥索引：");
-        String keyId = scanner.next();
-
-        System.out.println("algorithm: " + algorithm);
-        System.out.println("keyId: " + keyId);
-
-
-        String plainStr = null;
-        int precision = 5;
-        int scale = 2;
-
-        if ("SM4_ECB".equals(algorithm) || "SM4_ECB_V2".equals(algorithm) || "SM4_CTR".equals(algorithm) || "SM4_GCM".equals(algorithm) || "SM4_CBC".equals(algorithm) || "SM3".equals(algorithm) || "SM2".equals(algorithm) || "AES_ECB".equals(algorithm) || "AES_CTR".equals(algorithm) || "AES_GCM".equals(algorithm) || "AES_CBC".equals(algorithm) || "RSA".equals(algorithm)) {
-            plainStr = "Hello World";
-        } else if ("FPE_EMAIL".equals(algorithm)) {
-            plainStr = "hello@163.com";
-        } else if ("FPE_PHONE_NUM".equals(algorithm)) {
-            plainStr = "13581690000";
-        } else if ("FPE_OFFICER_CARD".equals(algorithm)) {
-            plainStr = "军字第00111206号";
-        } else if ("FPE_ID_CARD".equals(algorithm)) {
-            plainStr = "110103201206200910";
-        } else if ("FPE_PASSPORT".equals(algorithm)) {
-            plainStr = "E12345678";
-        } else if ("FPE_IDENTIFICATION".equals(algorithm)) {//未知证件
-            plainStr = "110103201206200910";
-        } else if ("FPE_NUMBER".equals(algorithm)) {
-            plainStr = "123.45";
-        } else if ("FPE_NUMBER_TWEAK".equals(algorithm)) {
-            plainStr = "123.45";
-        } else {
-            System.out.println("算法不合法");
-            return;
-        }
-
-        System.out.println("plainData: " + plainStr);
-
-        byte[] plainData = plainStr.getBytes("UTF-8");
-
-        String ivStr = "123456789abcdefg";
-        System.out.println("iv: " + ivStr);
-        byte[] iv = ivStr.getBytes("UTF-8");
-
-        String metadata = "ABCDBF";
-        System.out.println("metadata: " + metadata);
-
-        if ("FPE_NUMBER".equals(algorithm) || "FPE_NUMBER_TWEAK".equals(algorithm)) {
-            System.out.println("precision:" + precision);
-            System.out.println("scale:" + scale);
-        }
-
-        long beginTime = System.currentTimeMillis();
-
-        if ("FPE_NUMBER".equals(algorithm) || "FPE_NUMBER_TWEAK".equals(algorithm)) {
-            byte[] cipherData_fpe = CipherSuiteUtils.fpeNumberEncrypt(plainData, algorithm, keyId, metadata, iv, precision, scale);
-            System.out.println("encrypt result: " + new String(cipherData_fpe));
-            byte[] decryptData_fpe = CipherSuiteUtils.fpeNumberDecrypt(cipherData_fpe, algorithm, keyId, metadata, iv, precision, scale);
-            System.out.println("decrypt result: " + new String(decryptData_fpe));
-            long endTime = System.currentTimeMillis() - beginTime;
-            if (plainStr.equals(new String(decryptData_fpe))) {
-                System.out.println("encrypt and decrypt success. time:" + endTime + "ms.");
-            }
-        } else if ("SM3".equals(algorithm)) {
-            byte[] digestData = CipherSuiteUtils.digest(plainData, algorithm, keyId, metadata);
-            System.out.println("digest result: " + DatatypeConverter.printHexBinary(digestData));
-            long endTime = System.currentTimeMillis() - beginTime;
-            System.out.println("SM3 digest success. time:" + endTime + "ms.");
-        } else {
-            // 加密
-            byte[] cipherData = CipherSuiteUtils.encrypt(plainData, algorithm, keyId, metadata, iv);
-            if (algorithm.startsWith("FPE")) {
-                System.out.println("encrypt result: " + new String(cipherData));
-            } else {
-                System.out.println("encrypt result: " + DatatypeConverter.printHexBinary(cipherData));
-            }
-
-            // 解密
-            byte[] decryptData = CipherSuiteUtils.decrypt(cipherData, algorithm, keyId, metadata, iv);
-            System.out.println("decrypt result: " + new String(decryptData));
-
-            long endTime = System.currentTimeMillis() - beginTime;
-            if (plainStr.equals(new String(decryptData))) {
-                System.out.println("encrypt and decrypt success. time:" + endTime + "ms.");
-            }
-        }
-
-    }
-}

src/main/java/com/jkcredit/traffic/record/test/Test.java

@@ -1,117 +0,0 @@
-package com.jkcredit.traffic.record.test;
-
-import java.io.*;
-import java.nio.ByteBuffer;
-import java.nio.channels.FileChannel;
-import java.util.ArrayList;
-import java.util.List;
-
-public class Test {
-
-    public static void splitFile(String filePath, int fileCount, String tempFilePath) throws IOException {
-        FileInputStream fis = new FileInputStream(filePath);
-
-        FileChannel inputChannel = fis.getChannel();
-        final long fileSize = inputChannel.size();
-        long average = fileSize / fileCount;//平均值
-        long bufferSize = 2048; //缓存块大小，自行调整
-        ByteBuffer byteBuffer = ByteBuffer.allocate(Integer.parseInt(bufferSize + "")); // 申请一个缓存区
-        long startPosition = 0; //子文件开始位置
-        long endPosition = average < bufferSize ? 0 : average - bufferSize;//子文件结束位置
-        for (int i = 0; i < fileCount; i++) {
-            if (i + 1 != fileCount) {
-                int read = inputChannel.read(byteBuffer, endPosition);// 读取数据
-                readW:
-                while (read != -1) {
-                    byteBuffer.flip();//切换读模式
-                    byte[] array = byteBuffer.array();
-                    for (int j = 0; j < array.length; j++) {
-                        byte b = array[j];
-                        if (b == 10 || b == 13) { //判断\n\r
-                            endPosition += j;
-                            break readW;
-                        }
-                    }
-                    endPosition += bufferSize;
-                    byteBuffer.clear(); //重置缓存块指针
-                    read = inputChannel.read(byteBuffer, endPosition);
-                }
-            }else{
-                endPosition = fileSize; //最后一个文件直接指向文件末尾
-            }
-
-            File tempFile = new File(tempFilePath);
-            if (!tempFile.exists()) {
-                tempFile.mkdir();
-            }
-            FileOutputStream fos = new FileOutputStream(tempFilePath + "\\temp_" + (i + 1) + ".csv");
-            FileChannel outputChannel = fos.getChannel();
-            inputChannel.transferTo(startPosition, endPosition - startPosition, outputChannel);//通道传输文件数据
-            outputChannel.close();
-            fos.close();
-            startPosition = endPosition + 1;
-            endPosition += average;
-        }
-        inputChannel.close();
-        fis.close();
-    }
-
-    public static void createBigFile() throws IOException {
-        File file = new File("C:\\Users\\lin\\Desktop\\test\\originalFile.csv");
-        FileWriter fileWriter = new FileWriter(file);
-        BufferedWriter bufferedWriter = new BufferedWriter(fileWriter);
-        String str = "202104,京A123C48_1,8790.0,87392.0,2638403.0,2638403.0,2638403.0,2638403.0,2638403.0,2638403.0";
-        for (int i = 0; i < 10000000; i++) {
-            bufferedWriter.write(str);
-            bufferedWriter.newLine();
-        }
-        bufferedWriter.flush();
-        bufferedWriter.close();
-    }
-
-    public static void readSplitFiles(String filePath) throws IOException{
-        File baseFile = new File(filePath);
-        File[] files = baseFile.listFiles();
-        List<String> result = new ArrayList<>(1200000);
-
-
-        for (File file : files) {
-            FileInputStream fis = new FileInputStream(file);
-            FileChannel inputChannel = fis.getChannel();
-            long bufferSize = 2048; //缓存块大小，自行调整
-            ByteBuffer byteBuffer = ByteBuffer.allocate(Integer.parseInt(bufferSize + "")); // 申请一个缓存区
-            int read = inputChannel.read(byteBuffer);// 读取数据
-            readW:
-            while (read != -1) {
-                System.out.println(byteBuffer.toString());
-//                result.add(byteBuffer.toString());
-//            byteBuffer.flip();//切换读模式
-//            byte[] array = byteBuffer.array();
-//            for (int j = 0; j < array.length; j++) {
-//                byte b = array[j];
-//                if (b == 10 || b == 13) { //判断\n\r
-//                    break readW;
-//                }
-//            }
-//            byteBuffer.clear(); //重置缓存块指针
-//            read = inputChannel.read(byteBuffer);
-            }
-            System.out.println(result.size());
-        }
-
-    }
-
-    public static void main(String[] args) {
-        try {
-            splitFile("/Users/jkxy/Desktop/test/test/part-00001.csv", 10, "/Users/jkxy/Desktop/test/test/temp");
-        } catch (Exception e) {
-            e.printStackTrace();
-        }
-
-//        try {
-//            readSplitFiles("C:\\Users\\lin\\Desktop\\test\\temp");
-//        } catch (Exception e) {
-//            e.printStackTrace();
-//        }
-    }
-}

src/main/java/com/jkcredit/traffic/record/test/SparkTool.java

@@ -1,4 +1,4 @@
-package com.jkcredit.traffic.record.test;
+package com.jkcredit.traffic.record.util;
 
 import com.jkcredit.traffic.record.model.MonthResult;
 import org.apache.spark.SparkConf;