增加定时器，定时推送数据

pom.xml

@@ -127,6 +127,11 @@
             <artifactId>netty</artifactId>
             <version>3.9.9.Final</version>
         </dependency>
+        <dependency>
+            <groupId>org.quartz-scheduler</groupId>
+            <artifactId>quartz</artifactId>
+            <version>2.3.0</version>
+        </dependency>
     </dependencies>
     <build>
         <plugins>

src/main/java/com/jkcredit/traffic/record/TrafficRecordApplication.java

@@ -2,7 +2,9 @@ package com.jkcredit.traffic.record;
 
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.scheduling.annotation.EnableScheduling;
 
+@EnableScheduling
 @SpringBootApplication
 public class TrafficRecordApplication {
 

src/main/java/com/jkcredit/traffic/record/service/TrafficRecordsAnalyseServiceImpl.java

@@ -23,6 +23,7 @@ import org.springframework.stereotype.Service;
 
 import javax.xml.bind.DatatypeConverter;
 import java.io.File;
+import java.io.IOException;
 import java.io.UnsupportedEncodingException;
 import java.net.SocketTimeoutException;
 import java.util.ArrayList;
@@ -85,18 +86,20 @@ public class TrafficRecordsAnalyseServiceImpl implements TrafficRecordsAnalyseSe
             try {
                 // 大文件拆分，放入临时文件夹
                 CsvUtil.splitFile(files.get(i).getAbsolutePath(), 30, tempFolder);
-                for (int j = 1; j <= 30; j++) {
-                    // 数据脱敏
-                    List<MonthResult> maskResults = desensitize(j + "", tempFolder + "/temp_" + j + ".csv");
+            } catch (IOException e) {
+                log.error("splitFileError:{}", files.get(i).getAbsolutePath(), e);
+                return new CommonResponseObject().failed();
+            }
+
+            for (int j = 1; j <= 30; j++) {
+                // 数据脱敏
+                List<MonthResult> maskResults = desensitize(j + "", tempFolder + "/temp_" + j + ".csv");
 
-                    // 数据分割，推送数据
-                    push(maskResults);
-                    boolean deleteResult = FileUtil.INSTANCE.deleteFile(tempFolder + "/temp_" + j + ".csv");
-                    log.info("deleteTempFile:{}, deleteResult:{}", tempFolder + "/temp_" + j + ".csv", deleteResult);
-                    maskResults = null;
-                }
-            } catch (Exception e) {
-                e.printStackTrace();
+                // 数据分割，推送数据
+                push(maskResults);
+                boolean deleteResult = FileUtil.INSTANCE.deleteFile(tempFolder + "/temp_" + j + ".csv");
+                log.info("deleteTempFile:{}, deleteResult:{}", tempFolder + "/temp_" + j + ".csv", deleteResult);
+                maskResults = null;
             }
             boolean deleteResult = FileUtil.INSTANCE.deleteDirectory(tempFolder);
             log.info("deleteTempDir:{}, deleteResult:{}", tempFolder, deleteResult);
@@ -107,6 +110,8 @@ public class TrafficRecordsAnalyseServiceImpl implements TrafficRecordsAnalyseSe
 
     @Override
     public CommonResponseObject pushErrorRecords(String filePath) {
+        log.info("pushErrorRecordsBegin");
+        long startTime = System.currentTimeMillis();
         List<File> files = new ArrayList<>();
         File folder = new File(filePath);
         File[] tempList = folder.listFiles();
@@ -125,6 +130,7 @@ public class TrafficRecordsAnalyseServiceImpl implements TrafficRecordsAnalyseSe
             List<MonthResult> errorRecords = getMonthResult("ErrorRecord" + i, files.get(i).getAbsolutePath());
             push(errorRecords);
         }
+        log.info("pushErrorRecordsEnd, costTime:{}", System.currentTimeMillis() - startTime);
         return new CommonResponseObject().success();
     }
 
@@ -256,8 +262,8 @@ public class TrafficRecordsAnalyseServiceImpl implements TrafficRecordsAnalyseSe
                 .build();
         OkHttpClient client = okHttpClient.newBuilder()
                 .connectTimeout(1000, TimeUnit.MILLISECONDS)
-                .readTimeout(10000, TimeUnit.MILLISECONDS)
-                .writeTimeout(10000, TimeUnit.MILLISECONDS)
+                .readTimeout(5000, TimeUnit.MILLISECONDS)
+                .writeTimeout(5000, TimeUnit.MILLISECONDS)
                 .build();
         String responseContext = "";
         List<MonthResult> errorList = new ArrayList<>();

src/main/java/com/jkcredit/traffic/record/task/PushRecordsTask.java

@@ -0,0 +1,26 @@
+package com.jkcredit.traffic.record.task;
+
+import com.jkcredit.traffic.record.service.TrafficRecordsAnalyseService;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+/**
+ * @author xusonglin
+ * @version V1.0
+ **/
+@Component
+@Slf4j
+public class PushRecordsTask {
+    @Value("${originalRecords.path}")
+    private String originalRecordsPath;
+    @Autowired
+    TrafficRecordsAnalyseService trafficRecordsAnalyseService;
+
+    @Scheduled(cron = "0 0 18 * * ?")
+    public void push() {
+        trafficRecordsAnalyseService.pushTrafficRecords(originalRecordsPath);
+    }
+}

src/main/resources/application-dev.yml

@@ -14,3 +14,5 @@ errorRecords:
   path: /Users/jkxy/Desktop/test/output/
 tempFolder:
   path: /Users/jkxy/Desktop/test/temp
+originalRecords:
+  path: /home/jkxy-01/data/naruto

src/main/resources/application-prod.yml

@@ -14,3 +14,5 @@ errorRecords:
   path: /data/pushErrorRecords/
 tempFolder:
   path: /data/tmp
+originalRecords:
+  path: /home/jkxy-01/data/naruto

src/main/resources/application.yml

@@ -2,6 +2,6 @@ spring:
   application:
     name: traffic-record
   profiles:
-    active: dev
+    active: prod
 server:
   port: 28081

src/main/resources/logback-spring.xml

@@ -22,9 +22,8 @@
     <appender name="errorLog" class="ch.qos.logback.core.rolling.RollingFileAppender">
         <file>${log.path}/error/error.log</file>
         <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
-            <fileNamePattern>${log.path}/error/error-%d{yyyy-MM-dd}.%i.log</fileNamePattern>
+            <fileNamePattern>${log.path}/error/%d{yyyy-MM,aux}/error-%d{yyyy-MM-dd}.%i.log.gz</fileNamePattern>
             <maxFileSize>50MB</maxFileSize>
-            <maxHistory>30</maxHistory>
         </rollingPolicy>
         <encoder>
             <pattern>%date [%thread] %-5level [%logger{50}] %file:%line - %msg%n</pattern>
@@ -44,9 +43,8 @@
         </filter>
         <file>${log.path}/source/source.log</file>
         <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
-            <fileNamePattern>${log.path}/source/source-%d{yyyy-MM-dd}.%i.log</fileNamePattern>
+            <fileNamePattern>${log.path}/source/%d{yyyy-MM,aux}/source-%d{yyyy-MM-dd}.%i.log.gz</fileNamePattern>
             <maxFileSize>50MB</maxFileSize>
-            <maxHistory>30</maxHistory>
         </rollingPolicy>
         <encoder>
             <pattern>%date [%thread] %-5level [%logger{50}] %file:%line - %msg%n</pattern>