调试

src/main/java/com/jkcredit/info/query/config/ThreadPoolConfig.java

@@ -0,0 +1,42 @@
+package com.jkcredit.info.query.config;
+
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.scheduling.annotation.EnableAsync;
+
+import java.util.concurrent.*;
+
+/**
+ * @author xusonglin
+ * @version V1.0
+ **/
+@Configuration
+@EnableAsync
+public class ThreadPoolConfig {
+    @Value("${thread.pool.coreSize}")
+    private Integer coreSize;
+
+    @Value("${thread.pool.maximumPoolSize}")
+    private Integer maximumPoolSize;
+
+    @Value("${thread.pool.queueSize}")
+    private Integer queueSize;
+
+    @Bean("mainThreadPoolExecutor")
+    @Qualifier("executor")
+    public ThreadPoolExecutor mainThreadPoolExecutor() {
+        LinkedBlockingDeque<Runnable> deque = new LinkedBlockingDeque<>(queueSize);
+
+        return new ThreadPoolExecutor(coreSize,
+                maximumPoolSize, 10,
+                TimeUnit.MINUTES, deque);
+    }
+
+    @Bean
+    public ExecutorService executorService() {
+        return Executors.newWorkStealingPool();
+    }
+
+}

src/main/java/com/jkcredit/info/query/constant/CommonConstant.java

@@ -1,6 +1,5 @@
 package com.jkcredit.info.query.constant;
 
-import java.util.List;
-
 public class CommonConstant {
+    public static final String COOKIE = "cookie";
 }

src/main/java/com/jkcredit/info/query/controller/InfoQueryController.java

@@ -3,20 +3,14 @@ package com.jkcredit.info.query.controller;
 import com.alibaba.fastjson.JSONObject;
 import com.jkcredit.common.model.CommonResponseObject;
 import com.jkcredit.info.query.service.InformationQueryService;
-import com.netflix.discovery.DiscoveryManager;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.cloud.client.ServiceInstance;
 import org.springframework.cloud.client.discovery.DiscoveryClient;
 import org.springframework.cloud.client.loadbalancer.LoadBalancerClient;
-import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RestController;
 
-import javax.servlet.http.HttpServletResponse;
-import java.util.HashMap;
-
 @RestController
 @Slf4j
 public class InfoQueryController {

src/main/java/com/jkcredit/info/query/enums/base/IEnum.java

@@ -2,5 +2,6 @@ package com.jkcredit.info.query.enums.base;
 
 public interface IEnum<T> {
     T getValue();
+
     String getDesc();
 }

src/main/java/com/jkcredit/info/query/service/impl/InformationQueryServiceImpl.java

@@ -6,6 +6,7 @@ import com.jkcredit.common.constant.CommonConstant;
 import com.jkcredit.common.model.CommonResponseObject;
 import com.jkcredit.info.query.enums.CatalogEnum;
 import com.jkcredit.info.query.service.InformationQueryService;
+import com.jkcredit.info.query.task.AsyncQuery;
 import com.jkcredit.info.query.util.CatalogConcurrentHashMapUtil;
 import com.jkcredit.info.query.util.CookieUtil;
 import com.jkcredit.info.query.util.HttpUtil;
@@ -16,15 +17,17 @@ import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 import org.jsoup.select.Elements;
+import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
 import java.io.UnsupportedEncodingException;
-import java.net.InetAddress;
-import java.net.URLEncoder;
-import java.net.UnknownHostException;
+import java.net.*;
 import java.text.SimpleDateFormat;
 import java.util.*;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
 
 /**
  * @author xusonglin
@@ -43,6 +46,8 @@ public class InformationQueryServiceImpl implements InformationQueryService {
     private String queryResultUrl;
     @Value("${supplier.hostUrl}")
     private String hostUrl;
+    @Autowired
+    AsyncQuery asyncQuery;
 
     @Override
     public CommonResponseObject query(String keyWord, String type) {
@@ -70,11 +75,9 @@ public class InformationQueryServiceImpl implements InformationQueryService {
             CatalogConcurrentHashMapUtil.getInstance().putAll(catalogIdMap);
         }
 
-        Map<String, String> detailInfo;
         String id = "";
         String queryId = "";
         for (Map.Entry<String, JSONObject> entry : catalogIdMap.entrySet()) {
-            long detailStartTime = System.currentTimeMillis();
             String catalogName = entry.getKey();
             if (type.equals(catalogName)) {
                 JSONObject catalogIdObject = entry.getValue();
@@ -83,7 +86,6 @@ public class InformationQueryServiceImpl implements InformationQueryService {
             } else {
                 continue;
             }
-            log.info("detailCostTime:{}", System.currentTimeMillis() - detailStartTime);
         }
         String encodeKeyWord = "";
         try {
@@ -97,10 +99,10 @@ public class InformationQueryServiceImpl implements InformationQueryService {
         stringBuffer.append("&reportQuery.id=").append(queryId);
         stringBuffer.append("&resource.id=").append(id);
         String resultContent = HttpUtil.doGet(stringBuffer.toString(), cookie);
-        detailInfo = getDetailInfo(resultContent);
-        log.info("totalCostTime:{}", (System.currentTimeMillis() - startTime));
-        if (detailInfo != null && detailInfo.size() != 0) {
-            CommonResponseObject result = CommonResponseObject.success(detailInfo, getLocalHost(), requestId);
+        List<Map<String, String>> detailInfos = getDetailInfo(resultContent);
+
+        if (detailInfos.size() != 0) {
+            CommonResponseObject result = CommonResponseObject.success(detailInfos, getLocalHost(), requestId);
             log.info("keyWord:{}, type:{}, requestId:{}, result:{}, costTime:{}"
                     , keyWord, type, requestId, JSON.toJSONString(result), (System.currentTimeMillis() - startTime));
             return result;
@@ -112,27 +114,49 @@ public class InformationQueryServiceImpl implements InformationQueryService {
         }
     }
 
-    // todo 处理列表多条返回的情况
-    private Map<String, String> getDetailInfo(String content) {
+    // 处理列表多条返回的情况
+    private List<Map<String, String>> getDetailInfo(String content) {
+        List<Map<String, String>> results = new ArrayList<>();
+
         Document doc = Jsoup.parse(content);
         // 获取详情请求url
-        String url = doc.select("a").attr("url").replace("../", hostUrl);
-        if (StringUtils.isBlank(url)) {
-            return new HashMap<>();
+        List<String> urls = doc.select("a").eachAttr("url");
+        CountDownLatch countDownLatch = new CountDownLatch(urls.size());
+        List<String> detailContents = new ArrayList<>();
+        for (String url : urls) {
+            url = url.replace("../", hostUrl);
+            try {
+                Future<String> detailInfo = asyncQuery.detailInfoQuery(url);
+                String detailInfoContent = detailInfo.get();
+                detailContents.add(detailInfoContent);
+            } catch (InterruptedException ie) {
+                log.error("getDetailInfo.InterruptedException:", ie);
+            } catch (ExecutionException e) {
+                log.error("getDetailInfo.ExecutionException:", e);
+            } finally {
+                countDownLatch.countDown();
+            }
         }
-        System.out.println(url);
-        // 请求详情信息
-        String detailInfoContent = HttpUtil.doGet(url, CookieUtil.getAuthenticationCookie());
-        Document detailDoc = Jsoup.parse(detailInfoContent);
-        Elements elements = detailDoc.select(".detail-info-list");
-        Map<String, String> detailInfo = new HashMap<>();
-        for (int i = 0; i < elements.size(); i++) {
-            String dataName = detailDoc.select(".detail-info-list").get(i).select("span").get(1).attr("data-name");
-            String dataValue = detailDoc.select(".detail-info-list").get(i).select("span").get(1).text();
-            detailInfo.put(dataName, dataValue);
+        try {
+            countDownLatch.await();
+        } catch (InterruptedException ie) {
+            log.error("getDetailInfo.InterruptedException:", ie);
+        }
+
+        for (String detailInfoContent : detailContents) {
+            Document detailDoc = Jsoup.parse(detailInfoContent);
+            Elements elements = detailDoc.select(".detail-info-list");
+            Map<String, String> detailMap = new HashMap<>();
+            for (int i = 0; i < elements.size(); i++) {
+                String dataName = detailDoc.select(".detail-info-list").get(i).select("span").get(1).attr("data-name");
+                String dataValue = detailDoc.select(".detail-info-list").get(i).select("span").get(1).text();
+                detailMap.put(dataName, dataValue);
+            }
+            if (detailMap.size() != 0) {
+                results.add(detailMap);
+            }
         }
-        System.out.println(detailInfo.toString());
-        return detailInfo;
+        return results;
     }
 
     private Map<String, JSONObject> getCatalogIds(String catalogContent) {
@@ -188,7 +212,7 @@ public class InformationQueryServiceImpl implements InformationQueryService {
         if (StringUtils.isBlank(cookie)) {
             return CommonResponseObject.failed("查询失败", CommonConstant.NETWORK_DISABLED, getLocalHost(), requestId);
         } else {
-            return CommonResponseObject.success(new HashMap<>(), getLocalHost(), requestId);
+            return CommonResponseObject.success(new ArrayList<>(), getLocalHost(), requestId);
         }
     }
 

src/main/java/com/jkcredit/info/query/task/AsyncQuery.java

@@ -0,0 +1,26 @@
+package com.jkcredit.info.query.task;
+
+import com.jkcredit.info.query.util.CookieUtil;
+import com.jkcredit.info.query.util.HttpUtil;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.scheduling.annotation.Async;
+import org.springframework.scheduling.annotation.AsyncResult;
+import org.springframework.scheduling.annotation.EnableAsync;
+import org.springframework.stereotype.Service;
+
+import java.util.concurrent.Future;
+
+/**
+ * @author xusonglin
+ * @version V1.0
+ **/
+@Service
+@Slf4j
+@EnableAsync
+public class AsyncQuery {
+    @Async("executor")
+    public Future<String> detailInfoQuery(String url) throws InterruptedException {
+        String detailInfoContent = HttpUtil.doGet(url, CookieUtil.getAuthenticationCookie());
+        return new AsyncResult<>(detailInfoContent);
+    }
+}

src/main/java/com/jkcredit/info/query/util/CatalogConcurrentHashMapUtil.java

@@ -1,9 +1,7 @@
 package com.jkcredit.info.query.util;
 
 import com.alibaba.fastjson.JSONObject;
-import okhttp3.Cookie;
 
-import java.util.List;
 import java.util.concurrent.ConcurrentHashMap;
 
 /**

src/main/java/com/jkcredit/info/query/util/CookieConcurrentHashMapUtil.java

@@ -12,7 +12,7 @@ import java.util.concurrent.ConcurrentHashMap;
 public enum CookieConcurrentHashMapUtil {
     INSTANCE;
 
-    public static ConcurrentHashMap<String, List<Cookie>>  concurrentHashMap = new ConcurrentHashMap<>();
+    public static ConcurrentHashMap<String, List<Cookie>> concurrentHashMap = new ConcurrentHashMap<>();
 
     public static ConcurrentHashMap<String, List<Cookie>> getInstance() {
         return concurrentHashMap;

src/main/java/com/jkcredit/info/query/util/CookieUtil.java

@@ -1,13 +1,12 @@
 package com.jkcredit.info.query.util;
 
+import com.jkcredit.info.query.constant.CommonConstant;
 import lombok.extern.slf4j.Slf4j;
 import okhttp3.Cookie;
 import org.apache.commons.lang3.StringUtils;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 
-import java.net.InetAddress;
-import java.net.UnknownHostException;
 import java.util.List;
 import java.util.concurrent.ConcurrentHashMap;
 
@@ -21,42 +20,30 @@ public class CookieUtil {
     public void setCookieUrl(String cookieUrl) {
         CookieUtil.cookieUrl = cookieUrl;
     }
+
     @Value("${supplier.authenticationCookieUrl}")
     public void setAuthenticationCookieUrl(String authenticationCookieUrl) {
         CookieUtil.authenticationCookieUrl = authenticationCookieUrl;
     }
 
     public static String getAuthenticationCookie() {
-        String host = "";
-        InetAddress address = null;
-        try {
-            address = InetAddress.getLocalHost();
-        } catch (UnknownHostException e) {
-            e.printStackTrace();
-        }
-        if (address != null) {
-            host = address.getHostAddress();
-        }
-        log.info("host:{}", host);
         ConcurrentHashMap<String, List<Cookie>> cookieStore = CookieConcurrentHashMapUtil.getInstance();
-        // todo
-        host = "51.1.0.61";
-        if (cookieStore.get(host) != null) {
-            return getCookie(host);
+        if (cookieStore.get(CommonConstant.COOKIE) != null) {
+            return getCookie();
         }
-        // todo 此处需要判断获取token是否成功
         if (StringUtils.isNotBlank(HttpUtil.doGetLoadCookie(cookieUrl))) {
-            HttpUtil.doGet(authenticationCookieUrl, getCookie(host));
-        };
-        return getCookie(host);
+            HttpUtil.doGet(authenticationCookieUrl, getCookie());
+        }
+        ;
+        return getCookie();
     }
 
-    public static String getCookie(String host) {
+    public static String getCookie() {
         String cookie = "";
         ConcurrentHashMap<String, List<Cookie>> cookieStore = CookieConcurrentHashMapUtil.getInstance();
-        if (cookieStore.get(host) != null) {
-            cookie = cookieStore.get(host).get(0).name()
-                    + "=" + cookieStore.get(host).get(0).value();
+        if (cookieStore.get(CommonConstant.COOKIE) != null) {
+            cookie = cookieStore.get(CommonConstant.COOKIE).get(0).name()
+                    + "=" + cookieStore.get(CommonConstant.COOKIE).get(0).value();
         }
         return cookie;
     }

src/main/java/com/jkcredit/info/query/util/HttpUtil.java

@@ -1,5 +1,6 @@
 package com.jkcredit.info.query.util;
 
+import com.jkcredit.info.query.constant.CommonConstant;
 import lombok.extern.slf4j.Slf4j;
 import okhttp3.*;
 
@@ -18,7 +19,6 @@ public class HttpUtil {
     private static OkHttpClient okHttpClient = new OkHttpClient.Builder().build();
 
     public static String doPost(String url, FormBody formBody, String cookie) {
-        long startTime = System.currentTimeMillis();
         Request okRequest = new Request.Builder()
                 .post(formBody)
                 .url(url)
@@ -28,7 +28,7 @@ public class HttpUtil {
                 .connectTimeout(3000, TimeUnit.MILLISECONDS)
                 .readTimeout(3000, TimeUnit.MILLISECONDS)
                 .writeTimeout(3000, TimeUnit.MILLISECONDS)
-//                .sslSocketFactory(SSLSocketClient.getSocketFactory(), SSLSocketClient.getX509TrustManager())
+                .sslSocketFactory(SSLSocketClient.getSocketFactory(), SSLSocketClient.getX509TrustManager())
                 .build();
         String responseContext = "";
         try {
@@ -38,14 +38,12 @@ public class HttpUtil {
             }
             response.close();
         } catch (IOException ioe) {
-            ioe.printStackTrace();
+            log.error("doPost.IOException:", ioe);
         }
-        log.info("url:{}, costTime:{}, response:{}", url, System.currentTimeMillis() - startTime, responseContext);
         return responseContext;
     }
 
     public static String doGet(String url, String cookie) {
-        long startTime = System.currentTimeMillis();
         Request okRequest = new Request.Builder().url(url)
                 .header("Cookie", cookie)
                 .build();
@@ -54,8 +52,8 @@ public class HttpUtil {
                 .connectTimeout(3000, TimeUnit.MILLISECONDS)
                 .readTimeout(3000, TimeUnit.MILLISECONDS)
                 .writeTimeout(3000, TimeUnit.MILLISECONDS)
-//                .sslSocketFactory(SSLSocketClient.getSocketFactory())
-//                .hostnameVerifier(SSLSocketClient.getHostnameVerifier())
+                .sslSocketFactory(SSLSocketClient.getSocketFactory())
+                .hostnameVerifier(SSLSocketClient.getHostnameVerifier())
                 .build();
         String responseContext = "";
         try {
@@ -65,14 +63,12 @@ public class HttpUtil {
             }
             response.close();
         } catch (IOException ioe) {
-            ioe.printStackTrace();
+            log.error("doGet.IOException:", ioe);
         }
-        log.info("url:{}, costTime:{}, response:{}", url, System.currentTimeMillis() - startTime, responseContext);
         return responseContext;
     }
 
     public static String doGetLoadCookie(String url) {
-        long startTime = System.currentTimeMillis();
         ConcurrentHashMap<String, List<Cookie>> cookieStore = CookieConcurrentHashMapUtil.getInstance();
         Request okRequest = new Request.Builder().url(url)
                 .build();
@@ -84,17 +80,17 @@ public class HttpUtil {
                 .cookieJar(new CookieJar() {
                     @Override
                     public void saveFromResponse(HttpUrl httpUrl, List<Cookie> list) {
-                        cookieStore.put(httpUrl.host(), list);
+                        cookieStore.put(CommonConstant.COOKIE, list);
                     }
 
                     @Override
                     public List<Cookie> loadForRequest(HttpUrl httpUrl) {
-                        List<Cookie> cookies = cookieStore.get(httpUrl.host());
+                        List<Cookie> cookies = cookieStore.get(CommonConstant.COOKIE);
                         return cookies != null ? cookies : new ArrayList<Cookie>();
                     }
                 })
-//                .sslSocketFactory(SSLSocketClient.getSocketFactory(), SSLSocketClient.getX509TrustManager())
-//                .hostnameVerifier(SSLSocketClient.getHostnameVerifier())
+                .sslSocketFactory(SSLSocketClient.getSocketFactory(), SSLSocketClient.getX509TrustManager())
+                .hostnameVerifier(SSLSocketClient.getHostnameVerifier())
                 .build();
         String responseContext = "";
         try {
@@ -104,11 +100,10 @@ public class HttpUtil {
             }
             response.close();
         } catch (IOException ioe) {
-//            ioe.printStackTrace();
+            log.error("doGetLoadCookie.IOException:", ioe);
         } catch (Exception e) {
-//            e.printStackTrace();
+            log.error("doGetLoadCookie.Exception:", e);
         }
-        log.info("url:{}, costTime:{}, response:{}", url, System.currentTimeMillis() - startTime, responseContext);
         return responseContext;
     }
 

src/main/java/com/jkcredit/info/query/util/SSLSocketClient.java

@@ -17,6 +17,7 @@ public class SSLSocketClient {
         }
         return socketFactory;
     }
+
     public static X509TrustManager getX509TrustManager() {
         X509TrustManager x509TrustManager = new X509TrustManager() {
             @Override
@@ -36,8 +37,9 @@ public class SSLSocketClient {
         };
         return x509TrustManager;
     }
+
     public static TrustManager[] getTrustManager() {
-        TrustManager[] trustManagers = new TrustManager[] {
+        TrustManager[] trustManagers = new TrustManager[]{
                 new X509TrustManager() {
                     @Override
                     public void checkClientTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {

src/main/resources/application.yml

@@ -9,17 +9,22 @@ eureka:
       defaultZone: http://51.1.0.61:28081/eureka
   instance:
     instance-id: ${spring.application.name}:${spring.cloud.client.ip-address}:${server.port}
-#supplier:
-#  cookieUrl: https://jwzhcx.ga/sevenRule.action
-#  authenticationCookieUrl: https://jwzhcx.ga/pkiLogin.action
-#  mainPageRequestUrl: http://localhost:8080/cloudSearch/goSearch.action
-#  catalogUrl: https://jwzhcx.ga/cloudSearch/goSearch.action
-#  queryResultUrl: https://jwzhcx.ga/cloudSearch/ajaxQueryResult.action
-#  hostUrl: https://jwzhcx.ga/
 supplier:
-  cookieUrl: http://51.1.0.61:28087/sevenRule.action
-  authenticationCookieUrl: http://51.1.0.61:28087/pkiLogin.action
-  mainPageRequestUrl: http://51.1.0.61:28087/cloudSearch/goSearch.action
-  catalogUrl: http://51.1.0.61:28087/cloudSearch/goSearch.action
-  queryResultUrl: http://51.1.0.61:28087/cloudSearch/ajaxQueryResult.action
-  hostUrl: http://51.1.0.61:28087/
+  cookieUrl: https://jwzhcx.ga/sevenRule.action
+  authenticationCookieUrl: https://jwzhcx.ga/pkiLogin.action
+  mainPageRequestUrl: http://localhost:8080/cloudSearch/goSearch.action
+  catalogUrl: https://jwzhcx.ga/cloudSearch/goSearch.action
+  queryResultUrl: https://jwzhcx.ga/cloudSearch/ajaxQueryResult.action
+  hostUrl: https://jwzhcx.ga/
+#supplier:
+#  cookieUrl: http://51.1.0.61:28087/sevenRule.action
+#  authenticationCookieUrl: http://51.1.0.61:28087/pkiLogin.action
+#  mainPageRequestUrl: http://51.1.0.61:28087/cloudSearch/goSearch.action
+#  catalogUrl: http://51.1.0.61:28087/cloudSearch/goSearch.action
+#  queryResultUrl: http://51.1.0.61:28087/cloudSearch/ajaxQueryResult.action
+#  hostUrl: http://51.1.0.61:28087/
+thread:
+  pool:
+    coreSize: 10
+    maximumPoolSize: 20
+    queueSize: 1000

src/main/resources/logback-spring.xml

@@ -22,7 +22,7 @@
     <appender name="errorLog" class="ch.qos.logback.core.rolling.RollingFileAppender">
         <file>${log.path}/error/error.log</file>
         <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
-            <fileNamePattern>${log.path}/error/%d{yyyy-MM,aux}/error-%d{yyyy-MM-dd}.%i.log</fileNamePattern>
+            <fileNamePattern>${log.path}/error/%d{yyyy-MM,aux}/error-%d{yyyy-MM-dd}.%i.log.gz</fileNamePattern>
             <maxFileSize>50MB</maxFileSize>
         </rollingPolicy>
         <encoder>
@@ -44,7 +44,7 @@
         </filter>
         <file>${log.path}/source/source.log</file>
         <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
-            <fileNamePattern>${log.path}/source/%d{yyyy-MM,aux}/source-%d{yyyy-MM-dd}.%i.log</fileNamePattern>
+            <fileNamePattern>${log.path}/source/%d{yyyy-MM,aux}/source-%d{yyyy-MM-dd}.%i.log.gz</fileNamePattern>
             <maxFileSize>50MB</maxFileSize>
         </rollingPolicy>
         <encoder>