Merge remote-tracking branch 'origin/master'

platform-admin/src/main/java/com/platform/entity/TbProductMissEntity.java

@@ -25,6 +25,9 @@ public class TbProductMissEntity implements Serializable {
      * 失败类型【1：已存在】【2：出现了异常】
      */
     private Integer missType;
+    
+    private String missReason;
+    
     /**
      * 页数(每250个)
      */
@@ -99,4 +102,12 @@ public class TbProductMissEntity implements Serializable {
     public Date getCreateTime() {
         return createTime;
     }
+    
+    public String getMissReason() {
+        return missReason;
+    }
+    
+    public void setMissReason(String missReason) {
+        this.missReason = missReason;
+    }
 }

platform-admin/src/main/resources/com/platform/dao/TbCfStationItemDao.xml

@@ -229,7 +229,7 @@
 			#{itemUrl},
 			#{itemImg},
 			#{itemTags},
-			#{itemLabel}，
+			#{itemLabel},
 			#{itemNum},
 			#{itemCount},
 			#{itemSku},

platform-admin/src/main/resources/com/platform/dao/TbProductMissDao.xml

@@ -60,12 +60,14 @@
 			`id`,
 			`product_id`,
 			`miss_type`,
+			`miss_reason`,
 			`page_size`,
 			`create_time`)
 		values(
 			#{id},
 			#{productId},
 			#{missType},
+			#{missReason},
 			#{pageSize},
 			#{createTime})
 	</insert>

platform-admin/src/main/test/java/test/ConcurrencyApi.java

+package test;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+
+import java.util.concurrent.*;
+
+/**
+ * 并发测试
+ *
+ * @author 爱酱油不爱醋
+ * @version 1.0
+ */
+@RunWith(SpringJUnit4ClassRunner.class)
+@ContextConfiguration(locations = {"classpath*:/applicationContext-test.xml"})
+public class ConcurrencyApi {
+	
+	private final int corePoolSize = 15;
+	
+	private final int maximumPoolSize = 35;
+	
+	@Test
+	public void test() {
+		ExecutorService executor = Executors.newCachedThreadPool();
+		int i = 0;
+		while (true) {
+			RunningTest test = new RunningTest(i);
+			executor.submit(test);
+			System.out.println("正在执行的线程编号：" + i);
+			i++;
+		}
+		
+	}
+	
+	
+}

platform-admin/src/main/test/java/test/ImpartCode.java

 package test;
 
-import cn.hutool.core.lang.Console;
 import cn.hutool.core.util.IdUtil;
 import cn.hutool.http.HttpRequest;
 import com.platform.dao.*;
@@ -23,6 +22,8 @@ import java.net.URL;
 import java.util.Arrays;
 import java.util.Date;
 import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 
 /**
  * 商品导入功能
@@ -64,6 +65,11 @@ public class ImpartCode {
 	 */
 	private final String API_URL = "https://mollykitty.myshopify.com/admin/api/" + VERSION + "/products.json";
 
+	/**
+	 * 任务用线程池
+	 */
+	ExecutorService executorService = Executors.newCachedThreadPool();
+
 	/**
 	 * 根据分类进行导入
 	 */
@@ -71,31 +77,31 @@ public class ImpartCode {
 	public void goImport() {
 		List<String> list = Arrays.asList("Men", "Women", "Children", "Afri Home", "Cosmetics", "Hair", "Sportswear", "Electronics");
 		for (String product_type : list) {
-			// 记录商品数量
-			int productCount = 0;
-			// 记录页数
-			int pageSize = 1;
+			
 			// 测试数量
-			String link = "https://mollykitty.myshopify.com/admin/api/" + VERSION + "/products/count.json" + "?&product_type=" + product_type;
+			String link = "https://mollykitty.myshopify.com/admin/api/" + VERSION + "/products/count.json" + "?product_type=" + product_type;
 			String data = HttpRequest.get(link)
 					.header("Authorization", "Basic N2YwZTA0OGFjNDAxNmI5MzU3YmIxZWIyMjE3ZTQyMDE6MWM5N2ExMjIyYzViNDBlMDU5N2M4MTZjNmRmZWNhYzA=")
 					.execute().body();
 			System.err.println(product_type + "：" + data);
 			// 导入请求
-			link = API_URL + "?&product_type=" + product_type;
+			link = "https://mollykitty.myshopify.com/admin/api/" + VERSION + "/products.json" + "?product_type=" + product_type + "&limit=250";
 			// 开始第一次导入商品
-			importMethod(link, pageSize, productCount);
+			importMethod(link);
 			// 获取第一次循环的请求头
 			String headerLink = HttpRequest.get(link)
 					.header("Authorization", "Basic N2YwZTA0OGFjNDAxNmI5MzU3YmIxZWIyMjE3ZTQyMDE6MWM5N2ExMjIyYzViNDBlMDU5N2M4MTZjNmRmZWNhYzA=")
 					.execute().header("link");
 			// 获取第一次循环后的请求
+			try {
 				headerLink = headerLink.split(";")[0].replaceAll("<", "").replaceAll(">", "");
+			} catch (Exception e) {
+				continue;
+			}
 			// 无限导入
 			while (true) {
-				// 导入商品
-				productCount++;
-				importMethod(headerLink, pageSize, productCount);
+				
+				importMethod(headerLink);
 				
 				// 获取下一页的链接
 				try {
@@ -103,13 +109,11 @@ public class ImpartCode {
 							.header("Authorization", "Basic N2YwZTA0OGFjNDAxNmI5MzU3YmIxZWIyMjE3ZTQyMDE6MWM5N2ExMjIyYzViNDBlMDU5N2M4MTZjNmRmZWNhYzA=")
 							.execute().header("link")
 							.split(",")[1];
-					Console.error(headerLink);
 					headerLink = headerLink.substring(2, headerLink.indexOf(">"));
 				} catch (Exception e) {
 					// 死循环的终点为获取下一页链接抛出数组越界异常
 					break;
 				}
-				
 			}
 		}
 		
@@ -118,39 +122,76 @@ public class ImpartCode {
 	/**
 	 * 商品导入处理
 	 *
-	 * @param data 接口返回数据
-	 * @param pageSize 正在导入的页数
-	 * @param productCount 正在导入的商品数量
+	 * @param link 请求链接
 	 */
-	private void importMethod(String data, int pageSize, int productCount) {
+	private void importMethod(String link) {
+		// 请求
+		String data = HttpRequest.get(link)
+				.header("Authorization", "Basic N2YwZTA0OGFjNDAxNmI5MzU3YmIxZWIyMjE3ZTQyMDE6MWM5N2ExMjIyYzViNDBlMDU5N2M4MTZjNmRmZWNhYzA=")
+				.execute().body();
 		// 格式化
 		JSONObject jsonObject = JSONObject.fromObject(data);
 		// 取每个数组中的 products 数组节点
 		JSONArray productsArr = jsonObject.getJSONArray("products");
+		// 记录线程数
+		int threadCount = 0;
 
 		for (int i = 0; i < productsArr.size(); i++) {
-			
 			JSONObject productsObj = productsArr.getJSONObject(i);
 			
-			productCount++;
-			Console.log("当前正在导入的商品个数为：" + productCount);
-			
 			// 商品主体信息的商品 id
 			String itemId = IdUtil.simpleUUID();
 			// 复用的日期
 			Date date = new Date();
 			// 商品的平台 id
 			String id = productsObj.getString("id");
-			try {
+			
 			// 查询该商品的 id 是否已保存
 			if (tbCfStationItemDao.queryByCode(id) != null) {
-					// 继续循环
+				System.err.println("已存在的商品，跳过导入的商品id为：" + id);
+				// 跳过循环
 				continue;
 			}
-				// 用于记录 option 的属性名
+			
+ 			// 线程任务
+			int finalThreadCount = threadCount;
+			threadCount++;
+			Runnable runnable = () -> {
+				try {
+					// 商品详情
 					int optionNum = 0;
-				// 下方商品主体信息需要的商品属性，以"/"隔开
 					StringBuilder optionBuilder = new StringBuilder();
+					StringBuilder imageBuilder = new StringBuilder();
+					JSONArray imagesArr = productsObj.getJSONArray("images");
+					for (int j = 0; j < imagesArr.size(); j++) {
+						JSONObject imageObj = imagesArr.getJSONObject(j);
+						// 上传图片
+						String skuSrc = imageObj.getString("src");
+						InputStream skuInputStream = getImageStream(skuSrc);
+						byte[] skuBytes = toByteArray(skuInputStream);
+						String url = OssUtil.upload(skuBytes, id + "_sku_" + j + ".jpg", "image");
+						// 拼接富文本的 HTML
+						imageBuilder.append("<p><img src=\"")
+								.append(url).append("\" title=\"")
+								.append(IdUtil.randomUUID()).append("_350x350")
+								.append(IdUtil.simpleUUID()).append(".jpg\"/></p>");
+						System.err.println("富文本图片内容：" + imageBuilder.toString());
+					}
+					TbCfItemDescEntity itemDescEntity = tbCfItemDescDao.queryObject(itemId);
+					if (itemDescEntity == null) {
+						itemDescEntity = new TbCfItemDescEntity();
+						itemDescEntity.setItemId(itemId);
+						itemDescEntity.setItemDesc(imageBuilder.toString());
+						itemDescEntity.setCreateTime(date);
+						tbCfItemDescDao.save(itemDescEntity);
+					}
+					
+					// 上传图片获取主图
+					String src = productsObj.getJSONObject("image").getString("src");
+					InputStream inputStream = getImageStream(src);
+					byte[] bytes = toByteArray(inputStream);
+					String url = OssUtil.upload(bytes, id + "_station.jpg", "image");
+					System.err.println("主图的链接为： " + url);
 					
 					// 商品的属性
 					JSONArray optionsArr = productsObj.getJSONArray("options");
@@ -207,10 +248,15 @@ public class ImpartCode {
 							tbCfOptionDao.save(optionEntity);
 						}
 						categoryEntity.setCategoryDesc(categoryBuilder.toString());
+						try {
 							tbCfCategoryDao.save(categoryEntity);
+						} catch (Exception e) {
+							tbCfCategoryDao.update(categoryEntity);
+						}
 					}
 					
-				TbCfStationItemEntity tbCfStationItem = new TbCfStationItemEntity();    // 商品主体信息实体类
+					// 商品主体信息实体类
+					TbCfStationItemEntity tbCfStationItem = new TbCfStationItemEntity();
 
 					// 保存商品的变体数据
 					JSONArray variantsArr = productsObj.getJSONArray("variants");
@@ -227,7 +273,7 @@ public class ImpartCode {
 						skusEntity.setCreateTime(date);
 						skusEntity.setUpdateTime(date);
 						skusEntity.setSkuDesc(variantsObj.getString("title"));
-					
+						System.out.println(optionBuilder.toString());
 						skusEntity.setSkuName(optionBuilder.toString());
 						skusEntity.setSkuPrice(price);
 						skusEntity.setSkuCount(quantity);
@@ -235,7 +281,13 @@ public class ImpartCode {
 						tbCfItemSkusDao.save(skusEntity);
 
 						tbCfStationItem.setItemCount(quantity.longValue());
+						// 商品的原价
+						String compare_at_price = variantsObj.getString("compare_at_price");
+						if (!"null".equals(compare_at_price)) {
+							tbCfStationItem.setItemPrice(new BigDecimal(compare_at_price));
+						} else  {
 							tbCfStationItem.setItemPrice(null);
+						}
 						// 商品的折扣价
 						tbCfStationItem.setDiscountPrice(price);
 					}
@@ -244,7 +296,7 @@ public class ImpartCode {
 					String product_type = productsObj.getString("product_type");    // 商品的分类
 
 					// 保存一级分类名
-				String goods_type_id = IdUtil.simpleUUID();                         // 商品分类id
+					String goods_type_id = IdUtil.simpleUUID();
 					if (tbCfGoodstypeDao.queryByTitle(product_type) == null) {          // 如果分类名已存在则不保存
 						TbCfGoodstypeEntity tbCfGoodstypeEntity = new TbCfGoodstypeEntity();
 						tbCfGoodstypeEntity.setGoodstypeId(goods_type_id);
@@ -259,12 +311,10 @@ public class ImpartCode {
 						goods_type_id = tbCfGoodstypeEntity.getGoodstypeId();
 					}
 					
-				// 二级分类名
+					// 解析二三级分类名
 					String tags = "";
-				// 三级分类名
 					String three = "";
 					String[] s = productsObj.getString("tags").split(",");
-				Console.error(Arrays.toString(s));
 					for (String str : s) {
 						if (str.indexOf("#") == 0) {
 							tags = str.replaceAll("#", "");
@@ -274,7 +324,7 @@ public class ImpartCode {
 					}
 
 					// 保存二级分类名
-				String goodsTwoId = IdUtil.simpleUUID();                            // 商品二级分类id
+					String goodsTwoId = IdUtil.simpleUUID();
 					if (tbCfGoodstwotypeDao.queryTitle(tags) == null) {
 						TbCfGoodstwotypeEntity tbCfGoodstwotypeEntity = new TbCfGoodstwotypeEntity();
 						tbCfGoodstwotypeEntity.setGoodstwotypeId(goodsTwoId);
@@ -287,7 +337,7 @@ public class ImpartCode {
 					}
 
 					// 保存三级分类名(品名)
-				String goodsThreeId = IdUtil.simpleUUID();                          // 商品三级分类id
+					String goodsThreeId = IdUtil.simpleUUID();
 					if (tbCfDescripitonDao.queryByDescriptionName(three) == null) {
 						TbCfDescripitonEntity tbCfDescripitonEntity = new TbCfDescripitonEntity();
 						tbCfDescripitonEntity.setDescripitionId(goodsThreeId);
@@ -314,43 +364,10 @@ public class ImpartCode {
 					tbCfStationItem.setItemDescritionId(goodsThreeId);
 					tbCfStationItem.setItemTags(product_type + "," + tags + "," + three);
 					tbCfStationItem.setItemTop("N");
-				
-				// 上传图片
-				String src = productsObj.getJSONObject("image").getString("src");
-				InputStream inputStream = getImageStream(src);
-				byte[] bytes = toByteArray(inputStream);
-				String url = OssUtil.upload(bytes, id + "_station.jpg", "zion");
-				
 					tbCfStationItem.setItemImg(url);
 					tbCfStationItem.setEnableFlag(2);
 					tbCfStationItemDao.save(tbCfStationItem);
 					
-				// 商品的富文本描述
-				StringBuilder imageBuilder = new StringBuilder();
-				JSONArray imagesArr = productsObj.getJSONArray("images");
-				for (int j = 0; j < imagesArr.size(); j++) {
-					JSONObject imageObj = imagesArr.getJSONObject(j);
-					// 上传图片
-					src = imageObj.getString("src");
-					inputStream = getImageStream(src);
-					bytes = toByteArray(inputStream);
-					url = OssUtil.upload(bytes, id + "_sku_" + j + ".jpg", "zion");
-					
-					// 拼接富文本的 HTML
-					imageBuilder.append("<p><img src=\"")
-							.append(url).append("\" title=\"")
-							.append(IdUtil.randomUUID()).append("_350x350")
-							.append(IdUtil.simpleUUID()).append(".jpg\"/></p>");
-				}
-				// 商品详情
-				TbCfItemDescEntity itemDescEntity = tbCfItemDescDao.queryObject(itemId);
-				if (itemDescEntity == null) {
-					itemDescEntity = new TbCfItemDescEntity();
-					itemDescEntity.setItemId(itemId);
-					itemDescEntity.setItemDesc(imageBuilder.toString());
-					itemDescEntity.setCreateTime(date);
-					tbCfItemDescDao.save(itemDescEntity);
-				}
 				} catch (Exception e) {
 					e.printStackTrace();
 					// 记录失败的产品记录
@@ -358,12 +375,18 @@ public class ImpartCode {
 					tbProductMissEntity.setId(IdUtil.simpleUUID());
 					tbProductMissEntity.setProductId(id);
 					tbProductMissEntity.setMissType(2);
-				tbProductMissEntity.setPageSize(pageSize);
+					tbProductMissEntity.setMissReason(e.getMessage());
+					tbProductMissEntity.setPageSize(1);
 					tbProductMissEntity.setCreateTime(new Date());
 					tbProductMissDao.save(tbProductMissEntity);
 				}
-		}
+				System.err.println("线程号：" + finalThreadCount + "，已执行完成！");
+			};
 			
+			// 提交任务执行
+			executorService.submit(runnable);
+			System.err.println("线程号：" + finalThreadCount + "，开始提交任务！");
+		}
 	}
 	
 	/**
@@ -376,8 +399,8 @@ public class ImpartCode {
 	public static InputStream getImageStream(String url) {
 		try {
 			HttpURLConnection connection = (HttpURLConnection) new URL(url).openConnection();
-			connection.setReadTimeout(5000);
-			connection.setConnectTimeout(5000);
+			connection.setReadTimeout(10000);
+			connection.setConnectTimeout(10000);
 			connection.setRequestMethod("GET");
 			if (connection.getResponseCode() == HttpURLConnection.HTTP_OK) {
 				return connection.getInputStream();

platform-admin/src/main/test/java/test/ImportTask.java

+package test;
+
+/**
+ * 商品导入的任务块
+ *
+ * @author 爱酱油不爱醋
+ * @version 1.0
+ */
+public class ImportTask implements Runnable {
+
+
+    @Override
+    public void run() {
+
+    }
+}

platform-admin/src/main/test/java/test/RunningTest.java

+package test;
+
+/**
+ * @author 爱酱油不爱醋
+ * @version 1.0
+ */
+public class RunningTest implements Runnable {
+	
+	private int taskNum;
+	
+	public RunningTest(int num) {
+		this.taskNum = num;
+	}
+	
+	@Override
+	public void run() {
+		System.out.println("正在执行任务：" + taskNum);
+		try {
+			Thread.sleep(1000);
+		} catch (InterruptedException e) {
+			e.printStackTrace();
+		}
+		System.out.println("执行完毕：" + taskNum);
+	}
+}

platform-admin/src/main/test/java/test/TestApi.java

@@ -3,6 +3,7 @@ package test;
 import cn.hutool.core.lang.Console;
 import cn.hutool.core.util.IdUtil;
 import cn.hutool.http.HttpRequest;
+import com.google.gson.annotations.JsonAdapter;
 import com.platform.controller.api.ProductExitException;
 import com.platform.dao.*;
 import com.platform.entity.*;
@@ -66,7 +67,6 @@ public class TestApi {
      */
     private final String API_URL = "https://mollykitty.myshopify.com/admin/api/" + VERSION + "/products.json";
     
-    
     @Test
     public void host() {
         String link = API_URL + "?limit=250&page_info=eyJwcm9kdWN0X3R5cGUiOiJXb21lbiIsImxhc3RfaWQiOjQ0OTc2Nzg5NTg2ODksImxhc3RfdmFsdWUiOiJDYXN1YWwgU2xpbSBTd2VhdGVycyIsImRpcmVjdGlvbiI6Im5leHQifQ";
@@ -114,21 +114,36 @@ public class TestApi {
      */
     @Test
     public void linkTest() {
-        String link = "https://mollykitty.myshopify.com/admin/api/2020-01/products.json?vendor=%E6%B3%B0%E9%82%A6%E6%9C%8D%E9%A5%B0&limit=1&vendor=戴菊";
+        String link = "https://mollykitty.myshopify.com/admin/api/2020-01/products.json?vendor=%E6%B3%B0%E9%82%A6%E6%9C%8D%E9%A5%B0&limit=100&vendor=戴菊";
         link = HttpRequest.get(link)
                 .header("Authorization", "Basic N2YwZTA0OGFjNDAxNmI5MzU3YmIxZWIyMjE3ZTQyMDE6MWM5N2ExMjIyYzViNDBlMDU5N2M4MTZjNmRmZWNhYzA=")
                 .execute().body();
         
         JSONObject json = JSONObject.fromObject(link);
-        JSONObject product = json.getJSONArray("products").getJSONObject(0);
-        String tags = product.getString("tags");
+        JSONArray productArr = json.getJSONArray("products");
+        for (int j = 0; j < productArr.size(); j++) {
+            JSONObject product = productArr.getJSONObject(j);
+
+            JSONArray variantsArr = product.getJSONArray("variants");
+            for (int i = 0; i < variantsArr.size(); i++) {
+                JSONObject variantsObj = variantsArr.getJSONObject(i);
+                String compare_at_price = variantsObj.getString("compare_at_price");
+                if (!"null".equals(compare_at_price)) {
+                    System.out.println(compare_at_price);
+                }
+            }
+
+        }
+
+
+        /*String tags = product.getString("tags");
         String[] s = tags.split(",");
         Console.error(Arrays.toString(s));
         for (String i : s) {
             if (i.indexOf("#") == 0) {
                 System.out.println(i);
             }
-        }
+        }*/
     }
     
     /**
@@ -704,12 +719,6 @@ public class TestApi {
         return output.toByteArray();
     }
     
-    public static void main(String[] args) throws Exception {
-        String link = "https://cdn.shopify.com/s/files/1/0079/8330/0705/products/0_-9_aee971dc-5661-46bb-9d7e-11f0a3ee5b35.jpg?v=1580133539";
-        InputStream inputStream = getImageStream(link);
-        byte[] bytes = toByteArray(inputStream);
-        String url = OssUtil.upload(bytes, IdUtil.simpleUUID() + ".jpg", "zion");
-        System.out.println(url);
-    }
+    
     
 }

platform-admin/src/main/test/resource/log4j.properties

@@ -59,8 +59,8 @@ log4j.appender.slowsql.layout=org.apache.log4j.PatternLayout
 log4j.appender.slowsql.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss SSS}|%5p|%F.%M:%L|%m%n
 #控制台输出所有SQL
 #便于调试  生产环境注释
-log4j.logger.com.platform.dao=DEBUG,sql
-log4j.appender.sql=org.apache.log4j.ConsoleAppender
-log4j.appender.sql.Target=System.out
-log4j.appender.sql.layout=org.apache.log4j.PatternLayout
-log4j.appender.sql.layout.ConversionPattern=%m %n
+#log4j.logger.com.platform.dao=DEBUG,sql
+#log4j.appender.sql=org.apache.log4j.ConsoleAppender
+#log4j.appender.sql.Target=System.out
+#log4j.appender.sql.layout=org.apache.log4j.PatternLayout
+#log4j.appender.sql.layout.ConversionPattern=%m %n

platform-admin/src/main/test/sql/Test.sql

+SELECT count(*) FROM tb_cf_station_item WHERE item_category = '2fe6be991b4941608c7ebcea7091288f';
+SELECT count(*) FROM tb_cf_station_item WHERE item_category = '7c0d04650ad649fd9bf6b9476947c118';
+
+SELECT count(*) FROM tb_cf_station_item WHERE item_category = '03f2d9ab2ea44893a27d54c280233c9f';
+SELECT count(*) FROM tb_cf_station_item WHERE item_category = '59a9e2c64d2a4a1696a06045f39c0166';
+SELECT count(*) FROM tb_cf_station_item WHERE item_category = 'f0100ca392074027bb99da3a93625d63';
+
+SELECT count(*) FROM tb_cf_station_item WHERE item_category = 'a777a18b4fc4423082fa843231cfc7bc';