Skip to content

Z
zion
提交 299d967d authored 作者: zhengfg's avatar zhengfg
浏览文件
操作

更新

上级 a0551f6c
全部展开 隐藏空白字符变更
内嵌 并排
正在显示 包含 994 行增加 和 509 行删除
pom.xml
浏览文件 @ 299d967d
......@@ -38,6 +38,11 @@
</exclusion>
</exclusions>-->
</dependency>
<!-- https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-security -->
<!-- <dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-security</artifactId>
</dependency>-->
<!--aop-->
<dependency>
<groupId>org.springframework.boot</groupId>
......
src/main/java/com/diaoyun/zion/chinafrica/bis/IItemSpider.java
浏览文件 @ 299d967d
package com.diaoyun.zion.chinafrica.bis;
import net.sf.json.JSONObject;
import java.io.IOException;
import java.net.URISyntaxException;
import java.util.Map;
......@@ -15,6 +17,6 @@ public interface IItemSpider {
* @param targetUrl
* @return
*/
Map<String,Object> captureItem(String targetUrl) throws URISyntaxException, IOException, ExecutionException, InterruptedException;
JSONObject captureItem(String targetUrl) throws URISyntaxException, IOException, ExecutionException, InterruptedException;
}
src/main/java/com/diaoyun/zion/chinafrica/bis/impl/EmptyItemSpider.java
浏览文件 @ 299d967d
package com.diaoyun.zion.chinafrica.bis.impl;
import com.diaoyun.zion.chinafrica.bis.IItemSpider;
import net.sf.json.JSONObject;
import org.springframework.stereotype.Component;
import java.util.HashMap;
......@@ -12,8 +13,8 @@ import java.util.Map;
@Component("emptyItemSpider")
public class EmptyItemSpider implements IItemSpider {
@Override
public Map<String, Object> captureItem(String targetUrl) {
Map<String, Object> resultMap=new HashMap<>();
public JSONObject captureItem(String targetUrl) {
JSONObject resultMap=new JSONObject();
resultMap.put("code","-1");
resultMap.put("message","找不到此类网址的数据爬虫！");
return resultMap;
......
src/main/java/com/diaoyun/zion/chinafrica/bis/impl/NetWorkSpider.java
浏览文件 @ 299d967d
......@@ -28,7 +28,7 @@ public class NetWorkSpider {
*/
public static BigDecimal getRateFromHexun(String code) throws IOException, URISyntaxException {
String apiUrl=exchangeRateUrl+"code="+code+"&column=Price,Code,Name,UpdownRate,PriceWeight";
String dataUrl=HttpClientUtil.getContentByUrl(apiUrl);
String dataUrl=HttpClientUtil.getContentByUrl(apiUrl,"");
int firstBrackets=dataUrl.indexOf("(");
int lastBrackets=dataUrl.lastIndexOf(")");
String jsonStr=dataUrl.substring(firstBrackets+1,lastBrackets);
......
src/main/java/com/diaoyun/zion/chinafrica/bis/impl/TbItemSpider.java
浏览文件 @ 299d967d
package com.diaoyun.zion.chinafrica.bis.impl;
import com.diaoyun.zion.chinafrica.bis.IItemSpider;
import com.diaoyun.zion.chinafrica.enums.PlatformEnum;
import com.diaoyun.zion.master.bo.TencentTranslateParam;
import com.diaoyun.zion.master.thread.TaskLimitSemaphore;
import com.diaoyun.zion.master.thread.TranslateCallable;
import com.diaoyun.zion.master.util.HttpClientUtil;
import com.diaoyun.zion.master.util.JsoupUtil;
import com.diaoyun.zion.master.util.TranslateHelper;
import com.diaoyun.zion.master.util.ValidateUtils;
import net.sf.json.JSONObject;
import org.apache.commons.lang3.StringUtils;
......@@ -23,6 +25,7 @@ import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
......@@ -39,23 +42,26 @@ import java.util.concurrent.Future;
public class TbItemSpider implements IItemSpider {
private static Logger logger = LoggerFactory.getLogger(TbItemSpider.class);
//默认20条线程跑翻译
private final TaskLimitSemaphore taskLimitSemaphore=new TaskLimitSemaphore(20);
//淘宝商品详情
private static final String taobaoUrl="https://item.taobao.com/item.htm?";
@Override
public Map<String, Object> captureItem(String targetUrl) throws URISyntaxException, IOException, ExecutionException, InterruptedException {
public JSONObject captureItem(String targetUrl) throws URISyntaxException, IOException, ExecutionException, InterruptedException {
//主要是判断淘宝的url是手机端还是PC端，如果是手机端，那么还需要提取相关参数，组成新的url；还有去除链接中一些会引起请求错误的参数
targetUrl=processUrl(targetUrl);
List<Map<String, Object>> futureList= new ArrayList<>();
//获取url中的网页内容
String content = HttpClientUtil.getContentByUrl(targetUrl);
String content = HttpClientUtil.getContentByUrl(targetUrl,PlatformEnum.TB.getValue());
//获取商品相关信息，比如详情url
Map<String,String> infoMap= JsoupUtil.getUsefulInfo(content);
Map<String,String> infoMap= JsoupUtil.getTbItemDetail(content);
String usableSibUrl="https:"+infoMap.get("sibUrl");
//解析商品sku信息
Map<String,Object> propMap=JsoupUtil.getPropMap(content);
//调用腾讯ai，翻译规格
translateProp(futureList,propMap);
if(propMap.get("hasSku")==null||(boolean)propMap.get("hasSku")) {
//调用腾讯ai，翻译规格
translateProp(futureList,propMap);
}
/* ****************获取商品详情******************* */
//删除需要登录的模块
......@@ -73,30 +79,18 @@ public class TbItemSpider implements IItemSpider {
logger.info("获取详情时间(毫秒)："+(c-b));*/
//等待翻译结果
for(Map<String,Object> futureMap:futureList) {
Future<Map<String,Object>> future= (Future<Map<String, Object>>) futureMap.get("future");
Map<String,Object> valeMap = (Map<String,Object> ) futureMap.get("value");
while(!future.isDone());//Future返回如果没有完成，则一直循环等待，直到Future返回完成
Map<String,Object> resultMap=future.get();
String targetText="unknow";
if(resultMap!=null&&(int)resultMap.get("ret")==0) {
Map<String,Object> dataMap=(Map<String,Object>)resultMap.get("data");
targetText= (String) dataMap.get("target_text");
valeMap.put("translate",targetText);
} else {
logger.error("翻译出错");
}
}
TranslateHelper.waitForResult(futureList);
//关闭线程池 不能关闭，否则下次调用不了
//taskLimitSemaphore.shutdown();
/* Long d=System.currentTimeMillis();
/* Long d=System.currentTimeMillis();
logger.info("翻译时间(毫秒)："+(d-c));
logger.info("爬取数据总共耗费时间(毫秒)："+(d-a));*/
Map sibMap= JSONObject.fromObject(sibContent);
sibMap.put("Jprop",propMap);
sibMap.put("itemProp",infoMap);
return sibMap;
JSONObject returnJson= JSONObject.fromObject(sibContent);
returnJson.put("Jprop",propMap);
returnJson.put("itemProp",infoMap);
returnJson.put("platform", PlatformEnum.TB.getValue());
return returnJson;
}
......@@ -113,8 +107,8 @@ public class TbItemSpider implements IItemSpider {
String key=entry.getKey();
Map <String,Object> value= (Map<String, Object>) entry.getValue();
//翻译属性名
if(ValidateUtils.isChinese(key)) {
translateText(futureList,value,key);
if(ValidateUtils.isContainChinese(key)) {
TranslateHelper.translateText(futureList,value,key);
}
//翻译sku title
if(value!=null&&value.size()>0) {
......@@ -138,7 +132,7 @@ public class TbItemSpider implements IItemSpider {
if (StringUtils.isNotBlank(title)) {
//翻译属性名
if (ValidateUtils.isContainChinese(title)) {
translateText(futureList,value,title);
TranslateHelper.translateText(futureList,value,title);
//value.put("translate", translate);
}
}
......@@ -146,26 +140,7 @@ public class TbItemSpider implements IItemSpider {
}
}
/**
* 翻译文本
* @param valeMap
* @param text
* @throws ExecutionException
* @throws InterruptedException
*/
private void translateText(List<Map<String, Object>> futureList,Map<String,Object> valeMap,String text) throws ExecutionException, InterruptedException {
TencentTranslateParam tencentTranslateParam =new TencentTranslateParam(text);
Future<Map<String,Object>> future = null;
try {
future = taskLimitSemaphore.submit(new TranslateCallable(tencentTranslateParam));
Map<String,Object> map=new HashMap<>();
map.put("future",future);
map.put("value",valeMap);
futureList.add(map);
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* 去除需要登录或者不需要返回的参数
......@@ -183,4 +158,30 @@ public class TbItemSpider implements IItemSpider {
return usableSibUrl;
}
/**
* 主要是提取相关参数，组成新的url
*
* @param targetUrl
* @return
*/
private String processUrl(String targetUrl) throws URISyntaxException, MalformedURLException {
String newUrl=taobaoUrl;
//if(targetUrl.contains("h5.m.taobao.com")) {
Map<String,String> paramMap=HttpClientUtil.getParamMap(targetUrl);
//目前淘宝需要四个参数 spm id scm pvid
//引起错误的 参数 ali_refid
StringBuffer paramBuffer=new StringBuffer();
for(Map.Entry<String,String> entry:paramMap.entrySet()) {
if("ali_refid".equals(entry.getKey())||"track_params".equals(entry.getKey())||"utparam".equals(entry.getKey())
||"rmdChannelCode".equals(entry.getKey())||"locate".equals(entry.getKey())) {
} else {
paramBuffer.append(entry.getKey()+"="+entry.getValue()+"&");
}
}
return newUrl+paramBuffer.toString();
}
}
src/main/java/com/diaoyun/zion/chinafrica/bis/impl/TmItemSpider.java 0 → 100644
浏览文件 @ 299d967d
package com.diaoyun.zion.chinafrica.bis.impl;
import com.diaoyun.zion.chinafrica.bis.IItemSpider;
import com.diaoyun.zion.chinafrica.enums.PlatformEnum;
import com.diaoyun.zion.master.thread.TaskLimitSemaphore;
import com.diaoyun.zion.master.util.HttpClientUtil;
import com.diaoyun.zion.master.util.JsoupUtil;
import com.diaoyun.zion.master.util.TranslateHelper;
import com.diaoyun.zion.master.util.ValidateUtils;
import net.sf.json.JSONArray;
import net.sf.json.JSONNull;
import net.sf.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
import java.io.IOException;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
/**
* 天猫数据爬虫
*/
@Component("tmItemSpider")
public class TmItemSpider implements IItemSpider {
private static Logger logger = LoggerFactory.getLogger(TmItemSpider.class);
//天猫链接
private static final String tmallUrl="https://detail.m.tmall.com/item.htm?";
@Override
public JSONObject captureItem(String targetUrl) throws URISyntaxException, IOException, ExecutionException, InterruptedException {
//主要获取id参数，组成新链接
//targetUrl=processUrl(targetUrl);
List<Map<String, Object>> futureList= new ArrayList<>();
//获取url中的网页内容
String content = HttpClientUtil.getContentByUrl(targetUrl,PlatformEnum.TM.getValue());
//获取商品详情
JSONObject infoMap= JsoupUtil.getTmItemDetail(content);
JSONObject skuBaseMap= (JSONObject) infoMap.get("skuBase");
if(!(skuBaseMap.get("props") instanceof JSONNull)) {
JSONArray propsArray= (JSONArray) skuBaseMap.get("props");
for(int i=0;i<propsArray.size();i++) {
JSONObject propMap=propsArray.getJSONObject(i);
String propName= (String) propMap.get("name");
JSONArray valueArray=propMap.getJSONArray("values");
//翻译属性名
if(ValidateUtils.isContainChinese(propName)) {
TranslateHelper.translateText(futureList,propMap,propName);
}
//翻译sku title
if(valueArray!=null&&valueArray.size()>0) {
for(int j=0;j<valueArray.size();j++) {
JSONObject valueMap=valueArray.getJSONObject(j);
if(valueMap!=null&&valueMap.size()>0) {
String skuName= (String) valueMap.get("name");
//翻译属性名
if(ValidateUtils.isContainChinese(skuName)) {
TranslateHelper.translateText(futureList,valueMap,skuName);
}
}
}
}
}
}
//等待翻译结果
TranslateHelper.waitForResult(futureList);
//只取其中部分信息 TODO 后续可以继续封装
JSONObject returnJson=new JSONObject();
returnJson.put("seller",infoMap.get("seller"));
returnJson.put("item",infoMap.get("item"));
returnJson.put("mock",infoMap.get("mock"));
returnJson.put("skuBase",infoMap.get("skuBase"));
returnJson.put("platform", PlatformEnum.TM.getValue());
return returnJson;
}
/**
* 主要是提取相关参数，组成新的url
* @param targetUrl
* @return
*/
/*@Deprecated
private String processUrl(String targetUrl) throws URISyntaxException, MalformedURLException {
String newUrl=tmallUrl;
//if(targetUrl.contains("h5.m.taobao.com")) {
//替换会影响参数的字符
targetUrl= targetUrl.replaceAll("\\{","");
targetUrl= targetUrl.replaceAll("\\}","");
Map<String,String> paramMap=HttpClientUtil.getParamMap(targetUrl);
//目前淘宝需要四个参数 spm id scm pvid
//引起错误的 参数 ali_refid
*//*for(Map.Entry<String,String> entry:paramMap.entrySet()) {
if("id".equals(entry.getKey())) {
newUrl=newUrl.replace("itemId",entry.getValue());
break;
}
}*//*
StringBuffer paramBuffer=new StringBuffer();
for(Map.Entry<String,String> entry:paramMap.entrySet()) {
if("ali_refid".equals(entry.getKey())||"track_params".equals(entry.getKey())||"utparam".equals(entry.getKey())
||"rmdChannelCode".equals(entry.getKey())||"locate".equals(entry.getKey())) {
} else {
paramBuffer.append(entry.getKey()+"="+entry.getValue()+"&");
}
}
return newUrl+paramBuffer.toString();
}*/
}
src/main/java/com/diaoyun/zion/chinafrica/client/TokenVerification.java
浏览文件 @ 299d967d
......@@ -34,6 +34,8 @@ public class TokenVerification {
//@Pointcut("!execution(* com.diaoyun.zion.chinafrica.controller.LoginController*.*(..))&&within(com.diaoyun.zion.chinafrica.controller.*)")
@Pointcut("within(com.diaoyun.zion.chinafrica.controller.*)" +
"&&!execution(* com.diaoyun.zion.chinafrica.controller.SpiderController.*(..))" +
"&&!execution(* com.diaoyun.zion.chinafrica.controller.TbCfPlatformController.*(..))" +
"&&!execution(* com.diaoyun.zion.chinafrica.controller.TbCfStationItemController.*(..))" +
"&&!execution(* com.diaoyun.zion.chinafrica.controller.LoginController.*(..))" +
"&&!execution(* com.diaoyun.zion.chinafrica.controller.TbCfUserInfoController.getUserIdentifyCode(..))")
public void controllerAspect() {
......
src/main/java/com/diaoyun/zion/chinafrica/controller/SpiderController.java
浏览文件 @ 299d967d
......@@ -6,12 +6,14 @@ import com.diaoyun.zion.master.base.Result;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam;
import net.sf.json.JSONObject;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.*;
import java.io.IOException;
import java.net.URISyntaxException;
import java.net.URLDecoder;
import java.util.Map;
import java.util.concurrent.ExecutionException;
......@@ -30,10 +32,11 @@ public class SpiderController {
private SpiderService spiderService;
@ApiOperation("获取商品详情")
@GetMapping("/item/detail")
@PostMapping("/item/detail")
@ResponseBody
public Result getItemDetail(@ApiParam @RequestBody DetailParamVo detailParamVo) throws InterruptedException, ExecutionException, URISyntaxException, IOException {
Map<String,Object> itemMap=spiderService.getItemDetail(detailParamVo.getTargetUrl());
public Result getItemDetail(@ApiParam("targetUrl 需要先进行url编码") @RequestBody DetailParamVo detailParamVo) throws InterruptedException, ExecutionException, URISyntaxException, IOException {
String targetUrl= URLDecoder.decode(detailParamVo.getTargetUrl(),"UTF-8");
JSONObject itemMap=spiderService.getItemDetail(targetUrl);
Result result=new Result(itemMap,"商品规格信息");
return result;
}
......
src/main/java/com/diaoyun/zion/chinafrica/controller/TbCfAddressController.java
浏览文件 @ 299d967d
......@@ -33,13 +33,6 @@ public class TbCfAddressController {
}
@ApiOperation("增加用户地址信息")
@PostMapping
@ResponseBody
......
src/main/java/com/diaoyun/zion/chinafrica/controller/TbCfItemDetailController.java
浏览文件 @ 299d967d
......@@ -64,7 +64,7 @@ public class TbCfItemDetailController {
}
@ApiOperation("从购物车删除商品")
@DeleteMapping("/cart")
@PostMapping("/cart/delete")
@ResponseBody
public Result deleteItems(@ApiParam("商品Id数组") @RequestBody String[] cartRecordIds) {
Result result =tbCfCartRecordRService.deleteItems(cartRecordIds);
......
src/main/java/com/diaoyun/zion/chinafrica/controller/TbCfOrderController.java
浏览文件 @ 299d967d
......@@ -17,6 +17,7 @@ import java.io.IOException;
import java.net.URISyntaxException;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
/**
* Controller
......@@ -34,7 +35,7 @@ public class TbCfOrderController {
@ApiOperation("用户结算，返回订单")
@GetMapping("/settle")
@ResponseBody
public Result<TbCfOrderVo> settleAccount() throws IOException, URISyntaxException {
public Result<TbCfOrderVo> settleAccount() throws IOException, URISyntaxException, ExecutionException, InterruptedException {
Result result = tbCfOrderService.settleAccount();
return result;
}
......@@ -42,7 +43,7 @@ public class TbCfOrderController {
@ApiOperation("用户确定下单")
@PostMapping("/place")
@ResponseBody
public Result placeOrder(@ApiParam("订单详情") @RequestBody TbCfOrderVo tbCfOrderVo) throws IOException, URISyntaxException {
public Result placeOrder(@ApiParam("订单详情") @RequestBody TbCfOrderVo tbCfOrderVo) throws IOException, URISyntaxException, ExecutionException, InterruptedException {
Result result = tbCfOrderService.placeOrder(tbCfOrderVo);
return result;
}
......
src/main/java/com/diaoyun/zion/chinafrica/controller/TbCfPlatformController.java
浏览文件 @ 299d967d
package com.diaoyun.zion.chinafrica.controller;
import com.diaoyun.zion.chinafrica.service.TbCfPlatformService;
import com.diaoyun.zion.master.base.Result;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
/**
* 平台管理Controller
......@@ -11,10 +18,26 @@ import org.springframework.web.bind.annotation.RequestMapping;
* @author lipengjun
* @date 2019-09-05 16:51:07
*/
@Api(tags = "独立站")
@Controller
@RequestMapping("tbcfplatform")
@RequestMapping("platform")
public class TbCfPlatformController {
@Autowired
private TbCfPlatformService tbCfPlatformService;
@ApiOperation("获取平台独立站")
@GetMapping
@ResponseBody
public Result getPlatformList(@ApiParam(value = "页数 默认1") @RequestParam(required = false) Integer pageNum,
@ApiParam(value ="每页大小 默认10") @RequestParam(required = false) Integer pageSize) {
if (pageNum == null) {
pageNum = 1;
}
if (pageSize == null) {
pageSize = 10;
}
Result result = tbCfPlatformService.getPlatformList(pageNum,pageSize);
return result;
}
}
src/main/java/com/diaoyun/zion/chinafrica/controller/TbCfStationItemController.java
浏览文件 @ 299d967d
package com.diaoyun.zion.chinafrica.controller;
import com.diaoyun.zion.chinafrica.service.TbCfStationItemService;
import com.diaoyun.zion.master.base.Result;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import io.swagger.annotations.ApiParam;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
/**
* 站点商品Controller
......@@ -11,11 +18,26 @@ import org.springframework.web.bind.annotation.RequestMapping;
* @author lipengjun
* @date 2019-09-05 16:51:07
*/
@Api(tags="商品独立站")
@Controller
@RequestMapping("tbcfstationitem")
@RequestMapping("itemStation")
public class TbCfStationItemController {
@Autowired
private TbCfStationItemService tbCfStationItemService;
@ApiOperation("获取商品独立站")
@GetMapping
@ResponseBody
public Result getItemStationList(@ApiParam(value = "页数 默认1") @RequestParam(required = false) Integer pageNum,
@ApiParam(value ="每页大小 默认10") @RequestParam(required = false) Integer pageSize) {
if (pageNum == null) {
pageNum = 1;
}
if (pageSize == null) {
pageSize = 10;
}
Result result = tbCfStationItemService.getItemStationList(pageNum,pageSize);
return result;
}
}
src/main/java/com/diaoyun/zion/chinafrica/dao/TbCfPlatformDao.java
浏览文件 @ 299d967d
......@@ -3,6 +3,8 @@ package com.diaoyun.zion.chinafrica.dao;
import com.diaoyun.zion.chinafrica.entity.TbCfPlatformEntity;
import com.diaoyun.zion.master.dao.BaseDao;
import java.util.List;
/**
* 平台管理Dao
*
......@@ -11,4 +13,9 @@ import com.diaoyun.zion.master.dao.BaseDao;
*/
public interface TbCfPlatformDao extends BaseDao<TbCfPlatformEntity> {
/**
* 获取平台独立站
* @return
*/
List<TbCfPlatformEntity> getPlatformList();
}
src/main/java/com/diaoyun/zion/chinafrica/dao/TbCfStationItemDao.java
浏览文件 @ 299d967d
package com.diaoyun.zion.chinafrica.dao;
import com.diaoyun.zion.chinafrica.entity.TbCfPlatformEntity;
import com.diaoyun.zion.chinafrica.entity.TbCfStationItemEntity;
import com.diaoyun.zion.master.dao.BaseDao;
import java.util.List;
/**
* 站点商品Dao
*
......@@ -11,4 +14,9 @@ import com.diaoyun.zion.master.dao.BaseDao;
*/
public interface TbCfStationItemDao extends BaseDao<TbCfStationItemEntity> {
/**
* 获取商品独立站
* @return
*/
List<TbCfPlatformEntity> getItemStationList();
}
src/main/java/com/diaoyun/zion/chinafrica/factory/ItemSpiderFactory.java
浏览文件 @ 299d967d
......@@ -16,6 +16,10 @@ public class ItemSpiderFactory {
iItemSpider= (IItemSpider) SpringContextUtil.getBean("tbItemSpider");
break;
}
case "tm":{
iItemSpider= (IItemSpider) SpringContextUtil.getBean("tmItemSpider");
break;
}
default:{
iItemSpider= (IItemSpider) SpringContextUtil.getBean("emptyItemSpider");
break;
......
src/main/java/com/diaoyun/zion/chinafrica/service/SpiderService.java
浏览文件 @ 299d967d
package com.diaoyun.zion.chinafrica.service;
import net.sf.json.JSONObject;
import java.io.IOException;
import java.math.BigDecimal;
import java.net.URISyntaxException;
......@@ -12,7 +14,7 @@ public interface SpiderService {
* @param targetUrl
* @return
*/
Map<String, Object> getItemDetail(String targetUrl) throws InterruptedException, IOException, ExecutionException, URISyntaxException;
JSONObject getItemDetail(String targetUrl) throws InterruptedException, IOException, ExecutionException, URISyntaxException;
/**
......
src/main/java/com/diaoyun/zion/chinafrica/service/TbCfOrderService.java
浏览文件 @ 299d967d
......@@ -9,6 +9,7 @@ import java.io.IOException;
import java.net.URISyntaxException;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
/**
* Service接口
......@@ -79,7 +80,7 @@ public interface TbCfOrderService {
*
* @return
*/
Result settleAccount() throws IOException, URISyntaxException;
Result settleAccount() throws IOException, URISyntaxException, ExecutionException, InterruptedException;
/**
* 用户下单
......@@ -87,7 +88,7 @@ public interface TbCfOrderService {
* @param tbCfOrderVo
* @return
*/
Result placeOrder(TbCfOrderVo tbCfOrderVo) throws IOException, URISyntaxException;
Result placeOrder(TbCfOrderVo tbCfOrderVo) throws IOException, URISyntaxException, ExecutionException, InterruptedException;
/**
* 获取用户订单列表
......
src/main/java/com/diaoyun/zion/chinafrica/service/TbCfPlatformService.java
浏览文件 @ 299d967d
package com.diaoyun.zion.chinafrica.service;
import com.diaoyun.zion.chinafrica.entity.TbCfPlatformEntity;
import com.diaoyun.zion.master.base.Result;
import java.util.List;
import java.util.Map;
......@@ -16,7 +17,7 @@ public interface TbCfPlatformService {
/**
* 根据主键查询实体
*
* @param id 主键
* @param platformId 主键
* @return 实体
*/
TbCfPlatformEntity queryObject(String platformId);
......@@ -68,4 +69,12 @@ public interface TbCfPlatformService {
* @return 删除条数
*/
int deleteBatch(String[] platformIds);
/**
* 获取平台独立站
* @param pageNum
* @param pageSize
* @return
*/
Result getPlatformList(Integer pageNum, Integer pageSize);
}
src/main/java/com/diaoyun/zion/chinafrica/service/TbCfStationItemService.java
浏览文件 @ 299d967d
package com.diaoyun.zion.chinafrica.service;
import com.diaoyun.zion.chinafrica.entity.TbCfStationItemEntity;
import com.diaoyun.zion.master.base.Result;
import java.util.List;
import java.util.Map;
......@@ -16,7 +17,7 @@ public interface TbCfStationItemService {
/**
* 根据主键查询实体
*
* @param id 主键
* @param itemId 主键
* @return 实体
*/
TbCfStationItemEntity queryObject(String itemId);
......@@ -68,4 +69,12 @@ public interface TbCfStationItemService {
* @return 删除条数
*/
int deleteBatch(String[] itemIds);
/**
* 获取商品独立站
* @param pageNum
* @param pageSize
* @return
*/
Result getItemStationList(Integer pageNum, Integer pageSize);
}
src/main/java/com/diaoyun/zion/chinafrica/service/impl/SpiderServiceImpl.java
浏览文件 @ 299d967d
......@@ -5,6 +5,7 @@ import com.diaoyun.zion.chinafrica.bis.impl.NetWorkSpider;
import com.diaoyun.zion.chinafrica.enums.PlatformEnum;
import com.diaoyun.zion.chinafrica.factory.ItemSpiderFactory;
import com.diaoyun.zion.chinafrica.service.SpiderService;
import net.sf.json.JSONObject;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.NameValuePair;
import org.apache.http.client.utils.URLEncodedUtils;
......@@ -20,17 +21,13 @@ import java.util.concurrent.ExecutionException;
@Service("spiderService")
public class SpiderServiceImpl implements SpiderService {
//淘宝商品详情
private static final String taobaoUrl="https://item.taobao.com/item.htm?";
@Override
public Map<String, Object> getItemDetail(String targetUrl) throws InterruptedException, IOException, ExecutionException, URISyntaxException {
//主要是判断淘宝的url是手机端还是PC端，如果是手机端，那么还需要提取相关参数，组成新的url；还有去除链接中一些会引起请求错误的参数
targetUrl=processUrl(targetUrl);
public JSONObject getItemDetail(String targetUrl) throws InterruptedException, IOException, ExecutionException, URISyntaxException {
//判断链接属于哪个平台
PlatformEnum platformEnum=judgeUrlType(targetUrl);
IItemSpider iItemSpider=ItemSpiderFactory.getSpider(platformEnum);
Map<String,Object> itemMap=iItemSpider.captureItem(targetUrl);
JSONObject itemMap=iItemSpider.captureItem(targetUrl);
return itemMap;
}
......@@ -45,50 +42,12 @@ public class SpiderServiceImpl implements SpiderService {
return exchangeRate;
}
/**
* 主要是提取相关参数，组成新的url
*
* @param targetUrl
* @return
*/
private String processUrl(String targetUrl) throws URISyntaxException {
String newUrl=taobaoUrl;
//if(targetUrl.contains("h5.m.taobao.com")) {
Map<String,String> paramMap=getParamMap(targetUrl);
//目前淘宝需要四个参数 spm id scm pvid
//引起错误的 参数 ali_refid
StringBuffer paramBuffer=new StringBuffer();
for(Map.Entry<String,String> entry:paramMap.entrySet()) {
if("ali_refid".equals(entry.getKey())) {
} else {
paramBuffer.append(entry.getKey()+"="+entry.getValue()+"&");
}
}
return newUrl+paramBuffer.toString();
}
/**
* 获取连接中的参数
* @param targetUrl
* @return
*/
private Map<String, String> getParamMap(String targetUrl) throws URISyntaxException {
List<NameValuePair> params = URLEncodedUtils.parse(new URI(targetUrl), Charset.forName("UTF-8"));
Map<String,String> paramMap=new HashMap<>();
for (NameValuePair param : params) {
paramMap.put(param.getName(),param.getValue());
}
return paramMap;
}
private PlatformEnum judgeUrlType(String targetUrl) {
PlatformEnum platformEnum = PlatformEnum.UN;
if(targetUrl.contains("taobao.com")) {
if(targetUrl.contains("taobao.com")&&(targetUrl.contains("item.htm")||targetUrl.contains("detail.htm"))) {
platformEnum=PlatformEnum.TB;
} else if(targetUrl.contains("tmall.com/item.htm")) {
platformEnum=PlatformEnum.TM;
}
return platformEnum;
}
......
src/main/java/com/diaoyun/zion/chinafrica/service/impl/TbCfOrderServiceImpl.java
浏览文件 @ 299d967d
......@@ -21,6 +21,7 @@ import com.diaoyun.zion.master.config.DomainProperties;
import com.diaoyun.zion.master.util.AESUtils;
import com.diaoyun.zion.master.util.CookieUtils;
import com.diaoyun.zion.master.util.IdUtil;
import com.diaoyun.zion.master.util.WordposHelper;
import com.diaoyun.zion.master.validator.Validator;
import com.github.pagehelper.PageHelper;
import com.github.pagehelper.PageInfo;
......@@ -41,6 +42,7 @@ import java.lang.reflect.Array;
import java.math.BigDecimal;
import java.net.URISyntaxException;
import java.util.*;
import java.util.concurrent.ExecutionException;
/**
* Service实现类
......@@ -126,7 +128,7 @@ public class TbCfOrderServiceImpl implements TbCfOrderService {
}
@Override
public Result settleAccount() throws IOException, URISyntaxException {
public Result settleAccount() throws IOException, URISyntaxException, ExecutionException, InterruptedException {
Result result = new Result();
//获取用户
String token = CookieUtils.getCookie(request, TokenManager.TOKEN);
......@@ -145,7 +147,7 @@ public class TbCfOrderServiceImpl implements TbCfOrderService {
}
@Override
public Result placeOrder(TbCfOrderVo pageOrder) throws IOException, URISyntaxException {
public Result placeOrder(TbCfOrderVo pageOrder) throws IOException, URISyntaxException, ExecutionException, InterruptedException {
Result result = new Result();
//获取用户
String token = CookieUtils.getCookie(request, TokenManager.TOKEN);
......@@ -176,13 +178,13 @@ public class TbCfOrderServiceImpl implements TbCfOrderService {
//获取用户
String token = CookieUtils.getCookie(request, TokenManager.TOKEN);
TbCfUserInfoVo tbCfUserInfoVo = tokenManager.validate(token);
PageHelper.startPage(pageNum, pageSize);
//订单数据
List<TbCfOrderEntity> orderList = tbCfOrderDao.getUserOrderList(tbCfUserInfoVo.getUserId());
PageHelper.startPage(pageNum, pageSize);
PageInfo<TbCfOrderEntity> pageInfo = new PageInfo<>(orderList);
List<TbCfOrderVo> tbCfOrderVoList = new ArrayList<>();
//获取订单后，再获取订单内商品
List<TbCfOrderEntity> pagingOrderList = pageInfo.getList();
List<TbCfOrderVo> tbCfOrderVoList = new ArrayList<>();
if (pagingOrderList != null && pagingOrderList.size() > 0) {
for (TbCfOrderEntity order : pagingOrderList) {
TbCfOrderVo orderVo = new TbCfOrderVo();
......@@ -426,7 +428,7 @@ public class TbCfOrderServiceImpl implements TbCfOrderService {
* @throws IOException
* @throws URISyntaxException
*/
private TbCfOrderVo ensureOrder(TbCfUserInfoVo tbCfUserInfoVo, TbCfOrderVo pageOrder, List<TbCfCartItemDetailVo> tbCfCartItemDetailList) throws IOException, URISyntaxException {
private TbCfOrderVo ensureOrder(TbCfUserInfoVo tbCfUserInfoVo, TbCfOrderVo pageOrder, List<TbCfCartItemDetailVo> tbCfCartItemDetailList) throws IOException, URISyntaxException, ExecutionException, InterruptedException {
TbCfOrderVo definiteOrder = getGenericOrder(tbCfCartItemDetailList);
//获取可以使用的优惠券，后续还有判断此订单是否可以使用
List<TbCfCouponEntity> tbCfCouponList = tbCfCouponDao.queryUserAvailableCoupon(tbCfUserInfoVo.getUserId(), new Date());
......@@ -478,14 +480,18 @@ public class TbCfOrderServiceImpl implements TbCfOrderService {
*
* @return
*/
private TbCfOrderVo getGenericOrder(List<TbCfCartItemDetailVo> tbCfCartItemDetailList) throws IOException, URISyntaxException {
private TbCfOrderVo getGenericOrder(List<TbCfCartItemDetailVo> tbCfCartItemDetailList) throws IOException, URISyntaxException, ExecutionException, InterruptedException {
TbCfOrderVo genericOrder = new TbCfOrderVo();
genericOrder.setItemDetailList(tbCfCartItemDetailList);
//初始商品总价
BigDecimal itemsPrice = BigDecimal.valueOf(0.00);
BigDecimal totalExpressCost = BigDecimal.ZERO;
for (TbCfCartItemDetailVo tbCfCartItemDetailVo : tbCfCartItemDetailList) {
BigDecimal itemNum = BigDecimal.valueOf(tbCfCartItemDetailVo.getItemNum());
itemsPrice = itemsPrice.add(tbCfCartItemDetailVo.getItemPrice().multiply(itemNum));
//计算运费
BigDecimal expressCost = countExpressCost(tbCfCartItemDetailVo.getItemTitle());
totalExpressCost=totalExpressCost.add(expressCost);
}
/*获取人民币汇率 1美元换取人民币*/
BigDecimal rate = spiderService.getExchangeRate(null);
......@@ -494,21 +500,42 @@ public class TbCfOrderServiceImpl implements TbCfOrderService {
//System.out.println(itemsPrice);
//计算手续费
BigDecimal fee = countFee(itemsPrice);
//税费
BigDecimal tax = countTax(itemsPrice);
//计算运费 TODO
BigDecimal expressCost = new BigDecimal("0.01");
//商品总价
BigDecimal totalPrice = itemsPrice.add(fee).add(tax).add(expressCost);
BigDecimal totalPrice = itemsPrice.add(fee).add(tax).add(totalExpressCost);
genericOrder.setTotalPrice(totalPrice);
genericOrder.setItemsPrice(itemsPrice);
genericOrder.setExpressCost(expressCost);
genericOrder.setExpressCost(totalExpressCost);
genericOrder.setTax(tax);
genericOrder.setFee(fee);
return genericOrder;
}
/**
* 计算运费 TODO
* @param itemTitle
* @return
*/
private BigDecimal countExpressCost(String itemTitle) throws ExecutionException, InterruptedException {
//Map <String,Object> wordResult=separateText(itemTitle);
return BigDecimal.ZERO;
}
/**
* 分词
* @param text TODO
* @return
*/
/* public Map<String, Object> separateText(String text) throws ExecutionException, InterruptedException {
List<Map<String, Object>> futureList= new ArrayList<>();
Map<String,Object> titleMap=new HashMap();
titleMap.put("text",text);
WordposHelper.separeteText(futureList,titleMap,text);
WordposHelper.waitForResult(futureList);
return titleMap;
}*/
/**
* 验证订单优惠券
*
......@@ -528,7 +555,7 @@ public class TbCfOrderServiceImpl implements TbCfOrderService {
}
private TbCfOrderVo getOrderData(TbCfUserInfoVo tbCfUserInfoVo, List<TbCfCartItemDetailVo> tbCfCartItemDetailList) throws IOException, URISyntaxException {
private TbCfOrderVo getOrderData(TbCfUserInfoVo tbCfUserInfoVo, List<TbCfCartItemDetailVo> tbCfCartItemDetailList) throws IOException, URISyntaxException, ExecutionException, InterruptedException {
TbCfOrderVo tbCfOrder = getGenericOrder(tbCfCartItemDetailList);
//获取可以使用的优惠券，后续还有判断此订单是否可以使用
List<TbCfCouponEntity> tbCfCouponList = tbCfCouponDao.queryUserAvailableCoupon(tbCfUserInfoVo.getUserId(), new Date());
......
src/main/java/com/diaoyun/zion/chinafrica/service/impl/TbCfPlatformServiceImpl.java
浏览文件 @ 299d967d
......@@ -3,7 +3,10 @@ package com.diaoyun.zion.chinafrica.service.impl;
import com.diaoyun.zion.chinafrica.dao.TbCfPlatformDao;
import com.diaoyun.zion.chinafrica.entity.TbCfPlatformEntity;
import com.diaoyun.zion.chinafrica.service.TbCfPlatformService;
import com.diaoyun.zion.master.base.Result;
import com.diaoyun.zion.master.util.IdUtil;
import com.github.pagehelper.PageHelper;
import com.github.pagehelper.PageInfo;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
......@@ -56,4 +59,15 @@ public class TbCfPlatformServiceImpl implements TbCfPlatformService {
public int deleteBatch(String[] platformIds) {
return tbCfPlatformDao.deleteBatch(platformIds);
}
@Override
public Result getPlatformList(Integer pageNum, Integer pageSize) {
Result result = new Result();
PageHelper.startPage(pageNum,pageSize);
List<TbCfPlatformEntity> tbCfPlatformList=tbCfPlatformDao.getPlatformList();
PageInfo<TbCfPlatformEntity> pageInfo = new PageInfo<>(tbCfPlatformList);
//List<TbCfPlatformEntity> pagingPlatformList=pageInfo.getList();
result.setData(pageInfo);
return result;
}
}
src/main/java/com/diaoyun/zion/chinafrica/service/impl/TbCfStationItemServiceImpl.java
浏览文件 @ 299d967d
package com.diaoyun.zion.chinafrica.service.impl;
import com.diaoyun.zion.chinafrica.dao.TbCfStationItemDao;
import com.diaoyun.zion.chinafrica.entity.TbCfPlatformEntity;
import com.diaoyun.zion.chinafrica.entity.TbCfStationItemEntity;
import com.diaoyun.zion.chinafrica.service.TbCfStationItemService;
import com.diaoyun.zion.master.base.Result;
import com.diaoyun.zion.master.util.IdUtil;
import com.github.pagehelper.PageHelper;
import com.github.pagehelper.PageInfo;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
......@@ -56,4 +60,15 @@ public class TbCfStationItemServiceImpl implements TbCfStationItemService {
public int deleteBatch(String[] itemIds) {
return tbCfStationItemDao.deleteBatch(itemIds);
}
@Override
public Result getItemStationList(Integer pageNum, Integer pageSize) {
Result result = new Result();
PageHelper.startPage(pageNum,pageSize);
List<TbCfPlatformEntity> tbCfPlatformList=tbCfStationItemDao.getItemStationList();
PageInfo<TbCfPlatformEntity> pageInfo = new PageInfo<>(tbCfPlatformList);
//List<TbCfPlatformEntity> pagingPlatformList=pageInfo.getList();
result.setData(pageInfo);
return result;
}
}
src/main/java/com/diaoyun/zion/master/config/WebMvcConfig.java
浏览文件 @ 299d967d
......@@ -12,6 +12,7 @@ import org.springframework.http.converter.HttpMessageConverter;
import org.springframework.http.converter.StringHttpMessageConverter;
import org.springframework.http.converter.json.GsonHttpMessageConverter;
import org.springframework.http.converter.xml.SourceHttpMessageConverter;
import org.springframework.web.servlet.config.annotation.CorsRegistry;
import org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry;
import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
import springfox.documentation.spring.web.json.Json;
......@@ -65,4 +66,20 @@ public class WebMvcConfig implements WebMvcConfigurer {
converters.add(gsonHttpMessageConverter);
}
/**
* 跨域访问
* @param registry
*/
@Override
public void addCorsMappings(CorsRegistry registry) {
registry.addMapping("/**")
//.allowedOrigins("http://localhost:8095", "http://localhost:8080")
.allowedMethods("GET","POST","PUT","DELETE","HEAD","OPTIONS","PATCH")
.allowedHeaders("Content-Type")
.allowCredentials(true)
.maxAge(60000);
}
}
src/main/java/com/diaoyun/zion/master/exception/ExceptionResolver.java
浏览文件 @ 299d967d
......@@ -34,20 +34,23 @@ public class ExceptionResolver implements HandlerExceptionResolver {
result = new Result(ae.getCode(),ae.getMessage());
}
else {
result = new Result(ResultCode.ERROR,"未知错误");
result = new Result(ResultCode.ERROR,exception.getMessage());
logger.error(exception.getMessage(), exception);
}
response.setContentType("application/json;charset=UTF-8");
response.setStatus(HttpStatus.OK.value());
PrintWriter writer=null;
try {
PrintWriter writer = response.getWriter();
writer = response.getWriter();
writer.write(JSON.toJSONString(result));
writer.flush();
writer.close();
}
catch (IOException ie) {
logger.error("Failed to serialize the object to json for exception resolver!", ie);
} finally {
writer.flush();
writer.close();
}
return new ModelAndView();
}
......
src/main/java/com/diaoyun/zion/master/thread/TaskLimitSemaphore.java
浏览文件 @ 299d967d
......@@ -23,7 +23,7 @@ public class TaskLimitSemaphore {
public TaskLimitSemaphore(int limit) {
corePoolSize=limit;
this.maximumPoolSize = limit+20;
this.semaphore = new Semaphore(limit);;
this.semaphore = new Semaphore(limit);
this.executor = new ThreadPoolExecutor(corePoolSize, maximumPoolSize,
keepAliveTime, TimeUnit.SECONDS, new SynchronousQueue<Runnable>());
}
......
src/main/java/com/diaoyun/zion/master/util/EmailHelper.java
浏览文件 @ 299d967d
package com.diaoyun.zion.master.util;
import com.diaoyun.zion.master.bo.EmailTemplateBo;
import com.diaoyun.zion.master.config.DomainProperties;
import freemarker.template.Configuration;
import freemarker.template.Template;
import freemarker.template.TemplateException;
import freemarker.template.TemplateExceptionHandler;
import org.apache.commons.mail.EmailException;
import org.apache.commons.mail.HtmlEmail;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.ui.freemarker.FreeMarkerTemplateUtils;
......@@ -20,6 +23,10 @@ import java.util.Map;
public class EmailHelper {
@Autowired
private Configuration configuration;
@Autowired
private DomainProperties domainProperties;
private static Logger logger = LoggerFactory.getLogger(EmailHelper.class);
public Integer sendIdentifyEmail(String toEmail, String templateName, EmailTemplateBo emailTemplateBo) throws IOException, TemplateException, EmailException {
/*Configuration cfg = new Configuration(Configuration.VERSION_2_3_28);
......@@ -33,15 +40,16 @@ public class EmailHelper {
model.put("nick", emailTemplateBo.getNick());
String html = FreeMarkerTemplateUtils.processTemplateIntoString(t, model);
HtmlEmail email = new HtmlEmail();//创建一个HtmlEmail实例对象
email.setHostName("smtp.yeah.net");//邮箱的SMTP服务器，一般123邮箱的是smtp.123.com,qq邮箱为smtp.qq.com
email.setHostName(domainProperties.getProperty("email.hostName"));//邮箱的SMTP服务器，一般123邮箱的是smtp.123.com,qq邮箱为smtp.qq.com
email.setCharset("utf-8");//设置发送的字符类型
//email.addTo("zhengfanguang@163.com");//设置收件人
email.addTo(toEmail);//设置收件人
email.setFrom("chinafrica@yeah.net", "chinafrica");//发送人的邮箱为自己的，用户名可以随便填
email.setAuthentication("chinafrica@yeah.net", "diaoyun2019");//设置发送人的邮箱和用户名和授权码(授权码是自己设置的)
email.setSubject("测试验证码");//设置发送主题
email.setFrom(domainProperties.getProperty("email.from.email"), domainProperties.getProperty("email.from.name"));//发送人的邮箱为自己的，用户名可以随便填
email.setAuthentication(domainProperties.getProperty("email.authentication.userName"), domainProperties.getProperty("email.authentication.password"));//设置发送人的邮箱和用户名和授权码(授权码是自己设置的)
email.setSubject(domainProperties.getProperty("email.subject"));//设置发送主题
email.setMsg(html);//设置发送内容
email.send();
String res=email.send();
//logger.info("邮件发送结果："+res);
return emailTemplateBo.getIdentifyCode();
}
}
src/main/java/com/diaoyun/zion/master/util/HttpClientUtil.java
浏览文件 @ 299d967d
package com.diaoyun.zion.master.util;
import com.diaoyun.zion.chinafrica.enums.PlatformEnum;
import com.google.common.collect.Lists;
import org.apache.http.*;
import org.apache.http.client.CookieStore;
......@@ -14,39 +15,71 @@ import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicHeader;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class HttpClientUtil {
private static Logger logger = LoggerFactory.getLogger(HttpClientUtil.class);
/**
* 获取url链接中的content 网页内容
* @param url 目标url
* 获取url链接中的content 内容
* @param sourceUrl 目标url
* @param sourceType
* @return
* @throws URISyntaxException
* @throws IOException
*/
public static String getContentByUrl(String url) throws URISyntaxException, IOException {
public static String getContentByUrl(String sourceUrl, String sourceType) throws URISyntaxException, IOException {
URL url = new URL(sourceUrl);
//构建URI
URI uri=new URIBuilder(url).build();
URI uri=new URI(url.getProtocol(), url.getHost(), url.getPath(), url.getQuery(), null);
//创建一个HttpContext对象，用来保存Cookie
//HttpClientContext httpClientContext = HttpClientContext.create();
//构造自定义的HttpClient对象
CloseableHttpClient httpClient = createBrowserClient();
CloseableHttpClient httpClient;
if(PlatformEnum.TB.getValue().equals(sourceType)) {
httpClient = createBrowserClient();
} else if(PlatformEnum.TM.getValue().equals(sourceType)) {
//天猫用手机端访问拿的数据 香港服务器不可行
httpClient= createIphoneClient();
//httpClient= createBrowserClient();
} else {
httpClient = createBrowserClient();
}
/* 构造请求对象 */
HttpUriRequest httpUriRequest = RequestBuilder.get().setUri(uri).build();
/* ****************获取商品详情链接******************* */
//执行请求，传入HttpContext，将会得到请求结果的信息
//httpClient.execute(httpUriRequest,httpClientContext)
HttpResponse httpResponse=httpClient.execute(httpUriRequest);
//拿到返回的HttpResponse的"实体"
HttpEntity result = httpResponse.getEntity();
String content = EntityUtils.toString(result);
//System.getProperties().list(System.out);//输出当前环境属性 55555555
/*if(result.getContentEncoding()!=null) {
if ("gzip".equalsIgnoreCase(result.getContentEncoding().getValue())) {
result = new GzipDecompressingEntity(result);
} else if ("deflate".equalsIgnoreCase(result.getContentEncoding().getValue())) {
result = new DeflateDecompressingEntity(result);
}
}*/
String defaultCharset="UTF-8";
if(PlatformEnum.TM.getValue().equals(sourceType)) {
defaultCharset="UTF-8";
}
String content = EntityUtils.toString(result, defaultCharset);
//logger.info(content);
//关闭连接
httpClient.close();
return content;
......@@ -85,13 +118,13 @@ public class HttpClientUtil {
public static CloseableHttpClient createBrowserClient() {
//构造自定义Header信息
List<Header> headerList = Lists.newArrayList();
headerList.add(new BasicHeader(HttpHeaders.ACCEPT, "text/html,application/xhtml+xml,application/xml;q=0.9," +
"image/webp,image/apng,*/*;q=0.8"));
headerList.add(new BasicHeader(HttpHeaders.USER_AGENT, "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36"));
headerList.add(new BasicHeader(HttpHeaders.ACCEPT, "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3"));
headerList.add(new BasicHeader(HttpHeaders.USER_AGENT, "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.100 Safari/537.36"));
headerList.add(new BasicHeader(HttpHeaders.ACCEPT_ENCODING, "gzip, deflate, br"));
headerList.add(new BasicHeader(HttpHeaders.CACHE_CONTROL, "max-age=0"));
headerList.add(new BasicHeader(HttpHeaders.CONNECTION, "keep-alive"));
headerList.add(new BasicHeader(HttpHeaders.ACCEPT_LANGUAGE, "zh-CN,zh;q=0.9"));
headerList.add(new BasicHeader(HttpHeaders.CONTENT_TYPE, "charset=utf-8"));
//cookie
CookieStore cookieStore = new BasicCookieStore();
return HttpClients.custom().setDefaultHeaders(headerList).setDefaultCookieStore(cookieStore).build();
......@@ -117,6 +150,25 @@ public class HttpClientUtil {
return HttpClients.custom().setDefaultHeaders(headerList).setDefaultCookieStore(cookieStore).build();
}
/**
* 创建带苹果手机头部的连接
* @return
*/
public static CloseableHttpClient createIphoneClient() {
//构造自定义Header信息
List<Header> headerList = Lists.newArrayList();
headerList.add(new BasicHeader(HttpHeaders.ACCEPT, "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3"));
headerList.add(new BasicHeader(HttpHeaders.USER_AGENT, "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1"));
//headerList.add(new BasicHeader(HttpHeaders.ACCEPT_ENCODING, "gzip, deflate, br"));
headerList.add(new BasicHeader(HttpHeaders.CACHE_CONTROL, "max-age=0"));
headerList.add(new BasicHeader(HttpHeaders.CONNECTION, "keep-alive"));
headerList.add(new BasicHeader(HttpHeaders.ACCEPT_LANGUAGE, "zh-CN,zh;q=0.9"));
//cookie
CookieStore cookieStore = new BasicCookieStore();
return HttpClients.custom().setDefaultHeaders(headerList).setDefaultCookieStore(cookieStore).build();
}
//发送请求
public static String createConnection(String url,Map<String, Object> paramMap,String charencode) throws IOException {
List<NameValuePair> formparams = setHttpParams(paramMap);
......@@ -137,4 +189,21 @@ public class HttpClientUtil {
}
return list;
}
/**
* 获取连接中的参数
* @param targetUrl
* @return
*/
public static Map<String, String> getParamMap(String targetUrl) throws URISyntaxException, MalformedURLException {
URL url = new URL(targetUrl);
//构建URI
URI uri=new URI(url.getProtocol(), url.getHost(), url.getPath(), url.getQuery(), null);
List<NameValuePair> params = URLEncodedUtils.parse(uri, Charset.forName("UTF-8"));
Map<String,String> paramMap=new HashMap<>();
for (NameValuePair param : params) {
paramMap.put(param.getName(),param.getValue());
}
return paramMap;
}
}
src/main/java/com/diaoyun/zion/master/util/TranslateHelper.java 0 → 100644
浏览文件 @ 299d967d
package com.diaoyun.zion.master.util;
import com.diaoyun.zion.master.bo.TencentTranslateParam;
import com.diaoyun.zion.master.thread.TaskLimitSemaphore;
import com.diaoyun.zion.master.thread.TranslateCallable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
/**
* 腾讯翻译
*/
public class TranslateHelper {
private static Logger logger = LoggerFactory.getLogger(TranslateHelper.class);
//默认20条线程跑翻译
private static final TaskLimitSemaphore taskLimitSemaphore=new TaskLimitSemaphore(20);
/**
* 翻译文本
* @param futureList
* @param valeMap
* @param text
*/
public static void translateText(List<Map<String, Object>> futureList, Map<String,Object> valeMap, String text){
TencentTranslateParam tencentTranslateParam =new TencentTranslateParam(text);
Future<Map<String,Object>> future = null;
try {
future = taskLimitSemaphore.submit(new TranslateCallable(tencentTranslateParam));
Map<String,Object> map=new HashMap<>();
map.put("future",future);
map.put("value",valeMap);
futureList.add(map);
} catch (Exception e) {
logger.error(e.getMessage(),e);
}
}
/**
* 等待翻译结果
* @param futureList
*/
public static void waitForResult(List<Map<String, Object>> futureList) throws ExecutionException, InterruptedException {
for(Map<String,Object> futureMap:futureList) {
Future<Map<String,Object>> future= (Future<Map<String, Object>>) futureMap.get("future");
Map<String,Object> valeMap = (Map<String,Object> ) futureMap.get("value");
while(!future.isDone());//Future返回如果没有完成，则一直循环等待，直到Future返回完成
Map<String,Object> resultMap=future.get();
String targetText="unknow";
if(resultMap!=null&&(int)resultMap.get("ret")==0) {
Map<String,Object> dataMap=(Map<String,Object>)resultMap.get("data");
targetText= (String) dataMap.get("target_text");
valeMap.put("translate",targetText);
} else {
logger.error("翻译出错");
}
}
}
}
src/main/java/com/diaoyun/zion/master/util/WordposHelper.java 0 → 100644
浏览文件 @ 299d967d
package com.diaoyun.zion.master.util;
import com.diaoyun.zion.master.bo.TencentTranslateParam;
import com.diaoyun.zion.master.bo.TencentWordsegParam;
import com.diaoyun.zion.master.thread.TaskLimitSemaphore;
import com.diaoyun.zion.master.thread.TranslateCallable;
import com.diaoyun.zion.master.thread.WordposCallable;
import net.sf.json.JSONArray;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
/**
* 分词
*/
public class WordposHelper {
private static Logger logger = LoggerFactory.getLogger(WordposHelper.class);
//默认10条线程跑分词
private static final TaskLimitSemaphore taskLimitSemaphore=new TaskLimitSemaphore(10);
/**
* 翻译文本
* @param valeMap
* @param text
* @throws ExecutionException
* @throws InterruptedException
*/
public static void separeteText(List<Map<String, Object>> futureList, Map<String,Object> valeMap, String text) {
TencentWordsegParam param=new TencentWordsegParam(text);
Future<Map<String,Object>> future = null;
try {
future = taskLimitSemaphore.submit(new WordposCallable(param));
Map<String,Object> map=new HashMap<>();
map.put("future",future);
map.put("value",valeMap);
futureList.add(map);
} catch (Exception e) {
logger.error(e.getMessage(),e);
}
}
/**
* 等待翻译结果
* @param futureList
*/
public static void waitForResult(List<Map<String, Object>> futureList) throws ExecutionException, InterruptedException {
for(Map<String,Object> futureMap:futureList) {
Future<Map<String,Object>> future= (Future<Map<String, Object>>) futureMap.get("future");
Map<String,Object> valeMap = (Map<String,Object> ) futureMap.get("value");
while(!future.isDone());//Future返回如果没有完成，则一直循环等待，直到Future返回完成
Map<String,Object> resultMap=future.get();
//logger.info(resultMap.toString());
if(resultMap!=null&&(int)resultMap.get("ret")==0) {
Map<String,Object> dataMap=(Map<String,Object>)resultMap.get("data");
JSONArray tokens = (JSONArray) dataMap.get("base_tokens");
valeMap.put("base_tokens",tokens);
logger.info(tokens.toString());
} else {
logger.error("翻译出错");
}
}
}
}
src/main/java/com/diaoyun/zion/master/validator/Validator.java
浏览文件 @ 299d967d
......@@ -30,7 +30,7 @@ public enum Validator {
@Override
public void validate(String name, String value) throws ValidateException {
if (!ValidateUtils.isChinese(value)) {
if (!ValidateUtils.isContainChinese(value)) {
throw new ValidateException(name + "必须为中文！");
}
}
......
src/main/resources/logback-spring.xml
浏览文件 @ 299d967d
......@@ -28,7 +28,7 @@
<encoder>
<Pattern>${CONSOLE_LOG_PATTERN}</Pattern>
<!-- 本地打印 设置字符集 -->
<charset>GBK</charset>
<charset>UTF-8</charset>
<!-- 服务器打印 设置字符集 -->
<!--<charset>UTF-8</charset>-->
......
src/main/resources/mapper/TbCfPlatformDao.xml
浏览文件 @ 299d967d
......@@ -110,4 +110,9 @@
</foreach>
</delete>
<!--获取平台独立站-->
<select id="getPlatformList" resultType="com.diaoyun.zion.chinafrica.entity.TbCfPlatformEntity">
select * from tb_cf_platform where enable_flag=1 order by create_time desc
</select>
</mapper>
\ No newline at end of file
src/main/resources/mapper/TbCfStationItemDao.xml
浏览文件 @ 299d967d
......@@ -128,4 +128,11 @@
</foreach>
</delete>
<!--获取商品独立站-->
<select id="getItemStationList" resultType="com.diaoyun.zion.chinafrica.entity.TbCfStationItemEntity">
select * from tb_cf_station_item where enable_flag=1 order by create_time desc
</select>
</mapper>
\ No newline at end of file
src/main/resources/properties/domain.properties
浏览文件 @ 299d967d
......@@ -4,6 +4,23 @@
user.avatar=https://dev.diaosaas.com/upload/chinafrica/user/avatar.png
################################################################################
###################################邮箱###############################################
#网易
#email.hostName=smtp.yeah.net
#email.from.email=chinafrica@yeah.net
#email.from.name=chinafrica
#email.authentication.userName=chinafrica@yeah.net
#email.authentication.password=diaoyun2019
#腾讯企业邮箱
email.hostName=smtp.exmail.qq.com
email.from.email=joe.chow@diaocode.com
email.from.name=chinafrica
email.authentication.userName=joe.chow@diaocode.com
email.authentication.password=Yinggainuli.8
#发送主题
email.subject=Verification Code from Chinafrica
###################################邮箱 END###############################################
################redis有效时间###################
#登录token second 默认30分 一个星期
redis.token.expiredTime=604800
......
src/main/resources/static/js/angularjs/js/login/LoginController.js
浏览文件 @ 299d967d
......@@ -82,4 +82,61 @@ loginApp.controller('loginController', function ($scope, $q, $http) {
//console.log(selectTemplate);
}
var handler=function(d) {
console.log(d);
}
var xhr = new XMLHttpRequest();
var url = 'https://africa.gzdazt.com/zion/login';//?account=1244612031@qq.com&password=123456
xhr.open('POST', url, true);
xhr.withCredentials = true;
xhr.onreadystatechange = handler;
xhr.send();
/*var dataParam ={
targetUrl:"https://h5.m.taobao.com/awp/core/detail.htm?spm=a2141.9304519.0.0&id=9679381011&pvid=9e42d83a-82c7-43ac-a89f-199a74d48c2f&scene=3489&scm=1007.13489.98648.tab%3A0_id%3A9679381011_reason%3Ashoal_ranklist_rn%3A9e42d83a-82c7-43ac-a89f-199a74d48c2f"
};
$.ajax({
url: "https://africa.gzdazt.com/zion/spider/item/detail",
type: "post",
dataType: "json",
data: JSON.stringify(dataParam),
headers: {'Content-Type': 'application/json'},
success: function (res) {
console.log(res)
}
})*/
/*$.ajax({
url:"http://159.138.33.0/zion/spider/item/detail", //去请求项目二中的url
data:dataParam,
method:"GET",
contentType: "application/json;",
success:function (data) {
console.log(data)
}
})*/
/*var xhr = new XMLHttpRequest();
var url = ' http://159.138.33.0/zion/address/default/1c0dd6fcefb64b4ebe5febb8e42adfc0';
xhr.open('PUT', url, true);
xhr.withCredentials = true;
xhr.onreadystatechange = handler;
xhr.send();*/
/*$http({
method: "POST",
url: 'http://159.138.33.0/zion/login',
}).success(function (d) {
console.log(d)
});*/
});
src/main/resources/static/test.js
浏览文件 @ 299d967d
{
"code"
:
0, "page"
:
{
"currPage"
:
1, "list"
:
[{
"addressDetail": null, "areaCode": null, "cadre": null, "cityCode": null, "createTime": "2019-08-09
20: 24
:
05
","
latitude
":"
23.15693324928724
","
longitude
":"
112.88783401250839
","
name
":"
地图
","
organizationId
":"
49
ac972adcf148f4a5826eb9384f5aaf
","
phone
":null,"
provinceCode
":null},{"
addressDetail
":null,"
areaCode
":null,"
cadre
":null,"
cityCode
":null,"
createTime
":"
2019 - 0
8 - 0
9
20
:
11
:
28
","
latitude
":"
23.15282958501666
","
longitude
":"
113.28711837530136
","
name
":"
66666666666
","
organizationId
":"
361
c43f41afb4d329c8ea5951162aff8
","
phone
":null,"
provinceCode
":null},{"
addressDetail
":null,"
areaCode
":null,"
cadre
":null,"
cityCode
":null,"
createTime
":"
2019 - 0
8 - 0
9
20
:
11
:
18
","
latitude
":"
23.129467871785995
","
longitude
":"
113.3854803442955
","
name
":"
777
","
organizationId
":"
247e06
bb4cf749daade15c9986c3088f
","
phone
":null,"
provinceCode
":null},{"
addressDetail
":null,"
areaCode
":null,"
cadre
":null,"
cityCode
":null,"
createTime
":"
2019 - 0
8 - 0
9
20
:
0
8
:
56
","
latitude
":"
23.126626304711035
","
longitude
":"
113.38376373052597
","
name
":"
88
","
organizationId
":"
1
b79627ef649442b802dd03f28322daa
","
phone
":null,"
provinceCode
":null},{"
addressDetail
":"
广东省
","
areaCode
":null,"
cadre
":"
小明
","
cityCode
":null,"
createTime
":"
2019 - 0
8 - 0
9
20
:
0
8
:
15
","
latitude
":"
23.118416995082544
","
longitude
":"
112.73831695318222
","
name
":"
测试
","
organizationId
":"
e9b99909b3d2459b9934e5c606d67d7c
","
phone
":"
22222
","
provinceCode
":null}],"
pageSize
":177,"
totalCount
":5,"
totalPage
":1}}
\ No newline at end of file
location / {
if($request_method = 'OPTIONS') {
add_header
'Access-Control-Allow-Origin'
'*';
add_header
'Access-Control-Allow-Methods'
'GET, POST, OPTIONS';
#
# Custom
headers
and
headers
various
browsers * should * be
OK
with but aren
't
#
add_header
'Access-Control-Allow-Headers'
'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
#
# Tell
client
that
this
pre - flight
info
is
valid
for 20 days
#
add_header
'Access-Control-Max-Age'
1728000;
add_header
'Content-Type'
'text/plain; charset=utf-8';
add_header
'Content-Length'
0;
return 204;
}
if($request_method = 'POST') {
add_header
'Access-Control-Allow-Origin'
'*';
add_header
'Access-Control-Allow-Methods'
'GET, POST, OPTIONS';
add_header
'Access-Control-Allow-Headers'
'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
add_header
'Access-Control-Expose-Headers'
'Content-Length,Content-Range';
}
if($request_method = 'GET') {
add_header
'Access-Control-Allow-Origin'
'*';
add_header
'Access-Control-Allow-Methods'
'GET, POST, OPTIONS';
add_header
'Access-Control-Allow-Headers'
'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range';
add_header
'Access-Control-Expose-Headers'
'Content-Length,Content-Range';
}
}
\ No newline at end of file
src/main/resources/templates/email-template.ftl
浏览文件 @ 299d967d
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
<title>Java Techie Mail</title>
<title>Chinafrica Verification</title>
</head>
<body>
......@@ -21,8 +21,7 @@
display: none !important;
}
} </style>
<div style="font-size: 1px; color: #ffffff; display: none;">即将完成。欢迎使用 Creative Cloud，&nbsp;。在开始使用之前，我们需要快速验证您的电子邮件地址。
</div>
<table bgcolor="#ffffff" border="0" cellpadding="0" cellspacing="0" style="background-color:#ffffff;" width="100%">
<tbody>
<tr>
......@@ -37,22 +36,12 @@
width="90%">
<table align="center" border="0" cellpadding="0" cellspacing="0" width="100%">
<tbody>
<tr>
<td width="24">&nbsp;</td>
<td style="padding-top:16px;"><img height="19"
src="http://success.adobe.com/assets/en/images/email/templates/adobe.corp.email.ccm.transactional.transparent.344x38.png"
style="display:block; vertical-align:top;" width="172"></td>
<td align="right" style="padding-top:10px;"><img height="44"
src="http://success.adobe.com/assets/en/images/email/templates/adobe.corp.email.ccm.transactional.transparent.70x88.png"
style="display:block; vertical-align:top;"
width="35"></td>
<td width="24">&nbsp;</td>
</tr>
<tr>
<td width="24">&nbsp;</td>
<td colspan="2"
style="color:#333333; font-family:Arial, Helvetica, sans-serif; font-size:24px; line-height:26px; padding-top:18px;">
即将完成
Continue...
</td>
<td width="24">&nbsp;</td>
</tr>
......@@ -60,7 +49,7 @@
<td width="24">&nbsp;</td>
<td colspan="2"
style="color:#858585; font-family:Arial, Helvetica, sans-serif; font-size:14px; line-height:20px; padding-top:18px;">
${nick}欢迎使用 Chinafrica ！。在开始使用之前，我们需要快速验证您的电子邮件地址。
${nick},welcome to Chinafrica!Before we get started, we need to quickly verify your email address.
</td>
<td width="24">&nbsp;</td>
</tr>
......@@ -68,7 +57,7 @@
<td width="24">&nbsp;</td>
<td colspan="2"
style="color:#858585; font-family:Arial, Helvetica, sans-serif; font-size:14px; line-height:20px; padding-top:18px;">
你的验证码为：
Your verification code is:
</td>
<td width="24">&nbsp;</td>
</tr>
......@@ -76,31 +65,18 @@
<td width="24">&nbsp;</td>
<td colspan="2"
style="color:#858585; font-family:Arial, Helvetica, sans-serif; font-size:14px; line-height:20px; padding-top:18px;">
<span style="color:#2b9af3;text-decoration:none;font-weight:bold" rel="noopener" >${identifyCode}</a></span>
<span style="color:#2b9af3;text-decoration:none;font-weight:bold"
rel="noopener">${identifyCode}</a></span>
</td>
<td width="24">&nbsp;</td>
</tr>
<tr>
<td width="24">&nbsp;</td>
<td colspan="2"
style="color:#858585; font-family:Arial, Helvetica, sans-serif; font-size:14px; line-height:20px; padding-top:18px;">
电子邮件验证完毕后，请<a href="https://creative.adobe.com/"
style="color: #2b9af3; text-decoration: none; font-weight: bold;" rel="noopener"
target="_blank">登录</a>Chinafrica 以开始愉悦的购物之旅。&nbsp;
</td>
<td width="24">&nbsp;</td>
</tr>
<tr>
<td style="padding-top:18px; padding-bottom:32px; border-bottom:1px solid #e1e1e1;" width="24">
&nbsp;
</td>
<td colspan="2"
style="color:#858585; font-family:Arial, Helvetica, sans-serif; font-size:14px; line-height:20px; padding-top:18px; padding-bottom:32px; border-bottom:1px solid #e1e1e1;">
很高兴您使用我们的产品。<br> Chinafrica 团队
</td>
<td style="padding-top:18px; padding-bottom:32px; border-bottom:1px solid #e1e1e1;" width="24">
&nbsp;
By Chinafrica
</td>
</tr>
</tbody>
</table>
......
src/test/java/com/diaoyun/zion/ZionApplicationTests.java
浏览文件 @ 299d967d
package com.diaoyun.zion;
import com.diaoyun.zion.chinafrica.entity.TbCfCouponEntity;
import com.diaoyun.zion.chinafrica.service.SpiderService;
import com.diaoyun.zion.chinafrica.service.TbCfCouponService;
import com.diaoyun.zion.master.bo.TencentTranslateParam;
import com.diaoyun.zion.master.bo.TencentWordsegParam;
import com.diaoyun.zion.chinafrica.service.impl.TbCfOrderServiceImpl;
import com.diaoyun.zion.master.thread.TaskLimitSemaphore;
import com.diaoyun.zion.master.thread.TranslateCallable;
import com.diaoyun.zion.master.thread.WordposCallable;
import com.diaoyun.zion.master.util.*;
import com.google.gson.Gson;
import com.stripe.Stripe;
import com.diaoyun.zion.master.util.WordposHelper;
import com.stripe.exception.StripeException;
import com.stripe.model.Charge;
import freemarker.template.Configuration;
import net.sf.json.JSONObject;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.text.StringEscapeUtils;
import org.apache.http.*;
import org.apache.http.client.CookieStore;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.methods.RequestBuilder;
import org.apache.http.client.utils.URIBuilder;
import org.apache.http.impl.client.*;
import org.apache.http.message.BasicHeader;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
......@@ -34,11 +18,10 @@ import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.test.context.junit4.SpringRunner;
import java.io.*;
import java.math.BigDecimal;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
@RunWith(SpringRunner.class)
@SpringBootTest
......@@ -56,7 +39,7 @@ public class ZionApplicationTests {
private RedisTemplate redisTemplate;
private TaskLimitSemaphore taskLimitSemaphore = new TaskLimitSemaphore(12);
private List<Map<String, Object>> futureList = new ArrayList<Map<String, Object>>();
//private List<Map<String, Object>> futureList = new ArrayList<Map<String, Object>>();
@Test
public void contextLoads() throws InterruptedException, URISyntaxException, IOException, ExecutionException, StripeException {
......@@ -127,7 +110,7 @@ public class ZionApplicationTests {
HttpEntity result = httpResponse.getEntity();//拿到返回的HttpResponse的"实体"
String content = EntityUtils.toString(result);
//获取商品详情url
Map<String,String> infoMap= JsoupUtil.getUsefulInfo(content);
Map<String,String> infoMap= JsoupUtil.getTbItemDetail(content);
String usableSibUrl=infoMap.get("sibUrl");
//解析商品sku信息
Map<String,Object> propMap=JsoupUtil.getPropMap(content);
......@@ -274,6 +257,44 @@ public class ZionApplicationTests {
/*批量爬取*/
/*天猫爬取*/
/* String url="https://detail.tmall.com/item.htm?spm=a220m.1000858.1000725.71.5eac174fM6rzkB&id=570475903407&skuId=4611686588903291311&user_id=1689254786&cat_id=2&is_b=1&rn=cf06f08b8565fadd6e8f12c0c5bf581a";
JSONObject json=spiderService.getItemDetail(url);
System.out.println(json);
*/
/*计算运费*/
ExecutorService threadPool = Executors.newFixedThreadPool(20);
TbCfOrderServiceImpl tbCfOrderServiceImpl =new TbCfOrderServiceImpl();
long a=System.currentTimeMillis();
List<Map<String, Object>> futureList= new ArrayList<>();
for(int i=0;i<10;i++) {
Map<String, Object> titleMap = new HashMap();
String text = "夏季短袖衬衫男韩版修身免烫中袖衬衣男潮流帅气休闲五分袖黑色寸";
titleMap.put("text", text);
WordposHelper.separeteText(futureList, titleMap, text);
}
/*for(int i=0;i<10;i++) {
threadPool.submit(()->{
try {
//Map<String, Object> res=tbCfOrderServiceImpl.separateText("夏季短袖衬衫男韩版修身免烫中袖衬衣男潮流帅气休闲五分袖黑色寸");
//System.out.println(res);
} catch (ExecutionException e) {
e.printStackTrace();
} catch (InterruptedException e) {
e.printStackTrace();
}
});
}*/
WordposHelper.waitForResult(futureList);
long b=System.currentTimeMillis();
System.out.println(b-a);
while (true) {
}
}
......@@ -284,7 +305,7 @@ public class ZionApplicationTests {
String key=entry.getKey();
Map <String,Object> value= (Map<String, Object>) entry.getValue();
//翻译属性名
if(ValidateUtils.isChinese(key)) {
if(ValidateUtils.isContainChinese(key)) {
translateText(value,key);
//value.put("translate",translate);
}
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论