From 6975878debf1bad1b17feac6608bb0c5c8650a24 Mon Sep 17 00:00:00 2001 From: wangfukang <15630117759@163.com> Date: Tue, 14 May 2024 10:13:42 +0800 Subject: [PATCH] =?UTF-8?q?ai=E8=AF=86=E5=88=AB=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- hiver-modules/hiver-mall/pom.xml | 2 +- .../mall/config/thread/AiPurchaseThread.java | 8 +- .../mall/pojo/vo/CustomerBuyProductLogVo.java | 3 + .../mall/pojo/vo/SupplierBuyProductLogVo.java | 3 + .../impl/PurchaseOcrPictureServiceImpl.java | 5 +- .../vo/PurchaseOciPictureAddVo.java | 3 + .../purchaseocr/vo/PurchaseOcrExample.java | 14 + .../mybatis/PurchaseDetailServiceImpl.java | 6 +- .../java/cc/hiver/mall/utils/AliOcrUtil.java | 439 +++++++++++++++++- .../main/resources/mapper/PurchaseMapper.xml | 3 +- .../src/main/resources/mapper/SaleMapper.xml | 16 +- 11 files changed, 468 insertions(+), 34 deletions(-) create mode 100644 hiver-modules/hiver-mall/src/main/java/cc/hiver/mall/purchaseocr/vo/PurchaseOcrExample.java diff --git a/hiver-modules/hiver-mall/pom.xml b/hiver-modules/hiver-mall/pom.xml index 00cd6ca2..4fff5c97 100644 --- a/hiver-modules/hiver-mall/pom.xml +++ b/hiver-modules/hiver-mall/pom.xml @@ -20,7 +20,7 @@ com.alibaba dashscope-sdk-java - 2.12.0 + 2.14.1 org.slf4j diff --git a/hiver-modules/hiver-mall/src/main/java/cc/hiver/mall/config/thread/AiPurchaseThread.java b/hiver-modules/hiver-mall/src/main/java/cc/hiver/mall/config/thread/AiPurchaseThread.java index 5992a5ea..f41ba809 100644 --- a/hiver-modules/hiver-mall/src/main/java/cc/hiver/mall/config/thread/AiPurchaseThread.java +++ b/hiver-modules/hiver-mall/src/main/java/cc/hiver/mall/config/thread/AiPurchaseThread.java @@ -3,6 +3,7 @@ package cc.hiver.mall.config.thread; import cc.hiver.mall.common.constant.PurchaseConstant; import cc.hiver.mall.purchaseocr.entity.PurchaseOcrPicture; import cc.hiver.mall.purchaseocr.service.PurchaseOcrPictureService; +import cc.hiver.mall.purchaseocr.vo.PurchaseOcrExample; import cc.hiver.mall.service.mybatis.ProductService; import cc.hiver.mall.service.mybatis.PurchaseDetailService; import cc.hiver.mall.service.mybatis.PurchaseService; @@ -29,13 +30,15 @@ public class AiPurchaseThread implements Runnable { private PurchaseService purchaseService; private PurchaseDetailService purchaseDetailService; private PurchaseOcrPicture purchaseOcrPicture; + private PurchaseOcrExample purchaseOcrExample; public AiPurchaseThread() { } - public AiPurchaseThread(String purchaseId, PurchaseOcrPicture purchaseOcrPicture, PurchaseOcrPictureService purchaseOcrPictureService, ProductService productService, PurchaseDetailService purchaseDetailService, PurchaseService purchaseService) { + public AiPurchaseThread(String purchaseId, PurchaseOcrPicture purchaseOcrPicture,PurchaseOcrExample purchaseOcrExample, PurchaseOcrPictureService purchaseOcrPictureService, ProductService productService, PurchaseDetailService purchaseDetailService, PurchaseService purchaseService) { this.purchaseId = purchaseId; this.purchaseOcrPicture = purchaseOcrPicture; + this.purchaseOcrExample = purchaseOcrExample; this.purchaseOcrPictureService = purchaseOcrPictureService; this.productService = productService; this.purchaseDetailService = purchaseDetailService; @@ -66,7 +69,7 @@ public class AiPurchaseThread implements Runnable { log.info("当前线程名称：------------>>>>>" + Thread.currentThread().getName()); try { log.info("正在ocr识别：" + purchaseOcrPicture.getOcrPicture() + "
线程池名称：" + Thread.currentThread().getName()); - jsonObject = AliOcrUtil.multiRoundConversationCall(purchaseOcrPicture, productService, purchaseDetailService); + jsonObject = AliOcrUtil.multiRoundConversationCall(purchaseOcrPicture,purchaseOcrExample, productService, purchaseDetailService); // 得到处理结果之后，开始新增库存的详细信息 purchaseOcrPicture.setOcrMsg(jsonObject.getString("resultContent")); purchaseOcrPicture.setOcrStatus(PurchaseConstant.OCR_STATUS[1]); @@ -78,6 +81,7 @@ public class AiPurchaseThread implements Runnable { purchaseOcrPictureService.updateStatus(purchaseOcrPicture); jsonObject.put("code", 500); jsonObject.put("msg", e.getMessage()); + log.error(e.getMessage(),e); // throw new RuntimeException(e); } // 尝试更新主表识别状态 diff --git a/hiver-modules/hiver-mall/src/main/java/cc/hiver/mall/pojo/vo/CustomerBuyProductLogVo.java b/hiver-modules/hiver-mall/src/main/java/cc/hiver/mall/pojo/vo/CustomerBuyProductLogVo.java index b5440dac..d5e83b07 100644 --- a/hiver-modules/hiver-mall/src/main/java/cc/hiver/mall/pojo/vo/CustomerBuyProductLogVo.java +++ b/hiver-modules/hiver-mall/src/main/java/cc/hiver/mall/pojo/vo/CustomerBuyProductLogVo.java @@ -19,6 +19,9 @@ public class CustomerBuyProductLogVo implements Serializable { @ApiModelProperty(value = "商品名称") private String productName; + @ApiModelProperty(value = "商品货号") + private String productSn; + @ApiModelProperty(value = "历次销售单") private List customerBuySaleVoList; } diff --git a/hiver-modules/hiver-mall/src/main/java/cc/hiver/mall/pojo/vo/SupplierBuyProductLogVo.java b/hiver-modules/hiver-mall/src/main/java/cc/hiver/mall/pojo/vo/SupplierBuyProductLogVo.java index bf5a111b..662c8eab 100644 --- a/hiver-modules/hiver-mall/src/main/java/cc/hiver/mall/pojo/vo/SupplierBuyProductLogVo.java +++ b/hiver-modules/hiver-mall/src/main/java/cc/hiver/mall/pojo/vo/SupplierBuyProductLogVo.java @@ -19,6 +19,9 @@ public class SupplierBuyProductLogVo implements Serializable { @ApiModelProperty(value = "商品名称") private String productName; + @ApiModelProperty(value = "商品货号") + private String productSn; + @ApiModelProperty(value = "历次采购单") private List supplierBuySaleVos; } diff --git a/hiver-modules/hiver-mall/src/main/java/cc/hiver/mall/purchaseocr/service/impl/PurchaseOcrPictureServiceImpl.java b/hiver-modules/hiver-mall/src/main/java/cc/hiver/mall/purchaseocr/service/impl/PurchaseOcrPictureServiceImpl.java index 4597374a..08b2a17d 100644 --- a/hiver-modules/hiver-mall/src/main/java/cc/hiver/mall/purchaseocr/service/impl/PurchaseOcrPictureServiceImpl.java +++ b/hiver-modules/hiver-mall/src/main/java/cc/hiver/mall/purchaseocr/service/impl/PurchaseOcrPictureServiceImpl.java @@ -12,6 +12,7 @@ import cc.hiver.mall.purchaseocr.mapper.PurchaseOcrPictureMapper; import cc.hiver.mall.purchaseocr.service.PurchaseOcrPictureService; import cc.hiver.mall.purchaseocr.vo.PurchaseOciPictureAddVo; import cc.hiver.mall.purchaseocr.vo.PurchaseOcrCountVo; +import cc.hiver.mall.purchaseocr.vo.PurchaseOcrExample; import cc.hiver.mall.service.ShopService; import cc.hiver.mall.service.mybatis.ProductService; import cc.hiver.mall.service.mybatis.PurchaseDetailService; @@ -113,9 +114,11 @@ public class PurchaseOcrPictureServiceImpl implements PurchaseOcrPictureService purchaseOcrPictureMapper.batchSave(purchaseOcrPictureAddList); // 异步处理ocr识别 try { + // 获取参数示例 + PurchaseOcrExample purchaseOcrExample = purchaseOciPictureAddVo.getPurchaseOcrExample(); AiPurchaseThread timerThread; for (PurchaseOcrPicture purchaseOcrPicture : purchaseOcrPictureAddList) { - timerThread = new AiPurchaseThread(purchaseId, purchaseOcrPicture, purchaseOcrPictureService, productService, purchaseDetailService, purchaseService); + timerThread = new AiPurchaseThread(purchaseId, purchaseOcrPicture,purchaseOcrExample, purchaseOcrPictureService, productService, purchaseDetailService, purchaseService); // 这里可以使用线程池，也可以使用CompletionService处理，运行任务需要是callable的，需要最终结果。 threadPoolConfiguration.threadPoolTaskExecutor().execute(timerThread); } diff --git a/hiver-modules/hiver-mall/src/main/java/cc/hiver/mall/purchaseocr/vo/PurchaseOciPictureAddVo.java b/hiver-modules/hiver-mall/src/main/java/cc/hiver/mall/purchaseocr/vo/PurchaseOciPictureAddVo.java index 66ced0d1..329692c6 100644 --- a/hiver-modules/hiver-mall/src/main/java/cc/hiver/mall/purchaseocr/vo/PurchaseOciPictureAddVo.java +++ b/hiver-modules/hiver-mall/src/main/java/cc/hiver/mall/purchaseocr/vo/PurchaseOciPictureAddVo.java @@ -14,4 +14,7 @@ public class PurchaseOciPictureAddVo { @ApiModelProperty(value = "Oci图片信息") private List purchaseOcrPictureList; + + @ApiModelProperty("本次识别示例") + private PurchaseOcrExample purchaseOcrExample; } diff --git a/hiver-modules/hiver-mall/src/main/java/cc/hiver/mall/purchaseocr/vo/PurchaseOcrExample.java b/hiver-modules/hiver-mall/src/main/java/cc/hiver/mall/purchaseocr/vo/PurchaseOcrExample.java new file mode 100644 index 00000000..77a341eb --- /dev/null +++ b/hiver-modules/hiver-mall/src/main/java/cc/hiver/mall/purchaseocr/vo/PurchaseOcrExample.java @@ -0,0 +1,14 @@ +package cc.hiver.mall.purchaseocr.vo; + +import lombok.Data; + +@Data +public class PurchaseOcrExample { + + private String productSn; + private String productName; + private String price; + private String productCount; + private String color; + private String size; +} diff --git a/hiver-modules/hiver-mall/src/main/java/cc/hiver/mall/serviceimpl/mybatis/PurchaseDetailServiceImpl.java b/hiver-modules/hiver-mall/src/main/java/cc/hiver/mall/serviceimpl/mybatis/PurchaseDetailServiceImpl.java index ab660349..92702807 100644 --- a/hiver-modules/hiver-mall/src/main/java/cc/hiver/mall/serviceimpl/mybatis/PurchaseDetailServiceImpl.java +++ b/hiver-modules/hiver-mall/src/main/java/cc/hiver/mall/serviceimpl/mybatis/PurchaseDetailServiceImpl.java @@ -34,7 +34,7 @@ public class PurchaseDetailServiceImpl extends ServiceImpl purchaseDetails) { // 根据货号获取商品的详细信息是否存在 - final List addPurchaseDetails = new ArrayList<>(); + /*final List addPurchaseDetails = new ArrayList<>(); final List updatePurchaseDetails = new ArrayList<>(); purchaseDetails.forEach(purchaseDetail -> { final String productSn = purchaseDetail.getProductSn(); @@ -61,8 +61,10 @@ public class PurchaseDetailServiceImpl extends ServiceImpl stockLogList = new ArrayList<>(); for (PurchaseDetail purchaseDetail : purchaseDetails) { diff --git a/hiver-modules/hiver-mall/src/main/java/cc/hiver/mall/utils/AliOcrUtil.java b/hiver-modules/hiver-mall/src/main/java/cc/hiver/mall/utils/AliOcrUtil.java index f2334bbf..ae015ca5 100644 --- a/hiver-modules/hiver-mall/src/main/java/cc/hiver/mall/utils/AliOcrUtil.java +++ b/hiver-modules/hiver-mall/src/main/java/cc/hiver/mall/utils/AliOcrUtil.java @@ -5,6 +5,7 @@ import cc.hiver.mall.config.aliocr.AliOcrConfig; import cc.hiver.mall.entity.*; import cc.hiver.mall.pojo.vo.*; import cc.hiver.mall.purchaseocr.entity.PurchaseOcrPicture; +import cc.hiver.mall.purchaseocr.vo.PurchaseOcrExample; import cc.hiver.mall.service.mybatis.*; import cn.hutool.core.date.StopWatch; import com.alibaba.dashscope.aigc.generation.Generation; @@ -13,6 +14,7 @@ import com.alibaba.dashscope.aigc.generation.models.QwenParam; import com.alibaba.dashscope.aigc.multimodalconversation.*; import com.alibaba.dashscope.common.Message; import com.alibaba.dashscope.common.MessageManager; +import com.alibaba.dashscope.common.MultiModalMessage; import com.alibaba.dashscope.common.Role; import com.alibaba.dashscope.exception.ApiException; import com.alibaba.dashscope.exception.InputRequiredException; @@ -22,16 +24,14 @@ import com.alibaba.dashscope.utils.Constants; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONObject; +import io.reactivex.Flowable; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.jetbrains.annotations.NotNull; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import java.util.Arrays; -import java.util.Collections; -import java.util.Date; -import java.util.Map; +import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; import java.util.regex.Pattern; @@ -49,7 +49,7 @@ public class AliOcrUtil { private static final Pattern SYCM = Pattern.compile("所有尺码", Pattern.LITERAL); /** - * AI入库图像识别 + * AI入库图像识别-多轮对话 * * @param purchaseOcrPicture * @param productService @@ -58,9 +58,10 @@ public class AliOcrUtil { * @author 王富康 * @date 2024/3/31 */ - public static JSONObject multiRoundConversationCall(PurchaseOcrPicture purchaseOcrPicture, ProductService productService, PurchaseDetailService purchaseDetailService) throws ApiException, NoApiKeyException, UploadFileException { + public static JSONObject multiRoundConversationCall(PurchaseOcrPicture purchaseOcrPicture, PurchaseOcrExample purchaseOcrExample, ProductService productService, PurchaseDetailService purchaseDetailService) throws ApiException, NoApiKeyException, UploadFileException { final JSONObject jsonObject = new JSONObject(); final String picturePath = purchaseOcrPicture.getOcrPicture(); + final String parentPicturePath = purchaseOcrPicture.getParentPicturePath(); final Integer count = purchaseOcrPicture.getCount(); final String orderId = purchaseOcrPicture.getOrderId(); final String shopId = purchaseOcrPicture.getShopId(); @@ -74,7 +75,16 @@ public class AliOcrUtil { .role(Role.SYSTEM.getValue()).content(Collections.singletonList(systemText)).build(); final MultiModalMessageItemImage userImage = new MultiModalMessageItemImage(picturePath); - final String firstQuestionMsg = "将图中内容以JSON格式输出，JSON格式为：{ \"productSn\": \"货号\", \"productName\": \"商品名称\",\"price\": \"单价\", \"productCount\": \"数量\", \"attributeList\": \"{'color':'红色','size':'均码'}\" }, 请帮我返回" + count + "条不同规格的JSON数据"; + final String firstQuestionMsg = "请把图片中的内容按照商品的不同规格拆分，返回" + count + "条JSON数据。\n" + + " JSON示例:[{ \"productSn\": \"" + purchaseOcrExample.getProductSn() + "\", \"productName\":\"" + purchaseOcrExample.getProductName() + "\"， \"price\": \"" + purchaseOcrExample.getPrice() + "\", \"productCount\": \"" + purchaseOcrExample.getProductCount() + "\", \"attributeList\": \"{'color':'" + purchaseOcrExample.getColor() + "','size':'" + purchaseOcrExample.getSize() + "'}\" }]。" + + "以下是几点要求： " + + "1.JSON中“productSn”字段可能由字母或数字或特殊符号组成，“color”字段如果图片中没有内容，则赋值“均色”，“size”字段如果图片中没有内容，则赋值“均码”。" + + "2.严格按照JSON示例的格式和字段名返回！ " + + "3.严格按照图中内容值返回！ " + + "4.每个JSON对象必须包含这7个字段。" + + "5.如果图中内容没有productSn和productName信息，则使用JSON示例中的值填充。 " + + "6.请注意返回JSON符号解析格式正确 ,确保java程序能够正确解析。" + + "7.严格返回" + count + "条JSON数据，如果数量有偏差按照实际条数返回，不能自动省略！"; MultiModalMessageItemText userText = new MultiModalMessageItemText(firstQuestionMsg); final MultiModalConversationMessage userMessage = MultiModalConversationMessage.builder().role(Role.USER.getValue()) @@ -95,7 +105,8 @@ public class AliOcrUtil { stopWatch.stop(); multiModalConversationResults.add(result); // 解析结果 - final String text = result.getOutput().getChoices().get(0).getMessage().getContent().get(0).get("text").toString(); + String text = result.getOutput().getChoices().get(0).getMessage().getContent().get(0).get("text").toString(); + log.info("一轮对话-的json======"+ text); // 根据{}截取数据 final int startIndex = text.indexOf('{'); final int endIndex = text.lastIndexOf('}'); @@ -103,8 +114,119 @@ public class AliOcrUtil { // 没有查询到想要的数据，那么就报错返回 throw new RuntimeException("识别数据失败，未找到指定标识符：识别结果为：" + text); } - final String jsonStr = '[' + text.substring(startIndex, endIndex + 1) + ']'; - JSONArray json = JSON.parseArray(jsonStr); + // attributeList 有时候解析的不是json格式，特殊处理一下 + String jsonStr = '[' + text.substring(startIndex, endIndex + 1) + ']'; + // 有时候返回的是productCounts，改为productCount + jsonStr = fixProductCounts(jsonStr); + JSONArray json = new JSONArray(); + try{ + json = JSON.parseArray(jsonStr); + }catch (Exception e){ + log.error(Thread.currentThread().getName()+"-一轮对话--->json识别报错喽，没关系，我把错误信息输出出来："+e.getMessage(), e); + // 如果格式化报错，特殊处理下，再次进行转义。 + final String message = e.getMessage(); + boolean needAgain = true; + // 以下是对目前可能出现的问题进行过滤处理 + // attributeList格式不对 + if(message.contains("attributeList")){ + needAgain = false; + log.error(Thread.currentThread().getName()+"-一轮对话--->json识别报错喽，没关系，attributeList格式有问题，我来替换一下！"); + // 如果是attributeList出现问题，那么尝试修复。 + jsonStr = fixAttributeList(jsonStr); + try{ + json = JSON.parseArray(jsonStr); + }catch (Exception replaceOneException){ + log.error(Thread.currentThread().getName()+"-一轮对话--->json识别报错喽，没关系，attributeList替换了也没好使！"); + needAgain = true; + } + + + } + // 识别多了个“}” + if(message.contains("not close json text")){ + needAgain = false; + log.error(Thread.currentThread().getName()+"-一轮对话--->json识别报错喽，没关系，可能是多了“}”，我来去掉试一下一下！"); + // + jsonStr = jsonStr.substring(0, jsonStr.length()-1); + try{ + json = JSON.parseArray(jsonStr); + }catch (Exception replaceOneException){ + log.error(Thread.currentThread().getName()+"-一轮对话--->json识别报错喽，没关系，可能是多了“}”，去掉了也没好使！"); + needAgain = true; + } + + } + if(needAgain){ + log.error(Thread.currentThread().getName()+"-一轮对话--->json识别报错喽，没关系，我来重新识别一下！"); + //尝试重新识别一次 + final String errorJsonQuestionMsg = "返回的json格式有误，请重新返回。"; + final MultiModalMessageItemText assistentText = new MultiModalMessageItemText( + text); + final MultiModalConversationMessage assistentMessage = MultiModalConversationMessage.builder() + .role(Role.ASSISTANT.getValue()).content(Collections.singletonList(assistentText)).build(); + messages.add(assistentMessage); + userText = new MultiModalMessageItemText(errorJsonQuestionMsg); + messages.add(MultiModalConversationMessage.builder().role(Role.USER.getValue()) + .content(Collections.singletonList(userText)).build()); + param.setMessages(new CopyOnWriteArrayList(messages)); + stopWatch.start("一轮对话json格式错误重新识别！"); + result = conv.call(param); + multiModalConversationResults.add(result); + stopWatch.stop(); + // 解析结果 + text = result.getOutput().getChoices().get(0).getMessage().getContent().get(0).get("text").toString(); + log.info("一轮对话--json格式错误，重新识别的结果======"+ text); + // 根据{}截取数据 + final int secondStartIndex = text.indexOf('{'); + final int secondEndIndex = text.lastIndexOf('}'); + // attributeList 有时候解析的不是json格式，特殊处理一下 + String errorJsonJsonStr = '[' + text.substring(secondStartIndex, secondEndIndex + 1) + ']'; + // 有时候返回的是productCounts，改为productCount + errorJsonJsonStr = fixProductCounts(errorJsonJsonStr); + + try{ + // + json = JSON.parseArray(errorJsonJsonStr); + }catch (Exception errorJsonException){ + log.error(Thread.currentThread().getName()+"-一轮对话--->json识别报错喽，重新识别又报错了，没关系，我把错误信息输出出来："+errorJsonException.getMessage(), errorJsonException); + // 如果格式化报错，特殊处理下，再次进行转义。 + final String errorJsonExceptionMessage = errorJsonException.getMessage(); + boolean firstCheckResult = true; + if(errorJsonExceptionMessage.contains("attributeList")){ + firstCheckResult = false; + log.error(Thread.currentThread().getName()+"-一轮对话--->json识别报错喽，重新识别又报错了，没关系，attributeList格式有问题，我来替换一下！"); + // 如果是attributeList出现问题，那么尝试修复。 + errorJsonJsonStr = fixAttributeList(errorJsonJsonStr); + try{ + json = JSON.parseArray(errorJsonJsonStr); + }catch (Exception errorJsonJsonException){ + log.error(Thread.currentThread().getName()+"-一轮对话--->json识别报错喽，重新识别又报错了，attributeList替换了也没好使！"); + firstCheckResult = true; + } + + } + // 识别多了个“}” + if(errorJsonExceptionMessage.contains("not close json text")){ + firstCheckResult = false; + log.error(Thread.currentThread().getName()+"-一轮对话--->json识别报错喽，重新识别又报错了，没关系，可能是多了“}”，我来去掉试一下一下！"); + // + errorJsonJsonStr = errorJsonJsonStr.substring(0, errorJsonJsonStr.length()-1); + try{ + json = JSON.parseArray(errorJsonJsonStr); + }catch (Exception replaceOneException){ + log.error(Thread.currentThread().getName()+"-一轮对话--->json识别报错喽，重新识别又报错了，可能是多了“}”，去掉了也没好使！"); + firstCheckResult = true; + } + + } + if(firstCheckResult){ + log.error(Thread.currentThread().getName()+"-一轮对话--->json识别报错喽，，重新识别又报错了，不识别了，直接毁灭吧！"); + throw errorJsonException; + } + } + } + } + final int fristResultCount = json.size(); // 我会在这里对结果进行解析，然后，判断要不要走第二次回话 final int missingCount = count - fristResultCount; @@ -124,17 +246,127 @@ public class AliOcrUtil { multiModalConversationResults.add(result); stopWatch.stop(); // 解析结果 - final String secondText = result.getOutput().getChoices().get(0).getMessage().getContent().get(0).get("text").toString(); + String secondText = result.getOutput().getChoices().get(0).getMessage().getContent().get(0).get("text").toString(); + log.info("二轮对话识别结果："+secondText); // 根据{}截取数据 final int secondStartIndex = secondText.indexOf('{'); final int secondEndIndex = secondText.lastIndexOf('}'); - final String secondJsonStr = '[' + secondText.substring(secondStartIndex, secondEndIndex + 1) + ']'; - json = JSON.parseArray(secondJsonStr); + if (secondStartIndex == -1 || secondEndIndex == -1) { + // 二轮对话，没有查询到想要的数据，那么就使用一轮的结果 + log.info(Thread.currentThread().getName()+"-二轮对话--->识别数据失败，使用第一次的识别结果未找到指定标识符：识别结果为：" + text); + }else{ + String secondJsonStr = '[' + secondText.substring(secondStartIndex, secondEndIndex + 1) + ']'; + // 有时候返回的是productCounts，改为productCount + secondJsonStr = fixProductCounts(secondJsonStr); + try{ + json = JSON.parseArray(secondJsonStr); + }catch (Exception e){ + boolean needAgain = true; + log.error(Thread.currentThread().getName()+"-二轮对话--->json识别报错喽，没关系，我把错误信息输出出来："+e.getMessage(),e); + // 如果格式化报错，特殊处理下，再次进行转义。 + final String message = e.getMessage(); + if(message.contains("attributeList")){ + needAgain = false; + log.error(Thread.currentThread().getName()+"-二轮对话--->json识别报错喽，没关系，attributeList格式有问题，我来替换一下！"); + // 如果是attributeList出现问题，那么尝试修复。 + secondJsonStr = fixAttributeList(secondJsonStr); + try{ + json = JSON.parseArray(secondJsonStr); + }catch (Exception exception){ + log.error(Thread.currentThread().getName()+"-二轮对话--->json识别报错喽，attributeList替换了也没好使！"); + needAgain = true; + } + } + // 识别多了个“}” + if(message.contains("not close json text")){ + needAgain = false; + log.error(Thread.currentThread().getName()+"-二轮对话--->json识别报错喽，没关系，可能是多了“}”，我来去掉试一下一下！"); + // + secondJsonStr = secondJsonStr.substring(0, secondJsonStr.length()-1); + try{ + json = JSON.parseArray(secondJsonStr); + }catch (Exception replaceOneException){ + log.error(Thread.currentThread().getName()+"-二轮对话--->json识别报错喽，可能是多了“}”，去掉了也没好使！"); + needAgain = true; + } + + } + if(needAgain){ + log.error(Thread.currentThread().getName()+"-二轮对话--->json识别报错喽，没关系，我来重新识别一下！"); + //尝试重新识别一次 + final String errorJsonQuestionMsg = "返回的json格式有误，请重新返回。"; + final MultiModalMessageItemText secondJsonErrorAssistentText = new MultiModalMessageItemText( + secondText); + final MultiModalConversationMessage secondJsonErrorAssistentMessage = MultiModalConversationMessage.builder() + .role(Role.ASSISTANT.getValue()).content(Collections.singletonList(secondJsonErrorAssistentText)).build(); + messages.add(secondJsonErrorAssistentMessage); + userText = new MultiModalMessageItemText(errorJsonQuestionMsg); + messages.add(MultiModalConversationMessage.builder().role(Role.USER.getValue()) + .content(Collections.singletonList(userText)).build()); + param.setMessages(new CopyOnWriteArrayList(messages)); + stopWatch.start("二轮会话json格式错误，重新识别！"); + result = conv.call(param); + multiModalConversationResults.add(result); + stopWatch.stop(); + // 解析结果 + secondText = result.getOutput().getChoices().get(0).getMessage().getContent().get(0).get("text").toString(); + log.info("二轮对话-json格式错误，重新识别的结果======"+ secondText); + // 根据{}截取数据 + final int errorSsecondStartIndex = secondText.indexOf('{'); + final int errorSecondEndIndex = secondText.lastIndexOf('}'); + // attributeList 有时候解析的不是json格式，特殊处理一下 + String errorJsonJsonStr = '[' + secondText.substring(errorSsecondStartIndex, errorSecondEndIndex + 1) + ']'; + // 有时候返回的是productCounts，改为productCount + errorJsonJsonStr = fixProductCounts(errorJsonJsonStr); + try{ + // + json = JSON.parseArray(errorJsonJsonStr); + }catch (Exception errorJsonException){ + boolean secondCheckResult = true; + final String errorJsonExceptionMessage = errorJsonException.getMessage(); + log.error(Thread.currentThread().getName()+"-二轮对话--->json识别报错喽，重新识别又报错了，没关系，我把错误信息输出出来："+ errorJsonExceptionMessage, errorJsonException); + if(errorJsonExceptionMessage.contains("attributeList")){ + secondCheckResult = false; + log.error(Thread.currentThread().getName()+"-二轮对话--->json识别报错喽，，重新识别又报错了，没关系，，attributeList格式有问题，我来替换一下！"); + // 如果是attributeList出现问题，那么尝试修复。 + secondJsonStr = fixAttributeList(secondJsonStr); + try{ + json = JSON.parseArray(secondJsonStr); + }catch (Exception secondCheckException){ + log.error(Thread.currentThread().getName()+"-二轮对话--->json识别报错喽，，重新识别又报错了，attributeList替换了也没好使！"); + secondCheckResult = true; + } + } + if(errorJsonExceptionMessage.contains("not close json text")){ + secondCheckResult = false; + log.error(Thread.currentThread().getName()+"-二轮对话--->json识别报错喽，重新识别又报错了，没关系，可能是多了“}”，我来去掉试一下一下！"); + // 如果是attributeList出现问题，那么尝试修复。 + secondJsonStr = secondJsonStr.substring(0, secondJsonStr.length()-1); + try{ + json = JSON.parseArray(secondJsonStr); + }catch (Exception secondCheckException){ + log.error(Thread.currentThread().getName()+"-二轮对话--->json识别报错喽，重新识别又报错了，可能是多了“}”，去掉了也没好使！"); + secondCheckResult = true; + } + } + + if(secondCheckResult){ + log.error(Thread.currentThread().getName()+"-二轮对话--->json识别报错喽，，重新识别又报错了，不识别了，直接毁灭吧！"); + throw errorJsonException; + } + } + } + } + } } log.info(JSON.toJSONString(json)); stopWatch.start("解析结果=="); // 解析结果 - getPurchaseDetailMap(purchaseDetailService,productService, orderId, shopId, picturePath, json); + // 入库的时候子图直接存放主图的路径 + /*if (StringUtils.isNotEmpty(parentPicturePath)) { + picturePath = parentPicturePath; + }*/ + getPurchaseDetailMap(purchaseDetailService, productService, orderId, shopId, picturePath, json); stopWatch.stop(); // 将map 转为list 进行落库操作 /*stopWatch.start("批量插入=="); @@ -151,6 +383,153 @@ public class AliOcrUtil { return jsonObject; } + /** + * 1. attributeList 有时候解析的不是json格式，特殊处理一下 + * 2. 有时候返回的是productCounts，改为productCount + * + * @param attributeList + * @return String + * @author 王富康 + * @date 2024/5/12 + */ + public static String fixAttributeList(String attributeList) { + // 将字符串转换为正确的json格式 + attributeList = attributeList.replace("\"color\":\"", "'color':'"); + attributeList = attributeList.replace("\",\"size\":\"", "','size':'"); + // 正确的是以\"结尾的，不需要转换 + if (!attributeList.contains("\\\"}\"")) { + attributeList = attributeList.replace("\"}\"", "'}\""); + } + + attributeList = attributeList.replace("productCounts", "productCount"); + return attributeList; + } + + /** + * 有时候返回的是productCounts，改为productCount + * + * @param str + * @return String + * @author 王富康 + * @date 2024/5/12 + */ + public static String fixProductCounts(String str) { + str = str.replace("productCounts", "productCount"); + return str; + } + + /*public static void main(String[] args) throws NoApiKeyException, UploadFileException { + PurchaseOcrPicture purchaseOcrPicture = new PurchaseOcrPicture(); + purchaseOcrPicture.setOcrPicture("https://jewel-shop.oss-cn-beijing.aliyuncs.com/f51c446ef147437c9a8200ed2b43ec7f.jpg"); + purchaseOcrPicture.setCount(22); + streamCall(purchaseOcrPicture); + }*/ + + /** + * 图片识别-流式输出 + * + * @author 王富康 + * @date 2024/5/10 + */ + public static JSONObject streamCall(PurchaseOcrPicture purchaseOcrPicture, PurchaseOcrExample purchaseOcrExample, ProductService productService, PurchaseDetailService purchaseDetailService) + throws ApiException, NoApiKeyException, UploadFileException { + final JSONObject jsonObject = new JSONObject(); + final String picturePath = purchaseOcrPicture.getOcrPicture(); + final String parentPicturePath = purchaseOcrPicture.getParentPicturePath(); + + final Integer count = purchaseOcrPicture.getCount(); + final String orderId = purchaseOcrPicture.getOrderId(); + final String shopId = purchaseOcrPicture.getShopId(); + + final StopWatch stopWatch = new StopWatch("Ai回答计时"); + Constants.apiKey = "sk-bcfa4865b89548acb8225f910f13d682"; + // final String firstQuestionMsg = "将图中内容以JSON格式输出，JSON格式为：{ \"productSn\": \"货号\", \"productName\": \"商品名称\",\"price\": \"单价\", \"productCount\": \"数量\", \"attributeList\": \"{'color':'红色','size':'均码'}\" }, 请帮我返回" + count + "条不同规格的JSON数据"; + final String firstQuestionMsg = "请把图片中的内容按照商品的不同规格拆分，返回" + count + "条JSON数据。\n" + + " JSON示例:[{ \"productSn\": \"" + purchaseOcrExample.getProductSn() + "\", \"productName\":\"" + purchaseOcrExample.getProductName() + "\"， \"price\": \"" + purchaseOcrExample.getPrice() + "\", \"productCount\": \"" + purchaseOcrExample.getProductCount() + "\", \"attributeList\": \"{'color':'" + purchaseOcrExample.getColor() + "','size':'" + purchaseOcrExample.getSize() + "'}\" }]。" + + "以下是几点要求： " + + "1.JSON中“productSn”字段可能由字母或数字或特殊符号组成，“color”字段如果图片中没有内容，则赋值“均色”，“size”字段如果图片中没有内容，则赋值“均码”。" + + "2.严格按照JSON示例的格式和字段名返回！ " + + "3.严格按照图中内容值返回！ " + + "4.每个JSON对象必须包含这7个字段。" + + "5.如果图中内容没有productSn和productName信息，则使用JSON示例中的值填充。 " + + "6. 请注意返回JSON符号解析格式正确 。" + + "7.严格返回" + count + "条JSON数据，如果数量有偏差按照实际条数返回，不能自动省略！"; + final MultiModalConversation conv = new MultiModalConversation(); + // must create mutable map. + final String finalPicturePath = picturePath; + final MultiModalMessage userMessage = MultiModalMessage.builder().role(Role.USER.getValue()) + .content(Arrays.asList(new HashMap() { + private static final long serialVersionUID = 6779588534269073602L; + + { + put("image", finalPicturePath); + } + }, + new HashMap() { + private static final long serialVersionUID = 6788635831260953019L; + + { + put("text", firstQuestionMsg); + } + })).build(); + final MultiModalConversationParam param = MultiModalConversationParam.builder() + .model("qwen-vl-max") + .message(userMessage) + /*.stream(true) + .temperature(1F) + // 生成时，核采样方法的概率阈值。例如，取值为0.8时，仅保留累计概率之和大于等于0.8的概率分布中的token，作为随机采样的候选集。取值范围为(0,1.0)，取值越大，生成的随机性越高；取值越低，生成的随机性越低。默认值 0.8。注意，取值不要大于等于1 + .topP(0.1) + // 生成时，采样候选集的大小。例如，取值为50时，仅将单次生成中得分最高的50个token组成随机采样的候选集。取值越大，生成的随机性越高；取值越小，生成的确定性越高。注意：如果top_k的值大于100，top_k将采用默认值100 + .topK(1) + // 生成时，随机数的种子，用于控制模型生成的随机性。如果使用相同的种子，每次运行生成的结果都将相同；当需要复现模型的生成结果时，可以使用相同的种子。seed参数支持无符号64位整数类型。默认值 1234 + .seed(500)*/ + // 是否使用增量输出。当使用增量输出时每次流式返回的序列仅包含最新生成的增量内容，默认值为false，即输出完整的全量内容 + .incrementalOutput(true) + .build(); + stopWatch.start("开始识别！"); + final Flowable result = conv.streamCall(param); + // 拼接结果集 + final StringBuffer resultStr = new StringBuffer(); + result.blockingForEach(item -> { + // System.out.println(item); + final List> content = item.getOutput().getChoices().get(0).getMessage().getContent(); + if (!content.isEmpty()) { + final String string = content.get(0).get("text").toString(); + resultStr.append(string); + } + + }); + stopWatch.stop(); + final String text = resultStr.toString(); + log.info(text); + // 根据{}截取数据 + final int startIndex = text.indexOf('{'); + final int endIndex = text.lastIndexOf('}'); + if (startIndex == -1 || endIndex == -1) { + // 没有查询到想要的数据，那么就报错返回 + throw new RuntimeException("识别数据失败，未找到指定标识符：识别结果为：" + text); + } + final String jsonStr = '[' + text.substring(startIndex, endIndex + 1) + ']'; + final JSONArray json = JSON.parseArray(jsonStr); + log.info(JSON.toJSONString(json)); + stopWatch.start("解析结果=="); + // 解析结果 + // 入库的时候子图直接存放主图的路径 + /*if (StringUtils.isNotEmpty(parentPicturePath)) { + picturePath = parentPicturePath; + }*/ + getPurchaseDetailMap(purchaseDetailService, productService, orderId, shopId, picturePath, json); + stopWatch.stop(); + + log.info(stopWatch.prettyPrint()); + jsonObject.put("resultContent", json); + /*jsonObject.put("tokenCount", result.getUsage()); + jsonObject.put("requestId", result.getRequestId());*/ + jsonObject.put("cod", 200); + jsonObject.put("ocrCount", json.size()); + return jsonObject; + } + /** * 处理数据 * @@ -179,11 +558,32 @@ public class AliOcrUtil { purchaseOcrVo.setProductCount(jsonObject.getInteger("productCount")); final String attributeListStr = jsonObject.getString("attributeList"); final JSONObject attributeListJson = JSON.parseObject(attributeListStr); - final PurchaseOcrDetailVo purchaseOcrDetailVo = new PurchaseOcrDetailVo(); - purchaseOcrDetailVo.setColor(attributeListJson.getString("color")); - purchaseOcrDetailVo.setSize(attributeListJson.getString("size")); + final CopyOnWriteArrayList attributeList = new CopyOnWriteArrayList<>(); - attributeList.add(purchaseOcrDetailVo); + final String size = attributeListJson.getString("size"); + if(size.contains(",")){ + final String[] sizeArray = size.split(","); + for (String s : sizeArray) { + final PurchaseOcrDetailVo purchaseOcrDetailVo = new PurchaseOcrDetailVo(); + purchaseOcrDetailVo.setColor(attributeListJson.getString("color")); + purchaseOcrDetailVo.setSize(s); + attributeList.add(purchaseOcrDetailVo); + } + }else if(size.contains("-")){ + final String[] sizeArray = size.split("-"); + for (String s : sizeArray) { + final PurchaseOcrDetailVo purchaseOcrDetailVo = new PurchaseOcrDetailVo(); + purchaseOcrDetailVo.setColor(attributeListJson.getString("color")); + purchaseOcrDetailVo.setSize(s); + attributeList.add(purchaseOcrDetailVo); + } + }else{ + final PurchaseOcrDetailVo purchaseOcrDetailVo = new PurchaseOcrDetailVo(); + purchaseOcrDetailVo.setColor(attributeListJson.getString("color")); + purchaseOcrDetailVo.setSize(size); + attributeList.add(purchaseOcrDetailVo); + } + purchaseOcrVo.setAttributeList(attributeList); purchaseOcrVos.add(purchaseOcrVo); @@ -297,7 +697,7 @@ public class AliOcrUtil { purchaseDetail.setOcrPicturePath(ocrPicturePath); } else { final PurchaseDetail purchaseDetail = new PurchaseDetail(); - final StockLog stockLog = new StockLog(); + final CopyOnWriteArrayList stockLogList = new CopyOnWriteArrayList<>(); purchaseDetail.setPurchaseId(orderId); purchaseDetail.setCreateTime(new Date()); @@ -308,6 +708,7 @@ public class AliOcrUtil { // 采购价 purchaseDetail.setPurchasePrice(purchaseOcrVo.getPrice()); purchaseDetail.setOcrPicturePath(ocrPicturePath); + final StockLog stockLog = new StockLog(); stockLog.setOrderId(orderId); stockLog.setCreateTime(new Date()); stockLog.setDetailId(purchaseDetail.getId()); diff --git a/hiver-modules/hiver-mall/src/main/resources/mapper/PurchaseMapper.xml b/hiver-modules/hiver-mall/src/main/resources/mapper/PurchaseMapper.xml index d6da019b..2429f6d8 100644 --- a/hiver-modules/hiver-mall/src/main/resources/mapper/PurchaseMapper.xml +++ b/hiver-modules/hiver-mall/src/main/resources/mapper/PurchaseMapper.xml @@ -438,7 +438,8 @@ from t_purchase where del_flag ='0' - and id in (select purchase_id from t_purchase_detail where del_flag ='0' and in_storage_status = 1 and product_id =#{productId}) + and in_storage_status = 1 + and id in (select purchase_id from t_purchase_detail where del_flag ='0' and product_id =#{productId}) order by create_time desc diff --git a/hiver-modules/hiver-mall/src/main/resources/mapper/SaleMapper.xml b/hiver-modules/hiver-mall/src/main/resources/mapper/SaleMapper.xml index d0f00fc4..6f1a42ce 100644 --- a/hiver-modules/hiver-mall/src/main/resources/mapper/SaleMapper.xml +++ b/hiver-modules/hiver-mall/src/main/resources/mapper/SaleMapper.xml @@ -769,14 +769,14 @@ trans_company, company_name, product_count, remark, sale_name, company_phone, cr @@ -813,16 +813,15 @@ trans_company, company_name, product_count, remark, sale_name, company_phone, cr @@ -835,7 +834,8 @@ trans_company, company_name, product_count, remark, sale_name, company_phone, cr AND t.product_id = s.product_id WHERE t.shop_id = #{shopId} - AND purchase_id IN ( SELECT id FROM t_purchase WHERE deflag = 0 and in_storage_status = 1 and supplier_id = #{supplierId} ) + and t.del_flag != 1 + AND purchase_id IN ( SELECT id FROM t_purchase WHERE del_flag = 0 and in_storage_status = 1 and supplier_id = #{supplierId} ) and t.product_id in #{listItem}