要素提取补充AI提取

BaseDataMaintenance/AIUtils/ChatAgent.py

@@ -0,0 +1,91 @@
+#coding:utf8
+
+import requests
+import time
+import traceback
+import json
+
+agent_id = "7458183497300492338"
+url = "https://api.coze.cn/v3/chat"
+Authorization = "Bearer pat_XXlExeHd1loVyHY1kV7Z38GjBERfmvOAvhfxCtXSYLOzNtqYXBqd9Sh3BYwJJzjw"
+user_id = "2103446848"
+
+
+headers = {"Authorization":Authorization,
+           "Content-Type":"application/json"}
+
+
+def chat_agent(msg,retry_time=3):
+
+    for _ in range(retry_time):
+        try:
+
+            data_raw = {"bot_id":agent_id,
+                        "user_id":user_id,
+                        "stream":True,
+                        "auto_save_history":True,
+                        "additional_messages":[
+                            {
+                                "role":"user",
+                                "content":msg,
+                                "content_type":"text"
+                            }
+                        ]}
+
+            resp = requests.post(url,headers=headers,json=data_raw,timeout=60)
+
+            _data = resp.content.decode("utf-8")
+
+            return decode_msg(_data)
+
+        except Exception as e:
+            traceback.print_exc()
+            time.sleep(1)
+
+
+msg_list_url = "https://api.coze.cn/v3/chat/message/list"
+
+
+def get_msg_list(conversation_id,chat_id,retry_time=3):
+
+    for _ in range(retry_time):
+        try:
+
+            data_raw = {"conversation_id":conversation_id,
+                        "chat_id":chat_id}
+
+            resp = requests.post(msg_list_url,headers=headers,json=data_raw)
+
+            _data = _result.content.decode("utf-8")
+
+            return decode_msg(_data)
+        except Exception as e:
+            traceback.print_exc()
+            time.sleep(1)
+
+
+def decode_msg(content):
+    list_msg = content.split("\n\n")
+    response = ""
+    for msg in list_msg:
+        try:
+            list_line = msg.split("\n")
+            if len(list_line) ==2:
+                msg_type = list_line[0]
+                msg_data = json.loads(list_line[1][5:])
+                if msg_type=="event:conversation.message.delta":
+                    response += msg_data.get("content")
+
+        except Exception as e:
+            pass
+    return response
+
+
+if __name__ == '__main__':
+    msg = '''
+    上海市青浦区华新镇社区事务受理服务中心的完整名称和统一信用代码是多少，还有对齐原因，以json格式给出准确的结果
+    '''
+    _result = chat_agent(msg)
+    print(_result)
+    # _result = get_msg_list('7458566549444739098','7458566549444722714')
+    # print(_result.content.decode("utf-8"))

BaseDataMaintenance/AIUtils/deepseek_request.py

@@ -0,0 +1,277 @@
+#coding:utf8
+
+import requests
+import time
+
+
+# API 端点
+url = "http://192.168.2.103:14514/api/generate"
+
+
+def request_deepseek(prompt,model="deepseek-r1:7b"):
+    # 请求数据
+    data = {
+        "model": model,
+        "prompt": prompt,
+        "stream": False,
+        "options": {
+            "temperature": 0,
+            "num_ctx": 10240,
+        },
+    }
+
+    # 发送 POST 请求
+    response = requests.post(url, json=data)
+
+    return_text = ""
+    # 检查响应状态码
+    if response.status_code == 200:
+        result = response.json()
+        # with open("a.json","w",encoding='utf-8') as f:
+        #     f.write(result["response"])
+        return_text = result["response"]
+    else:
+        print(f"请求失败，状态码: {response.status_code}，错误信息: {response.text}")
+        return_text = response.text
+    return response.status_code,return_text
+
+
+import requests
+
+free_url = "https://api.siliconflow.cn/v1/chat/completions"
+def request_deepseek_free(prompt,model="deepseek-ai/DeepSeek-R1-Distill-Llama-8B"):
+
+
+    payload = {
+        "model": model,
+        "messages": [
+            {
+                "role": "user",
+                "content": prompt
+            }
+        ],
+        "stream": False,
+        "options": {
+            "temperature": 0
+        },
+
+
+    }
+    headers = {
+        "Authorization": "Bearer sk-nmktkhqjuajlhyfythdinassblapkvznxbzxlmlimxmwzaal",
+        "Content-Type": "application/json"
+    }
+
+    response = requests.request("POST", free_url, json=payload, headers=headers)
+
+    return response.text
+
+if __name__ == '__main__':
+    start_time = time.time()
+    _prompt = '''
+    下面的html中文章的标题和正文的xpath是多少，按照实际html写出，格式如{"标题":"标题xpath","正文":"正文xpath"}
+    <html><head>
+    <meta charset="utf-8">
+    <meta content="width=device-width, initial-scale=1.0" name="viewport">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
+
+    <meta name="author" content="hngp.gov.cn">
+    <meta name="Description" content="惠济区政府采购网">
+    <meta name="Copyright" content="惠济区政府采购网 版权所有 请勿转载">
+    <meta name="KeyWords" content="惠济区政府采购网 电子化政府采购">
+    <title>惠济区政府采购网</title>
+    <link rel="icon" href="/resources/img/favicon.ico" types="image/x-icon">
+    <link rel="shortcut icon" href="/resources/img/favicon2.ico" types="image/x-icon">
+
+    <link href="/resources/style/animate.min.css" type="text/css" rel="stylesheet">
+    <link href="/resources/style/global.css" type="text/css" rel="stylesheet">
+    <link href="/resources/style/index.css" type="text/css" rel="stylesheet">
+    <script type="text/javascript" src="/resources/style/jquery.min.js"></script>
+    <title>惠济区政府采购网</title>
+    <style>
+        .htTable {
+            font-size: 12px;
+            text-decoration: none;
+            margin: 3px;
+            background-color: #cbcbcb;
+            color: #2b2b2b;
+        }
+
+        .htTable tr{
+            background-color: #FFFFFF;
+        }
+
+        .htTable td{
+            padding-left:5px;
+        }
+
+        .htLable {
+            line-height: 18px;
+            padding: 3px;
+            text-align: left;
+            vertical-align: middle;
+            padding: 5px;
+            background-color: #F4F4FF;
+        }
+
+        .htFileInfo{
+            margin: 0;
+            list-style: none;
+            cursor: default;
+            position: relative;
+            zoom: 1;
+            padding: 10px 20px 10px 35px;
+            border-bottom-width: 1px;
+            border-bottom-style: dotted;
+            border-color: #ccc;
+        }
+    </style>
+</head>
+<body>
+<!--Top Begin-->
+<div class="Header">
+    <!--Time Begin-->
+    <div class="W1200 Time Center">
+        <div class="Left">您好，欢迎访问惠济区政府采购网！</div>
+        <div class="Right Blue B">服务热线：0371-65808207、0371-65808480</div>
+        <div class="Clear"></div>
+    </div>
+    <!--Time end-->
+    <!--Logo Beign-->
+    <div class="shixianBg huiji">
+    <div class="W1200 Center">
+        <div class="Logo"><a href="#"><img src="/resources/images/huiji/logo.png"></a></div>
+        <form class="Search" action="/huiji/search" target="_blank">
+            <input type="submit" value="搜索" class="SeachMore Right">
+            <input type="hidden" name="ctk" value="4a56bf2f72cf49cfbecd9663da875a32">
+            <input name="q" type="text" class="TxtInput Right" placeholder="输入关键字，搜索相关信息">
+        </form>
+        <div class="Clear"></div>
+    </div>
+    </div>
+    <!--Login End-->
+</div>
+<div class="MenuBg">
+    <!--Nav Begin-->
+    <div class="W1200 Center nav  shixianNav">
+<ul>
+    <li><a href="/huiji" target="_self" class="selected">首页</a></li>
+    <li><a href="/huiji/list2?pageNo=1&amp;pageSize=16&amp;gglx=1&amp;bz=0" target="_self">工作动态</a>
+    </li>
+    <li><a href="/huiji/list2?pageNo=1&amp;pageSize=16&amp;gglx=2&amp;bz=0" target="_self">通知公告</a>
+    </li>
+    <li><a href="/huiji/ggcx" target="_self">采购信息</a>
+    </li>
+    <li><a href="https://zfcg.henan.gov.cn/gmfw/zz/" target="_self">购买服务</a></li>
+    <li><a href="/huiji/list2?pageNo=1&amp;pageSize=16&amp;gglx=5&amp;bz=0" target="_self">监督管理</a>
+    </li>
+    <li><a href="/huiji/list2?pageNo=1&amp;pageSize=16&amp;gglx=6&amp;bz=0" target="_self">政策法规</a>
+    </li>
+    <li><a href="/huiji/list2?pageNo=1&amp;pageSize=16&amp;gglx=10&amp;bz=0" target="_self">政策解读</a></li>
+    <li><a href="/huiji/list2?pageNo=1&amp;pageSize=16&amp;gglx=3&amp;bz=0" target="_self">操作指南</a>
+    </li>
+    <li><a href="/huiji/list2?pageNo=1&amp;pageSize=16&amp;gglx=4&amp;bz=0" target="_self">文件下载</a> </li>
+    <li><a href="/huiji/zxlist" target="_self">公众咨询</a></li>
+</ul>    </div>
+    <!--Nav End-->
+</div>
+<!--Top End-->
+<!--lmBanner Begin-->
+<div class="LmBanner">
+    <div class="W1200 Center">
+        <!--<h2>详细信息</h2>-->
+    </div>
+</div>
+<!--lmBanner end-->
+<div class="W1200 Center Top15 BorderEEE" id="print-content" style="min-height:600px;">
+    <h1 class="TxtCenter Top18 PaddingLR15 PaddingTop10">卫辉市沧河事务服务中心2025年2至3月政府采购意向</h1>
+    <div class="TxtCenter BorderTopDot Gray" style="margin:20px 30px 0px;">
+        发布机构：<span class="Gray">卫辉市沧河事务服务中心</span> &nbsp;&nbsp;
+        发布日期：<span class="Gray">2025-02-19 10:46</span>&nbsp;&nbsp;
+        访问次数：<span class="Gray">495</span>
+    </div>
+
+    <div class="Content">
+            <div id="content">
+
+
+  <meta charset="UTF-8">
+  <title>公告内容文档</title>
+
+
+<table class="Content">
+    <tbody>
+    <tr>
+        <td><p style="text-indent: 2em">为便于供应商及时了解政府采购信息，根据《河南省财政厅关于开展政府采购意向公开工作的通知》（豫财购〔2020〕8号）等有关规定，现将卫辉市沧河事务服务中心 2025年2（至）3月采购意向公开如下：</p></td>
+    </tr>
+    <tr>
+        <td class="suojin">
+            <table border="1" cellpadding="0" cellspacing="0" style="width: 90%;text-align: center;">
+                <tbody><tr style="height: 30px;line-height: 30px;">
+                    <td style="width:5%">序号</td>
+                    <td style="width:15%">采购单位名称</td>
+                    <td style="width:15%">采购项目名称</td>
+                    <td style="width:30%">采购需求概况</td>
+                    <td style="width:10%"><p class="hkys">预算金额</p><p class="hkys">（万元）</p></td>
+                    <td style="width:10%">预计采购时间</td>
+                    <td style="width:20%">备注</td>
+                </tr>
+                        <tr>
+                            <td>1</td>
+                            <td>卫辉市沧河事务服务中心</td>
+                            <td>卫辉市沧河事务服务中心卫辉市城乡供水一体化增补工程项目</td>
+                            <td><textarea class="TextAreaStyle" style="width: 90%; height: 100px;" readonly="">实施内容主要包括吴坡村工点(吴坡村西侧穿越京广高铁)、卫共大桥工点(京广高铁与比千大道交叉点东侧穿越京广高铁)、S226 工点(京广高铁与 S226 共产主义渠桥交叉点西侧穿越京广高铁)三处，每处顶管长度为82m，供水管道采用K9级DN800球墨铸铁给水管及管件，T型橡胶圈柔性接口，水泥砂浆衬里，重力流供水，长度 246m。</textarea></td>
+                            <td>1,299.00</td>
+                            <td> 2025年3月</td>
+                            <td><textarea class="TextAreaStyle" style="width: 90%; height: 100px;" readonly=""></textarea></td>
+                        </tr>
+            </tbody></table>
+        </td>
+    </tr>
+    <tr>
+        <td class="suojin"><p style="line-height:40px;font-weight:bolder;background:cornsilk;color:crimson;border:1px solid crimson;font-size:14px;text-align: center;width: 90%">本次公开的采购意向是本单位政府采购工作的初步安排，具体采购项目情况以相关采购公告和采购文件为准。</p></td>
+    </tr>
+    <tr><td style="padding-left: 70%">卫辉市沧河事务服务中心</td></tr>
+    <tr><td style="padding-left: 70%">2025年02月19日</td></tr>
+    </tbody>
+</table>
+
+</div>
+    </div>
+</div>
+
+<!--Bottom Begin-->
+<div class="Top15 SiteBottom">
+<div class="W1200 Center TxtCenter">
+        <span class="Right Top10"><a href="http://bszs.conac.cn/sitename?method=show&amp;id=AE64A9A5321A96D6E05310291AAC5767" target="_blank" class="dzjp"><img align="absmiddle" src="https://file.henan.gov.cn/4500000012/img/dzjg.png"></a><a href="https://zfwzzc.www.gov.cn/check_web/errorInfo/jcInfoNew?siteCode=4100000057&amp;url=http://www.hngp.gov.cn"><img align="absmiddle" src="/resources/images/zhaocuo.jpg"></a></span>
+        <p>版权所有 ©2007 河南省财政厅                网站标识码：4100000057  备案号：<a href="https://beian.miit.gov.cn/#/Integrated/ComplaintA" target="_blank">豫ICP备09005258号</a>  <a href="http://www.beian.gov.cn/portal/registerSystemInfo?recordcode=41010502006131" target="_blank"><img src="https://file.henan.gov.cn/4500000012/img/beian.jpg">豫公网安备 41010502006131号</a></p>
+        <p>地址：河南省郑州市经三路25号  邮编：450008  技术服务电话：0371-65808207、0371-65808480</p>
+        <div id="online">
+            <p>今日本站访问人数：5294，今日本站访问量：7916，今日全站访问量：13467346，累计全站访问量：28085607101</p>
+        </div>
+</div>
+</div>
+<!--Bottom End-->
+<script>
+    jQuery(document).ready(function () {
+        $.get("/cmsweb81e27e/nas/webfile2024//weihui/cgxx/cgyx/webinfo/2025/02/1737799.htm", function (data) {
+            var s = data;
+            
+        $("#content").html(s);
+        });
+    });
+</script>
+
+
+
+</body><div class="xl-chrome-ext-bar_4DB361DE-01F7-4376-B494-639E489D19ED" id="xl_chrome_ext_4DB361DE-01F7-4376-B494-639E489D19ED" data-v-app="" style="display: block;"><div class=""><div id="xl_chrome_ext_video_tag_wrapper_4DB361DE-01F7-4376-B494-639E489D19ED" config="[object Object]" uiversion="v2" exception="false" latestvideosrc="" isshowvideotag="false" isshowdownloadbar="true" isshowcloudaddbar="false"><div class="_video_op_wrapper_96mx8_1" config="[object Object]" uiversion="v2" style="display: none;"><ul class="_video_op_list_96mx8_17"><li class="_op_item_96mx8_39 _download_96mx8_45"><span class="_op_icon_96mx8_25"></span><span class="_op_text_96mx8_31">下载</span></li><li class="_op_item_96mx8_39 _screen_96mx8_51"><span class="_op_icon_96mx8_25"></span><span class="_op_text_96mx8_31">投屏</span></li></ul></div></div><div class="badge_wrapper_4DB361DE-01F7-4376-B494-639E489D19ED _badge_wrapper_eor63_1" id="badge_wrapper_4DB361DE-01F7-4376-B494-639E489D19ED"><div class="_logo_eor63_20"></div><span class="_line_eor63_27"></span><div class="_text_wrapper_eor63_33"><span class="_download_icon_eor63_39"></span><span class="_text_eor63_33">高速下载</span></div></div><!----><!----><!----><!----></div></div></html>
+
+    '''
+    new_prompt = _prompt
+    print("prompt length %d"%len(new_prompt))
+    status_code,resp_text = request_deepseek(new_prompt)
+    # resp_text = request_deepseek_free(_prompt,model='deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B')
+    # 修改标准输出的编码为 utf-8
+    resp_text.encode('gbk', 'ignore').decode('gbk')
+    print(resp_text)
+    print("cost",time.time()-start_time)

BaseDataMaintenance/AIUtils/html2text.py

@@ -0,0 +1,36 @@
+#coding:utf8
+
+from bs4 import BeautifulSoup
+import re
+
+def html2text_with_tablehtml(_html):
+    # 如果输入是字符串，使用 BeautifulSoup 解析
+    if isinstance(_html, str):
+        _soup = BeautifulSoup(_html, "lxml")
+    else:
+        _soup = _html
+
+    # 用于存储处理后的文本
+    result_parts = []
+
+    _find = False
+    # 遍历所有直接子元素
+    for child in _soup.find_all(recursive=False):
+        if child.name in ["table", "tbody"]:
+            # 如果是表格或表格主体，保留 HTML 代码
+            result_parts.append(str(child))
+        else:
+            # 递归处理其他元素并转换为文本
+            text = html2text_with_tablehtml(child)
+            result_parts.append(text)
+        _find = True
+    if not _find:
+        result_parts.append(str(_soup.get_text()))
+
+    # 将所有处理后的部分连接成一个字符串
+    result = "\n".join(result_parts)
+    return result
+
+if __name__ == '__main__':
+    _html = "<div><p>这是一个p</p><table><tr><td>这是一个td</td></tr></table></div>"
+    print(html2text_with_tablehtml(_html))

BaseDataMaintenance/AIUtils/prompts.py

@@ -0,0 +1,23 @@
+#coding:utf8
+
+
+
+def get_prompt_extract_role():
+    _prompt = '''
+    #角色
+    招投标信息提取专家
+    #工作流程1
+    1. 通读全文
+    2. 找到其中的招标人
+    3. 找到其中的中标信息，如中标人所在标段的中标金额
+    #规范1
+    1. 要求提取的招标人和中标人必须是机构实体，如不是，则为空
+    2. 招标人是招标公告的业主单位
+    3. 中标人必须是该标的中标企业
+    4. 标段号是指中标人所中标的包号：如A包则为A，1包则为1
+    5. 中标信息中的中标人名称、中标金额和标段号必须和文中的信息对应，如果有多个，则再数组中返回多个
+    #返回结果1
+    结果返回json格式{"招标人":"","中标信息":[{"中标人名称":"","中标金额":"":"标段号":""}]}
+    '''
+    return _prompt
+

BaseDataMaintenance/maintenance/2.py

@@ -1,14 +1,36 @@
+#coding:utf8
 
-from multiprocessing import Queue
+from bs4 import BeautifulSoup
+import re
 
+def html2text(_html):
+    # 如果输入是字符串，使用 BeautifulSoup 解析
+    if isinstance(_html, str):
+        _soup = BeautifulSoup(_html, "lxml")
+    else:
+        _soup = _html
 
-a = Queue()
+    # 用于存储处理后的文本
+    result_parts = []
 
-a.put("1")
+    _find = False
+    # 遍历所有直接子元素
+    for child in _soup.find_all(recursive=False):
+        if child.name in ["table", "tbody"]:
+            # 如果是表格或表格主体，保留 HTML 代码
+            result_parts.append(str(child))
+        else:
+            # 递归处理其他元素并转换为文本
+            text = html2text(child)
+            result_parts.append(text)
+        _find = True
+    if not _find:
+        result_parts.append(str(_soup.get_text()))
 
-while 1:
-    try:
-        b = a.get(block=False)
-        print(b)
-    except Exception as e:
-        break
+    # 将所有处理后的部分连接成一个字符串
+    result = "\n".join(result_parts)
+    return result
+
+if __name__ == '__main__':
+    _html = "<div><p>这是一个p</p><table><tr><td>这是一个td</td></tr></table></div>"
+    print(html2text(_html))

BaseDataMaintenance/maintenance/dataflow_mq.py

@@ -21,6 +21,10 @@ sys.setrecursionlimit(1000000)
 
 from multiprocessing import Process
 
+from BaseDataMaintenance.AIUtils.DoubaoUtils import chat_doubao,get_json_from_text
+from BaseDataMaintenance.AIUtils.html2text import html2text_with_tablehtml
+from BaseDataMaintenance.AIUtils.prompts import get_prompt_extract_role
+
 class ActiveMQListener():
 
     def __init__(self,conn,_queue,*args,**kwargs):
@@ -809,6 +813,7 @@ class Dataflow_ActivteMQ_extract(Dataflow_extract):
 
 
         self.mq_extract = "/queue/dataflow_extract"
+        self.mq_extract_ai = "/queue/dataflow_extract_AI"
         self.mq_extract_failed = "/queue/dataflow_extract_failed"
 
         self.whole_weight = 0
@@ -1074,6 +1079,8 @@ class Dataflow_ActivteMQ_extract(Dataflow_extract):
 
             _fingerprint = getFingerprint(str(data["title"])+str(data["content"]))+str(data["original_docchannel"])
 
+            to_ai = False
+
             if all_done>0:
                 _time = time.time()
                 # extract_json = self.getExtract_json_fromDB(_fingerprint)
@@ -1096,6 +1103,7 @@ class Dataflow_ActivteMQ_extract(Dataflow_extract):
                     else:
                         log("%s--%s"%(str(resp.status_code),resp.content.decode("utf8")))
                         all_done = -2
+                to_ai = self.should_to_extract_ai(extract_json)
             # if all_done>0:
             #     resp = self.request_industry_interface(json=data,headers=self.header)
             #     if (resp.status_code >=200 and resp.status_code<=213):
@@ -1169,6 +1177,10 @@ class Dataflow_ActivteMQ_extract(Dataflow_extract):
 
             if _to_ack:
                 ackMsg(conn,message_id,subscription)
+                if to_ai:
+                    #sent to ai
+                    item[document_extract2_extract_json] = _extract.getProperties().get(document_extract2_extract_json,"{}")
+                    send_msg_toacmq(self.pool_mq,json.dumps(item,ensure_ascii=False),self.mq_extract_ai)
             else:
                 item["extract_times"] -= 1
                 send_msg_toacmq(self.pool_mq,json.dumps(item,ensure_ascii=False),self.mq_extract)
@@ -1221,6 +1233,73 @@ class Dataflow_ActivteMQ_extract(Dataflow_extract):
                     ackMsg(conn,message_id,subscription)
 
 
+    def should_to_extract_ai(self,extract_json):
+        _extract = {}
+        if extract_json is not None:
+            try:
+                _extract = json.loads(extract_json)
+            except Exception as e:
+                pass
+
+        has_entity = False
+        has_tenderee = False
+        has_win_tenderer = False
+        docchannel = _extract.get("docchannel",{}).get("life_docchannel","")
+        if len(_extract.get("dict_enterprise",{}).keys())>0:
+            has_entity = True
+        prem = _extract.get("prem",{})
+        for k,v in prem.items():
+            for role in v.get("roleList",[]):
+                if role.get("role_name")=="tenderee":
+                    has_tenderee = True
+                if role.get("role_name")=="win_tenderer":
+                    has_win_tenderer = True
+
+        if has_entity:
+            if not has_tenderee:
+                return True
+            if not has_win_tenderer and docchannel in {"中标信息","候选人公示","合同公告","开标记录","验收合同"}:
+                return True
+
+
+    def extract_ai_handle(self,_dict,result_queue):
+        frame = _dict["frame"]
+        conn = _dict["conn"]
+        message_id = frame.headers["message-id"]
+        subscription = frame.headers.setdefault('subscription', None)
+        item = json.loads(frame.body)
+        dtmp = Document_tmp(item)
+
+        dhtml = Document_html({"partitionkey":item.get("partitionkey"),
+                               "docid":item.get("docid")})
+
+        extract_times = item.get("extract_times",0)+1
+        item["extract_times"] = extract_times
+
+        _extract_ai = {}
+        try:
+            _dochtmlcon = item.get(document_tmp_dochtmlcon,"")
+
+            _text = html2text_with_tablehtml(_dochtmlcon)
+
+            msg = "%s\n公告内容如下\n%s"%(get_prompt_extract_role(),_text)
+
+            result = chat_doubao(msg)
+
+            _json = get_json_from_text(result)
+
+            if _json is not None:
+                _extract_ai = json.loads(_json)
+        except Exception as e:
+            pass
+
+
+
+
+
+
+
+
     def delete_document_extract(self,save_count=70*10000):
         conn = self.pool_postgres.getConnector()
         try: