BIDI_EXTRACTION_AI/BiddingKG/dl/template_extract/table_extractor.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
@author: bidikeji
@time: 2025/10/11 11:48
"""
from lxml import etree
import copy
import re
from BiddingKG.dl.template_extract.abstract_template import table2list

def get_html_table(html):
    """
    解析html提取表格内容
    :param html:
    :return: 表格列表
    """
    table_list = []
    tree = etree.HTML(html)
    # 拆分正文和附件
    richText = tree.xpath('//div[@class="richTextFetch"]')
    if richText:
        parent = richText[0].getparent()
        richText_tree = copy.deepcopy(richText[0])
        parent.remove(richText[0])
        # richText = re.sub('\s', '', richText_tree.xpath('string(.)'))
        # if len(richText) < 2:
        #     print('异常附件：', docid)
        # print('richText:', richText[:20])
        # print(tree.xpath('//div[@class="richTextFetch"]'))
    tables = tree.xpath('//table')
    for table in tables:
        # 判断是否有嵌套表格
        inner_tb = table.xpath('././/table')
        filter_flag = False  # 嵌套表格行数大于等于2,不处理外部表格
        for tb in inner_tb:
            parent = tb.getparent()
            # print('嵌套表格列数：', len(tb.xpath('././/tr')))
            # 如果嵌套表格行数小于2，直接把表格内容赋予附标签
            if 0 < len(tb.xpath('././/tr')) < 2:
                parent.text = parent.text.strip() if parent.text else ''
                parent.text += ' '.join(tb.xpath('././/text()')).strip()
            else:
                filter_flag = True
            parent.remove(tb)
        # print('表格内容：', table.xpath('.//td//text()|.//th//text()'))
        if filter_flag:
            continue
        l = table2list(table)
        table_list.append(l)
    return table_list

def get_table_preceding_text(table):
    caption_text = table.xpath('string(./caption)').strip()
    if caption_text:
        return caption_text
    preceding_text = ''
    preceding_tags = table.xpath('./preceding-sibling::*')
    if preceding_tags == []:
        if table.getparent() is not None:
            preceding_tags = table.getparent().xpath('./preceding-sibling::*')
    block_num = 0
    for tag in reversed(preceding_tags):
        if tag.tag == 'table':
            break
        elif tag.tag == 'div' and (tag.xpath('.//table') or len(tag.xpath('p'))>2):
            break

        texts = tag.xpath('.//text()')
        current_text = ''
        for text in reversed(texts):
            text = text.strip()
            if text:
                current_text = text + current_text
        preceding_text = current_text + preceding_text
        if tag.tag in ['p', 'div', 'br', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6']:
            block_num += 1
            if block_num > 2:
                break
            if current_text.strip():
                break
        if len(preceding_text) > 100:
            break
    return preceding_text

def get_html_table_and_prev_text(html, return_richText=False):
    """
    解析html提取表格内容及表格前文
    :param html:
    :return: 表格列表
    """
    def extract_table(tree):
        '''
        提取所有表格并返回每个表格内容列表矩阵及表格前文
        :param tree: etree 根节点
        :return:
        '''
        tables = tree.xpath('//table')
        table_list = []
        tables_prev_text = []
        for table in tables:
            # 判断是否有嵌套表格
            inner_tb = table.xpath('././/table')
            filter_flag = False  # 嵌套表格行数大于等于2,不处理外部表格
            for tb in inner_tb:
                parent = tb.getparent()
                # print('嵌套表格列数：', len(tb.xpath('././/tr')))
                # 如果嵌套表格行数小于2，直接把表格内容赋予附标签
                if 0 < len(tb.xpath('././/tr')) < 2:
                    parent.text = parent.text.strip() if parent.text else ''
                    parent.text += ' '.join(tb.xpath('././/text()')).strip()
                else:
                    filter_flag = True
                parent.remove(tb)
            if filter_flag:
                continue
            # table_grid = table2list(table)
            table_grid = parse_table_to_grid(table)
            full_prev_text = get_table_preceding_text(table)
            table_list.append(table_grid)
            tables_prev_text.append(full_prev_text)
        return table_list, tables_prev_text

    tree = etree.HTML(html)
    table_list_richText = []
    tables_prev_text_richText = []
    # 拆分正文和附件
    richText = tree.xpath('//div[@class="richTextFetch"]')
    if richText:
        if return_richText:
            table_list_richText, tables_prev_text_richText = extract_table(richText[0])
        parent = richText[0].getparent()
        parent.remove(richText[0])

    table_list, tables_prev_text = extract_table(tree)
    if return_richText:
        return table_list, tables_prev_text, table_list_richText, tables_prev_text_richText
    else:
        return table_list, tables_prev_text


def normalize_text(text):
    """清理文本：去除多余空白，保留单个空格"""
    if text is None:
        return ""
    return re.sub(r'\s+', ' ', text.strip())

def parse_table_to_grid(table_element):
    """
    将 lxml 的 table 元素解析为二维网格（list of lists）
    处理 rowspan 和 colspan
    """
    rows = table_element.xpath('.//tr')
    if not rows:
        return []

    # 第一步：收集所有单元格及其位置信息
    grid = []  # 最终的二维网格
    row_index = 0

    for tr in rows:
        # 找到当前行实际起始列（跳过已被 rowspan 占用的格子）
        if len(grid) <= row_index:
            grid.append([])

        col_index = 0
        # 跳过已被上方 rowspan 占据的列
        while col_index < len(grid[row_index]) and grid[row_index][col_index] is not None:
            col_index += 1

        cells = tr.xpath('.//td | .//th')
        for cell in cells:
            # 跳过已被占据的位置
            while col_index < len(grid[row_index]) and grid[row_index][col_index] is not None:
                col_index += 1

            # 获取文本内容
            text = normalize_text(''.join(cell.xpath('.//text()')))
            # 处理省略号情况：如果有title属性且文本以...结尾，使用title内容
            title_attr = cell.get('title')
            if (title_attr and text.replace(' ', '').endswith('...') and
                    title_attr.replace(' ', '').startswith(text.replace(' ', '')[:-3])):
                text = title_attr
            text = re.sub('\s', '', text).replace('(', '（').replace(')', '）') # 修复 653042182 成交金额 被不同标签拆分中间有空格导致提取失败

            # 获取 colspan 和 rowspan（默认为1）
            colspan = int(cell.get('colspan', '1')) if cell.get('colspan', '1').isdigit() and int(cell.get('colspan', '1')) > 0 else 1 # 修复 234475886 colspan="0"异常情况
            rowspan = int(cell.get('rowspan', '1')) if cell.get('rowspan', '1').isdigit() and int(cell.get('rowspan', '1')) > 0 else 1

            # 确保网格足够大
            for r in range(row_index, row_index + rowspan):
                while len(grid) <= r:
                    grid.append([])
                while len(grid[r]) <= col_index + colspan - 1:
                    grid[r].append(None)

            # 填充主单元格
            grid[row_index][col_index] = text

            # 标记被 colspan 占据的位置（同一行）
            for c in range(col_index + 1, col_index + colspan):
                # grid[row_index][c] = ''  # 空字符串表示被合并
                grid[row_index][c] = text

            # 标记被 rowspan 占据的位置（下方行）
            for r in range(row_index + 1, row_index + rowspan):
                for c in range(col_index, col_index + colspan):
                    # grid[r][c] = ''  # 被上方 rowspan 占据
                    grid[r][c] = text

            col_index += colspan

        row_index += 1

    # 修复第一行前面有空格异常导致表头内容错位 例：715247564
    if len(grid) > 1 and len(grid[0]) > len(grid[1]):
        n = 0
        for it in grid[0]:
            if it == '':
                n += 1
            else:
                break
        if n > 0 and len(grid[0]) - n == len(grid[1]):
            grid[0] = grid[0][n:]

    # 清理：确保所有行长度一致（补 None）
    max_cols = max(len(row) for row in grid) if grid else 0
    for row in grid:
        while len(row) < max_cols:
            row.append(None)

    # 将 None 替换为空字符串（未定义的单元格）
    for i in range(len(grid)):
        for j in range(len(grid[i])):
            if grid[i][j] is None:
                grid[i][j] = ""

    # 移除重复的行
    filtered_grid = []
    for row in grid:
        if row not in filtered_grid:
            filtered_grid.append(row)

    return filtered_grid


if __name__ == "__main__":
    with open('d:/html/2.html', encoding='utf-8') as f:
        html = f.read()
        # table_list = get_html_table(html)
        # for table in table_list:
        #     for data in table:
        #         print(data)
        #     print([len(row) for row in table])

        table_list, tables_prev_text = get_html_table_and_prev_text(html)
        i = 0
        for table in table_list:
            print('表格前文：', tables_prev_text[i])
            i += 1
            for data in table:
                print(data)
            print([len(row) for row in table])