一、环境搭建

官方代码：GitHub - ymy-k/DPText-DETR: [AAAI'23 Oral] DPText-DETR: Towards Better Scene Text Detection with Dynamic Points in Transformer

建议使用 Anaconda 配置环境。推荐 Python 3.8 + PyTorch 1.9.1（或 1.9.0）+ CUDA 11.1 + Detectron2 (v0.6)。

按照readme.md安装，在运行python setup.py build develop之前没出现问题；编译时可能缺少部分所需头文件，查找原因是cuda环境路径问题，因实验室电脑不方便安装指定cuda，转向docker配置。

docker中编译时，有两个包numba，rapidfuzz版本不对，安装合适版本后编译完成。

下载权重文件，运行demo.py，rapidfuzz库部分函数改变，询问AI简单修改完成推理。

二、数据准备

Train  

标签搞成了一个大json，通过txt_to_16points.py将txt标签四点坐标转为模型输入要求的16点坐标。通过process_polygon_positional.py处理成作者论文说的从左上角开始的顺时针的16个点的标注，标注格式为COCO。点的存储路径annotations的polys下。

AI生成脚本txt_to_16points.py将txt标签四点坐标转为模型输入要求的16点坐标：

# txt_to_16points.py

import json
import os
import cv2
import numpy as np
from tqdm import tqdm


def convert_quad_to_bezier(quad_points):
    """
    将四边形点转换为Bezier控制点格式（生成16个控制点）
    输入: [x0,y0, x1,y1, x2,y2, x3,y3] (4个点，8个数值)
    输出: 32个控制点坐标值(16个点)
    """
    # 解包四边形点 (左上->右上->右下->左下)
    x0, y0, x1, y1, x2, y2, x3, y3 = quad_points

    # 计算顶部控制点 (左上->右上) - 8个点
    top_points = []
    for i in np.linspace(0, 1, 8):
        x = x0 + (x1 - x0) * i
        y = y0 + (y1 - y0) * i
        top_points.append([x, y])

    # 计算底部控制点 (右下->左下) - 8个点
    bottom_points = []
    for i in np.linspace(0, 1, 8):
        x = x2 + (x3 - x2) * i
        y = y2 + (y3 - y2) * i
        bottom_points.append([x, y])

    # 组合所有点 (先顶部后底部)
    all_points = top_points + bottom_points

    # 展平为32个数值的列表
    flattened = [coord for point in all_points for coord in point]
    return flattened

def calculate_axis_aligned_bbox(points):
    """
    计算旋转文本的轴对齐边界框
    输入: 四边形点列表 [x0,y0, x1,y1, x2,y2, x3,y3]
    输出: [x_min, y_min, x_max, y_max]
    """
    pts = np.array(points).reshape(4, 2)
    x_min, y_min = np.min(pts, axis=0)
    x_max, y_max = np.max(pts, axis=0)
    return [x_min, y_min, x_max, y_max]


def create_totaltext_dataset(images_dir, labels_dir, output_json):
    """
    将整个数据集目录转换为TotalText格式的JSON

    参数:
        images_dir: 包含图像的目录
        labels_dir: 包含标注文本文件的目录
        output_json: 输出JSON文件路径
    """
    # 获取所有标注文件
    label_files = [f for f in os.listdir(labels_dir) if f.endswith('.txt')]

    label_files.sort(key=lambda x: (int(os.path.basename(x)[:-4])if os.path.basename(x)[:-4].isdigit() else float('inf'), x))
    zero_file_path = os.path.join(labels_dir, '0000000.txt')
    if zero_file_path in label_files:
        label_files.insert(0, label_files.pop(label_files.index(zero_file_path)))

    # label_files = [f for f in os.listdir(labels_dir) if f.endswith('.txt')]

    # 创建基本数据结构
    dataset = {
        "images": [],
        "categories": [{"supercategories": "text", "id": 1, "name": "text"}],
        "annotations": []
    }

    annotation_id = 0

    # 处理每个文件


    for label_file in tqdm(label_files, desc="处理标注文件"):
        # 获取对应的图像文件
        image_file = label_file.replace('.txt', '.jpg')
        image_path = os.path.join(images_dir, os.path.basename(image_file))

        # 检查图像是否存在
        if not os.path.exists(image_path):
            print(f"警告: 图像文件 {image_file} 不存在，跳过 {label_file}")
            continue

        # 读取图像获取尺寸
        img = cv2.imread(image_path)
        if img is None:
            print(f"警告: 无法读取图像 {image_file}，跳过 {label_file}")
            continue

        height, width = img.shape[:2]

        # 创建图像条目
        image_id = len(dataset["images"])
        dataset["images"].append({
            "id": image_id,
            "width": width,
            "height": height,
            "file_name": image_file
        })

        # 读取标注文件
        label_path = os.path.join(labels_dir, label_file)
        try:
            with open(label_path, 'r', encoding='utf-8') as f:
                lines = f.readlines()
        except:
            print(f"警告: 无法读取标注文件 {label_file}，跳过")
            continue

        # 处理每个文本实例
        for line in lines:
            parts = line.strip().split(',')
            if len(parts) < 8:
                continue

            # 提取坐标和文本
            try:
                coords = list(map(float, parts[:8]))
                text = ','.join(parts[8:]).strip() if len(parts) > 8 else ""
            except:
                print(f"格式错误: {label_file} - {line}")
                continue

            # 转换为Bezier控制点 - 现在生成32个坐标值(16个点)
            bezier_pts = convert_quad_to_bezier(coords)

            # 检查点数量是否正确
            if len(bezier_pts) != 32:
                print(f"警告: {label_file} 中的实例生成点数错误: {len(bezier_pts)}")
                continue

            # 计算两种边界框
            x_min, y_min, x_max, y_max = calculate_axis_aligned_bbox(coords)
            aabb = [x_min, y_min, x_max - x_min, y_max - y_min]

            # 创建标注对象
            annotation = {
                "id": annotation_id,
                "image_id": image_id,
                "category_id": 1,
                "polys": bezier_pts,  # 32个数值
                "bbox": aabb,  # 轴对齐边界框
            }

            dataset["annotations"].append(annotation)
            annotation_id += 1

    # 保存JSON文件
    with open(output_json, 'w', encoding='utf-8') as f:
        json.dump(dataset, f, indent=2, ensure_ascii=False)

    print(f"\n转换完成! 共处理:")
    print(f"  - 图像: {len(dataset['images'])} 张")
    print(f"  - 文本实例: {len(dataset['annotations'])} 个")
    print(f"结果已保存至: {output_json}")


if __name__ == "__main__":
    # 使用示例
    IMAGES_DIR = ""  # 图像目录
    LABELS_DIR = ""  # 标注文本文件目录
    OUTPUT_JSON = "gt_poly.json"  # 输出JSON文件路径

    create_totaltext_dataset(IMAGES_DIR, LABELS_DIR, OUTPUT_JSON)

Eval

可以在训练时进行eval（训练时可以eval也可以不eval）；也可以训练完成后单独对测试集进行eval。

eval时除了text_poly_pos.json外还需要一个zip文件，zip中每个txt文件是一张图片的标注（注意：zip时不要把文件夹压缩进去，在文件夹内部进行打包）。eval时图像和txt的命名对应，txt命名一定是从0000000.txt、0000001.txt...开始，代码中有限制（训练集无限制）。

数据集文件目录如下:

|-datasets
   |-full_ocr
   | |-train_images
   | |-test_images
   | └─train_ploy_pos.json
   | └─test_ploy_pos.json
   |-evaluation
   | |-gt_train.zip
   | └─gt_test.zip

配置文件

训练时，当成是TotalText数据集，主要有以下几个配置文件：

configs/DPText_DETR/TotalText/R_50_poly.yaml
configs/DPText_DETR/Base.yaml
adet/data/builtin.py # 自定义数据集登记

# configs/DPText_DETR/TotalText/R_50_poly.yaml
_BASE_: "../Base.yaml"

DATASETS: # 键值对，builtin.py中指向了对应的图片及json的路径
  TRAIN: ("totaltext_poly_train_pos",)
  TEST: ("totaltext_poly_test",)

MODEL:
  WEIGHTS: "detectron2://ImageNetPretrained/torchvision/R-50.pkl"

SOLVER:
  IMS_PER_BATCH: 1 # batch_size
  BASE_LR: 4e-5 # lr 
  LR_BACKBONE: 4e-6
  WARMUP_ITERS: 3000
  STEPS: (320000,) # 学习率调整iter
  MAX_ITER: 400000 
  CHECKPOINT_PERIOD: 20000

TEST:
  EVAL_PERIOD: 10000

OUTPUT_DIR: "output/r_50_poly/pretrain" # 模型保存路径

# configs/DPText_DETR/Base.yaml
MODEL:
  META_ARCHITECTURE: "TransformerPureDetector"
  MASK_ON: False
  PIXEL_MEAN: [123.675, 116.280, 103.530]
  PIXEL_STD: [58.395, 57.120, 57.375]
  BACKBONE:
    NAME: "build_resnet_backbone"
  RESNETS:
    DEPTH: 50
    STRIDE_IN_1X1: False
    OUT_FEATURES: ["res3", "res4", "res5"]
  TRANSFORMER:
    ENABLED: True
    NUM_FEATURE_LEVELS: 4
    ENC_LAYERS: 6 # encoder 层数
    DEC_LAYERS: 6
    DIM_FEEDFORWARD: 1024
    HIDDEN_DIM: 256
    DROPOUT: 0.1
    NHEADS: 8
    NUM_QUERIES: 400 # 可学习的查询向量数量,切片的数量，限制输出检测框数量，需要根据场景调整
    ENC_N_POINTS: 4
    DEC_N_POINTS: 4
    USE_POLYGON: True
    NUM_CTRL_POINTS: 16
    EPQM: True
    EFSA: True
    INFERENCE_TH_TEST: 0.4 # 推理时输出bbox的阈值，这个值越小，输出的bbox越多，过小可能导致重叠的bbox

SOLVER:
  WEIGHT_DECAY: 1e-4
  OPTIMIZER: "ADAMW"
  LR_BACKBONE_NAMES: ['backbone.0']
  LR_LINEAR_PROJ_NAMES: ['reference_points', 'sampling_offsets']
  LR_LINEAR_PROJ_MULT: 0.1
  CLIP_GRADIENTS:
    ENABLED: True
    CLIP_TYPE: "full_model"
    CLIP_VALUE: 0.1
    NORM_TYPE: 2.0

INPUT:
  MIN_SIZE_TRAIN: (480, 512, 544, 576, 608, 640, 672, 704, 736, 768, 800, 832,) # 多尺度训练，训练时图像的短边随机选择这些值之一
  MAX_SIZE_TRAIN: 1600 # 长边不超过1600
  MIN_SIZE_TEST: 1000 # 短边=1000
  MAX_SIZE_TEST: 1800 # 长边<=1800
  CROP:
    ENABLED: True
    CROP_INSTANCE: False
    SIZE: [0.1, 0.1]
  FORMAT: "RGB"

TEST:
  DET_ONLY: True  # evaluate only detection metrics

# adet/data/builtin.py # 自定义数据集登记
import os

from detectron2.data.datasets.register_coco import register_coco_instances
from detectron2.data.datasets.builtin_meta import _get_builtin_metadata

from .datasets.text import register_text_instances

# register plane reconstruction

_PREDEFINED_SPLITS_PIC = {
    "pic_person_train": ("pic/image/train", "pic/annotations/train_person.json"),
    "pic_person_val": ("pic/image/val", "pic/annotations/val_person.json"),
}

metadata_pic = {
    "thing_classes": ["person"]
}

_PREDEFINED_SPLITS_TEXT = {
    # training sets with polygon annotations
    "syntext1_poly_train_pos": ("syntext1/train_images", "syntext1/train_poly_pos.json"),
    "syntext2_poly_train_pos": ("syntext2/train_images", "syntext2/train_poly_pos.json"),
    "mlt_poly_train_pos": ("mlt/train_images","mlt/train_poly_pos.json"),
    "totaltext_poly_train_ori": ("totaltext/train_images_rotate", "totaltext/train_poly_ori.json"),
    "totaltext_poly_train_pos": ("full_ocr/train_images", "full_ocr/train_poly_pos_datasets.json"),
    "totaltext_poly_train_rotate_ori": ("totaltext/train_images_rotate", "totaltext/train_poly_rotate_ori.json"),
    "totaltext_poly_train_rotate_pos": ("full_ocr/rotate_images", "full_ocr/train_rotate_poly.json"),
    "ctw1500_poly_train_rotate_pos": ("ctw1500/train_images_rotate", "ctw1500/train_poly_rotate_pos.json"),
    "lsvt_poly_train_pos": ("lsvt/train_images","lsvt/train_poly_pos.json"),
    "art_poly_train_pos": ("art/train_images_rotate","art/train_poly_pos.json"),
    "art_poly_train_rotate_pos": ("art/train_images_rotate","art/train_poly_rotate_pos.json"),
    #-------------------------------------------------------------------------------------------------------
    "totaltext_poly_test": ("full_ocr/test_images", "full_ocr/img_test_poly.json"),
    "totaltext_poly_test_rotate": ("totaltext/test_images_rotate", "totaltext/test_poly_rotate.json"),
    "ctw1500_poly_test": ("ctw1500/test_images","ctw1500/test_poly.json"),
    "art_test": ("art/test_images","art/test_poly.json"),
    "inversetext_test": ("inversetext/test_images","inversetext/test_poly.json"),
}

metadata_text = {
    "thing_classes": ["text"]
}


def register_all_coco(root="datasets"):
    for key, (image_root, json_file) in _PREDEFINED_SPLITS_PIC.items():
        # Assume pre-defined datasets live in `./datasets`.
        register_coco_instances(
            key,
            metadata_pic,
            os.path.join(root, json_file) if "://" not in json_file else json_file,
            os.path.join(root, image_root),
        )
    for key, (image_root, json_file) in _PREDEFINED_SPLITS_TEXT.items():
        # Assume pre-defined datasets live in `./datasets`.
        register_text_instances(
            key,
            metadata_text,
            os.path.join(root, json_file) if "://" not in json_file else json_file,
            os.path.join(root, image_root),
        )


register_all_coco()

评估时，需要修改的代码文件：

adet/evaluation/text_evaluation_det.py
adet/evaluation/text_evaluation.py # eval数据集登记

# use dataset_name to decide eval_gt_path
        if "rotate" in dataset_name:
            if "totaltext" in dataset_name:
                self._text_eval_gt_path = "datasets/evaluation/gt_totaltext_rotate.zip"
        elif "totaltext" in dataset_name:
            self._text_eval_gt_path = "/data/kechen.han/code/DPText-DETR-main/datasets/evaluation/gt_test.zip"
        elif "ctw1500" in dataset_name:
            self._text_eval_gt_path = "datasets/evaluation/gt_ctw1500.zip"
        elif "art" in dataset_name:
            self._text_eval_gt_path = None
            self.submit = True
        elif "inversetext" in dataset_name:
            self._text_eval_gt_path = "datasets/evaluation/gt_inversetext.zip"
        else:
            raise NotImplementedError

三、训练和评估

Training

 1. __Pre-train:__  为了在Total-Text上预训练模型，应该修改配置文件`/DPText_DETR/Pretrain/R_50_poly.yaml.`请根据你的情况调整GPU的数量。

python tools/train_net.py --config-file ${CONFIG_FILE} --num-gpus 4

 代码改动，训练时会报错ccw问题，即提示某个box的点不是顺时针排列的，但是print出的box又是顺时针的。参考另外一篇文章，可能是源码问题，修改：

# adet/evaluation/rrc_evaluation_funcs_det.py
    if not pRing.is_ccw:
        assert (0), (
            "Points are not clockwise. The coordinates of bounding quadrilaterals have to be given in clockwise order. Regarding the correct interpretation of 'clockwise' remember that the image coordinate system used is the standard one, with the image origin at the upper left, the X axis extending to the right and Y axis extending downwards.")

2. Fine-tune: 带有预训练模型，使用下面的指令去在目标基准上微调。预训练模型需要被提供，示例：

python tools/train_net.py --config-file configs/DPText_DETR/TotalText/R_50_poly.yaml --num-gpus 4

Evaluation

python tools/train_net.py --config-file ${CONFIG_FILE} --eval-only MODEL.WEIGHTS ${MODEL_PATH}

模型收敛速度很快，训练到260k就达到最高性能，得到precision、recall、hmean三个指标。

Inference & Visualization:

python demo/demo.py --config-file ${CONFIG_FILE} --input ${IMAGES_FOLDER_OR_ONE_IMAGE_PATH} --output ${OUTPUT_PATH} --opts MODEL.WEIGHTS <MODEL_PATH>

四、结果分析

根据自己的数据集调整配置文件中的NUM_QUERIES和INFERENCE_TH_TEST参数。

可视化结果分析：
模型优点：

边界框回归贴合，精确

可以检测弯曲、旋转文本

可以把一些特别近甚至有点重合的分开，也有可能回归成一个框（密集文本）

模型缺点：

阈值放太低可能会同个box重复多检，太高又会漏检，合适的值在0.3-0.4之间

无NMS，会出现较多重叠框

密集文本边界框回归冗余

有角度图片跨行回归错误，通过在旋转数据集上微调解决

在另一篇文章进行代码解读

参考文章：

DPText-DETR原理及源码解读（一）_dptext-detr: towards better scene text detection w-CSDN博客