修复附件识别和要素提取消费不足问题

BaseDataMaintenance/maintenance/3.py

@@ -0,0 +1,394 @@
+
+
+import re
+
+
+s = '''
+2023-11-21 15:53:32,858 - root - INFO - handler task queue size is 23 need_stop True thread_id:140523427829504
+2023-11-21 15:53:33,383 - root - INFO - handler task queue size is 22 need_stop True thread_id:140524518356736
+2023-11-21 15:53:33,602 - root - INFO - handler task queue size is 22 need_stop True thread_id:140532529489664
+2023-11-21 15:53:34,110 - root - INFO - handler task queue size is 22 need_stop True thread_id:140537277433600
+2023-11-21 15:53:35,926 - root - INFO - handler task queue size is 20 need_stop True thread_id:140527741531904
+2023-11-21 15:53:36,221 - root - INFO - handler task queue size is 19 need_stop True thread_id:140525885712128
+2023-11-21 15:53:37,094 - root - INFO - handler task queue size is 16 need_stop True thread_id:140521766893312
+2023-11-21 15:53:37,974 - root - INFO - handler task queue size is 17 need_stop True thread_id:140525843748608
+2023-11-21 15:53:38,058 - root - INFO - handler task queue size is 17 need_stop True thread_id:140522530268928
+2023-11-21 15:53:38,369 - root - INFO - handler task queue size is 17 need_stop True thread_id:140525105551104
+2023-11-21 15:53:38,725 - root - INFO - handler task queue size is 17 need_stop True thread_id:140521783678720
+2023-11-21 15:53:38,948 - root - INFO - handler task queue size is 17 need_stop True thread_id:140523973093120
+2023-11-21 15:53:39,541 - root - INFO - handler task queue size is 16 need_stop True thread_id:140532521096960
+2023-11-21 15:53:39,877 - root - INFO - handler task queue size is 17 need_stop True thread_id:140525139121920
+2023-11-21 15:53:40,215 - root - INFO - handler task queue size is 16 need_stop True thread_id:140532722423552
+2023-11-21 15:53:40,372 - root - INFO - handler task queue size is 15 need_stop True thread_id:140525684385536
+2023-11-21 15:53:41,349 - root - INFO - handler task queue size is 15 need_stop True thread_id:140522362480384
+2023-11-21 15:53:41,725 - root - INFO - handler task queue size is 14 need_stop True thread_id:140525868926720
+2023-11-21 15:53:42,519 - root - INFO - handler task queue size is 11 need_stop True thread_id:140525692778240
+2023-11-21 15:53:42,581 - root - INFO - handler task queue size is 12 need_stop True thread_id:140523427829504
+2023-11-21 15:53:43,127 - root - INFO - handler task queue size is 13 need_stop True thread_id:140537277433600
+2023-11-21 15:53:43,129 - root - INFO - handler task queue size is 13 need_stop True thread_id:140532529489664
+2023-11-21 15:53:43,963 - root - INFO - handler task queue size is 13 need_stop True thread_id:140522530268928
+2023-11-21 15:53:44,434 - root - INFO - handler task queue size is 14 need_stop True thread_id:140525105551104
+2023-11-21 15:53:44,584 - root - INFO - handler task queue size is 14 need_stop True thread_id:140527741531904
+2023-11-21 15:53:44,590 - root - INFO - handler task queue size is 14 need_stop True thread_id:140521783678720
+2023-11-21 15:53:44,652 - root - INFO - handler task queue size is 14 need_stop True thread_id:140525885712128
+2023-11-21 15:53:44,912 - root - INFO - handler task queue size is 13 need_stop True thread_id:140522370873088
+2023-11-21 15:53:45,587 - root - INFO - handler task queue size is 14 need_stop True thread_id:140521766893312
+2023-11-21 15:53:49,073 - root - INFO - handler task queue size is 15 need_stop True thread_id:140532722423552
+2023-11-21 15:53:50,773 - root - INFO - handler task queue size is 21 need_stop True thread_id:140523973093120
+2023-11-21 15:53:50,830 - root - INFO - handler task queue size is 21 need_stop True thread_id:140525843748608
+2023-11-21 15:53:51,208 - root - INFO - handler task queue size is 22 need_stop True thread_id:140522362480384
+2023-11-21 15:53:51,211 - root - INFO - handler task queue size is 22 need_stop True thread_id:140525684385536
+2023-11-21 15:53:51,950 - root - INFO - handler task queue size is 22 need_stop True thread_id:140525868926720
+2023-11-21 15:53:52,984 - root - INFO - handler task queue size is 21 need_stop True thread_id:140532521096960
+2023-11-21 15:53:53,051 - root - INFO - handler task queue size is 21 need_stop True thread_id:140523427829504
+2023-11-21 15:53:53,113 - root - INFO - handler task queue size is 21 need_stop True thread_id:140525692778240
+2023-11-21 15:53:54,097 - root - INFO - handler task queue size is 19 need_stop True thread_id:140525139121920
+2023-11-21 15:53:54,394 - root - INFO - handler task queue size is 18 need_stop True thread_id:140537277433600
+2023-11-21 15:53:54,519 - root - INFO - handler task queue size is 18 need_stop True thread_id:140532529489664
+2023-11-21 15:53:55,328 - root - INFO - handler task queue size is 19 need_stop True thread_id:140522530268928
+2023-11-21 15:53:56,059 - root - INFO - handler task queue size is 19 need_stop True thread_id:140527741531904
+2023-11-21 15:53:56,151 - root - INFO - handler task queue size is 18 need_stop True thread_id:140521783678720
+2023-11-21 15:53:56,348 - root - INFO - handler task queue size is 18 need_stop True thread_id:140525105551104
+2023-11-21 15:53:56,356 - root - INFO - handler task queue size is 17 need_stop True thread_id:140525885712128
+2023-11-21 15:53:56,749 - root - INFO - handler task queue size is 17 need_stop True thread_id:140522370873088
+2023-11-21 15:53:57,390 - root - INFO - handler task queue size is 14 need_stop True thread_id:140521766893312
+2023-11-21 15:53:59,687 - root - INFO - handler task queue size is 13 need_stop True thread_id:140525868926720
+2023-11-21 15:53:59,849 - root - INFO - handler task queue size is 13 need_stop True thread_id:140532722423552
+2023-11-21 15:54:00,499 - root - INFO - handler task queue size is 13 need_stop True thread_id:140525692778240
+2023-11-21 15:54:00,977 - root - INFO - handler task queue size is 16 need_stop True thread_id:140525843748608
+2023-11-21 15:54:01,142 - root - INFO - handler task queue size is 16 need_stop True thread_id:140522362480384
+2023-11-21 15:54:01,291 - root - INFO - handler task queue size is 16 need_stop True thread_id:140525684385536
+2023-11-21 15:54:02,617 - root - INFO - handler task queue size is 14 need_stop True thread_id:140523427829504
+2023-11-21 15:54:02,695 - root - INFO - handler task queue size is 14 need_stop True thread_id:140532521096960
+2023-11-21 15:54:02,878 - root - INFO - handler task queue size is 14 need_stop True thread_id:140521758500608
+2023-11-21 15:54:03,429 - root - INFO - handler task queue size is 12 need_stop True thread_id:140525139121920
+2023-11-21 15:54:03,663 - root - INFO - handler task queue size is 11 need_stop True thread_id:140521766893312
+2023-11-21 15:54:03,797 - root - INFO - handler task queue size is 12 need_stop True thread_id:140532529489664
+2023-11-21 15:54:04,122 - root - INFO - handler task queue size is 12 need_stop True thread_id:140537277433600
+2023-11-21 15:54:04,259 - root - INFO - handler task queue size is 12 need_stop True thread_id:140522530268928
+2023-11-21 15:54:04,496 - root - INFO - handler task queue size is 12 need_stop True thread_id:140527741531904
+2023-11-21 15:54:04,768 - root - INFO - handler task queue size is 12 need_stop True thread_id:140521783678720
+2023-11-21 15:54:04,772 - root - INFO - handler task queue size is 12 need_stop True thread_id:140525885712128
+2023-11-21 15:54:04,977 - root - INFO - handler task queue size is 9 need_stop True thread_id:140522370873088
+2023-11-21 15:54:06,735 - root - INFO - handler task queue size is 5 need_stop True thread_id:140523973093120
+2023-11-21 15:54:09,653 - root - INFO - handler task queue size is 6 need_stop True thread_id:140522362480384
+2023-11-21 15:54:09,740 - root - INFO - handler task queue size is 6 need_stop True thread_id:140525868926720
+2023-11-21 15:54:09,800 - root - INFO - handler task queue size is 6 need_stop True thread_id:140525684385536
+2023-11-21 15:54:11,229 - root - INFO - handler task queue size is 10 need_stop True thread_id:140525692778240
+2023-11-21 15:54:11,826 - root - INFO - handler task queue size is 11 need_stop True thread_id:140523427829504
+2023-11-21 15:54:11,955 - root - INFO - handler task queue size is 11 need_stop True thread_id:140532521096960
+2023-11-21 15:54:12,495 - root - INFO - handler task queue size is 8 need_stop True thread_id:140532722423552
+2023-11-21 15:54:12,907 - root - INFO - handler task queue size is 10 need_stop True thread_id:140532529489664
+2023-11-21 15:54:13,015 - root - INFO - handler task queue size is 10 need_stop True thread_id:140537277433600
+2023-11-21 15:54:13,351 - root - INFO - handler task queue size is 13 need_stop True thread_id:140522530268928
+2023-11-21 15:54:13,408 - root - INFO - handler task queue size is 13 need_stop True thread_id:140527741531904
+2023-11-21 15:54:13,630 - root - INFO - handler task queue size is 14 need_stop True thread_id:140525843748608
+2023-11-21 15:54:13,711 - root - INFO - handler task queue size is 14 need_stop True thread_id:140521783678720
+2023-11-21 15:54:13,793 - root - INFO - handler task queue size is 13 need_stop True thread_id:140525885712128
+2023-11-21 15:54:13,794 - root - INFO - handler task queue size is 13 need_stop True thread_id:140521758500608
+2023-11-21 15:54:13,870 - root - INFO - handler task queue size is 14 need_stop True thread_id:140525139121920
+2023-11-21 15:54:13,945 - root - INFO - handler task queue size is 13 need_stop True thread_id:140521766893312
+2023-11-21 15:54:13,948 - root - INFO - handler task queue size is 13 need_stop True thread_id:140522370873088
+2023-11-21 15:54:15,419 - root - INFO - handler task queue size is 10 need_stop True thread_id:140523973093120
+2023-11-21 15:54:15,790 - root - INFO - handler task queue size is 6 need_stop True thread_id:140522362480384
+2023-11-21 15:54:15,853 - root - INFO - handler task queue size is 6 need_stop True thread_id:140525868926720
+2023-11-21 15:54:17,887 - root - INFO - handler task queue size is 9 need_stop True thread_id:140525684385536
+2023-11-21 15:54:18,553 - root - INFO - handler task queue size is 12 need_stop True thread_id:140525692778240
+2023-11-21 15:54:20,553 - root - INFO - handler task queue size is 15 need_stop True thread_id:140523973093120
+2023-11-21 15:54:19,198 - root - INFO - handler task queue size is 14 need_stop True thread_id:140521758500608
+2023-11-21 15:54:19,205 - root - INFO - handler task queue size is 14 need_stop True thread_id:140525139121920
+2023-11-21 15:54:19,344 - root - INFO - handler task queue size is 15 need_stop True thread_id:140522370873088
+2023-11-21 15:54:19,348 - root - INFO - handler task queue size is 15 need_stop True thread_id:140532521096960
+2023-11-21 15:54:19,350 - root - INFO - handler task queue size is 15 need_stop True thread_id:140521766893312
+2023-11-21 15:54:19,351 - root - INFO - handler task queue size is 15 need_stop True thread_id:140523427829504
+2023-11-21 15:54:19,846 - root - INFO - handler task queue size is 15 need_stop True thread_id:140532722423552
+2023-11-21 15:54:20,183 - root - INFO - handler task queue size is 15 need_stop True thread_id:140537277433600
+2023-11-21 15:54:20,185 - root - INFO - handler task queue size is 15 need_stop True thread_id:140522530268928
+2023-11-21 15:54:20,260 - root - INFO - handler task queue size is 14 need_stop True thread_id:140527741531904
+2023-11-21 15:54:20,341 - root - INFO - handler task queue size is 14 need_stop True thread_id:140524535142144
+2023-11-21 15:54:20,548 - root - INFO - handler task queue size is 15 need_stop True thread_id:140525843748608
+2023-11-21 15:54:20,553 - root - INFO - handler task queue size is 15 need_stop True thread_id:140521783678720
+2023-11-21 15:54:20,637 - root - INFO - handler task queue size is 13 need_stop True thread_id:140525885712128
+2023-11-21 15:54:20,969 - root - INFO - handler task queue size is 9 need_stop True thread_id:140525868926720
+2023-11-21 15:54:22,709 - root - INFO - handler task queue size is 2 need_stop True thread_id:140522362480384
+2023-11-21 15:54:23,066 - root - INFO - handler task queue size is 2 need_stop True thread_id:140525684385536
+2023-11-21 15:54:23,683 - root - INFO - handler task queue size is 6 need_stop True thread_id:140525692778240
+2023-11-21 15:54:24,612 - root - INFO - handler task queue size is 7 need_stop True thread_id:140523973093120
+2023-11-21 15:54:24,618 - root - INFO - handler task queue size is 7 need_stop True thread_id:140521758500608
+2023-11-21 15:54:24,686 - root - INFO - handler task queue size is 7 need_stop True thread_id:140525139121920
+2023-11-21 15:54:24,753 - root - INFO - handler task queue size is 7 need_stop True thread_id:140522370873088
+2023-11-21 15:54:24,761 - root - INFO - handler task queue size is 7 need_stop True thread_id:140532521096960
+2023-11-21 15:54:24,831 - root - INFO - handler task queue size is 7 need_stop True thread_id:140521766893312
+2023-11-21 15:54:24,832 - root - INFO - handler task queue size is 7 need_stop True thread_id:140523427829504
+2023-11-21 15:54:25,281 - root - INFO - handler task queue size is 8 need_stop True thread_id:140532722423552
+2023-11-21 15:54:25,915 - root - INFO - handler task queue size is 8 need_stop True thread_id:140524518356736
+2023-11-21 15:54:28,695 - root - INFO - handler task queue size is 1 need_stop True thread_id:140522530268928
+2023-11-21 15:54:28,845 - root - INFO - handler task queue size is 1 need_stop True thread_id:140527741531904
+2023-11-21 15:54:29,197 - root - INFO - handler task queue size is 1 need_stop True thread_id:140524535142144
+2023-11-21 15:54:29,366 - root - INFO - handler task queue size is 1 need_stop True thread_id:140525843748608
+2023-11-21 15:54:29,502 - root - INFO - handler task queue size is 2 need_stop True thread_id:140521783678720
+2023-11-21 15:54:31,482 - root - INFO - handler task queue size is 0 need_stop True thread_id:140525885712128
+2023-11-21 15:54:32,506 - root - INFO - handler task queue size is 3 need_stop True thread_id:140525868926720
+2023-11-21 15:54:32,795 - root - INFO - handler task queue size is 4 need_stop True thread_id:140525692778240
+2023-11-21 15:54:32,974 - root - INFO - handler task queue size is 4 need_stop True thread_id:140522362480384
+2023-11-21 15:54:33,269 - root - INFO - handler task queue size is 5 need_stop True thread_id:140525684385536
+2023-11-21 15:54:33,538 - root - INFO - handler task queue size is 5 need_stop True thread_id:140521758500608
+2023-11-21 15:54:33,676 - root - INFO - handler task queue size is 5 need_stop True thread_id:140522370873088
+2023-11-21 15:54:33,679 - root - INFO - handler task queue size is 5 need_stop True thread_id:140525139121920
+2023-11-21 15:54:33,893 - root - INFO - handler task queue size is 6 need_stop True thread_id:140523427829504
+2023-11-21 15:54:34,029 - root - INFO - handler task queue size is 6 need_stop True thread_id:140532722423552
+2023-11-21 15:54:34,034 - root - INFO - handler task queue size is 6 need_stop True thread_id:140523973093120
+2023-11-21 15:54:34,608 - root - INFO - handler task queue size is 5 need_stop True thread_id:140524518356736
+2023-11-21 15:54:34,991 - root - INFO - handler task queue size is 4 need_stop True thread_id:140524535142144
+2023-11-21 15:54:35,606 - root - INFO - handler task queue size is 3 need_stop True thread_id:140525843748608
+2023-11-21 15:54:36,237 - root - INFO - handler task queue size is 0 need_stop True thread_id:140521783678720
+2023-11-21 15:54:37,044 - root - INFO - handler task queue size is 0 need_stop True thread_id:140522530268928
+2023-11-21 15:54:37,178 - root - INFO - handler task queue size is 0 need_stop True thread_id:140532521096960
+2023-11-21 15:54:37,248 - root - INFO - handler task queue size is 0 need_stop True thread_id:140527741531904
+2023-11-21 15:54:38,627 - root - INFO - handler task queue size is 0 need_stop True thread_id:140525868926720
+2023-11-21 15:54:38,968 - root - INFO - handler task queue size is 0 need_stop True thread_id:140525692778240
+2023-11-21 15:54:39,819 - root - INFO - handler task queue size is 0 need_stop True thread_id:140522362480384
+2023-11-21 15:54:40,252 - root - INFO - handler task queue size is 0 need_stop True thread_id:140525684385536
+2023-11-21 15:54:40,478 - root - INFO - handler task queue size is 0 need_stop True thread_id:140525885712128
+2023-11-21 15:54:41,215 - root - INFO - handler task queue size is 5 need_stop True thread_id:140523427829504
+2023-11-21 15:54:42,840 - root - INFO - handler task queue size is 1 need_stop True thread_id:140522370873088
+2023-11-21 15:54:42,842 - root - INFO - handler task queue size is 1 need_stop True thread_id:140525139121920
+2023-11-21 15:54:43,244 - root - INFO - handler task queue size is 2 need_stop True thread_id:140532722423552
+2023-11-21 15:54:43,640 - root - INFO - handler task queue size is 3 need_stop True thread_id:140523973093120
+2023-11-21 15:54:43,696 - root - INFO - handler task queue size is 3 need_stop True thread_id:140521758500608
+2023-11-21 15:54:43,815 - root - INFO - handler task queue size is 3 need_stop True thread_id:140537277433600
+2023-11-21 15:54:43,876 - root - INFO - handler task queue size is 3 need_stop True thread_id:140524518356736
+2023-11-21 15:54:44,407 - root - INFO - handler task queue size is 4 need_stop True thread_id:140525868926720
+2023-11-21 15:54:44,544 - root - INFO - handler task queue size is 4 need_stop True thread_id:140524535142144
+2023-11-21 15:54:44,688 - root - INFO - handler task queue size is 4 need_stop True thread_id:140525843748608
+2023-11-21 15:54:44,763 - root - INFO - handler task queue size is 4 need_stop True thread_id:140521783678720
+2023-11-21 15:54:45,889 - root - INFO - handler task queue size is 2 need_stop True thread_id:140522362480384
+2023-11-21 15:54:46,625 - root - INFO - handler task queue size is 2 need_stop True thread_id:140522530268928
+2023-11-21 15:54:47,964 - root - INFO - handler task queue size is 0 need_stop True thread_id:140525692778240
+2023-11-21 15:54:48,033 - root - INFO - handler task queue size is 0 need_stop True thread_id:140527741531904
+2023-11-21 15:54:49,062 - root - INFO - handler task queue size is 1 need_stop True thread_id:140532521096960
+2023-11-21 15:54:49,067 - root - INFO - handler task queue size is 1 need_stop True thread_id:140525105551104
+2023-11-21 15:54:49,216 - root - INFO - handler task queue size is 1 need_stop True thread_id:140525684385536
+2023-11-21 15:54:49,621 - root - INFO - handler task queue size is 1 need_stop True thread_id:140525885712128
+2023-11-21 15:54:49,983 - root - INFO - handler task queue size is 1 need_stop True thread_id:140523427829504
+2023-11-21 15:54:52,605 - root - INFO - handler task queue size is 0 need_stop True thread_id:140524535142144
+2023-11-21 15:54:52,674 - root - INFO - handler task queue size is 0 need_stop True thread_id:140525868926720
+2023-11-21 15:54:52,821 - root - INFO - handler task queue size is 0 need_stop True thread_id:140525139121920
+2023-11-21 15:54:53,052 - root - INFO - handler task queue size is 0 need_stop True thread_id:140532722423552
+2023-11-21 15:54:53,609 - root - INFO - handler task queue size is 0 need_stop True thread_id:140521758500608
+2023-11-21 15:54:53,755 - root - INFO - handler task queue size is 0 need_stop True thread_id:140522362480384
+2023-11-21 15:54:54,062 - root - INFO - handler task queue size is 0 need_stop True thread_id:140537277433600
+2023-11-21 15:54:54,160 - root - INFO - handler task queue size is 0 need_stop True thread_id:140524518356736
+2023-11-21 15:54:54,503 - root - INFO - handler task queue size is 0 need_stop True thread_id:140522530268928
+2023-11-21 15:54:55,053 - root - INFO - handler task queue size is 0 need_stop True thread_id:140525843748608
+2023-11-21 15:54:55,144 - root - INFO - handler task queue size is 0 need_stop True thread_id:140525692778240
+2023-11-21 15:54:55,617 - root - INFO - handler task queue size is 0 need_stop True thread_id:140521783678720
+2023-11-21 15:54:55,741 - root - INFO - handler task queue size is 0 need_stop True thread_id:140522370873088
+2023-11-21 15:54:57,356 - root - INFO - handler task queue size is 0 need_stop True thread_id:140527741531904
+2023-11-21 15:54:57,508 - root - INFO - handler task queue size is 0 need_stop True thread_id:140532521096960
+2023-11-21 15:54:57,899 - root - INFO - handler task queue size is 0 need_stop True thread_id:140525684385536
+2023-11-21 15:54:58,234 - root - INFO - handler task queue size is 0 need_stop True thread_id:140523427829504
+2023-11-21 15:54:58,327 - root - INFO - handler task queue size is 0 need_stop True thread_id:140524535142144
+2023-11-21 15:54:58,422 - root - INFO - handler task queue size is 0 need_stop True thread_id:140525868926720
+2023-11-21 15:54:58,556 - root - INFO - handler task queue size is 0 need_stop True thread_id:140525139121920
+2023-11-21 15:54:59,407 - root - INFO - handler task queue size is 0 need_stop True thread_id:140525105551104
+2023-11-21 15:54:59,563 - root - INFO - handler task queue size is 0 need_stop True thread_id:140532722423552
+2023-11-21 15:54:59,789 - root - INFO - handler task queue size is 0 need_stop True thread_id:140521758500608
+2023-11-21 15:55:00,015 - root - INFO - handler task queue size is 0 need_stop True thread_id:140522362480384
+2023-11-21 15:55:00,213 - root - INFO - handler task queue size is 0 need_stop True thread_id:140525885712128
+2023-11-21 15:55:01,819 - root - INFO - handler task queue size is 0 need_stop True thread_id:140537277433600
+2023-11-21 15:55:02,538 - root - INFO - handler task queue size is 0 need_stop True thread_id:140524518356736
+2023-11-21 15:55:02,871 - root - INFO - handler task queue size is 1 need_stop True thread_id:140522530268928
+2023-11-21 15:55:02,874 - root - INFO - handler task queue size is 1 need_stop True thread_id:140525684385536
+2023-11-21 15:55:02,878 - root - INFO - handler task queue size is 2 need_stop True thread_id:140523427829504
+2023-11-21 15:55:03,109 - root - INFO - handler task queue size is 3 need_stop True thread_id:140524535142144
+2023-11-21 15:55:03,508 - root - INFO - handler task queue size is 2 need_stop True thread_id:140525692778240
+2023-11-21 15:55:03,592 - root - INFO - handler task queue size is 2 need_stop True thread_id:140525868926720
+2023-11-21 15:55:03,698 - root - INFO - handler task queue size is 2 need_stop True thread_id:140525843748608
+2023-11-21 15:55:03,706 - root - INFO - handler task queue size is 2 need_stop True thread_id:140521783678720
+2023-11-21 15:55:03,811 - root - INFO - handler task queue size is 2 need_stop True thread_id:140527741531904
+2023-11-21 15:55:04,145 - root - INFO - handler task queue size is 3 need_stop True thread_id:140532521096960
+2023-11-21 15:55:04,150 - root - INFO - handler task queue size is 3 need_stop True thread_id:140525139121920
+2023-11-21 15:55:04,780 - root - INFO - handler task queue size is 3 need_stop True thread_id:140525105551104
+2023-11-21 15:55:05,222 - root - INFO - handler task queue size is 3 need_stop True thread_id:140532722423552
+2023-11-21 15:55:05,224 - root - INFO - handler task queue size is 3 need_stop True thread_id:140521758500608
+2023-11-21 15:55:05,639 - root - INFO - handler task queue size is 2 need_stop True thread_id:140532529489664
+2023-11-21 15:55:07,577 - root - INFO - handler task queue size is 4 need_stop True thread_id:140537277433600
+2023-11-21 15:55:08,898 - root - INFO - handler task queue size is 5 need_stop True thread_id:140524518356736
+2023-11-21 15:55:09,433 - root - INFO - handler task queue size is 5 need_stop True thread_id:140522362480384
+2023-11-21 15:55:09,660 - root - INFO - handler task queue size is 5 need_stop True thread_id:140525885712128
+2023-11-21 15:55:10,084 - root - INFO - handler task queue size is 5 need_stop True thread_id:140524535142144
+2023-11-21 15:55:12,985 - root - INFO - handler task queue size is 1 need_stop True thread_id:140532722423552
+2023-11-21 15:55:13,499 - root - INFO - handler task queue size is 1 need_stop True thread_id:140532529489664
+2023-11-21 15:55:13,581 - root - INFO - handler task queue size is 1 need_stop True thread_id:140521783678720
+2023-11-21 15:55:14,048 - root - INFO - handler task queue size is 1 need_stop True thread_id:140527741531904
+2023-11-21 15:55:14,224 - root - INFO - handler task queue size is 1 need_stop True thread_id:140525843748608
+2023-11-21 15:55:14,357 - root - INFO - handler task queue size is 1 need_stop True thread_id:140522530268928
+2023-11-21 15:55:14,358 - root - INFO - handler task queue size is 1 need_stop True thread_id:140523427829504
+2023-11-21 15:55:14,577 - root - INFO - handler task queue size is 1 need_stop True thread_id:140532521096960
+2023-11-21 15:55:14,664 - root - INFO - handler task queue size is 1 need_stop True thread_id:140525684385536
+2023-11-21 15:55:15,189 - root - INFO - handler task queue size is 1 need_stop True thread_id:140525139121920
+2023-11-21 15:55:15,260 - root - INFO - handler task queue size is 1 need_stop True thread_id:140525692778240
+2023-11-21 15:55:15,260 - root - INFO - handler task queue size is 1 need_stop True thread_id:140525868926720
+2023-11-21 15:55:16,097 - root - INFO - handler task queue size is 0 need_stop True thread_id:140525105551104
+2023-11-21 15:55:16,585 - root - INFO - handler task queue size is 0 need_stop True thread_id:140522362480384
+2023-11-21 15:55:16,850 - root - INFO - handler task queue size is 0 need_stop True thread_id:140525885712128
+2023-11-21 15:55:17,388 - root - INFO - handler task queue size is 0 need_stop True thread_id:140524535142144
+2023-11-21 15:55:18,860 - root - INFO - handler task queue size is 0 need_stop True thread_id:140524518356736
+2023-11-21 15:55:20,078 - root - INFO - handler task queue size is 0 need_stop True thread_id:140537277433600
+2023-11-21 15:55:20,470 - root - INFO - handler task queue size is 0 need_stop True thread_id:140532722423552
+2023-11-21 15:55:21,015 - root - INFO - handler task queue size is 0 need_stop True thread_id:140532529489664
+2023-11-21 15:55:21,682 - root - INFO - handler task queue size is 0 need_stop True thread_id:140527741531904
+2023-11-21 15:55:24,699 - root - INFO - handler task queue size is 0 need_stop True thread_id:140523427829504
+2023-11-21 15:55:23,485 - root - INFO - handler task queue size is 0 need_stop True thread_id:140521783678720
+2023-11-21 15:55:24,174 - root - INFO - handler task queue size is 0 need_stop True thread_id:140522530268928
+2023-11-21 15:55:24,789 - root - INFO - handler task queue size is 0 need_stop True thread_id:140525684385536
+2023-11-21 15:55:25,302 - root - INFO - handler task queue size is 0 need_stop True thread_id:140525139121920
+2023-11-21 15:55:25,390 - root - INFO - handler task queue size is 0 need_stop True thread_id:140525692778240
+2023-11-21 15:55:25,703 - root - INFO - handler task queue size is 0 need_stop True thread_id:140525105551104
+2023-11-21 15:55:26,171 - root - INFO - handler task queue size is 0 need_stop True thread_id:140525843748608
+2023-11-21 15:55:26,707 - root - INFO - handler task queue size is 0 need_stop True thread_id:140532521096960
+2023-11-21 15:55:26,794 - root - INFO - handler task queue size is 0 need_stop True thread_id:140521766893312
+2023-11-21 15:55:27,132 - root - INFO - handler task queue size is 0 need_stop True thread_id:140525868926720
+2023-11-21 15:55:27,617 - root - INFO - handler task queue size is 0 need_stop True thread_id:140522362480384
+2023-11-21 15:55:28,198 - root - INFO - handler task queue size is 1 need_stop True thread_id:140524535142144
+2023-11-21 15:55:30,340 - root - INFO - handler task queue size is 0 need_stop True thread_id:140537277433600
+2023-11-21 15:55:31,119 - root - INFO - handler task queue size is 3 need_stop True thread_id:140522530268928
+2023-11-21 15:55:31,496 - root - INFO - handler task queue size is 3 need_stop True thread_id:140532722423552
+2023-11-21 15:55:31,635 - root - INFO - handler task queue size is 3 need_stop True thread_id:140525684385536
+2023-11-21 15:55:33,222 - root - INFO - handler task queue size is 4 need_stop True thread_id:140527741531904
+2023-11-21 15:55:33,318 - root - INFO - handler task queue size is 4 need_stop True thread_id:140523427829504
+2023-11-21 15:55:33,752 - root - INFO - handler task queue size is 5 need_stop True thread_id:140525105551104
+2023-11-21 15:55:34,227 - root - INFO - handler task queue size is 4 need_stop True thread_id:140525843748608
+2023-11-21 15:55:34,235 - root - INFO - handler task queue size is 4 need_stop True thread_id:140532521096960
+2023-11-21 15:55:34,793 - root - INFO - handler task queue size is 3 need_stop True thread_id:140521766893312
+2023-11-21 15:55:35,082 - root - INFO - handler task queue size is 4 need_stop True thread_id:140525868926720
+2023-11-21 15:55:35,341 - root - INFO - handler task queue size is 4 need_stop True thread_id:140521783678720
+2023-11-21 15:55:35,678 - root - INFO - handler task queue size is 4 need_stop True thread_id:140522362480384
+2023-11-21 15:55:37,703 - root - INFO - handler task queue size is 2 need_stop True thread_id:140525139121920
+2023-11-21 15:55:38,944 - root - INFO - handler task queue size is 2 need_stop True thread_id:140525692778240
+2023-11-21 15:55:39,623 - root - INFO - handler task queue size is 4 need_stop True thread_id:140524535142144
+2023-11-21 15:55:41,258 - root - INFO - handler task queue size is 6 need_stop True thread_id:140524518356736
+2023-11-21 15:55:41,717 - root - INFO - handler task queue size is 6 need_stop True thread_id:140525885712128
+2023-11-21 15:55:42,806 - root - INFO - handler task queue size is 12 need_stop True thread_id:140521766893312
+2023-11-21 15:55:42,861 - root - INFO - handler task queue size is 12 need_stop True thread_id:140532722423552
+2023-11-21 15:55:43,173 - root - INFO - handler task queue size is 12 need_stop True thread_id:140525684385536
+2023-11-21 15:55:43,341 - root - INFO - handler task queue size is 11 need_stop True thread_id:140537277433600
+2023-11-21 15:55:43,822 - root - INFO - handler task queue size is 11 need_stop True thread_id:140525105551104
+2023-11-21 15:55:44,043 - root - INFO - handler task queue size is 11 need_stop True thread_id:140525843748608
+2023-11-21 15:55:44,117 - root - INFO - handler task queue size is 11 need_stop True thread_id:140523427829504
+2023-11-21 15:55:44,198 - root - INFO - handler task queue size is 11 need_stop True thread_id:140527741531904
+2023-11-21 15:55:44,270 - root - INFO - handler task queue size is 9 need_stop True thread_id:140532521096960
+2023-11-21 15:55:44,633 - root - INFO - handler task queue size is 11 need_stop True thread_id:140521783678720
+2023-11-21 15:55:44,803 - root - INFO - handler task queue size is 10 need_stop True thread_id:140522362480384
+2023-11-21 15:55:45,036 - root - INFO - handler task queue size is 10 need_stop True thread_id:140524535142144
+2023-11-21 15:55:46,255 - root - INFO - handler task queue size is 4 need_stop True thread_id:140525692778240
+2023-11-21 15:55:46,257 - root - INFO - handler task queue size is 4 need_stop True thread_id:140524518356736
+2023-11-21 15:55:46,626 - root - INFO - handler task queue size is 5 need_stop True thread_id:140525885712128
+2023-11-21 15:55:46,923 - root - INFO - handler task queue size is 4 need_stop True thread_id:140523973093120
+2023-11-21 15:55:47,468 - root - INFO - handler task queue size is 6 need_stop True thread_id:140522530268928
+2023-11-21 15:55:48,112 - root - INFO - handler task queue size is 7 need_stop True thread_id:140521766893312
+2023-11-21 15:55:48,112 - root - INFO - handler task queue size is 7 need_stop True thread_id:140532722423552
+2023-11-21 15:55:48,513 - root - INFO - handler task queue size is 7 need_stop True thread_id:140525684385536
+2023-11-21 15:55:48,951 - root - INFO - handler task queue size is 5 need_stop True thread_id:140537277433600
+2023-11-21 15:55:49,954 - root - INFO - handler task queue size is 4 need_stop True thread_id:140525139121920
+2023-11-21 15:55:50,016 - root - INFO - handler task queue size is 4 need_stop True thread_id:140525105551104
+2023-11-21 15:55:50,222 - root - INFO - handler task queue size is 5 need_stop True thread_id:140525843748608
+2023-11-21 15:55:50,293 - root - INFO - handler task queue size is 5 need_stop True thread_id:140523427829504
+2023-11-21 15:55:50,608 - root - INFO - handler task queue size is 5 need_stop True thread_id:140527741531904
+2023-11-21 15:55:50,961 - root - INFO - handler task queue size is 3 need_stop True thread_id:140532521096960
+2023-11-21 15:55:51,483 - root - INFO - handler task queue size is 5 need_stop True thread_id:140521783678720
+2023-11-21 15:55:51,561 - root - INFO - handler task queue size is 5 need_stop True thread_id:140522362480384
+2023-11-21 15:55:51,980 - root - INFO - handler task queue size is 4 need_stop True thread_id:140524535142144
+2023-11-21 15:55:54,346 - root - INFO - handler task queue size is 1 need_stop True thread_id:140525692778240
+2023-11-21 15:55:55,784 - root - INFO - handler task queue size is 5 need_stop True thread_id:140532722423552
+2023-11-21 15:55:57,198 - root - INFO - handler task queue size is 7 need_stop True thread_id:140525885712128
+2023-11-21 15:55:57,981 - root - INFO - handler task queue size is 8 need_stop True thread_id:140523973093120
+2023-11-21 15:55:58,258 - root - INFO - handler task queue size is 7 need_stop True thread_id:140532521096960
+2023-11-21 15:55:58,453 - root - INFO - handler task queue size is 8 need_stop True thread_id:140522530268928
+2023-11-21 15:55:58,708 - root - INFO - handler task queue size is 8 need_stop True thread_id:140521783678720
+2023-11-21 15:55:58,861 - root - INFO - handler task queue size is 8 need_stop True thread_id:140522362480384
+2023-11-21 15:55:58,863 - root - INFO - handler task queue size is 8 need_stop True thread_id:140525684385536
+2023-11-21 15:55:59,123 - root - INFO - handler task queue size is 8 need_stop True thread_id:140537277433600
+2023-11-21 15:55:59,127 - root - INFO - handler task queue size is 8 need_stop True thread_id:140524535142144
+2023-11-21 15:55:59,491 - root - INFO - handler task queue size is 9 need_stop True thread_id:140521766893312
+2023-11-21 15:55:59,844 - root - INFO - handler task queue size is 10 need_stop True thread_id:140525139121920
+2023-11-21 15:56:00,193 - root - INFO - handler task queue size is 10 need_stop True thread_id:140523427829504
+2023-11-21 15:56:00,276 - root - INFO - handler task queue size is 10 need_stop True thread_id:140525843748608
+2023-11-21 15:56:00,359 - root - INFO - handler task queue size is 10 need_stop True thread_id:140527741531904
+2023-11-21 15:56:01,105 - root - INFO - handler task queue size is 7 need_stop True thread_id:140525692778240
+2023-11-21 15:56:01,289 - root - INFO - handler task queue size is 7 need_stop True thread_id:140522370873088
+2023-11-21 15:56:01,651 - root - INFO - handler task queue size is 6 need_stop True thread_id:140524518356736
+2023-11-21 15:56:02,656 - root - INFO - handler task queue size is 3 need_stop True thread_id:140532722423552
+2023-11-21 15:56:03,883 - root - INFO - handler task queue size is 3 need_stop True thread_id:140525885712128
+2023-11-21 15:56:04,809 - root - INFO - handler task queue size is 3 need_stop True thread_id:140523973093120
+2023-11-21 15:56:05,038 - root - INFO - handler task queue size is 2 need_stop True thread_id:140532521096960
+2023-11-21 15:56:05,176 - root - INFO - handler task queue size is 2 need_stop True thread_id:140522530268928
+2023-11-21 15:56:05,586 - root - INFO - handler task queue size is 2 need_stop True thread_id:140521783678720
+2023-11-21 15:56:05,751 - root - INFO - handler task queue size is 2 need_stop True thread_id:140522362480384
+2023-11-21 15:56:05,763 - root - INFO - handler task queue size is 2 need_stop True thread_id:140525684385536
+2023-11-21 15:56:06,053 - root - INFO - handler task queue size is 3 need_stop True thread_id:140537277433600
+2023-11-21 15:56:06,116 - root - INFO - handler task queue size is 4 need_stop True thread_id:140524535142144
+2023-11-21 15:56:06,351 - root - INFO - handler task queue size is 4 need_stop True thread_id:140521766893312
+2023-11-21 15:56:06,613 - root - INFO - handler task queue size is 4 need_stop True thread_id:140523427829504
+2023-11-21 15:56:06,786 - root - INFO - handler task queue size is 3 need_stop True thread_id:140525843748608
+2023-11-21 15:56:06,853 - root - INFO - handler task queue size is 3 need_stop True thread_id:140527741531904
+2023-11-21 15:56:07,659 - root - INFO - handler task queue size is 0 need_stop True thread_id:140525692778240
+2023-11-21 15:56:08,069 - root - INFO - handler task queue size is 2 need_stop True thread_id:140524518356736
+2023-11-21 15:56:08,359 - root - INFO - handler task queue size is 1 need_stop True thread_id:140521758500608
+2023-11-21 15:56:08,592 - root - INFO - handler task queue size is 0 need_stop True thread_id:140532722423552
+2023-11-21 15:56:09,345 - root - INFO - handler task queue size is 3 need_stop True thread_id:140525139121920
+2023-11-21 15:56:09,503 - root - INFO - handler task queue size is 2 need_stop True thread_id:140525885712128
+2023-11-21 15:56:11,364 - root - INFO - handler task queue size is 1 need_stop True thread_id:140522370873088
+2023-11-21 15:56:11,833 - root - INFO - handler task queue size is 1 need_stop True thread_id:140524535142144
+2023-11-21 15:56:13,470 - root - INFO - handler task queue size is 6 need_stop True thread_id:140522530268928
+2023-11-21 15:56:14,101 - root - INFO - handler task queue size is 7 need_stop True thread_id:140532521096960
+2023-11-21 15:56:14,167 - root - INFO - handler task queue size is 7 need_stop True thread_id:140523973093120
+2023-11-21 15:56:14,225 - root - INFO - handler task queue size is 7 need_stop True thread_id:140521783678720
+2023-11-21 15:56:14,567 - root - INFO - handler task queue size is 6 need_stop True thread_id:140525684385536
+2023-11-21 15:56:14,725 - root - INFO - handler task queue size is 7 need_stop True thread_id:140522362480384
+2023-11-21 15:56:14,959 - root - INFO - handler task queue size is 7 need_stop True thread_id:140537277433600
+2023-11-21 15:56:15,033 - root - INFO - handler task queue size is 7 need_stop True thread_id:140521758500608
+2023-11-21 15:56:15,269 - root - INFO - handler task queue size is 7 need_stop True thread_id:140523427829504
+2023-11-21 15:56:15,888 - root - INFO - handler task queue size is 8 need_stop True thread_id:140525843748608
+2023-11-21 15:56:15,965 - root - INFO - handler task queue size is 8 need_stop True thread_id:140521766893312
+2023-11-21 15:56:16,834 - root - INFO - handler task queue size is 10 need_stop True thread_id:140527741531904
+2023-11-21 15:56:17,331 - root - INFO - handler task queue size is 8 need_stop True thread_id:140525139121920
+2023-11-21 15:56:17,500 - root - INFO - handler task queue size is 8 need_stop True thread_id:140525692778240
+2023-11-21 15:56:17,501 - root - INFO - handler task queue size is 8 need_stop True thread_id:140525885712128
+2023-11-21 15:56:18,395 - root - INFO - handler task queue size is 2 need_stop True thread_id:140532722423552
+2023-11-21 15:56:19,468 - root - INFO - handler task queue size is 1 need_stop True thread_id:140522370873088
+2023-11-21 15:56:20,042 - root - INFO - handler task queue size is 1 need_stop True thread_id:140524535142144
+2023-11-21 15:56:21,413 - root - INFO - handler task queue size is 0 need_stop True thread_id:140522530268928
+2023-11-21 15:56:21,921 - root - INFO - handler task queue size is 0 need_stop True thread_id:140532521096960
+2023-11-21 15:56:22,081 - root - INFO - handler task queue size is 0 need_stop True thread_id:140523973093120
+2023-11-21 15:56:22,317 - root - INFO - handler task queue size is 0 need_stop True thread_id:140521783678720
+2023-11-21 15:56:23,121 - root - INFO - handler task queue size is 0 need_stop True thread_id:140525684385536
+2023-11-21 15:56:23,618 - root - INFO - handler task queue size is 0 need_stop True thread_id:140537277433600
+2023-11-21 15:56:23,687 - root - INFO - handler task queue size is 0 need_stop True thread_id:140521758500608
+2023-11-21 15:56:23,698 - root - INFO - handler task queue size is 0 need_stop True thread_id:140523427829504
+2023-11-21 15:56:25,097 - root - INFO - handler task queue size is 0 need_stop True thread_id:140525139121920
+2023-11-21 15:56:25,551 - root - INFO - handler task queue size is 0 need_stop True thread_id:140522362480384
+2023-11-21 15:56:25,793 - root - INFO - handler task queue size is 0 need_stop True thread_id:140525843748608
+2023-11-21 15:56:26,270 - root - INFO - handler task queue size is 0 need_stop True thread_id:140527741531904
+2023-11-21 15:56:26,522 - root - INFO - handler task queue size is 0 need_stop True thread_id:140521766893312
+2023-11-21 15:56:27,902 - root - INFO - handler task queue size is 0 need_stop True thread_id:140525885712128
+2023-11-21 15:56:28,194 - root - INFO - handler task queue size is 0 need_stop True thread_id:140525692778240
+2023-11-21 15:56:28,803 - root - INFO - handler task queue size is 1 need_stop True thread_id:140522530268928
+2023-11-21 15:56:29,497 - root - INFO - handler task queue size is 2 need_stop True thread_id:140522370873088
+2023-11-21 15:56:29,651 - root - INFO - handler task queue size is 2 need_stop True thread_id:140524535142144
+2023-11-21 15:56:29,727 - root - INFO - handler task queue size is 2 need_stop True thread_id:140521783678720
+2023-11-21 15:56:29,730 - root - INFO - handler task queue size is 2 need_stop True thread_id:140525684385536
+2023-11-21 15:56:29,889 - root - INFO - handler task queue size is 1 need_stop True thread_id:140532529489664
+2023-11-21 15:56:31,469 - root - INFO - handler task queue size is 6 need_stop True thread_id:140521758500608
+2023-11-21 15:56:31,759 - root - INFO - handler task queue size is 5 need_stop True thread_id:140523973093120
+2023-11-21 15:56:31,835 - root - INFO - handler task queue size is 5 need_stop True thread_id:140532521096960
+'''
+
+set_a = set()
+for a in re.split("\n",s):
+    a = a.strip()
+    b = a.split(":")[-1]
+    print(b)
+    set_a.add(b)
+print(len(set_a))

BaseDataMaintenance/maintenance/dataflow.py

@@ -1817,8 +1817,11 @@ class Dataflow_attachment(Dataflow):
         for t in self.process_list_thread:
             t.start()
 
-        for t in self.process_list_thread:
-            t.join()
+        for _i in range(len(self.process_list_thread)):
+            t = self.process_list_thread[_i]
+            if not t.is_alive():
+                self.prcess_list_thread[_i] = Thread(target=self.process_comsumer_handle)
+                self.prcess_list_thread[_i].start()
 
 
     def process_comsumer_handle(self):

BaseDataMaintenance/maintenance/dataflow_mq.py

@@ -547,13 +547,14 @@ class Dataflow_ActivteMQ_extract(Dataflow_extract):
 
     class ExtractListener():
 
-        def __init__(self,conn,_func,*args,**kwargs):
+        def __init__(self,conn,_func,_idx,*args,**kwargs):
             self.conn = conn
             self._func = _func
+            self._idx = _idx
 
         def on_message(self, headers):
             try:
-                log("get message")
+                log("get message of idx:%d"%(self._idx))
                 message_id = headers.headers["message-id"]
                 body = headers.body
                 log("get message %s crtime:%s"%(message_id,json.loads(body).get("crtime","")))
@@ -601,7 +602,7 @@ class Dataflow_ActivteMQ_extract(Dataflow_extract):
 
         self.list_extract_comsumer = []
         for _i in range(self.comsumer_count):
-            listener_extract = self.ExtractListener(getConnect_activateMQ(),self.comsumer_handle)
+            listener_extract = self.ExtractListener(getConnect_activateMQ(),self.comsumer_handle,_i)
             createComsumer(listener_extract,self.mq_extract)
             self.list_extract_comsumer.append(listener_extract)