|
@@ -24,6 +24,7 @@ from BaseDataMaintenance.maintenance.product.product_dict import Product_Dict_Ma
|
|
|
from apscheduler.schedulers.blocking import BlockingScheduler
|
|
|
|
|
|
from BaseDataMaintenance.maintenance.product.make_brand_pattern import *
|
|
|
+from BaseDataMaintenance.maintenance.product.product_dict import IS_SYNCHONIZED
|
|
|
import logging
|
|
|
|
|
|
root = logging.getLogger()
|
|
@@ -177,227 +178,198 @@ class Product_Manager(Product_Dict_Manager):
|
|
|
# _dpd.update_row(self.ots_client)
|
|
|
break
|
|
|
if name_ots_id is not None:
|
|
|
+
|
|
|
if brand is not None and brand!="":
|
|
|
- s_brand = brand
|
|
|
- l_brand = [brand]
|
|
|
- brand_ch = get_chinese_string(brand)
|
|
|
- l_brand.extend(brand_ch)
|
|
|
-
|
|
|
- _find = False
|
|
|
- for brand in l_brand:
|
|
|
-
|
|
|
- brand_vector = request_embedding(brand)
|
|
|
- if brand_vector is not None:
|
|
|
- Coll,_ = self.get_collection(BRAND_GRADE)
|
|
|
- search_list = search_embedding(Coll,embedding_index_name,[brand_vector],self.search_params,output_fields,limit=60)
|
|
|
-
|
|
|
- # log("search brand %s"%(brand))
|
|
|
- for _search in search_list:
|
|
|
-
|
|
|
- ots_id = _search.entity.get("standard_name_id")
|
|
|
- ots_name = _search.entity.get("standard_name")
|
|
|
- ots_parent_id = _search.entity.get("ots_parent_id")
|
|
|
-
|
|
|
- # log("check brand %s and %s"%(brand,ots_name))
|
|
|
- if is_similar(brand,ots_name) or check_brand(brand,ots_name):
|
|
|
-
|
|
|
- # log("check brand similar succeed:%s and %s"%(brand,ots_name))
|
|
|
- new_brand = ots_name
|
|
|
-
|
|
|
- # judge if the brand which parent_id is name_ots_id exists,if not insert one else update alias
|
|
|
-
|
|
|
- if name_ots_id is not None:
|
|
|
- brand_ots_id = get_document_product_dict_id(name_ots_id,new_brand)
|
|
|
-
|
|
|
- _d_brand = {DOCUMENT_PRODUCT_DICT_ID:brand_ots_id,
|
|
|
- DOCUMENT_PRODUCT_DICT_NAME:new_brand,
|
|
|
- DOCUMENT_PRODUCT_DICT_ALIAS:"%s"%(str(brand).lower()),
|
|
|
- DOCUMENT_PRODUCT_DICT_GRADE:BRAND_GRADE,
|
|
|
- DOCUMENT_PRODUCT_DICT_STATUS:1,
|
|
|
- DOCUMENT_PRODUCT_DICT_PARENT_ID:name_ots_id,
|
|
|
- DOCUMENT_PRODUCT_DICT_CREATE_TIME:getCurrent_date(format="%Y-%m-%d %H:%M:%S"),
|
|
|
- DOCUMENT_PRODUCT_DICT_UPDATE_TIME:getCurrent_date(format="%Y-%m-%d %H:%M:%S"),
|
|
|
- }
|
|
|
- _dpd_brand = Document_product_dict(_d_brand)
|
|
|
- # _dpd_brand.updateAlias(str(new_brand).lower())
|
|
|
- if not _dpd_brand.exists_row(self.ots_client):
|
|
|
- # _dpd_brand.update_row(self.ots_client)
|
|
|
- _d_brand = {DOCUMENT_PRODUCT_DICT_INTERFACE_ID:uuid4().hex,
|
|
|
- DOCUMENT_PRODUCT_DICT_INTERFACE_NAME:new_brand,
|
|
|
- DOCUMENT_PRODUCT_DICT_INTERFACE_ALIAS:"%s"%(str(brand).lower()),
|
|
|
- DOCUMENT_PRODUCT_DICT_INTERFACE_GRADE:BRAND_GRADE,
|
|
|
- DOCUMENT_PRODUCT_DICT_INTERFACE_STATUS:1,
|
|
|
- DOCUMENT_PRODUCT_DICT_INTERFACE_PARENT_ID:name_ots_id,
|
|
|
- DOCUMENT_PRODUCT_DICT_INTERFACE_CREATE_TIME:getCurrent_date(format="%Y-%m-%d %H:%M:%S"),
|
|
|
- DOCUMENT_PRODUCT_DICT_INTERFACE_UPDATE_TIME:getCurrent_date(format="%Y-%m-%d %H:%M:%S"),
|
|
|
- DOCUMENT_PRODUCT_DICT_INTERFACE_ACTION:"insert"
|
|
|
- }
|
|
|
- dpdi = Document_product_dict_interface(_d_brand)
|
|
|
- dpdi.update_row(self.ots_client)
|
|
|
- else:
|
|
|
- pass
|
|
|
- # #update alias
|
|
|
- # _dpd = Document_product_dict({DOCUMENT_PRODUCT_DICT_ID:brand_ots_id})
|
|
|
- # _flag = _dpd.fix_columns(self.ots_client,[DOCUMENT_PRODUCT_DICT_ALIAS],True)
|
|
|
- # if _flag:
|
|
|
- # if _dpd.updateAlias(brand):
|
|
|
- # _dpd.update_row(self.ots_client)
|
|
|
-
|
|
|
- _find = True
|
|
|
- break
|
|
|
- else:
|
|
|
- # log("check brand similar failed:%s and %s"%(brand,ots_name))
|
|
|
- # add new brand?
|
|
|
- pass
|
|
|
- if _find:
|
|
|
- break
|
|
|
- if not _find:
|
|
|
+
|
|
|
+ #check ots
|
|
|
+ bool_query = BoolQuery(must_queries=[
|
|
|
+ TermQuery(DOCUMENT_PRODUCT_DICT_NAME,brand),
|
|
|
+ TermQuery(DOCUMENT_PRODUCT_DICT_GRADE,BRAND_GRADE)
|
|
|
+ ])
|
|
|
+ rows,next_token,total_count,is_all_succeed = self.ots_client.search("document_product_dict","document_product_dict_index",
|
|
|
+ SearchQuery(bool_query,get_total_count=True))
|
|
|
+ if total_count>0:
|
|
|
+ new_brand = brand
|
|
|
+ else:
|
|
|
+ s_brand = brand
|
|
|
+ l_brand = [brand]
|
|
|
+ l_brand.append(clean_product_brand(s_brand))
|
|
|
+ brand_ch = get_chinese_string(brand)
|
|
|
+ l_brand.extend(brand_ch)
|
|
|
+
|
|
|
+ _find = False
|
|
|
for brand in l_brand:
|
|
|
- if self.check_new_brand(brand):
|
|
|
- _d_brand = {DOCUMENT_PRODUCT_DICT_INTERFACE_ID:uuid4().hex,
|
|
|
- DOCUMENT_PRODUCT_DICT_INTERFACE_NAME:brand,
|
|
|
- DOCUMENT_PRODUCT_DICT_INTERFACE_ALIAS:"%s"%(str(brand).lower()),
|
|
|
- DOCUMENT_PRODUCT_DICT_INTERFACE_GRADE:BRAND_GRADE,
|
|
|
- DOCUMENT_PRODUCT_DICT_INTERFACE_STATUS:1,
|
|
|
- DOCUMENT_PRODUCT_DICT_INTERFACE_PARENT_ID:name_ots_id,
|
|
|
- DOCUMENT_PRODUCT_DICT_INTERFACE_CREATE_TIME:getCurrent_date(format="%Y-%m-%d %H:%M:%S"),
|
|
|
- DOCUMENT_PRODUCT_DICT_INTERFACE_UPDATE_TIME:getCurrent_date(format="%Y-%m-%d %H:%M:%S"),
|
|
|
- DOCUMENT_PRODUCT_DICT_INTERFACE_ACTION:"insert"
|
|
|
- }
|
|
|
- dpdi = Document_product_dict_interface(_d_brand)
|
|
|
- dpdi.update_row(self.ots_client)
|
|
|
|
|
|
+ brand_vector = request_embedding(brand)
|
|
|
+ if brand_vector is not None:
|
|
|
+ Coll,_ = self.get_collection(BRAND_GRADE)
|
|
|
+ search_list = search_embedding(Coll,embedding_index_name,[brand_vector],self.search_params,output_fields,limit=60)
|
|
|
|
|
|
- if specs is not None and specs!="":
|
|
|
- debug("getting sepcs %s"%(specs))
|
|
|
- list_specs = []
|
|
|
- c_specs = clean_product_specs(specs)
|
|
|
- list_specs.append(c_specs)
|
|
|
-
|
|
|
- for s in re.split("[\u4e00-\u9fff]",specs):
|
|
|
- if s!="" and len(s)>4:
|
|
|
- list_specs.append(s)
|
|
|
- similar_flag = None
|
|
|
- _index = 0
|
|
|
- break_flag = False
|
|
|
- for c_specs in list_specs:
|
|
|
- if break_flag:
|
|
|
- break
|
|
|
- _index += 1
|
|
|
- specs_vector = request_embedding(c_specs)
|
|
|
-
|
|
|
- if specs_vector is not None:
|
|
|
- Coll,_ = self.get_collection(SPECS_GRADE)
|
|
|
- search_list = search_embedding(Coll,embedding_index_name,[specs_vector],self.search_params,output_fields,limit=60)
|
|
|
-
|
|
|
- for _search in search_list:
|
|
|
-
|
|
|
- ots_id = _search.entity.get("standard_name_id")
|
|
|
- ots_name = _search.entity.get("standard_name")
|
|
|
- ots_parent_id = _search.entity.get("ots_parent_id")
|
|
|
-
|
|
|
- log("checking specs %s and %s"%(specs,ots_name))
|
|
|
- if is_similar(specs,ots_name):
|
|
|
- # log("specs is_similar")
|
|
|
- if check_specs(c_specs,ots_name):
|
|
|
- log("check_specs succeed")
|
|
|
- break_flag = True
|
|
|
- new_specs = ots_name
|
|
|
-
|
|
|
- # to update the document_product_dict which is builded for search
|
|
|
- if brand_ots_id is not None:
|
|
|
- # judge if the specs which parent_id is brand_ots_id exists,insert one if not exists else update alias
|
|
|
- specs_ots_id = get_document_product_dict_id(brand_ots_id,new_specs)
|
|
|
-
|
|
|
- _d_specs = {DOCUMENT_PRODUCT_DICT_ID:specs_ots_id,
|
|
|
- DOCUMENT_PRODUCT_DICT_NAME:new_specs,
|
|
|
- DOCUMENT_PRODUCT_DICT_ALIAS:"%s"%(str(specs).lower()),
|
|
|
- DOCUMENT_PRODUCT_DICT_GRADE:SPECS_GRADE,
|
|
|
+ # log("search brand %s"%(brand))
|
|
|
+ for _search in search_list:
|
|
|
+
|
|
|
+ ots_id = _search.entity.get("standard_name_id")
|
|
|
+ ots_name = _search.entity.get("standard_name")
|
|
|
+ ots_parent_id = _search.entity.get("ots_parent_id")
|
|
|
+
|
|
|
+ # log("check brand %s and %s"%(brand,ots_name))
|
|
|
+ if is_similar(brand,ots_name) or check_brand(brand,ots_name):
|
|
|
+
|
|
|
+ # log("check brand similar succeed:%s and %s"%(brand,ots_name))
|
|
|
+ new_brand = ots_name
|
|
|
+
|
|
|
+ log("checking brand %s succeed %s"%(brand,new_brand))
|
|
|
+ # judge if the brand which parent_id is name_ots_id exists,if not insert one else update alias
|
|
|
+
|
|
|
+ if name_ots_id is not None:
|
|
|
+ brand_ots_id = get_document_product_dict_id(name_ots_id,new_brand)
|
|
|
+
|
|
|
+ _d_brand = {DOCUMENT_PRODUCT_DICT_ID:brand_ots_id,
|
|
|
+ DOCUMENT_PRODUCT_DICT_NAME:new_brand,
|
|
|
+ DOCUMENT_PRODUCT_DICT_ALIAS:"%s"%(str(brand).lower()),
|
|
|
+ DOCUMENT_PRODUCT_DICT_GRADE:BRAND_GRADE,
|
|
|
DOCUMENT_PRODUCT_DICT_STATUS:1,
|
|
|
- DOCUMENT_PRODUCT_DICT_PARENT_ID:brand_ots_id,
|
|
|
+ DOCUMENT_PRODUCT_DICT_PARENT_ID:name_ots_id,
|
|
|
+ DOCUMENT_PRODUCT_DICT_IS_SYNCHONIZED:IS_SYNCHONIZED,
|
|
|
DOCUMENT_PRODUCT_DICT_CREATE_TIME:getCurrent_date(format="%Y-%m-%d %H:%M:%S"),
|
|
|
DOCUMENT_PRODUCT_DICT_UPDATE_TIME:getCurrent_date(format="%Y-%m-%d %H:%M:%S"),
|
|
|
}
|
|
|
- _dpd_specs = Document_product_dict(_d_specs)
|
|
|
- # _dpd_specs.updateAlias(str(new_specs).lower())
|
|
|
- if not _dpd_specs.exists_row(self.ots_client):
|
|
|
- # _dpd_specs.update_row(self.ots_client)
|
|
|
- # user interface to add
|
|
|
- _d = {DOCUMENT_PRODUCT_DICT_INTERFACE_ID:uuid4().hex,
|
|
|
- DOCUMENT_PRODUCT_DICT_INTERFACE_NAME:new_specs,
|
|
|
- DOCUMENT_PRODUCT_DICT_INTERFACE_ALIAS:"%s"%(new_specs.lower()),
|
|
|
- DOCUMENT_PRODUCT_DICT_INTERFACE_GRADE:SPECS_GRADE,
|
|
|
- DOCUMENT_PRODUCT_DICT_INTERFACE_STATUS:1,
|
|
|
- DOCUMENT_PRODUCT_DICT_INTERFACE_PARENT_ID:brand_ots_id,
|
|
|
- DOCUMENT_PRODUCT_DICT_INTERFACE_CREATE_TIME:getCurrent_date(format="%Y-%m-%d %H:%M:%S"),
|
|
|
- DOCUMENT_PRODUCT_DICT_INTERFACE_UPDATE_TIME:getCurrent_date(format="%Y-%m-%d %H:%M:%S"),
|
|
|
- DOCUMENT_PRODUCT_DICT_INTERFACE_ACTION:"insert"
|
|
|
- }
|
|
|
- _dpdi = Document_product_dict_interface(_d)
|
|
|
- _dpdi.update_row(self.ots_client)
|
|
|
+ _dpd_brand = Document_product_dict(_d_brand)
|
|
|
+ # _dpd_brand.updateAlias(str(new_brand).lower())
|
|
|
+ if not _dpd_brand.exists_row(self.ots_client):
|
|
|
+ _dpd_brand.update_row(self.ots_client)
|
|
|
|
|
|
else:
|
|
|
pass
|
|
|
# #update alias
|
|
|
- # _dpd = Document_product_dict({DOCUMENT_PRODUCT_DICT_ID:specs_ots_id})
|
|
|
+ # _dpd = Document_product_dict({DOCUMENT_PRODUCT_DICT_ID:brand_ots_id})
|
|
|
# _flag = _dpd.fix_columns(self.ots_client,[DOCUMENT_PRODUCT_DICT_ALIAS],True)
|
|
|
# if _flag:
|
|
|
- # if _dpd.updateAlias(specs):
|
|
|
+ # if _dpd.updateAlias(brand):
|
|
|
# _dpd.update_row(self.ots_client)
|
|
|
+
|
|
|
+ _find = True
|
|
|
break
|
|
|
else:
|
|
|
- if _index == 1:
|
|
|
- similar_flag = True
|
|
|
-
|
|
|
- if not break_flag and similar_flag:
|
|
|
- log("check_specs failed")
|
|
|
- new_specs = clean_product_specs(specs)
|
|
|
- # insert into document_product_dict a new record
|
|
|
- # to update the document_product_dict which is builded for search
|
|
|
- # add new specs
|
|
|
- if brand_ots_id is not None and name_ots_id is not None:
|
|
|
- # _md5 = get_document_product_dict_id(brand_ots_id,new_specs)
|
|
|
- # _d = {DOCUMENT_PRODUCT_DICT_ID:_md5,
|
|
|
- # DOCUMENT_PRODUCT_DICT_NAME:new_specs,
|
|
|
- # DOCUMENT_PRODUCT_DICT_ALIAS:"%s"%(new_specs.lower()),
|
|
|
- # DOCUMENT_PRODUCT_DICT_GRADE:SPECS_GRADE,
|
|
|
- # DOCUMENT_PRODUCT_DICT_STATUS:1,
|
|
|
- # DOCUMENT_PRODUCT_DICT_PARENT_ID:brand_ots_id,
|
|
|
- # DOCUMENT_PRODUCT_DICT_CREATE_TIME:getCurrent_date(format="%Y-%m-%d %H:%M:%S"),
|
|
|
- # DOCUMENT_PRODUCT_DICT_UPDATE_TIME:getCurrent_date(format="%Y-%m-%d %H:%M:%S"),
|
|
|
- # }
|
|
|
- # _dpd = Document_product_dict(_d)
|
|
|
- # # _dpd.updateAlias(new_specs)
|
|
|
- # _dpd.update_row(self.ots_client)
|
|
|
-
|
|
|
- # user interface to add
|
|
|
- _d = {DOCUMENT_PRODUCT_DICT_INTERFACE_ID:uuid4().hex,
|
|
|
- DOCUMENT_PRODUCT_DICT_INTERFACE_NAME:new_specs,
|
|
|
- DOCUMENT_PRODUCT_DICT_INTERFACE_ALIAS:"%s"%(specs,new_specs.lower()),
|
|
|
- DOCUMENT_PRODUCT_DICT_INTERFACE_GRADE:SPECS_GRADE,
|
|
|
- DOCUMENT_PRODUCT_DICT_INTERFACE_STATUS:1,
|
|
|
- DOCUMENT_PRODUCT_DICT_INTERFACE_PARENT_ID:brand_ots_id,
|
|
|
- DOCUMENT_PRODUCT_DICT_INTERFACE_CREATE_TIME:getCurrent_date(format="%Y-%m-%d %H:%M:%S"),
|
|
|
- DOCUMENT_PRODUCT_DICT_INTERFACE_UPDATE_TIME:getCurrent_date(format="%Y-%m-%d %H:%M:%S"),
|
|
|
- DOCUMENT_PRODUCT_DICT_INTERFACE_ACTION:"insert"
|
|
|
- }
|
|
|
- _dpdi = Document_product_dict_interface(_d)
|
|
|
- _dpdi.update_row(self.ots_client)
|
|
|
+ # log("check brand similar failed:%s and %s"%(brand,ots_name))
|
|
|
+ # add new brand?
|
|
|
+ pass
|
|
|
+ if _find:
|
|
|
+ break
|
|
|
+ if not _find:
|
|
|
+ for brand in l_brand:
|
|
|
+ if self.check_new_brand(brand):
|
|
|
+ new_brand = clean_product_brand(brand)
|
|
|
+ if new_brand=="":
|
|
|
+ continue
|
|
|
+ log("adding new brand %s"%(str(new_brand)))
|
|
|
+ _d_brand = {DOCUMENT_PRODUCT_DICT_INTERFACE_ID:uuid4().hex,
|
|
|
+ DOCUMENT_PRODUCT_DICT_INTERFACE_NAME:new_brand,
|
|
|
+ DOCUMENT_PRODUCT_DICT_INTERFACE_ALIAS:"%s"%(str(brand).lower()),
|
|
|
+ DOCUMENT_PRODUCT_DICT_INTERFACE_GRADE:BRAND_GRADE,
|
|
|
+ DOCUMENT_PRODUCT_DICT_INTERFACE_STATUS:1,
|
|
|
+ DOCUMENT_PRODUCT_DICT_INTERFACE_PARENT_ID:name_ots_id,
|
|
|
+ DOCUMENT_PRODUCT_DICT_INTERFACE_CREATE_TIME:getCurrent_date(format="%Y-%m-%d %H:%M:%S"),
|
|
|
+ DOCUMENT_PRODUCT_DICT_INTERFACE_UPDATE_TIME:getCurrent_date(format="%Y-%m-%d %H:%M:%S"),
|
|
|
+ DOCUMENT_PRODUCT_DICT_INTERFACE_ACTION:"insert"
|
|
|
+ }
|
|
|
+ dpdi = Document_product_dict_interface(_d_brand)
|
|
|
+ dpdi.update_row(self.ots_client)
|
|
|
+ break
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+ if specs is not None and specs!="":
|
|
|
+
|
|
|
+ #check ots
|
|
|
+ bool_query = BoolQuery(must_queries=[
|
|
|
+ TermQuery(DOCUMENT_PRODUCT_DICT_NAME,specs),
|
|
|
+ TermQuery(DOCUMENT_PRODUCT_DICT_GRADE,SPECS_GRADE)
|
|
|
+ ])
|
|
|
+ rows,next_token,total_count,is_all_succeed = self.ots_client.search("document_product_dict","document_product_dict_index",
|
|
|
+ SearchQuery(bool_query,get_total_count=True))
|
|
|
+ if total_count>0:
|
|
|
+ new_specs = specs
|
|
|
else:
|
|
|
- # add new specs?
|
|
|
- debug("specs not similar")
|
|
|
- if is_legal_specs(specs):
|
|
|
- debug("is_legal_specs")
|
|
|
+ debug("getting sepcs %s"%(specs))
|
|
|
+ list_specs = []
|
|
|
+ c_specs = clean_product_specs(specs)
|
|
|
+ list_specs.append(c_specs)
|
|
|
+
|
|
|
+ for s in re.split("[\u4e00-\u9fff]",specs):
|
|
|
+ if s!="" and len(s)>4:
|
|
|
+ list_specs.append(s)
|
|
|
+ similar_flag = None
|
|
|
+ _index = 0
|
|
|
+ break_flag = False
|
|
|
+ for c_specs in list_specs:
|
|
|
+ if break_flag:
|
|
|
+ break
|
|
|
+ _index += 1
|
|
|
+ specs_vector = request_embedding(c_specs)
|
|
|
+
|
|
|
+ if specs_vector is not None:
|
|
|
+ Coll,_ = self.get_collection(SPECS_GRADE)
|
|
|
+ search_list = search_embedding(Coll,embedding_index_name,[specs_vector],self.search_params,output_fields,limit=60)
|
|
|
+
|
|
|
+ for _search in search_list:
|
|
|
+
|
|
|
+ ots_id = _search.entity.get("standard_name_id")
|
|
|
+ ots_name = _search.entity.get("standard_name")
|
|
|
+ ots_parent_id = _search.entity.get("ots_parent_id")
|
|
|
+
|
|
|
+ debug("checking specs %s and %s"%(specs,ots_name))
|
|
|
+ if is_similar(specs,ots_name):
|
|
|
+ # log("specs is_similar")
|
|
|
+ if check_specs(c_specs,ots_name):
|
|
|
+ break_flag = True
|
|
|
+ new_specs = ots_name
|
|
|
+ log("check_specs %s succeed %s"%(specs,new_specs))
|
|
|
+
|
|
|
+ # to update the document_product_dict which is builded for search
|
|
|
+ if brand_ots_id is not None:
|
|
|
+ # judge if the specs which parent_id is brand_ots_id exists,insert one if not exists else update alias
|
|
|
+ specs_ots_id = get_document_product_dict_id(brand_ots_id,new_specs)
|
|
|
+
|
|
|
+ _d_specs = {DOCUMENT_PRODUCT_DICT_ID:specs_ots_id,
|
|
|
+ DOCUMENT_PRODUCT_DICT_NAME:new_specs,
|
|
|
+ DOCUMENT_PRODUCT_DICT_ALIAS:"%s"%(str(specs).lower()),
|
|
|
+ DOCUMENT_PRODUCT_DICT_GRADE:SPECS_GRADE,
|
|
|
+ DOCUMENT_PRODUCT_DICT_STATUS:1,
|
|
|
+ DOCUMENT_PRODUCT_DICT_PARENT_ID:brand_ots_id,
|
|
|
+ DOCUMENT_PRODUCT_DICT_IS_SYNCHONIZED:IS_SYNCHONIZED,
|
|
|
+ DOCUMENT_PRODUCT_DICT_CREATE_TIME:getCurrent_date(format="%Y-%m-%d %H:%M:%S"),
|
|
|
+ DOCUMENT_PRODUCT_DICT_UPDATE_TIME:getCurrent_date(format="%Y-%m-%d %H:%M:%S"),
|
|
|
+ }
|
|
|
+ _dpd_specs = Document_product_dict(_d_specs)
|
|
|
+ # _dpd_specs.updateAlias(str(new_specs).lower())
|
|
|
+ if not _dpd_specs.exists_row(self.ots_client):
|
|
|
+ _dpd_specs.update_row(self.ots_client)
|
|
|
+ # user interface to add
|
|
|
+ else:
|
|
|
+ pass
|
|
|
+ # #update alias
|
|
|
+ # _dpd = Document_product_dict({DOCUMENT_PRODUCT_DICT_ID:specs_ots_id})
|
|
|
+ # _flag = _dpd.fix_columns(self.ots_client,[DOCUMENT_PRODUCT_DICT_ALIAS],True)
|
|
|
+ # if _flag:
|
|
|
+ # if _dpd.updateAlias(specs):
|
|
|
+ # _dpd.update_row(self.ots_client)
|
|
|
+ break
|
|
|
+ else:
|
|
|
+ if _index == 1:
|
|
|
+ similar_flag = True
|
|
|
+
|
|
|
+ if not break_flag and similar_flag:
|
|
|
+ debug("check_specs failed")
|
|
|
new_specs = clean_product_specs(specs)
|
|
|
# insert into document_product_dict a new record
|
|
|
# to update the document_product_dict which is builded for search
|
|
|
# add new specs
|
|
|
- if brand_ots_id is not None and name_ots_id is not None:
|
|
|
- _md5 = get_document_product_dict_id(brand_ots_id,new_specs)
|
|
|
-
|
|
|
+ if brand_ots_id is not None and name_ots_id is not None and len(specs)<MAX_NAME_LENGTH:
|
|
|
+ # _md5 = get_document_product_dict_id(brand_ots_id,new_specs)
|
|
|
# _d = {DOCUMENT_PRODUCT_DICT_ID:_md5,
|
|
|
# DOCUMENT_PRODUCT_DICT_NAME:new_specs,
|
|
|
- # DOCUMENT_PRODUCT_DICT_ALIAS:"%s&&%s"%(specs,new_specs),
|
|
|
+ # DOCUMENT_PRODUCT_DICT_ALIAS:"%s"%(new_specs.lower()),
|
|
|
# DOCUMENT_PRODUCT_DICT_GRADE:SPECS_GRADE,
|
|
|
# DOCUMENT_PRODUCT_DICT_STATUS:1,
|
|
|
# DOCUMENT_PRODUCT_DICT_PARENT_ID:brand_ots_id,
|
|
@@ -405,12 +377,14 @@ class Product_Manager(Product_Dict_Manager):
|
|
|
# DOCUMENT_PRODUCT_DICT_UPDATE_TIME:getCurrent_date(format="%Y-%m-%d %H:%M:%S"),
|
|
|
# }
|
|
|
# _dpd = Document_product_dict(_d)
|
|
|
+ # # _dpd.updateAlias(new_specs)
|
|
|
# _dpd.update_row(self.ots_client)
|
|
|
|
|
|
# user interface to add
|
|
|
+ log("adding new specs %s"%(new_specs))
|
|
|
_d = {DOCUMENT_PRODUCT_DICT_INTERFACE_ID:uuid4().hex,
|
|
|
DOCUMENT_PRODUCT_DICT_INTERFACE_NAME:new_specs,
|
|
|
- DOCUMENT_PRODUCT_DICT_INTERFACE_ALIAS:"%s"%(new_specs.lower()),
|
|
|
+ DOCUMENT_PRODUCT_DICT_INTERFACE_ALIAS:"%s"%(specs),
|
|
|
DOCUMENT_PRODUCT_DICT_INTERFACE_GRADE:SPECS_GRADE,
|
|
|
DOCUMENT_PRODUCT_DICT_INTERFACE_STATUS:1,
|
|
|
DOCUMENT_PRODUCT_DICT_INTERFACE_PARENT_ID:brand_ots_id,
|
|
@@ -420,6 +394,44 @@ class Product_Manager(Product_Dict_Manager):
|
|
|
}
|
|
|
_dpdi = Document_product_dict_interface(_d)
|
|
|
_dpdi.update_row(self.ots_client)
|
|
|
+ else:
|
|
|
+ # add new specs?
|
|
|
+ debug("specs not similar")
|
|
|
+ if is_legal_specs(specs) and len(specs)<MAX_NAME_LENGTH:
|
|
|
+ debug("is_legal_specs")
|
|
|
+ new_specs = clean_product_specs(specs)
|
|
|
+ # insert into document_product_dict a new record
|
|
|
+ # to update the document_product_dict which is builded for search
|
|
|
+ # add new specs
|
|
|
+ if brand_ots_id is not None and name_ots_id is not None:
|
|
|
+ _md5 = get_document_product_dict_id(brand_ots_id,new_specs)
|
|
|
+
|
|
|
+ # _d = {DOCUMENT_PRODUCT_DICT_ID:_md5,
|
|
|
+ # DOCUMENT_PRODUCT_DICT_NAME:new_specs,
|
|
|
+ # DOCUMENT_PRODUCT_DICT_ALIAS:"%s&&%s"%(specs,new_specs),
|
|
|
+ # DOCUMENT_PRODUCT_DICT_GRADE:SPECS_GRADE,
|
|
|
+ # DOCUMENT_PRODUCT_DICT_STATUS:1,
|
|
|
+ # DOCUMENT_PRODUCT_DICT_PARENT_ID:brand_ots_id,
|
|
|
+ # DOCUMENT_PRODUCT_DICT_CREATE_TIME:getCurrent_date(format="%Y-%m-%d %H:%M:%S"),
|
|
|
+ # DOCUMENT_PRODUCT_DICT_UPDATE_TIME:getCurrent_date(format="%Y-%m-%d %H:%M:%S"),
|
|
|
+ # }
|
|
|
+ # _dpd = Document_product_dict(_d)
|
|
|
+ # _dpd.update_row(self.ots_client)
|
|
|
+
|
|
|
+ log("adding new specs %s"%(new_specs))
|
|
|
+ # user interface to add
|
|
|
+ _d = {DOCUMENT_PRODUCT_DICT_INTERFACE_ID:uuid4().hex,
|
|
|
+ DOCUMENT_PRODUCT_DICT_INTERFACE_NAME:new_specs,
|
|
|
+ DOCUMENT_PRODUCT_DICT_INTERFACE_ALIAS:"%s"%(new_specs.lower()),
|
|
|
+ DOCUMENT_PRODUCT_DICT_INTERFACE_GRADE:SPECS_GRADE,
|
|
|
+ DOCUMENT_PRODUCT_DICT_INTERFACE_STATUS:1,
|
|
|
+ DOCUMENT_PRODUCT_DICT_INTERFACE_PARENT_ID:brand_ots_id,
|
|
|
+ DOCUMENT_PRODUCT_DICT_INTERFACE_CREATE_TIME:getCurrent_date(format="%Y-%m-%d %H:%M:%S"),
|
|
|
+ DOCUMENT_PRODUCT_DICT_INTERFACE_UPDATE_TIME:getCurrent_date(format="%Y-%m-%d %H:%M:%S"),
|
|
|
+ DOCUMENT_PRODUCT_DICT_INTERFACE_ACTION:"insert"
|
|
|
+ }
|
|
|
+ _dpdi = Document_product_dict_interface(_d)
|
|
|
+ _dpdi.update_row(self.ots_client)
|
|
|
|
|
|
# judge if the product matches the standard product
|
|
|
if name_ots_id is not None:
|
|
@@ -485,6 +497,10 @@ class Product_Manager(Product_Dict_Manager):
|
|
|
save_product_tmp.update_row(self.ots_client)
|
|
|
|
|
|
def check_new_brand(self,brand):
|
|
|
+
|
|
|
+ _search = re.search("品牌[::;;](?P<brand>.{2,8}?)([.。、;::]|规格|型号|生产厂家|厂家)",brand)
|
|
|
+ if _search is not None:
|
|
|
+ brand = _search.groupdict().get("brand")
|
|
|
if brand is None or len(brand)<2:
|
|
|
return False
|
|
|
# check whether this brand exists in interface and action is delete
|
|
@@ -501,14 +517,14 @@ class Product_Manager(Product_Dict_Manager):
|
|
|
|
|
|
# check whether this brand exists in dict and grade=name_grade or grade=specs_grade
|
|
|
bool_query = BoolQuery(must_queries=[
|
|
|
- TermQuery(DOCUMENT_PRODUCT_DICT_NAME,name),
|
|
|
+ TermQuery(DOCUMENT_PRODUCT_DICT_NAME,brand),
|
|
|
BoolQuery(should_queries=[
|
|
|
TermQuery(DOCUMENT_PRODUCT_DICT_GRADE,NAME_GRADE),
|
|
|
TermQuery(DOCUMENT_PRODUCT_DICT_GRADE,SPECS_GRADE)
|
|
|
])
|
|
|
|
|
|
])
|
|
|
- rows,next_token,total_count,is_all_succeed = self.ots_client.search("document_product","document_product_index",
|
|
|
+ rows,next_token,total_count,is_all_succeed = self.ots_client.search("document_product_dict","document_product_dict_index",
|
|
|
SearchQuery(bool_query,get_total_count=True))
|
|
|
if total_count>0:
|
|
|
return False
|
|
@@ -535,9 +551,12 @@ class Product_Manager(Product_Dict_Manager):
|
|
|
|
|
|
if total_count>=5:
|
|
|
new_brand = re.sub("[^\u4e00-\u9fff]",'',brand)
|
|
|
- if re.search("详见|无|国产|null|其他|详细|废标|[0-9/]|品牌|文件") is None and len(brand)<=8:
|
|
|
+ if re.search("详见|无|国产|null|其他|详细|废标|[0-9/]|品牌|文件",brand) is None and len(brand)<=8:
|
|
|
return True
|
|
|
|
|
|
+ # extract the brand
|
|
|
+ # "品牌[::]?(<brand>.{2,5}([.。、::]|型号|生产厂家|厂家))"
|
|
|
+
|
|
|
|
|
|
|
|
|
@staticmethod
|
|
@@ -816,6 +835,11 @@ def fix_product_data():
|
|
|
mt.run()
|
|
|
|
|
|
def test_check_brand():
|
|
|
+ import logging
|
|
|
+ root = logging.getLogger()
|
|
|
+ root.setLevel(logging.DEBUG)
|
|
|
+ from queue import Queue
|
|
|
+
|
|
|
brand_path = "brand.txt"
|
|
|
list_brand = []
|
|
|
with open(brand_path,"r",encoding="utf8") as f:
|
|
@@ -827,6 +851,8 @@ def test_check_brand():
|
|
|
if len(line)>0:
|
|
|
brand = {"brand":line}
|
|
|
list_brand.append(brand)
|
|
|
+ # if len(list_brand)>100:
|
|
|
+ # break
|
|
|
task_queue = Queue()
|
|
|
for _d in list_brand:
|
|
|
task_queue.put(_d)
|
|
@@ -835,18 +861,22 @@ def test_check_brand():
|
|
|
|
|
|
def _handle(item,result_queue):
|
|
|
brand = item.get("brand")
|
|
|
+ new_brand = clean_product_brand(brand)
|
|
|
_f = pm.check_new_brand(brand)
|
|
|
item["f"] = _f
|
|
|
+ item["new_brand"] = new_brand
|
|
|
mt = MultiThreadHandler(task_queue,_handle,None,30,1)
|
|
|
mt.run()
|
|
|
list_legal_brand = []
|
|
|
list_illegal_brand = []
|
|
|
for _d in list_brand:
|
|
|
- brand = _d.get("brand")
|
|
|
- f = _d["f"]
|
|
|
+ f = _d.get("f")
|
|
|
+ log("brand %s flag %s"%(brand,str(f)))
|
|
|
if f:
|
|
|
+ brand = _d.get("new_brand")
|
|
|
list_legal_brand.append(brand)
|
|
|
else:
|
|
|
+ brand = _d.get("brand")
|
|
|
list_illegal_brand.append(brand)
|
|
|
with open("legal_brand.txt","w",encoding="utf8") as f:
|
|
|
for b in list_legal_brand:
|
|
@@ -855,12 +885,22 @@ def test_check_brand():
|
|
|
for b in list_illegal_brand:
|
|
|
f.write(b+"\n")
|
|
|
|
|
|
+def test_match():
|
|
|
+ a = "Mini-7"
|
|
|
+ vector = request_embedding(a)
|
|
|
+ pm = Product_Manager()
|
|
|
+ Coll,_ = pm.get_collection(NAME_GRADE)
|
|
|
+ output_fields = ['ots_id','ots_name',"ots_parent_id","standard_name","standard_name_id"]
|
|
|
+ search_list = search_embedding(Coll,embedding_index_name,[vector],pm.search_params,output_fields,limit=60)
|
|
|
+ print(search_list)
|
|
|
+
|
|
|
|
|
|
def test():
|
|
|
# pm = Product_Manager()
|
|
|
# pm.test()
|
|
|
- fix_product_data()
|
|
|
-
|
|
|
+ # fix_product_data()
|
|
|
+ # test_check_brand()
|
|
|
+ test_match()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|