|
@@ -113,53 +113,6 @@ class Product_Manager(Product_Dict_Manager):
|
|
|
self.standardize(item)
|
|
|
|
|
|
|
|
|
- def match_specs(self,specs):
|
|
|
- bool_query = BoolQuery(must_queries=[
|
|
|
- TermQuery(DOCUMENT_PRODUCT_DICT_NAME,specs),
|
|
|
- TermQuery(DOCUMENT_PRODUCT_DICT_GRADE,SPECS_GRADE)
|
|
|
- ])
|
|
|
- rows,next_token,total_count,is_all_succeed = self.ots_client.search("document_product_dict","document_product_dict_index",
|
|
|
- SearchQuery(bool_query,get_total_count=True))
|
|
|
- if total_count>0:
|
|
|
- new_specs = specs
|
|
|
- return new_specs
|
|
|
- else:
|
|
|
- debug("getting sepcs %s"%(specs))
|
|
|
- list_specs = []
|
|
|
- c_specs = clean_product_specs(specs)
|
|
|
- list_specs.append(c_specs)
|
|
|
-
|
|
|
- for s in re.split("[\u4e00-\u9fff]",specs):
|
|
|
- if s!="" and len(s)>4:
|
|
|
- list_specs.append(s)
|
|
|
- similar_flag = None
|
|
|
- _index = 0
|
|
|
- break_flag = False
|
|
|
- for c_specs in list_specs:
|
|
|
- if break_flag:
|
|
|
- break
|
|
|
- _index += 1
|
|
|
- specs_vector = request_embedding(c_specs)
|
|
|
-
|
|
|
- if specs_vector is not None:
|
|
|
- Coll,_ = self.get_collection(SPECS_GRADE)
|
|
|
- search_list = search_embedding(Coll,embedding_index_name,[specs_vector],self.search_params,output_fields,limit=60)
|
|
|
-
|
|
|
- for _search in search_list:
|
|
|
-
|
|
|
- ots_id = _search.entity.get("standard_name_id")
|
|
|
- ots_name = _search.entity.get("standard_name")
|
|
|
- ots_parent_id = _search.entity.get("ots_parent_id")
|
|
|
-
|
|
|
- debug("checking specs %s and %s"%(specs,ots_name))
|
|
|
- if is_similar(specs,ots_name):
|
|
|
- # log("specs is_similar")
|
|
|
- if check_specs(c_specs,ots_name):
|
|
|
- break_flag = True
|
|
|
- new_specs = ots_name
|
|
|
- return new_specs
|
|
|
-
|
|
|
-
|
|
|
|
|
|
def standardize(self,tmp_dict,output_fields = ['ots_id','ots_name',"ots_parent_id","standard_name","standard_name_id"]):
|
|
|
'''
|
|
@@ -229,7 +182,7 @@ class Product_Manager(Product_Dict_Manager):
|
|
|
name_vector = request_embedding(name)
|
|
|
if name_vector is not None:
|
|
|
Coll,_ = self.get_collection(NAME_GRADE)
|
|
|
- search_list = search_embedding(Coll,embedding_index_name,[name_vector],self.search_params,output_fields,limit=60)
|
|
|
+ search_list = search_embedding(Coll,embedding_index_name,[name_vector],self.search_params,output_fields,limit=20)
|
|
|
|
|
|
for _search in search_list:
|
|
|
ots_id = _search.entity.get("standard_name_id")
|
|
@@ -352,24 +305,33 @@ class Product_Manager(Product_Dict_Manager):
|
|
|
if _find:
|
|
|
break
|
|
|
l_brand = [brand]
|
|
|
- l_brand.append(clean_product_brand(s_brand))
|
|
|
+ l_brand.append(clean_product_brand(brand))
|
|
|
brand_ch = get_chinese_string(brand)
|
|
|
l_brand.extend(brand_ch)
|
|
|
|
|
|
for brand in l_brand:
|
|
|
if _find:
|
|
|
break
|
|
|
- brand_vector = request_embedding(brand)
|
|
|
+ start_time = time.time()
|
|
|
+ # brand_vector = request_embedding(brand)
|
|
|
+ brand_vector = get_embedding_request(brand)
|
|
|
+ log("get embedding for brand %s takes %.4fs"%(brand,time.time()-start_time))
|
|
|
if brand_vector is not None:
|
|
|
Coll,_ = self.get_collection(BRAND_GRADE)
|
|
|
- search_list = search_embedding(Coll,embedding_index_name,[brand_vector],self.search_params,output_fields,limit=60)
|
|
|
-
|
|
|
+ start_time = time.time()
|
|
|
+ # search_list = search_embedding(Coll,embedding_index_name,[brand_vector],self.search_params,output_fields,limit=10)
|
|
|
+ search_list = get_milvus_search(Coll,embedding_index_name,brand,[brand_vector],self.search_params,output_fields,limit=10)
|
|
|
+ log("get search_list for brand %s takes %.4fs"%(brand,time.time()-start_time))
|
|
|
# log("search brand %s"%(brand))
|
|
|
for _search in search_list:
|
|
|
|
|
|
- ots_id = _search.entity.get("standard_name_id")
|
|
|
- ots_name = _search.entity.get("standard_name")
|
|
|
- ots_parent_id = _search.entity.get("ots_parent_id")
|
|
|
+
|
|
|
+ # ots_id = _search.entity.get("standard_name_id")
|
|
|
+ # ots_name = _search.entity.get("standard_name")
|
|
|
+ # ots_parent_id = _search.entity.get("ots_parent_id")
|
|
|
+ ots_id = _search.get("standard_name_id")
|
|
|
+ ots_name = _search.get("standard_name")
|
|
|
+ ots_parent_id = _search.get("ots_parent_id")
|
|
|
|
|
|
# log("check brand %s and %s"%(brand,ots_name))
|
|
|
if is_similar(brand,ots_name,_radio=95) or check_brand(brand,ots_name):
|
|
@@ -419,6 +381,26 @@ class Product_Manager(Product_Dict_Manager):
|
|
|
SearchQuery(bool_query,get_total_count=True))
|
|
|
if total_count>0:
|
|
|
new_specs = specs
|
|
|
+
|
|
|
+ if brand_ots_id is not None:
|
|
|
+ # judge if the specs which parent_id is brand_ots_id exists,insert one if not exists else update alias
|
|
|
+ specs_ots_id = get_document_product_dict_id(brand_ots_id,new_specs)
|
|
|
+
|
|
|
+ _d_specs = {DOCUMENT_PRODUCT_DICT_ID:specs_ots_id,
|
|
|
+ DOCUMENT_PRODUCT_DICT_NAME:new_specs,
|
|
|
+ DOCUMENT_PRODUCT_DICT_ALIAS:"%s"%(str(specs).lower()),
|
|
|
+ DOCUMENT_PRODUCT_DICT_GRADE:SPECS_GRADE,
|
|
|
+ DOCUMENT_PRODUCT_DICT_STATUS:1,
|
|
|
+ DOCUMENT_PRODUCT_DICT_PARENT_ID:brand_ots_id,
|
|
|
+ DOCUMENT_PRODUCT_DICT_IS_SYNCHONIZED:IS_SYNCHONIZED,
|
|
|
+ DOCUMENT_PRODUCT_DICT_CREATE_TIME:getCurrent_date(format="%Y-%m-%d %H:%M:%S"),
|
|
|
+ DOCUMENT_PRODUCT_DICT_UPDATE_TIME:getCurrent_date(format="%Y-%m-%d %H:%M:%S"),
|
|
|
+ }
|
|
|
+ _dpd_specs = Document_product_dict(_d_specs)
|
|
|
+ # _dpd_specs.updateAlias(str(new_specs).lower())
|
|
|
+ if not _dpd_specs.exists_row(self.ots_client):
|
|
|
+ _dpd_specs.update_row(self.ots_client)
|
|
|
+ # user interface to add
|
|
|
else:
|
|
|
debug("getting sepcs %s"%(specs))
|
|
|
list_specs = []
|
|
@@ -566,9 +548,15 @@ class Product_Manager(Product_Dict_Manager):
|
|
|
for specs in list_candidates:
|
|
|
if _find:
|
|
|
break
|
|
|
- s = self.match_specs(specs)
|
|
|
- if s is not None:
|
|
|
- new_specs = s
|
|
|
+ bool_query = BoolQuery(must_queries=[
|
|
|
+ TermQuery(DOCUMENT_PRODUCT_DICT_NAME,specs),
|
|
|
+ TermQuery(DOCUMENT_PRODUCT_DICT_GRADE,SPECS_GRADE)
|
|
|
+ ])
|
|
|
+ rows,next_token,total_count,is_all_succeed = self.ots_client.search("document_product_dict","document_product_dict_index",
|
|
|
+ SearchQuery(bool_query,get_total_count=True))
|
|
|
+ if total_count>0:
|
|
|
+ new_specs = specs
|
|
|
+ _find = True
|
|
|
if brand_ots_id is not None:
|
|
|
# judge if the specs which parent_id is brand_ots_id exists,insert one if not exists else update alias
|
|
|
specs_ots_id = get_document_product_dict_id(brand_ots_id,new_specs)
|
|
@@ -587,8 +575,62 @@ class Product_Manager(Product_Dict_Manager):
|
|
|
# _dpd_specs.updateAlias(str(new_specs).lower())
|
|
|
if not _dpd_specs.exists_row(self.ots_client):
|
|
|
_dpd_specs.update_row(self.ots_client)
|
|
|
- _find = True
|
|
|
- break
|
|
|
+ # user interface to add
|
|
|
+ else:
|
|
|
+ debug("getting sepcs %s"%(specs))
|
|
|
+ list_specs = []
|
|
|
+ c_specs = clean_product_specs(specs)
|
|
|
+ list_specs.append(c_specs)
|
|
|
+
|
|
|
+ for s in re.split("[\u4e00-\u9fff]",specs):
|
|
|
+ if s!="" and len(s)>4:
|
|
|
+ list_specs.append(s)
|
|
|
+ similar_flag = None
|
|
|
+ _index = 0
|
|
|
+ for c_specs in list_specs:
|
|
|
+ if _find:
|
|
|
+ break
|
|
|
+ _index += 1
|
|
|
+ specs_vector = request_embedding(c_specs)
|
|
|
+
|
|
|
+ if specs_vector is not None:
|
|
|
+ Coll,_ = self.get_collection(SPECS_GRADE)
|
|
|
+ search_list = search_embedding(Coll,embedding_index_name,[specs_vector],self.search_params,output_fields,limit=20)
|
|
|
+
|
|
|
+ for _search in search_list:
|
|
|
+ if _find:
|
|
|
+ break
|
|
|
+
|
|
|
+ ots_id = _search.entity.get("standard_name_id")
|
|
|
+ ots_name = _search.entity.get("standard_name")
|
|
|
+ ots_parent_id = _search.entity.get("ots_parent_id")
|
|
|
+
|
|
|
+ debug("checking specs %s and %s"%(specs,ots_name))
|
|
|
+ if is_similar(specs,ots_name):
|
|
|
+ # log("specs is_similar")
|
|
|
+ if check_specs(c_specs,ots_name):
|
|
|
+ break_flag = True
|
|
|
+ new_specs = ots_name
|
|
|
+ if brand_ots_id is not None:
|
|
|
+ # judge if the specs which parent_id is brand_ots_id exists,insert one if not exists else update alias
|
|
|
+ specs_ots_id = get_document_product_dict_id(brand_ots_id,new_specs)
|
|
|
+
|
|
|
+ _d_specs = {DOCUMENT_PRODUCT_DICT_ID:specs_ots_id,
|
|
|
+ DOCUMENT_PRODUCT_DICT_NAME:new_specs,
|
|
|
+ DOCUMENT_PRODUCT_DICT_ALIAS:"%s"%(str(specs).lower()),
|
|
|
+ DOCUMENT_PRODUCT_DICT_GRADE:SPECS_GRADE,
|
|
|
+ DOCUMENT_PRODUCT_DICT_STATUS:1,
|
|
|
+ DOCUMENT_PRODUCT_DICT_PARENT_ID:brand_ots_id,
|
|
|
+ DOCUMENT_PRODUCT_DICT_IS_SYNCHONIZED:IS_SYNCHONIZED,
|
|
|
+ DOCUMENT_PRODUCT_DICT_CREATE_TIME:getCurrent_date(format="%Y-%m-%d %H:%M:%S"),
|
|
|
+ DOCUMENT_PRODUCT_DICT_UPDATE_TIME:getCurrent_date(format="%Y-%m-%d %H:%M:%S"),
|
|
|
+ }
|
|
|
+ _dpd_specs = Document_product_dict(_d_specs)
|
|
|
+ # _dpd_specs.updateAlias(str(new_specs).lower())
|
|
|
+ if not _dpd_specs.exists_row(self.ots_client):
|
|
|
+ _dpd_specs.update_row(self.ots_client)
|
|
|
+ _find = True
|
|
|
+ break
|
|
|
|
|
|
# judge if the product matches the standard product
|
|
|
if name_ots_id is not None:
|
|
@@ -612,27 +654,34 @@ class Product_Manager(Product_Dict_Manager):
|
|
|
|
|
|
|
|
|
if isinstance(unit_price,(float,int)) and isinstance(quantity,(float,int)) and isinstance(total_price,(float,int)):
|
|
|
- new_quantity = total_price/unit_price
|
|
|
- if new_quantity!=quantity:
|
|
|
- if new_quantity==total_price//unit_price:
|
|
|
- quantity = int(new_quantity)
|
|
|
- _product.setValue(DOCUMENT_PRODUCT_QUANTITY,quantity,True)
|
|
|
- else:
|
|
|
- is_legal_data = False
|
|
|
+ if unit_price>0:
|
|
|
+ new_quantity = total_price/unit_price
|
|
|
+ if new_quantity!=quantity:
|
|
|
+ if new_quantity==total_price//unit_price:
|
|
|
+ quantity = int(new_quantity)
|
|
|
+ _product.setValue(DOCUMENT_PRODUCT_QUANTITY,quantity,True)
|
|
|
+ else:
|
|
|
+ is_legal_data = False
|
|
|
+ elif quantity>0:
|
|
|
+ unit_price = total_price/quantity
|
|
|
+ _product.setValue(DOCUMENT_PRODUCT_UNIT_PRICE,unit_price,True)
|
|
|
+
|
|
|
elif isinstance(unit_price,(float,int)) and isinstance(quantity,(float,int)):
|
|
|
total_price = float("%.2f"%(unit_price*quantity))
|
|
|
_product.setValue(DOCUMENT_PRODUCT_TOTAL_PRICE,total_price,True)
|
|
|
elif isinstance(unit_price,(float,int)) and isinstance(total_price,(float,int)):
|
|
|
- quantity = int(total_price//unit_price)
|
|
|
- _product.setValue(DOCUMENT_PRODUCT_QUANTITY,quantity,True)
|
|
|
+ if unit_price>0:
|
|
|
+ quantity = int(total_price//unit_price)
|
|
|
+ _product.setValue(DOCUMENT_PRODUCT_QUANTITY,quantity,True)
|
|
|
elif isinstance(quantity,(float,int)) and isinstance(total_price,(float,int)):
|
|
|
- unit_price = float("%.2f"%(total_price/quantity))
|
|
|
- _product.setValue(DOCUMENT_PRODUCT_UNIT_PRICE,unit_price,True)
|
|
|
+ if quantity>0:
|
|
|
+ unit_price = float("%.2f"%(total_price/quantity))
|
|
|
+ _product.setValue(DOCUMENT_PRODUCT_UNIT_PRICE,unit_price,True)
|
|
|
elif isinstance(quantity,(float,int)) and quantity>10000:
|
|
|
is_legal_data = False
|
|
|
|
|
|
if isinstance(_product.getProperties().get(DOCUMENT_PRODUCT_TOTAL_PRICE),(float,int)) and isinstance(win_bid_price,(float,int)):
|
|
|
- if _product.getProperties().get(DOCUMENT_PRODUCT_TOTAL_PRICE)>win_bid_price:
|
|
|
+ if _product.getProperties().get(DOCUMENT_PRODUCT_TOTAL_PRICE)>win_bid_price*10:
|
|
|
is_legal_data = False
|
|
|
|
|
|
if isinstance(_product.getProperties().get(DOCUMENT_PRODUCT_UNIT_PRICE),(float,int)) and _product.getProperties().get(DOCUMENT_PRODUCT_UNIT_PRICE)>100000000:
|
|
@@ -956,8 +1005,10 @@ def fix_product_data():
|
|
|
:return:
|
|
|
'''
|
|
|
ots_client = getConnect_ots()
|
|
|
- bool_query = BoolQuery(must_queries=[RangeQuery("status",1)
|
|
|
- ])
|
|
|
+ bool_query = BoolQuery(must_queries=[
|
|
|
+ # RangeQuery("status",1)
|
|
|
+ TermQuery("docid",246032980)
|
|
|
+ ])
|
|
|
|
|
|
rows,next_token,total_count,is_all_succeed = ots_client.search("document_product","document_product_index",
|
|
|
SearchQuery(bool_query,sort=Sort(sorters=[FieldSort("status")]),limit=100,get_total_count=True),
|
|
@@ -1022,7 +1073,7 @@ def fix_product_data():
|
|
|
def handle(item,result_queue):
|
|
|
print("handle")
|
|
|
|
|
|
- mt = MultiThreadHandler(task_queue,handle,None,30,1)
|
|
|
+ mt = MultiThreadHandler(task_queue,deleteAndReprocess,None,30,1)
|
|
|
mt.run()
|
|
|
|
|
|
def test_check_brand():
|
|
@@ -1089,9 +1140,9 @@ def test_match():
|
|
|
def test():
|
|
|
# pm = Product_Manager()
|
|
|
# pm.test()
|
|
|
- # fix_product_data()
|
|
|
+ fix_product_data()
|
|
|
# test_check_brand()
|
|
|
- test_match()
|
|
|
+ # test_match()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|