DataMining/dataSource/source.py

#encoding:UTF8

from dataSource.setttings import *
import requests
import json
import pymysql
import pymongo
from py2neo import Graph,NodeMatcher
import tablestore
import psycopg2
from elasticsearch import Elasticsearch



def solrQuery(collection,args):
    if collection in solr_collections:
        _arg = ""
        for k,v in args.items():
            _arg += "&%s=%s"%(k,v)
        _arg = _arg[1:]
        url = "%s%s/select?%s"%(solr_collections[collection],collection,_arg)
        resp = requests.get(url)
        if resp.status_code==200:
            return json.loads(resp.content.decode())
        return None

def solrQuery_url(url):
    resp = requests.get(url)
    if resp.status_code==200:
        return json.loads(resp.content.decode())
    return None

def getConnection_mysql(db=None):
    if db is None:
        db = mysql_db
    connect = pymysql.Connect(host=mysql_host, port=mysql_port, db=db, user=mysql_user, passwd=mysql_pass)
    return connect

def getConnection_testmysql(db=None):
    if db is None:
        db = test_mysql_db
    connect = pymysql.Connect(host=test_mysql_host, port=test_mysql_port, db=db, user=test_mysql_user, passwd=test_mysql_pass)
    return connect

def getConnection_postgres(db):
    conn = psycopg2.connect(dbname=db,user="postgres",password="postgres",host="192.168.2.103")
    return conn

def getConnection_oracle():
    import cx_Oracle
    connect = cx_Oracle.connect(oracle_user,oracle_pass,'%s:%s/%s'%(oracle_host,oracle_port,oracle_db), encoding = "UTF-8", nencoding = "UTF-8")
    # connect = cx_Oracle.connect('%s/%s@%s:%s/%s'%(oracle_user,oracle_pass,oracle_host,oracle_port,oracle_db))
    return connect

def getConnect_mongodb():
    client = pymongo.MongoClient(mongo_host,mongo_port)
    db = client[mongo_db]
    db.authenticate(mongo_user,mongo_pass)
    return db

def make_elasticSearch(elasticSearch_url,query,auth):
    resp = requests.post(elasticSearch_url,json=query,auth=auth)
    print(resp.status_code)
    if resp.status_code==200:
        return json.loads(resp.content.decode())
    return None

def getConnect_neo4j():
    graph = Graph(host=neo4j_host,auth=(neo4j_user,neo4j_pass))
    return graph
    # finded = graph.run("MATCH (n:Organization)-[R:ZhaoBiaoRelation]->(p:Project) where n.name='昆山市周市基础建设开发有限公司的昆山市恒迪服装辅料公司'  RETURN p LIMIT 25")
    # print(json.loads(json.dumps(finded.data())))
    # print(finded)

import platform

import os
def check_net(testserver):
    try:
        response = os.system("ping -c 1 " + testserver)
        if response == 0:
            return True
    except:
        return False
    return False

import platform
print(platform.system())
if platform.system()=="Windows":
    OTS_URL = "https://bxkc-ots.cn-hangzhou.ots.aliyuncs.com"
    OTS_URL = "https://bxkc-ots.cn-hangzhou.vpc.tablestore.aliyuncs.com"
else:
    OTS_URL = "https://bxkc-ots.cn-hangzhou.vpc.tablestore.aliyuncs.com"
    check_url = "oss-cn-hangzhou-internal.aliyuncs.com"
    is_internal = True
    if not check_net(check_url):
        is_internal = False
        OTS_URL = "https://bxkc-ots.cn-hangzhou.ots.aliyuncs.com"

def getConnect_ots():
    # ots_client = tablestore.client.OTSClient(OTS_URL, 'LTAI4FyUT7ZcQFZPjVtw5y9b', '2zscfFTvy3JWavtCeCOthLxF8bDNH3',
    #                                          'bxkc-ots', logger_name = 'table_store.log',
    #                                          retry_policy = tablestore.WriteRetryPolicy(),socket_timeout=3000)
    ots_client = tablestore.client.OTSClient(OTS_URL, ots_AccessKeyId, ots_AccessKeySecret,
                                             'bxkc-ots', logger_name = 'table_store.log',
                                             retry_policy = tablestore.WriteRetryPolicy(),socket_timeout=3000)
    return ots_client

def getConnect_capacity():
    # ots_client = tablestore.client.OTSClient(OTS_URL, 'LTAI4FyUT7ZcQFZPjVtw5y9b', '2zscfFTvy3JWavtCeCOthLxF8bDNH3',
    #                                          'bxkc-ots', logger_name = 'table_store.log',
    #                                          retry_policy = tablestore.WriteRetryPolicy(),socket_timeout=3000)
    ots_client = tablestore.client.OTSClient(OTS_URL, ots_AccessKeyId, ots_AccessKeySecret,
                                             'bxkc-capacity', logger_name = 'table_store.log',
                                             retry_policy = tablestore.WriteRetryPolicy(),socket_timeout=3000)
    return ots_client

def getConnect_gdb():
    from gremlin_python.driver import client
    client = client.Client('ws://gds-bp130d7rgd9m7n61150070pub.graphdb.rds.aliyuncs.com:3734/gremlin', 'g', username="bxkc", password="k0n1bxkc!0K^Em%j")
    callback = client.submitAsync("g.V('北京赛克德利科贸有限公司').outE('ZhongBiaoRelation').inV().inE('ZhaoBiaoRelation').outV()")
    for result in callback.result():
        for item in result:
            print(item.id)
    return client
import oss2
def getAuth():
    auth = oss2.Auth(ots_AccessKeyId, ots_AccessKeySecret)
    return auth

import os
def check_net(testserver):
    try:
        response = os.system("ping -c 1 " + testserver)
        if response == 0:
            return True
    except:
        return False
    return False

if __name__=="__main__":
    # solrQuery("document",{"q":"*:*"})
    # getConnect_mongodb()
    # data = solrQuery_url('http://47.97.221.63:8983/solr/document/select?fq=(publishtime:[2020-01-01T00:00:00Z%20TO%202020-08-12T23:59:59Z])&q=dochtmlcon:"防盗门"')
    # data = solrQuery("document",{"q":'dochtmlcon:"防盗门"',"fq":'(publishtime:[2020-01-01T00:00:00Z%20TO%202020-08-12T23:59:59Z])',"fl":"city","rows":1})
    # data = make_elasticSearch({"query":{"bool":{"must":[{"wildcard":{"nicknames.keyword":"*服装*"}}],"must_not":[],"should":[]}},"from":0,"size":10,"sort":[],"aggs":{}})
    # print(data)
    # getConnect_neo4j()
    # conn = getConnection_oracle()

    # cursor = conn.cursor()
    # getConnect_gdb()
    es = Elasticsearch(hosts=ES_HOST,http_auth=(ES_USER, ES_PASSWORD))
    _q = '''{
          "track_total_hits": true,
          "size":1,
          "query": {
              "bool": {
                  "must":[
                      {"terms":{"contact_phone_type":[1,2]}},
                      {"match_phrase":{"city":"深圳"}}
                  ]
              }
          }
          }'''
    print(es)
    print(es.info())
    # print(es.search(body=_q,index="enterprise"))