BIDI_ML_INFO_EXTRACTION/BiddingKG/deepdive/test/getArticles.py

#coding:utf8

import requests

import json
import pandas as pd


id = []
l_content = []
l_tenderee = []
l_agency = []
l_win_tenderer = []
l_first_tenderer = []
l_second_tenderer = []
l_third_tenderer = []

for i in range(1):
    print(i)
    #type=tenderee,agency,tenderer,firsttenderer,secondtenderer,thirdtenderer
    page = requests.post("http://192.168.2.54:8081/data-bigdata-api-1.0.0/data/outPutData?num=3000&type=zhaobiao&param=tenderee,agency").text
    #page = requests.post("http://192.168.2.3:9090/data/outPutData?num=2&type=zhongbiao&param=tenderer").text
    #print(page)
    for item in json.loads(page,encoding="utf8"):
        id.append(item["document_id"])
        l_content.append(item["content"])
        l_tenderee.append(item["tenderee"])
        l_agency.append(item["agency"])
        l_win_tenderer.append("")
        l_first_tenderer.append("")
        l_second_tenderer.append("")
        l_third_tenderer.append("")
'''     
for i in range(1):
    print(i)
    #type=tenderee,agency,tenderer,firsttenderer,secondtenderer,thirdtenderer
    page = requests.post("http://192.168.2.54:8081/data-bigdata-api-1.0.0/data/outPutData?num=1500&type=zhongbiao&param=tenderee,agency,tenderer,firsttenderer,secondtenderer,thirdtenderer").text
    #page = requests.post("http://192.168.2.3:9090/data/outPutData?num=2&type=zhongbiao&param=tenderer").text
    #print(page)
    for item in json.loads(page,encoding="utf8"):
        id.append(item["document_id"])
        l_content.append(item["content"])
        l_tenderee.append(item["tenderee"])
        l_agency.append(item["agency"])
        l_win_tenderer.append(item["win_tenderer"])
        l_first_tenderer.append(item["first_tenderer"])
        l_second_tenderer.append(item["second_tenderer"])
        l_third_tenderer.append(item["third_tenderer"])
'''
'''
for i in range(1):
    print(i)
    #type=tenderee,agency,tenderer,firsttenderer,secondtenderer,thirdtenderer
    page = requests.post("http://192.168.2.54:8081/data-bigdata-api-1.0.0/data/outPutData?num=1000&not=true&type=zhongbiao&param=tenderer").text
    #page = requests.post("http://192.168.2.3:9090/data/outPutData?num=2&type=zhongbiao&param=tenderer").text
    #print(page)
    for item in json.loads(page,encoding="utf8"):
        id.append(item["document_id"])
        l_content.append(item["content"])
        l_tenderee.append("")
        l_agency.append("")
        l_win_tenderer.append("")
        l_first_tenderer.append("")
        l_second_tenderer.append("")
        l_third_tenderer.append("")
'''        
dataframe = pd.DataFrame({'id':id,'content':l_content,"tenderee":l_tenderee,"agency":l_agency,"win_tenderer":l_win_tenderer,"first_tenderer":l_first_tenderer,"second_tenderer":l_second_tenderer,"third_tenderer":l_third_tenderer})
columns = ['id','content',"tenderee","agency","win_tenderer","first_tenderer","second_tenderer","third_tenderer"]
dataframe.to_csv("articles.csv",index=False,header=False,sep=",",encoding="utf8",columns=columns)