twitter-ner-python/download_wiki.py at master · plotlabs/twitter-ner-python · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122


import requests
import json
import regex as re
from glob import glob
import json


query="""
SELECT ?itemLabel
WHERE {
  ?item wdt:P31 wd:Q5 . #instance of human
        ?item wdt:P106/wdt:P279 wd:Q639669 . #occupation a subclass of musician
  SERVICE wikibase:label {
    bd:serviceParam wikibase:language "en" .
   }
}

LIMIT 10
"""

QID_REGEX=re.compile(r'^Q[0-9]+$')


url = 'https://query.wikidata.org/bigdata/namespace/wdq/sparql'
def get_query(queryfile):
    query = ""
    with open(queryfile) as fp:
        query = fp.read().strip()
    return query

def download_data(query, data_format="json"):
    if data_format == "xml":
        data = requests.get(url, params={'query': query, 'format': 'xml'})
    else:
        data = requests.get(url, params={'query': query, 'format': 'json'}).json()
        print "Downloaded %s records" % (len(data["results"]["bindings"]))
    return data

def save_data(data, outfile, label="itemLabel", data_format="json"):
    with open(outfile, "wb+") as fp:
        processed, written = 0, 0
        if data_format == "xml":
            from bs4 import BeautifulSoup
            data_xml = BeautifulSoup(data.text, "lxml")
            results = data_xml.find_all("result")
        else:
            results = data["results"]["bindings"]

        for k in results:
            if data_format == "xml":
                line = k.text.strip()
            else:
                line=k[label]["value"]
            try:
                processed += 1
                if QID_REGEX.match(line):
                    continue
                print >> fp, line
                written += 1
            except:
                continue
    print "Processed: %s, Written: %s" % (processed, written)
    return True


data = download_data(query=query)
save_data(data, "data/wikidata/downloaded/temp.txt", label="itemLabel")

query_files = glob("data/wikidata/queries/*.txt")
print query_files

OUTDIR="./data/wikidata/downloaded/"


for q in ["data/wikidata/queries/companynames.txt"]:
    print "Processing %s" % q
    if "persons" in q or "movies" in q:
        data_format="xml"
    else:
        data_format="json"
    base_file=q.split("/")[-1]
    outfile="%s/%s.results.txt" % (OUTDIR, base_file)
    query = get_query(q)
    data = download_data(query=query, data_format=data_format)
    print "Saving to %s" % outfile
    save_data(data, outfile, label="itemLabel", data_format=data_format)


for q in ["data/wikidata/queries/buildings.txt"]:
    print "Processing %s" % q
    if "persons" in q or "movies" in q:
        data_format="xml"
    else:
        data_format="json"
    base_file=q.split("/")[-1]
    outfile="%s/%s.results.txt" % (OUTDIR, base_file)
    query = get_query(q)
    data = download_data(query=query, data_format=data_format)
    print "Saving to %s" % outfile
    save_data(data, outfile, label="itemLabel", data_format=data_format)


for q in query_files:
    print "Processing %s" % q
    if "persons" in q or "movies" in q:
        data_format="xml"
    else:
        data_format="json"
    base_file=q.split("/")[-1]
    outfile="%s/%s.results.txt" % (OUTDIR, base_file)
    query = get_query(q)
    data = download_data(query=query, data_format=data_format)
    print "Saving to %s" % outfile
    save_data(data, outfile, label="itemLabel", data_format=data_format)


data = requests.get(url, params={'query': query, 'format': 'xml'})
save_data(data, outfile=outfile, data_format="xml")