|
|
|
|
|
|
|
|
import requests, time, os, sys |
|
|
|
|
|
import pandas, msgpack |
|
|
|
|
|
|
|
|
import requests, time, os, sys, json, csv |
|
|
from bs4 import BeautifulSoup |
|
|
from bs4 import BeautifulSoup |
|
|
|
|
|
|
|
|
|
|
|
class DataPocket: |
|
|
|
|
|
def __init__(self): |
|
|
|
|
|
self.rows = {} |
|
|
|
|
|
|
|
|
|
|
|
def addRow(self, rownum, name=None, values=None): |
|
|
|
|
|
### values should be a list in column order |
|
|
|
|
|
rownum = int(rownum) |
|
|
|
|
|
d = {'name': name, |
|
|
|
|
|
'values': [] if values == None else [x for x in values] |
|
|
|
|
|
} |
|
|
|
|
|
self.rows[rownum] = d |
|
|
|
|
|
|
|
|
|
|
|
def addName(self, rownum, name): |
|
|
|
|
|
if int(rownum) in self.rows.keys(): |
|
|
|
|
|
self.rows[rownum]['name'] = name |
|
|
|
|
|
else: |
|
|
|
|
|
print("No such row.") |
|
|
|
|
|
|
|
|
|
|
|
def addValues(self, rownum, values): |
|
|
|
|
|
### values should be a list in column order |
|
|
|
|
|
if int(rownum) in self.rows.keys(): |
|
|
|
|
|
self.rows[rownum]['values'] = [x for x in values] |
|
|
|
|
|
else: |
|
|
|
|
|
print("No such row.") |
|
|
|
|
|
|
|
|
|
|
|
def setValue(self, rownum, value, col): |
|
|
|
|
|
### col should be a zero-indexed integer |
|
|
|
|
|
col = int(col) |
|
|
|
|
|
if int(rownum) in self.rows.keys(): |
|
|
|
|
|
try: |
|
|
|
|
|
self.rows[rownum]['values'][col] = value |
|
|
|
|
|
except IndexError: |
|
|
|
|
|
while len(self.rows[rownum]['values'] < col): |
|
|
|
|
|
self.rows[rownum]['values'].append(None) |
|
|
|
|
|
self.rows[rownum]['values'].append(value) |
|
|
|
|
|
else: |
|
|
|
|
|
print("No such row.") |
|
|
|
|
|
|
|
|
|
|
|
def printRow(self, rownum): |
|
|
|
|
|
if int(rownum) in self.rows.keys(): |
|
|
|
|
|
print("Row {} (\"{}\"): {}".format(rownum, self.rows[rownum]['name'], self.rows[rownum]['values'])) |
|
|
|
|
|
else: |
|
|
|
|
|
print("No such row.") |
|
|
|
|
|
|
|
|
|
|
|
def printData(self): |
|
|
|
|
|
for row in self.rows.keys(): |
|
|
|
|
|
self.printRow(row) |
|
|
|
|
|
|
|
|
class DataColl: |
|
|
class DataColl: |
|
|
def __init__(self): |
|
|
def __init__(self): |
|
|
self._year = None |
|
|
self._year = None |
|
|
|
|
|
|
|
|
df.fmt = "json" |
|
|
df.fmt = "json" |
|
|
print(df.endpoint) |
|
|
print(df.endpoint) |
|
|
|
|
|
|
|
|
# df.setFilename(False, "txt") |
|
|
|
|
|
df.setFilename(False, "json") |
|
|
|
|
|
|
|
|
df.setFilename(False, "csv") |
|
|
|
|
|
|
|
|
r = requests.get(df.endpoint) |
|
|
r = requests.get(df.endpoint) |
|
|
# t = msgpack.unpackb(r.content) |
|
|
|
|
|
# with open(df.filename, "w", newline='', encoding="utf-8") as file: |
|
|
|
|
|
# file.write(str(t[b'results'])) |
|
|
|
|
|
|
|
|
|
|
|
rs = r.json()["results"] |
|
|
|
|
|
|
|
|
|
|
|
for k,_v in rs.items(): |
|
|
|
|
|
print(k) |
|
|
|
|
|
#data = pandas.read_json(r.text, orient="columns") |
|
|
|
|
|
|
|
|
|
|
|
# with open(df.filename, "w", newline='') as file: |
|
|
|
|
|
# data.to_csv(file) |
|
|
|
|
|
#print(r.json()) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|