diff --git a/.gitignore b/.gitignore index a49c0c0..32698b7 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,5 @@ **/__pycache__/** **.pyc Pipfile.lock -config.json +config/config.json +config/pgdb.key \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 31a9b8c..9201fe3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,8 @@ FROM python:3.8.1-alpine RUN apk --no-cache add gcc \ musl-dev \ - python3-dev + python3-dev \ + postgresql-dev # Create a new user to run as and set the working directory ENV USER=covidbot @@ -29,6 +30,5 @@ RUN pip install pipenv RUN pipenv install Pipfile COPY covid_bot.py covid_bot.py -COPY config.json config.json ENTRYPOINT [ "pipenv", "run", "python", "covid_bot.py" ] \ No newline at end of file diff --git a/Dockerfile_report b/Dockerfile_report index 51bc983..755265a 100644 --- a/Dockerfile_report +++ b/Dockerfile_report @@ -3,7 +3,8 @@ FROM python:3.8.1-alpine RUN apk --no-cache add gcc \ musl-dev \ - python3-dev + python3-dev \ + postgresql-dev # Create a new user to run as and set the working directory ENV USER=covidreport @@ -29,6 +30,5 @@ RUN pip install pipenv RUN pipenv install Pipfile COPY covid_report.py covid_report.py -COPY config.json config.json ENTRYPOINT [ "pipenv", "run", "python", "covid_report.py" ] \ No newline at end of file diff --git a/Pipfile b/Pipfile index ababc56..cb0f6b6 100644 --- a/Pipfile +++ b/Pipfile @@ -14,6 +14,8 @@ requests = "*" discord = "*" beautifulsoup4 = "*" pytz = "*" +psycopg2-binary = "*" +sqlalchemy = "*" [required] python = "*" # Replace this with your project's stable Python version diff --git a/template_config.json b/config/template_config.json similarity index 100% rename from template_config.json rename to config/template_config.json diff --git a/docker-compose.yml b/docker-compose.yml index c5ac9b1..b452a06 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,9 +8,13 @@ services: image: covidbot:latest user: covidbot container_name: covidbot_container_service + networks: + - net + links: + - postgres volumes: - ./lib:/home/covidbot/lib - - ./.keys:/home/covidbot/.keys + - ./config:/home/covidbot/config covidreport: build: context: . @@ -18,6 +22,30 @@ services: image: covidreport:latest user: covidreport container_name: covidreport_container_service + networks: + - net + links: + - postgres volumes: - ./lib:/home/covidreport/lib - - ./.keys:/home/covidreport/.keys + - ./config:/home/covidreport/config + postgres: + image: postgres:alpine + networks: + - net + ports: + - 5432:5432 + volumes: + - ./pg_data:/var/lib/postgresql/data/pg_data + - ./config:/config + environment: + POSTGRES_PASSWORD_FILE: /config/pgdb.key + POSTGRES_DB: covid_prod + POSTGRES_USER: covidbot + POSTGRES_HOST: localhost + POSTGRES_PORT: 5432 + PGDATA: /var/lib/postgresql/data/pg_data +networks: + net: +volumes: + pg_data: diff --git a/lib/covidBot.py b/lib/covidBot.py index 7e9ee1d..9a51cf3 100644 --- a/lib/covidBot.py +++ b/lib/covidBot.py @@ -11,11 +11,15 @@ async def ping(ctx): @commands.command() async def report(ctx, arg): if(arg): - update_data() - if(arg == 'KEYS'): + # update_data() + if(arg.upper() == 'KEYS'): await ctx.send(covid_db.keys()) else: - await ctx.send(get_covid_data(arg.upper())) + print('Got command.') + try: + await ctx.send(get_covid_data(arg.upper())) + except Exception as e: + await ctx.send(f'{e}') @commands.command() diff --git a/lib/covidData.py b/lib/covidData.py index f74ebf1..8135644 100755 --- a/lib/covidData.py +++ b/lib/covidData.py @@ -1,36 +1,40 @@ -class covidData(): - def __init__(self, selection='', import_data=[]): - super().__init__() - if(selection and len(import_data) > 8): - self.set_data(selection, import_data) - else: - self.data = { - 'Selection': '', - 'Total Cases': '', - 'New Cases': '', - 'Total Deaths': '', - 'New Deaths': '', - 'Total Recovered': '', - 'Active Cases': '', - 'Serious/Critical': '', - 'Total Cases/1M Population': '' - } +from sqlalchemy import Column, Integer, String, create_engine +from sqlalchemy.ext.declarative import declarative_base +import json +from os.path import exists - def set_data(self, selection, import_data): - self.data = { - 'Selection': selection, - 'Total Cases': import_data[1].strip(), - 'New Cases': import_data[2].strip(), - 'Total Deaths': import_data[3].strip(), - 'New Deaths': import_data[4].strip(), - 'Total Recovered': import_data[5].strip(), - 'Active Cases': import_data[6].strip(), - 'Serious/Critical': import_data[7].strip(), - 'Total Cases/1M Population': import_data[8].strip() - } +Base = declarative_base() - def get_formatted_data(self): - output = '' - for key in self.data.keys(): - output += f'{key}: {self.data[key]}\n' - return output \ No newline at end of file + +class covidData(Base): + __tablename__ = 'data' + selection = Column(String(32), primary_key=True) + total_cases = Column(Integer) + new_cases = Column(Integer) + total_deaths = Column(Integer) + new_deaths = Column(Integer) + total_recovered = Column(Integer) + active_cases = Column(Integer) + serious_critical = Column(Integer) + total_cases_per_one_mil = Column(String(32)) + + +path = 'config/config.json' +if(exists(path)): + try: + with open(path) as config_file: + config_data = json.load(config_file) + except Exception as e: + print(f'There was some issue opening and loading the config.\n{e}') + # exit(1) +else: + print('Didn\'t find the config file.') + # exit(1) +try: + with open(config_data['postgres_pass']) as pgdb_pass: + engine = create_engine( + f"postgresql+psycopg2://{config_data['postgres_user']}:{pgdb_pass.readline().strip()}@{config_data['postgres_host']}:{config_data['postgres_port']}/{config_data['postgres_db']}") + Base.metadata.create_all(engine) +except Exception as e: + print( + f'There was an issue opening the config file for the postgres password.\n{e}') diff --git a/lib/covid_data_lib.py b/lib/covid_data_lib.py new file mode 100644 index 0000000..384378c --- /dev/null +++ b/lib/covid_data_lib.py @@ -0,0 +1,96 @@ +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from inspect import getmembers, isroutine +from lib.covidData import covidData, Base + + +def init_database(config_data): + try: + with open(config_data['postgres_pass']) as pgdb_pass: + engine = create_engine( + f"postgresql+psycopg2://{config_data['postgres_user']}:{pgdb_pass.readline().strip()}@{config_data['postgres_host']}:{config_data['postgres_port']}/{config_data['postgres_db']}") + Base.metadata.bind = engine + covidDataSession = sessionmaker(bind=engine) + return covidDataSession() + except Exception as e: + print( + f'There was an issue opening the config file for the postgres password.\n{e}') + exit(1) + + +def set_data(session, selection, import_data): + new_data = covidData(selection=selection.upper()) + for n in range(1, 8): + data = import_data[n].strip().replace( + ',', '').replace('+', '').replace('+', '') + if(data and not data == ''): + if(n == 1): + new_data.total_cases = int(data) + if(n == 2): + new_data.new_cases = int(data) + if(n == 3): + new_data.total_deaths = int(data) + if(n == 4): + new_data.new_deaths = int(data) + if(n == 5): + new_data.total_recovered = int(data) + if(n == 6): + new_data.active_cases = int(data) + if(n == 7): + new_data.serious_critical = int(data) + else: + if(n == 1): + new_data.total_cases = 0 + if(n == 2): + new_data.new_cases = 0 + if(n == 3): + new_data.total_deaths = 0 + if(n == 4): + new_data.new_deaths = 0 + if(n == 5): + new_data.total_recovered = 0 + if(n == 6): + new_data.active_cases = 0 + if(n == 7): + new_data.serious_critical = 0 + new_data.total_cases_per_one_mil = import_data[8].strip() + try: + session.query(covidData).filter(covidData.selection == selection).one() + try: + session.merge(new_data) + session.commit() + except Exception as e: + session.rollback() + print(f'There was an issue trying to add new data:\n{e}') + # exit(1) + except: + try: + session.add(new_data) + session.commit() + except Exception as e: + session.rollback() + print(f'There was an issue trying to add new data:\n{e}') + # exit(1) + + +def get_formatted_data(session, selection): + print('Formatting data.') + columns = sorted([i for i in covidData.__dict__.keys() if i[:1] != '_']) + columns = [' '.join([d.capitalize() for d in c.replace( + 'per', '/').split('_')]) for c in columns] + all_data_query = session.query(covidData).filter( + covidData.selection == selection).all() + output = '' + for data in all_data_query: + get_values = [attr for attr in getmembers(data, lambda a:not( + isroutine(a))) if not(attr[0].startswith('__') and attr[0].endswith('__')) and not attr[0].startswith('_') and not attr[0] == 'metadata'] + output += f'{columns[3]}: {get_values[3][1]}\n' + output += f'{columns[5]}: {get_values[5][1]}\n' + output += f'{columns[1]}: {get_values[1][1]}\n' + output += f'{columns[7]}: {get_values[7][1]}\n' + output += f'{columns[2]}: {get_values[2][1]}\n' + output += f'{columns[8]}: {get_values[8][1]}\n' + output += f'{columns[0]}: {get_values[0][1]}\n' + output += f'{columns[4]}: {get_values[4][1]}\n' + output += f'{columns[6]}: {get_values[6][1]}\n' + return output diff --git a/lib/parse_data.py b/lib/parse_data.py index 8343ea8..c067dff 100755 --- a/lib/parse_data.py +++ b/lib/parse_data.py @@ -2,9 +2,9 @@ import requests from bs4 import BeautifulSoup -from lib.covidData import covidData import json from os.path import exists +from lib.covid_data_lib import init_database, set_data, get_formatted_data # temporary database covid_db = {} @@ -13,7 +13,7 @@ covid_db = {} sa_data_url = 'https://www.sanantonio.gov/health/news/alerts/coronavirus' -def import_config(path='config.json'): +def import_config(path='config/config.json'): if(exists(path)): try: with open(path) as config_file: @@ -28,6 +28,13 @@ def import_config(path='config.json'): def update_data(): + try: + print('Creating session.') + session = init_database(import_config()) + except Exception as e: + session.rollback() + print(f'There was an error trying to create a database session:\n{e}') + # exit(1) data_html = requests.get('https://www.worldometers.info/coronavirus/') if(data_html.status_code == '200' or data_html.status_code == 200): parsed_html = BeautifulSoup(data_html.text, features='html.parser') @@ -35,17 +42,17 @@ def update_data(): for row in table.findAll('tr'): if(row and row.findAll('td')): if(row.find('a')): - covid_db[row.find('a').text.upper()] = covidData( - row.find('a').text, [r.text for r in row.findAll('td')]) + covid_db[row.find('a').text.upper()] = set_data( + session, row.find('a').text, [r.text for r in row.findAll('td')]) elif(row.findAll('td')[0] and row.findAll('td')[0].text): - covid_db[row.findAll('td')[0].text.replace(':', '').upper()] = covidData( - row.findAll('td')[0].text.replace(':', ''), [r.text for r in row.findAll('td')]) + covid_db[row.findAll('td')[0].text.replace(':', '').upper()] = set_data( + session, row.findAll('td')[0].text.replace(':', ''), [r.text for r in row.findAll('td')]) def get_covid_data(selection): - if(not covid_db): - update_data() - return covid_db[selection].get_formatted_data() + print('Updating data.') + update_data() + return get_formatted_data(init_database(import_config()), selection) def get_top_data(number):