Able to get reporting to semi-work with database input. Need to fix forced capitalization, move away from postgres to sqlite to minimize local debugging setup.

.gitignore

@@ -3,4 +3,5 @@
 **/__pycache__/**
 **.pyc
 Pipfile.lock
-config.json
+config/config.json
+config/pgdb.key

Dockerfile

@@ -3,7 +3,8 @@ FROM python:3.8.1-alpine
 
 RUN apk --no-cache add gcc \
     musl-dev \
-    python3-dev
+    python3-dev \
+    postgresql-dev
 
 # Create a new user to run as and set the working directory
 ENV USER=covidbot
@@ -29,6 +30,5 @@ RUN pip install pipenv
 RUN pipenv install Pipfile
 
 COPY covid_bot.py covid_bot.py
-COPY config.json config.json
 
 ENTRYPOINT [ "pipenv", "run", "python", "covid_bot.py" ]

Dockerfile_report

@@ -3,7 +3,8 @@ FROM python:3.8.1-alpine
 
 RUN apk --no-cache add gcc \
     musl-dev \
-    python3-dev
+    python3-dev \
+    postgresql-dev
 
 # Create a new user to run as and set the working directory
 ENV USER=covidreport
@@ -29,6 +30,5 @@ RUN pip install pipenv
 RUN pipenv install Pipfile
 
 COPY covid_report.py covid_report.py
-COPY config.json config.json
 
 ENTRYPOINT [ "pipenv", "run", "python", "covid_report.py" ]

Pipfile

@@ -14,6 +14,8 @@ requests = "*"
 discord = "*"
 beautifulsoup4 = "*"
 pytz = "*"
+psycopg2-binary = "*"
+sqlalchemy = "*"
 
 [required]
 python = "*" # Replace this with your project's stable Python version

docker-compose.yml

@@ -8,9 +8,13 @@ services:
     image: covidbot:latest
     user: covidbot
     container_name: covidbot_container_service
+    networks:
+      - net
+    links:
+      - postgres
     volumes:
       - ./lib:/home/covidbot/lib
-      - ./.keys:/home/covidbot/.keys
+      - ./config:/home/covidbot/config
   covidreport:
     build:
       context: .
@@ -18,6 +22,30 @@ services:
     image: covidreport:latest
     user: covidreport
     container_name: covidreport_container_service
+    networks:
+      - net
+    links:
+      - postgres
     volumes:
       - ./lib:/home/covidreport/lib
-      - ./.keys:/home/covidreport/.keys
+      - ./config:/home/covidreport/config
+  postgres:
+    image: postgres:alpine
+    networks:
+      - net
+    ports:
+      - 5432:5432
+    volumes:
+      - ./pg_data:/var/lib/postgresql/data/pg_data
+      - ./config:/config
+    environment:
+      POSTGRES_PASSWORD_FILE: /config/pgdb.key
+      POSTGRES_DB: covid_prod
+      POSTGRES_USER: covidbot
+      POSTGRES_HOST: localhost
+      POSTGRES_PORT: 5432
+      PGDATA: /var/lib/postgresql/data/pg_data
+networks:
+  net:
+volumes:
+  pg_data:

lib/covidBot.py

@@ -11,11 +11,15 @@ async def ping(ctx):
 @commands.command()
 async def report(ctx, arg):
     if(arg):
-        update_data()
+        # update_data()
-        if(arg == 'KEYS'):
+        if(arg.upper() == 'KEYS'):
             await ctx.send(covid_db.keys())
         else:
-            await ctx.send(get_covid_data(arg.upper()))
+            print('Got command.')
+            try:
+                await ctx.send(get_covid_data(arg.upper()))
+            except Exception as e:
+                await ctx.send(f'{e}')
 
 
 @commands.command()

lib/covidData.py

@@ -1,36 +1,40 @@
-class covidData():
+from sqlalchemy import Column, Integer, String, create_engine
-    def __init__(self, selection='', import_data=[]):
+from sqlalchemy.ext.declarative import declarative_base
-        super().__init__()
+import json
-        if(selection and len(import_data) > 8):
+from os.path import exists
-            self.set_data(selection, import_data)
-        else:
-            self.data = {
-                'Selection': '',
-                'Total Cases': '',
-                'New Cases': '',
-                'Total Deaths': '',
-                'New Deaths': '',
-                'Total Recovered': '',
-                'Active Cases': '',
-                'Serious/Critical': '',
-                'Total Cases/1M Population': ''
-            }
 
-    def set_data(self, selection, import_data):
+Base = declarative_base()
-        self.data = {
-            'Selection': selection,
-            'Total Cases': import_data[1].strip(),
-            'New Cases': import_data[2].strip(),
-            'Total Deaths': import_data[3].strip(),
-            'New Deaths': import_data[4].strip(),
-            'Total Recovered': import_data[5].strip(),
-            'Active Cases': import_data[6].strip(),
-            'Serious/Critical': import_data[7].strip(),
-            'Total Cases/1M Population': import_data[8].strip()
-        }
 
-    def get_formatted_data(self):
+
-        output = ''
+class covidData(Base):
-        for key in self.data.keys():
+    __tablename__ = 'data'
-            output += f'{key}: {self.data[key]}\n'
+    selection = Column(String(32), primary_key=True)
-        return output
+    total_cases = Column(Integer)
+    new_cases = Column(Integer)
+    total_deaths = Column(Integer)
+    new_deaths = Column(Integer)
+    total_recovered = Column(Integer)
+    active_cases = Column(Integer)
+    serious_critical = Column(Integer)
+    total_cases_per_one_mil = Column(String(32))
+
+
+path = 'config/config.json'
+if(exists(path)):
+    try:
+        with open(path) as config_file:
+            config_data = json.load(config_file)
+    except Exception as e:
+        print(f'There was some issue opening and loading the config.\n{e}')
+        # exit(1)
+else:
+    print('Didn\'t find the config file.')
+    # exit(1)
+try:
+    with open(config_data['postgres_pass']) as pgdb_pass:
+        engine = create_engine(
+            f"postgresql+psycopg2://{config_data['postgres_user']}:{pgdb_pass.readline().strip()}@{config_data['postgres_host']}:{config_data['postgres_port']}/{config_data['postgres_db']}")
+        Base.metadata.create_all(engine)
+except Exception as e:
+    print(
+        f'There was an issue opening the config file for the postgres password.\n{e}')

lib/covid_data_lib.py

@@ -0,0 +1,96 @@
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+from inspect import getmembers, isroutine
+from lib.covidData import covidData, Base
+
+
+def init_database(config_data):
+    try:
+        with open(config_data['postgres_pass']) as pgdb_pass:
+            engine = create_engine(
+                f"postgresql+psycopg2://{config_data['postgres_user']}:{pgdb_pass.readline().strip()}@{config_data['postgres_host']}:{config_data['postgres_port']}/{config_data['postgres_db']}")
+            Base.metadata.bind = engine
+            covidDataSession = sessionmaker(bind=engine)
+            return covidDataSession()
+    except Exception as e:
+        print(
+            f'There was an issue opening the config file for the postgres password.\n{e}')
+        exit(1)
+
+
+def set_data(session, selection, import_data):
+    new_data = covidData(selection=selection.upper())
+    for n in range(1, 8):
+        data = import_data[n].strip().replace(
+            ',', '').replace('+', '').replace('+', '')
+        if(data and not data == ''):
+            if(n == 1):
+                new_data.total_cases = int(data)
+            if(n == 2):
+                new_data.new_cases = int(data)
+            if(n == 3):
+                new_data.total_deaths = int(data)
+            if(n == 4):
+                new_data.new_deaths = int(data)
+            if(n == 5):
+                new_data.total_recovered = int(data)
+            if(n == 6):
+                new_data.active_cases = int(data)
+            if(n == 7):
+                new_data.serious_critical = int(data)
+        else:
+            if(n == 1):
+                new_data.total_cases = 0
+            if(n == 2):
+                new_data.new_cases = 0
+            if(n == 3):
+                new_data.total_deaths = 0
+            if(n == 4):
+                new_data.new_deaths = 0
+            if(n == 5):
+                new_data.total_recovered = 0
+            if(n == 6):
+                new_data.active_cases = 0
+            if(n == 7):
+                new_data.serious_critical = 0
+    new_data.total_cases_per_one_mil = import_data[8].strip()
+    try:
+        session.query(covidData).filter(covidData.selection == selection).one()
+        try:
+            session.merge(new_data)
+            session.commit()
+        except Exception as e:
+            session.rollback()
+            print(f'There was an issue trying to add new data:\n{e}')
+            # exit(1)
+    except:
+        try:
+            session.add(new_data)
+            session.commit()
+        except Exception as e:
+            session.rollback()
+            print(f'There was an issue trying to add new data:\n{e}')
+            # exit(1)
+
+
+def get_formatted_data(session, selection):
+    print('Formatting data.')
+    columns = sorted([i for i in covidData.__dict__.keys() if i[:1] != '_'])
+    columns = [' '.join([d.capitalize() for d in c.replace(
+        'per', '/').split('_')]) for c in columns]
+    all_data_query = session.query(covidData).filter(
+        covidData.selection == selection).all()
+    output = ''
+    for data in all_data_query:
+        get_values = [attr for attr in getmembers(data, lambda a:not(
+            isroutine(a))) if not(attr[0].startswith('__') and attr[0].endswith('__')) and not attr[0].startswith('_') and not attr[0] == 'metadata']
+        output += f'{columns[3]}: {get_values[3][1]}\n'
+        output += f'{columns[5]}: {get_values[5][1]}\n'
+        output += f'{columns[1]}: {get_values[1][1]}\n'
+        output += f'{columns[7]}: {get_values[7][1]}\n'
+        output += f'{columns[2]}: {get_values[2][1]}\n'
+        output += f'{columns[8]}: {get_values[8][1]}\n'
+        output += f'{columns[0]}: {get_values[0][1]}\n'
+        output += f'{columns[4]}: {get_values[4][1]}\n'
+        output += f'{columns[6]}: {get_values[6][1]}\n'
+    return output

lib/parse_data.py

@@ -2,9 +2,9 @@
 
 import requests
 from bs4 import BeautifulSoup
-from lib.covidData import covidData
 import json
 from os.path import exists
+from lib.covid_data_lib import init_database, set_data, get_formatted_data
 
 # temporary database
 covid_db = {}
@@ -13,7 +13,7 @@ covid_db = {}
 sa_data_url = 'https://www.sanantonio.gov/health/news/alerts/coronavirus'
 
 
-def import_config(path='config.json'):
+def import_config(path='config/config.json'):
     if(exists(path)):
         try:
             with open(path) as config_file:
@@ -28,6 +28,13 @@ def import_config(path='config.json'):
 
 
 def update_data():
+    try:
+        print('Creating session.')
+        session = init_database(import_config())
+    except Exception as e:
+        session.rollback()
+        print(f'There was an error trying to create a database session:\n{e}')
+        # exit(1)
     data_html = requests.get('https://www.worldometers.info/coronavirus/')
     if(data_html.status_code == '200' or data_html.status_code == 200):
         parsed_html = BeautifulSoup(data_html.text, features='html.parser')
@@ -35,17 +42,17 @@ def update_data():
         for row in table.findAll('tr'):
             if(row and row.findAll('td')):
                 if(row.find('a')):
-                    covid_db[row.find('a').text.upper()] = covidData(
+                    covid_db[row.find('a').text.upper()] = set_data(
-                        row.find('a').text, [r.text for r in row.findAll('td')])
+                        session, row.find('a').text, [r.text for r in row.findAll('td')])
                 elif(row.findAll('td')[0] and row.findAll('td')[0].text):
-                    covid_db[row.findAll('td')[0].text.replace(':', '').upper()] = covidData(
+                    covid_db[row.findAll('td')[0].text.replace(':', '').upper()] = set_data(
-                        row.findAll('td')[0].text.replace(':', ''), [r.text for r in row.findAll('td')])
+                        session, row.findAll('td')[0].text.replace(':', ''), [r.text for r in row.findAll('td')])
 
 
 def get_covid_data(selection):
-    if(not covid_db):
+    print('Updating data.')
-        update_data()
+    update_data()
-    return covid_db[selection].get_formatted_data()
+    return get_formatted_data(init_database(import_config()), selection)
 
 
 def get_top_data(number):