#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import zipfile
from datetime import date
from datetime import datetime
from datetime import timedelta
from io import BytesIO
from io import StringIO
from typing import Dict
from typing import Tuple

import pandas
from aws_xray_sdk.core import xray_recorder as xray
from matplotlib.pyplot import axis
from matplotlib.pyplot import pie
from matplotlib.pyplot import savefig

from iobot.utils import init_config
from psi_logger import logger


@xray.capture()
class Qualys(object):
    """ Initialize a copy of psi_athena_api.Qualys() for use

    Parameters
    ----------
    log : logging.Logger()
        Traditional python logger

    session : boto3.session.Session()
        Boto3 Session Object

    config : configparser.ConfigParser()
        Per https://docs.python.org/2.0/ref/function.html set config
        explicitly to None and evaluate how it should be set if no value
        is passed. See iobot.utils for more information.

    config_file: str
        Defaults to iobot.ini
        If config is not passed, we will attempt to load config from this file.
    """

    @xray.capture()
    @logger.log
    def __init__(self, log: Dict, session: Dict, config: Dict = None, config_file: str = "iobot.ini") -> None:
        """ Create a Qualys object and assert its config

        Testing
        -------

        >>> os.environ['STACK_NAME'] = 'iobots-stack'

        >>> testing.export_secret_arns(os.environ['STACK_NAME'])

        >>> client = Qualys(log,boto_session,config_file='environments/unit.ini')

        >>> client.config['Qualys']['output_format']
        'csv'

        >>> testing.unexport_secret_arns()

        >>> del os.environ['STACK_NAME']
        """
        self._log = log
        self._session = session
        if config:
            self.config = config
        else:
            self.config = init_config(
                config_file=config_file, env=dict(os.environ))

    @xray.capture()
    @logger.log
    def get_report_index_for_date(self, bucket="qualys-rollout-reports", report_prefix: str = 'qualys-rollout-report', mask: str = f'{date.today().year}-{str(date.today().month).zfill(2)}-{str(date.today().day).zfill(2)}') -> Tuple:
        client = self._session.client('s3')

        self._log.debug(f'searching date mask: "{mask}"')
        response = client.list_objects_v2(
            Bucket=bucket,
            EncodingType='url',
            Prefix=f'{report_prefix}_{mask}'
        )

        if not response.get('Contents'):
            return tuple()

        return response.get('Contents')

    @xray.capture()
    @logger.log
    def get_basic_report_data(self, key: str, bucket: str = "qualys-rollout-reports") -> str:
        client = self._session.client('s3')
        obj = client.get_object(Bucket=bucket, Key=key)

        with zipfile.ZipFile(BytesIO(obj['Body'].read())) as report:
            mask = key.split('.')[0].split('_')[1]

            return report.read(f"reports/basic-qualys-rollout-report_{mask}.csv").decode('utf-8')

    @xray.capture()
    @staticmethod
    @logger.log
    def render_basic_report_data_as_graph(data: str):
        """Ingest qualys basic compliance report and render a pie chart graphic

        Parameters
        ----------
        data : str
            CSV report data that can be parsed by pandas

        Returns
        -------
        buffer : io.BytesIO
            PNG report data that has been graphed by matplotlib

        Testing
        -------

        Bootstrap libraries for doctest
        >>> skip('deprecated package code failing - drop or resolve')

        >>> from iobot.slack import Agent

        >>> from psi_athena_api import Qualys

        >>> os.environ['STACK_NAME'] = 'iobots-stack'

        >>> testing.export_secret_arns('iobots-stack')

        >>> data: str='Account ID,t_AppID,Instances with an agent,Instances without an agent,% of instances with agents\\n635712145102,SVC02422,0,3,0.000000\\n213313366758,untagged,1,1,0.500000'

        >>> graph = Qualys.render_basic_report_data_as_graph(data)

        >>> assert (len(graph.getbuffer()) > 1000), f'expected at greater than 1000 bytes, got {len(graph.getbuffer())} bytes'

        >>> slack_client = slack_sdk.WebClient(token=testing.slack_bearer_token())

        >>> agent = Agent(log,boto_session,slack_client,config_file='environments/unit.ini', entitlement_level=Agent.entitlement_levels().UNRESTRICTED)

        >>> result = agent.upload_data_to_channel_as_file(graph.getvalue(), 'CJGRXLNRK', content_type='png', title='basic-qualys-test-graph-v1.png')

        >>> assert (result['ok']), f'should not complain about an error {result}'

        >>> testing.unexport_secret_arns()
        """
        csv = pandas.read_csv(StringIO(data))
        # sums = csv.groupby(csv['Account ID'])['Instances without an agent'].sum()
        sums = csv.groupby(['Account ID', 't_AppID'])[
            'Instances without an agent'].sum().nlargest(10)
        axis('equal')
        pie(sums, labels=sums.index)
        buffer = BytesIO(b'')  # blank buffer

        savefig(buffer, bbox_inches='tight')
        return buffer

    @xray.capture()
    @staticmethod
    @logger.log
    def render_basic_report_data_as_graph_v2(data: str):
        """Ingest qualys basic compliance report and render a pie chart graphic

        Parameters
        ----------
        data : list
            CSV report data that can be parsed by pandas

        Returns
        -------
        buffer : io.BytesIO
            PNG report data that has been graphed by matplotlib

        Testing
        -------

        Bootstrap libraries for doctest
        >>> from iobot.slack import Agent

        >>> from psi_athena_api import Qualys

        >>> os.environ['STACK_NAME'] = 'iobots-stack'

        >>> testing.export_secret_arns('iobots-stack')

        >>> data: str='instance_id,account_id,region,t_appid,t_environment,t_dcl,instance_state,auto_scaling_group_name,ssm_patch_compliance_status,qualys_agent_status,vulnerabilities_count,report_date\\ni-06fe60cd4def4c2de,504377606089,us-east-1,SVC01990,nonprod,1,Running,,Unknown,STATUS_NO_AGENT,Unknown,2020-08-06T01-03\\ni-0243b4abd11712297,504377606089,us-east-1,SVC01990,poc,1,Running,accu-poc-master,Unknown,STATUS_ACTIVE,62,2020-08-06T01-03\\ni-ec05dd75,012597783113,us-east-1,SVC01990,untagged,untagged,Running,,Unknown,STATUS_ACTIVE,409,2020-08-06T01-03\\n'

        >>> graph = Qualys.render_basic_report_data_as_graph_v2(data)

        >>> assert (len(graph.getbuffer()) > 1000), f'expected at greater than 1000 bytes, got {len(graph.getbuffer())} bytes'

        >>> slack_client = slack_sdk.WebClient(token=testing.slack_bearer_token())

        >>> agent = Agent(log,boto_session,slack_client,config_file='environments/unit.ini', entitlement_level=Agent.entitlement_levels().UNRESTRICTED)

        >>> result = agent.upload_data_to_channel_as_file(graph.getvalue(), 'CJGRXLNRK', content_type='png', title='basic-qualys-test-graph-v2.png')

        >>> assert (result['ok']), f'should not complain about an error {result}'

        >>> t = Qualys(log,boto_session,config_file='environments/unit.ini')

        >>> new_data = t.get_query_results_from_s3('john.torres@pearson.com/ec18656e-05a7-4993-9b95-af38f78e5a32').decode('utf-8')

        >>> big_graph = Qualys.render_basic_report_data_as_graph_v2(new_data)

        >>> result = agent.upload_data_to_channel_as_file(big_graph.getvalue(), 'CJGRXLNRK', content_type='png', title='big-result-qualys-test-graph-v2.png')

        >>> assert (result['ok']), f'should not complain about an error {result}'

        >>> testing.unexport_secret_arns()
        """
        buffer: StringIO = StringIO(data)
        csv = pandas.read_csv(buffer)
        agent_status = {'STATUS_NO_AGENT': 1,
                        'STATUS_ACTIVE': 0, 'STATUS_INACTIVE': 1}
        sums = csv.replace({'qualys_agent_status': agent_status}).groupby(
            ['account_id', 't_appid'])['qualys_agent_status'].sum().nlargest(10)
        axis('equal')
        pie(sums, labels=sums.index)
        image = BytesIO(b'')  # blank buffer

        savefig(image, bbox_inches='tight')
        return image

    @xray.capture()
    @logger.log
    def get_detailed_report_data(self, key: str, bucket: str = "qualys-rollout-reports") -> str:
        client = self._session.client('s3')
        obj = client.get_object(Bucket=bucket, Key=key)

        with zipfile.ZipFile(BytesIO(obj['Body'].read())) as report:
            mask = key.split('.')[0].split('_')[1]

            return report.read(f"reports/detailed-qualys-rollout-report_{mask}.csv").decode('utf-8')

    @xray.capture()
    # TODO: deprecated, remove
    @logger.log
    def get_basic_vulnerability_report_data_from_s3(self, key: str, bucket: str = "qualys-rollout-reports", report_prefix: str = "basic-qualys-rollout-report") -> str:
        client = self._session.client('s3')
        obj = client.get_object(Bucket=bucket, Key=key)

        with zipfile.ZipFile(BytesIO(obj['Body'].read())) as report:
            mask = key.split('.')[0].split('_')[1]

            return report.read(f"reports/{report_prefix}-{mask}.csv").decode('utf-8')

    @xray.capture()
    # TODO: deprecated, remove
    @logger.log
    def get_detailed_vulnerability_report_data_from_s3(self, key: str, bucket: str = "qualys-rollout-reports", report_prefix: str = "detailed-qualys-rollout-report") -> str:
        client = self._session.client('s3')
        obj = client.get_object(Bucket=bucket, Key=key)

        with zipfile.ZipFile(BytesIO(obj['Body'].read())) as report:
            mask = key.split('.')[0].split('_')[1]

            return report.read(f"reports/{report_prefix}-{mask}.csv").decode('utf-8')

    @xray.capture()
    @logger.log
    def start_query_execution(self, query: str, parameters: Dict = {}, filter_date: str = "",
                              database: str = 'vulnerability_management', catalog: str = 'AwsDataCatalog',
                              workgroup: str = 'vulnerability-management', limit_results: int = 0) -> str:
        """ Validate historical vulnerability reports can be started

        Parameters
        ----------
        query : str
            SQL query to execute on athena

        parameters : Dict
            Map of query parameter filters.
            E.g. {'id':'svc02222','environment':'prd','account': '012597783113'}

        filter_date : str
            Format YYYY-MM-DD as generated by datetime.date.today()

        database : str
            SQL query against this database

        catalog : str
            SQL query against this catalog

        workgroup : str
            SQL query against this workgroup

        limit_results : int
            Number of records to return

        Returns
        -------
        QueryExecutionId : str
            Athena query execution id

        Testing
        -------

        >>> os.environ['STACK_NAME'] = 'iobots-stack'

        >>> testing.export_secret_arns(os.environ['STACK_NAME'])

        >>> t = Qualys(log,boto_session,config_file='environments/unit.ini')

        >>> query_id = t.start_query_execution('SELECT * FROM vulnerability_management.swa_current_instances_vw',parameters={'id':'SVC01188','environment':'prd','account': '012597783113'}, filter_date='2020-08-05', limit_results=1)

        >>> assert (query_id is not None), f'expected query_id to not be None, got {query_id}'

        >>> testing.unexport_secret_arns()

        >>> del os.environ['STACK_NAME']
        """
        athena_client = self._session.client('athena')
        response = athena_client.start_query_execution(
            QueryString=Qualys.get_query_string(
                parameters=parameters, filter_date=filter_date, query=query, limit_results=limit_results),
            QueryExecutionContext={'Database': database, 'Catalog': catalog},
            WorkGroup=workgroup
        )
        return response['QueryExecutionId']

    @xray.capture()
    @logger.log
    def start_current_instances_query_execution(self, parameters: Dict = {},
                                                filter_date: str = f"{date.today()}", database: str = 'vulnerability_management',
                                                catalog: str = 'AwsDataCatalog', workgroup: str = 'vulnerability-management', limit_results: int = 0) -> str:
        """ Validate current instance reports can be started

        Parameters
        ----------

        parameters : Dict
            Map of query parameter filters.
            E.g. {'id':'svc02222','environment':'prd','account': '012597783113'}

        filter_date : str
            Format YYYY-MM-DD as generated by datetime.date.today().
            Defaults to datetime.date.today().

        database : str
            SQL query against this database.
            Defaults to 'vulnerability_management'. Note the underscore.

        catalog : str
            SQL query against this catalog
            Defaults to 'AwsDataCatalog'. Note the underscore.

        workgroup : str
            SQL query against this workgroup
            Defaults to 'vulnerability-management'. Note the hyphen.

        limit_results : int
            Number of records to return

        Returns
        -------
        QueryExecutionId : str
            Athena query execution id

        Testing
        -------

        >>> os.environ['STACK_NAME'] = 'iobots-stack'

        >>> testing.export_secret_arns(os.environ['STACK_NAME'])

        >>> t = Qualys(log,boto_session,config_file='environments/unit.ini')

        >>> query_id = t.start_current_instances_query_execution(parameters={'id':'SVC01188','environment':'prd','account': '012597783113'}, filter_date='2020-08-05')

        >>> assert (query_id is not None), f'expected query_id to not be None, got {query_id}'

        >>> testing.unexport_secret_arns()

        >>> del os.environ['STACK_NAME']
        """
        return self.start_query_execution('SELECT * FROM vulnerability_management.swa_current_instances_vw',
                                          parameters=parameters, filter_date=filter_date, database=database, catalog=catalog, workgroup=workgroup, limit_results=limit_results)

    @xray.capture()
    @logger.log
    def start_current_vulnerabilities_query_execution(self, parameters: Dict = {},
                                                      filter_date: str = f"{date.today()}", database: str = 'vulnerability_management',
                                                      catalog: str = 'AwsDataCatalog', workgroup: str = 'vulnerability-management', limit_results: int = 0) -> str:
        """ Validate current instance reports can be started
        >>> os.environ['STACK_NAME'] = 'iobots-stack'
        >>> testing.export_secret_arns(os.environ['STACK_NAME'])
        >>> t = Qualys(log,boto_session,config_file='environments/unit.ini')
        >>> query_id = t.start_current_vulnerabilities_query_execution(parameters={'id':'SVC01188','environment':'prd','account': '012597783113'}, filter_date='2020-08-05')
        >>> assert (query_id is not None), f'expected query_id to not be None, got {query_id}'
        >>> testing.unexport_secret_arns()

        >>> del os.environ['STACK_NAME']
        """
        return self.start_query_execution('SELECT * FROM vulnerability_management.swa_current_vulnerabilities_vw',
                                          parameters=parameters,
                                          filter_date=filter_date,
                                          database=database,
                                          catalog=catalog,
                                          workgroup=workgroup,
                                          limit_results=limit_results)

    @xray.capture()
    @logger.log
    def start_historical_instances_query_execution(self,
                                                   parameters: dict = {},
                                                   filter_date: str = "",
                                                   database: str = 'vulnerability_management',
                                                   catalog: str = 'AwsDataCatalog',
                                                   workgroup: str = 'vulnerability-management',
                                                   limit_results: int = 0) -> str:
        """ Validate historical instances reports can be started

        Parameters
        ----------

        parameters : Dict
            Map of query parameter filters.
            E.g. {'id':'svc02222','environment':'prd','account': '012597783113'}

        filter_date : str
            Format YYYY-MM-DD as generated by datetime.date.today().
            Defaults to datetime.date.today().

        database : str
            SQL query against this database.
            Defaults to 'vulnerability_management'. Note the underscore.

        catalog : str
            SQL query against this catalog
            Defaults to 'AwsDataCatalog'. Note the underscore.

        workgroup : str
            SQL query against this workgroup
            Defaults to 'vulnerability-management'. Note the hyphen.

        limit_results : int
            Number of records to return

        Returns
        -------
        QueryExecutionId : str
            Athena query execution id

        Testing
        -------

        >>> os.environ['STACK_NAME'] = 'iobots-stack'

        >>> testing.export_secret_arns(os.environ['STACK_NAME'])

        >>> t = Qualys(log,boto_session,config_file='environments/unit.ini')

        >>> query_id = t.start_historical_instances_query_execution(parameters={'id':'SVC01188','environment':'prd','account': '012597783113'}, filter_date='2020-08-05')

        >>> assert (query_id is not None), f'expected query_id to not be None, got {query_id}'

        >>> testing.unexport_secret_arns()

        >>> del os.environ['STACK_NAME']
        """
        return self.start_query_execution('SELECT * FROM vulnerability_management.swa_historical_instances_vw',
                                          parameters=parameters,
                                          filter_date=filter_date,
                                          database=database,
                                          catalog=catalog,
                                          workgroup=workgroup,
                                          limit_results=limit_results)

    @xray.capture()
    @logger.log
    def start_historical_vulnerabilities_query_execution(self,
                                                         parameters: dict = None,
                                                         filter_date: str = "",
                                                         database: str = 'vulnerability_management',
                                                         catalog: str = 'AwsDataCatalog',
                                                         workgroup: str = 'vulnerability-management',
                                                         limit_results: int = 0) -> str:
        """ Validate historical vulnerabilities reports can be started

        Parameters
        ----------

        parameters : Dict
            Map of query parameter filters.
            E.g. {'id':'svc02222','environment':'prd','account': '012597783113'}

        filter_date : str
            Format YYYY-MM-DD as generated by datetime.date.today().
            Defaults to datetime.date.today().

        database : str
            SQL query against this database.
            Defaults to 'vulnerability_management'. Note the underscore.

        catalog : str
            SQL query against this catalog
            Defaults to 'AwsDataCatalog'. Note the underscore.

        workgroup : str
            SQL query against this workgroup
            Defaults to 'vulnerability-management'. Note the hyphen.

        limit_results : int
            Number of records to return

        Returns
        -------
        QueryExecutionId : str
            Athena query execution id

        Testing
        -------

        >>> os.environ['STACK_NAME'] = 'iobots-stack'

        >>> testing.export_secret_arns(os.environ['STACK_NAME'])

        >>> t = Qualys(log,boto_session,config_file='environments/unit.ini')

        >>> query_id = t.start_historical_vulnerabilities_query_execution(parameters={'id':'SVC01188','environment':'prd','account': '012597783113'}, filter_date='2020-08-05')

        >>> assert (query_id is not None), f'expected query_id to not be None, got {query_id}'

        >>> testing.unexport_secret_arns()

        >>> del os.environ['STACK_NAME']
        """
        return self.start_query_execution('SELECT * FROM vulnerability_management.swa_historical_vulnerabilities_vw',
                                          parameters=parameters,
                                          filter_date=filter_date,
                                          database=database,
                                          catalog=catalog,
                                          workgroup=workgroup,
                                          limit_results=limit_results)

    @xray.capture()
    @logger.log
    def start_vpd_trends_query_execution(self,
                                         query: str = 'SELECT * FROM swa_vpd_30day_change_vw',
                                         parameters: Dict = {},
                                         filter_date: datetime = None,
                                         start_range: int = 30,
                                         end_range: int = 1,
                                         database: str = 'vulnerability_management', catalog: str = 'AwsDataCatalog',
                                         workgroup: str = 'vulnerability-management', limit_results: int = 0) -> str:
        """ Validate historical vulnerability reports can be started

        Parameters
        ----------
        query : str
            BASE SQL 'FROM' query claus to execute on athena. Will be combined
            with parameters and filter_date.

        parameters : Dict
            Map of query parameter filters.
            E.g. {'ids':['svc02222'],'environments':['prd'],'accounts': ['012597783113']}

        filter_date : str
            Format YYYY-MM-DD as generated by datetime.date.today()

        start_range : Dict
            Beginning of date range to measure VPD over

            Defaults to -30 or thirty days prior to today's date.
            Used to calculate end date in the formula: (filter_date + start_range)

        end_range : Dict
            Ending of date range to measure VPD over

            Defaults to -1 or one day prior to today's date.
            Used to calculate end date in the formula: (filter_date + end_range)

        database : str
            SQL query against this database.
            Defaults to 'vulnerability_management'. Note the underscore.

        catalog : str
            SQL query against this catalog
            Defaults to 'AwsDataCatalog'. Note the underscore.

        workgroup : str
            SQL query against this workgroup
            Defaults to 'vulnerability-management'. Note the hyphen.

        limit_results : int
            Number of records to return

        Returns
        -------
        QueryExecutionId : str
            Athena query execution id

        Testing
        -------

        >>> os.environ['STACK_NAME'] = 'iobots-stack'

        >>> testing.export_secret_arns(os.environ['STACK_NAME'])

        >>> t = Qualys(log,boto_session,config_file='environments/unit.ini')

        >>> query_id = t.start_vpd_trends_query_execution(parameters={'ids':['SVC0765']})

        >>> assert (query_id is not None), f'expected query_id to not be None, got {query_id}'

        >>> testing.unexport_secret_arns()

        >>> del os.environ['STACK_NAME']
        """
        athena_client = self._session.client('athena')
        response = athena_client.start_query_execution(
            QueryString=Qualys.get_vpd_trends_query(query=query,
                                                    parameters=parameters,
                                                    filter_date=filter_date,
                                                    start_range=start_range,
                                                    end_range=end_range,
                                                    limit_results=limit_results),
            QueryExecutionContext={'Database': database, 'Catalog': catalog},
            WorkGroup=workgroup
        )
        return response['QueryExecutionId']

    @xray.capture()
    @staticmethod
    @logger.log
    def get_vpd_trends_query(query: str = "SELECT * FROM swa_vpd_30day_change_vw",
                             parameters: dict = None,
                             filter_date: datetime = None,
                             start_range: int = 31,
                             end_range: int = 1,
                             limit_results: int = 0) -> str:
        """ Render historical trends report query

        Parameters
        ----------
        query : str
            BASE SQL 'FROM' query claus to execute on athena. Will be combined
            with parameters and filter_date.

        parameters : Dict
            Map of query parameter filters.
            E.g. {'ids':['svc02222'],'environments':['prd'],'accounts': ['012597783113']}

        filter_date : str
            Format YYYY-MM-DD as generated by datetime.date.today()

        start_range : Dict
            Beginning of date range to measure VPD over

            Defaults to 30 or thirty days prior to today's date.
            Used to calculate end date in the formula: (filter_date - start_range)

        end_range : Dict
            Ending of date range to measure VPD over

            Defaults to 1 or one day prior to today's date.
            Used to calculate end date in the formula: (filter_date - end_range)

        limit_results : int
            Number of records to return

        Returns
        -------
        QueryExecutionId : str
            Athena query execution id

        Testing
        -------

        >>> os.environ['STACK_NAME'] = 'iobots-stack'

        >>> testing.export_secret_arns(os.environ['STACK_NAME'])

        >>> s = Qualys.get_vpd_trends_query(parameters={'ids': ['SVC02109','SVC00618'],
        ...                                             'environments': ['prd'],
        ...                                             'accounts': ['012597783113']},
        ...                                 filter_date=datetime.today())

        >>> yesterday: str = f'{(datetime.today() - timedelta(days=1)).date()}'
        >>> monthago: str = f'{(datetime.today() - timedelta(days=31)).date()}'

        >>> yesterday in s
        True

        >>> monthago in s
        True

        >>> Qualys.get_vpd_trends_query(parameters={'ids': ['SVC02109']})
        "SELECT * FROM swa_vpd_30day_change_vw WHERE t_appid IN ('SVC02109') ORDER BY t_appid ASC"

        >>> Qualys.get_vpd_trends_query(parameters={'ids': ['SVC02109','SVC00618']})
        "SELECT * FROM swa_vpd_30day_change_vw WHERE t_appid IN ('SVC02109','SVC00618') ORDER BY t_appid ASC"

        >>> Qualys.get_vpd_trends_query(parameters={'ids': ['SVC02109','SVC00618'],
        ...                                         'environments': ['prd']})
        "SELECT * FROM swa_vpd_30day_change_vw WHERE t_appid IN ('SVC02109','SVC00618') AND t_environment IN ('prd') ORDER BY t_appid ASC"

        >>> Qualys.get_vpd_trends_query(parameters={'ids': ['SVC02109','SVC00618'],
        ...                                             'environments': ['prd'],
        ...                                             'accounts': ['012597783113']})
        "SELECT * FROM swa_vpd_30day_change_vw WHERE t_appid IN ('SVC02109','SVC00618') AND account IN ('012597783113') AND t_environment IN ('prd') ORDER BY t_appid ASC"

        >>> testing.unexport_secret_arns()

        >>> del os.environ['STACK_NAME']
        """

        if any([parameters.get('ids'), parameters.get('accounts'), parameters.get('environments')]):  # noqa E501
            if 'WHERE' not in query.upper():
                query += " WHERE"

        id_string: str = ""
        if parameters.get('ids'):
            id_string = ','.join(f"'{id}'" for id in parameters['ids'])
            query += f" t_appid IN ({id_string})"

        account_string: str = ""
        if parameters.get('accounts'):
            account_string = ','.join(f"'{account}'" for account in parameters['accounts'])  # noqa E501
            query += f" AND account IN ({account_string})"

        environment_string: str = ""
        if parameters.get('environments'):
            environment_string = ','.join(
                f"'{environment}'" for environment in parameters['environments'])
            query += f" AND t_environment IN ({environment_string})"

        if filter_date:
            query += f" AND ( report_date = CAST('{(filter_date - timedelta(days=start_range)).date()}' AS DATE)"
            query += f" OR report_date = CAST('{(filter_date - timedelta(days=end_range)).date()}' AS DATE) )"

        query += " ORDER BY t_appid ASC"

        if limit_results:
            query += f' LIMIT {limit_results}'

        return query

    @xray.capture()
    @logger.log
    def query_execution_status(self, execution_id: str):
        """ Validate we can fetch query execution status

        Parameters
        ----------
        execution_id : str
            Athena query execution id

        Returns
        -------
        State : str
            Athena query execution status

        Testing
        -------

        >>> os.environ['STACK_NAME'] = 'iobots-stack'

        >>> testing.export_secret_arns(os.environ['STACK_NAME'])

        >>> t = Qualys(log,boto_session,config_file='environments/unit.ini')

        >>> query_id = t.start_historical_vulnerabilities_query_execution(parameters={'id':'SVC01188','environment':'prd','account': '012597783113'}, filter_date='2020-08-05', limit_results=1)

        >>> assert (query_id is not None), f'expected query_id to not be None, got {query_id}'

        >>> from time import sleep

        >>> sleep(8)

        >>> t.query_execution_status(query_id)
        'SUCCEEDED'

        >>> query_id = t.start_query_execution('SELECT foo FROM vulnerability_management.swa_current_instances_vw',parameters={'id':'SVC01188','environment':'prd','account': '012597783113'}, filter_date='2020-08-05', limit_results=1)

        >>> sleep(2)

        >>> t.query_execution_status(query_id)
        'FAILED'

        >>> testing.unexport_secret_arns()

        >>> del os.environ['STACK_NAME']
        """
        athena_client = self._session.client('athena')
        response = athena_client.get_query_execution(
            QueryExecutionId=execution_id
        )
        return response['QueryExecution']['Status']['State']

    @xray.capture()
    @logger.log
    def get_query_results(self,
                          execution_id,
                          max_records_per_page: int = 1000,
                          max_items_total: int = -1) -> list:
        """ Validate we can fetch query execution status

        Parameters
        ----------
        execution_id : str
            Athena query execution id

        max_records_per_page : int
            Number of rows per page returned. Defaults to 1000.

        max_items_total : int
            Number of total records to return initially.
            Defaults to `-1` or not set.

        Returns
        -------
        results : List[str]
            Athena query execution results

        Testing
        -------

        >>> os.environ['STACK_NAME'] = 'iobots-stack'

        >>> testing.export_secret_arns(os.environ['STACK_NAME'])

        >>> t = Qualys(log,boto_session,config_file='environments/unit.ini')

        >>> query_id = t.start_historical_vulnerabilities_query_execution(parameters={'id':'SVC02430','environment':'dev','account': '747135266249'}, filter_date='2020-11-24', limit_results=1)

        >>> assert (query_id is not None), f'expected query_id to not be None, got {query_id}'

        >>> from time import sleep

        >>> attempts: int = 0

        >>> status: str = ""

        >>> while status not in ['SUCCEEDED', 'FAILED', 'CANCELLED']:
        ...     if attempts > 20:
        ...         status = 'FAILED'
        ...         break
        ...     sleep(1)
        ...     status = t.query_execution_status(query_id)
        ...     attempts += 1

        >>> results = t.get_query_results(query_id)

        >>> results[1].startswith('74713')
        True

        >>> query_id = t.start_historical_vulnerabilities_query_execution(parameters={'id':'','environment':'','account': ''}, filter_date='2020-11-26', limit_results=1)

        >>> assert (query_id is not None), f'expected query_id to not be None, got {query_id}'

        >>> attempts: int = 0

        >>> status: str = ""

        >>> while status not in ['SUCCEEDED', 'FAILED', 'CANCELLED']:
        ...     if attempts > 20:
        ...         status = 'FAILED'
        ...         break
        ...     sleep(1)
        ...     status = t.query_execution_status(query_id)
        ...     attempts += 1

        >>> results = t.get_query_results(query_id)

        >>> len(results)
        2

        >>> results[0].startswith('account_id')
        True

        >>> filter_date: datetime = datetime.strptime('2020-11-26', '%Y-%m-%d')

        >>> query_id = t.start_vpd_trends_query_execution('SELECT * FROM swa_vpd_30day_change_vw',parameters={'ids':['SVC02430']})

        >>> assert (query_id is not None), f'expected query_id to not be None, got {query_id}'

        >>> attempts: int = 0

        >>> status: str = ""

        >>> while status not in ['SUCCEEDED', 'FAILED', 'CANCELLED']:
        ...     if attempts > 20:
        ...         status = 'FAILED'
        ...         break
        ...     sleep(1)
        ...     status = t.query_execution_status(query_id)
        ...     attempts += 1

        >>> results = t.get_query_results(query_id)

        >>> len(results)
        2

        >>> results[0].startswith('t_appid')
        True

        >>> testing.unexport_secret_arns()

        >>> del os.environ['STACK_NAME']
        """
        athena_client = self._session.client('athena')
        response = athena_client.get_paginator('get_query_results')

        query_options: Dict = {
            'PageSize': max_records_per_page
        }

        if max_items_total >= 0:
            query_options['MaxItems'] = max_items_total

        pages = response.paginate(
            QueryExecutionId=execution_id,
            PaginationConfig=query_options
        )
        rows = []
        for page in pages:
            gen_rows = [
                Qualys.data_to_row(data)
                for data in page['ResultSet']['Rows']
            ]

            for row in gen_rows:
                rows.append(row)

        # TODO: validate this format with pandas...
        return list(','.join(row) for row in rows)

    @xray.capture()
    @logger.log
    def get_query_results_from_s3(self,
                                  execution_id: str,
                                  bucket: str = "") -> str:
        """ Retrieve athena query results from S3 for improved performance
            over pagination.

        Parameters
        ----------
        execution_id : str
            Athena query execution id

        bucket : str
            Location where to fetch athena results.
            Defaults to <environment>.ini:Qualys->athena_bucket.

        Returns
        -------
        results : str
            Athena query execution results as a CSV string

        Testing
        -------

        >>> os.environ['STACK_NAME'] = 'iobots-stack'

        >>> testing.export_secret_arns(os.environ['STACK_NAME'])

        >>> t = Qualys(log,boto_session,config_file='environments/unit.ini')

        >>> report = t.get_query_results_from_s3('john.torres@pearson.com/ec18656e-05a7-4993-9b95-af38f78e5a32')

        >>> import sys

        >>> sys.getsizeof(report)
        931

        >>> testing.unexport_secret_arns()

        >>> del os.environ['STACK_NAME']
        """
        if not bucket:
            bucket = self.config['Qualys']['athena_bucket']
            self._log.debug(f'no bucket configured, defaulting to {bucket}')

        client = self._session.client('s3')

        object_key: str = f'athena/{execution_id}.csv'
        self._log.debug(f"fetching object: {bucket}/{object_key}")
        obj = client.get_object(
            Bucket=bucket, Key=object_key)

        return obj['Body'].read()

    @xray.capture()
    @staticmethod
    @logger.log
    def get_query_string(parameters: dict,
                         filter_date: str = f'{date.today()}',
                         query: str = 'SELECT * FROM "vulnerability_management"."swa_current_vulnerabilities_vw"',
                         limit_results: int = 0) -> str:
        """ Validate our SQL looks right

        Parameters
        ----------
        parameters : Dict
            Map of query parameter filters.
            E.g. {'id':'svc02222','environment':'prd','account': '012597783113'}

        filter_date : str
            Format YYYY-MM-DD as generated by datetime.date.today()

        query : str
            BASE SQL 'FROM' query claus to execute on athena. Will be combined
            with parameters and filter_date.

        Returns
        -------
        results : List[str]
            Athena query execution results

        Testing
        -------

        >>> os.environ['STACK_NAME'] = 'iobots-stack'

        >>> testing.export_secret_arns(os.environ['STACK_NAME'])

        >>> s = Qualys.get_query_string({'id':'svc02222','environment':'','account': '' })

        >>> assert (str(date.today()) in s), f'expected "{date.today()}", got "{s}"'

        >>> Qualys.get_query_string({'id':'','environment':'','account': '' }, filter_date='2020-08-05', limit_results=1)
        'SELECT * FROM "vulnerability_management"."swa_current_vulnerabilities_vw" WHERE report_date = DATE(\\'2020-08-05\\') LIMIT 1;'

        >>> Qualys.get_query_string({'id':'','environment':'','account': '' }, filter_date='2020-08-05')
        'SELECT * FROM "vulnerability_management"."swa_current_vulnerabilities_vw" WHERE report_date = DATE(\\'2020-08-05\\');'

        >>> Qualys.get_query_string({'id':'svc02222','environment':'','account': '' }, filter_date='2020-08-05')
        'SELECT * FROM "vulnerability_management"."swa_current_vulnerabilities_vw" WHERE report_date = DATE(\\'2020-08-05\\') AND t_appid = \\'svc02222\\';'

        >>> Qualys.get_query_string({'id':'svc02222','environment':'prd','account': '' }, filter_date='2020-08-05')
        'SELECT * FROM "vulnerability_management"."swa_current_vulnerabilities_vw" WHERE report_date = DATE(\\'2020-08-05\\') AND t_appid = \\'svc02222\\' AND t_environment LIKE \\'%prd%\\';'

        >>> Qualys.get_query_string({'id':'svc02222','environment':'prd','account': '012597783113' }, filter_date='2020-08-05')
        'SELECT * FROM "vulnerability_management"."swa_current_vulnerabilities_vw" WHERE report_date = DATE(\\'2020-08-05\\') AND t_appid = \\'svc02222\\' AND t_environment LIKE \\'%prd%\\' AND account_id = \\'012597783113\\';'

        >>> testing.unexport_secret_arns()

        >>> del os.environ['STACK_NAME']
        """
        query += f' WHERE report_date = DATE(\'{filter_date}\')'
        if parameters.get("id"):
            query += f' AND t_appid = \'{parameters.get("id")}\''

        if parameters.get("environment"):
            query += f' AND t_environment LIKE \'%{parameters.get("environment")}%\''

        if parameters.get("account"):
            query += f' AND account_id = \'{parameters.get("account")}\''

        if limit_results:
            query += f' LIMIT {limit_results}'

        return f'{query};'

    @xray.capture()
    @staticmethod
    @logger.log
    def data_to_row(data) -> Tuple:
        row = [
            datum.get('VarCharValue', '')
            for datum in data['Data']
        ]
        return row
