#!/usr/bin/env python
# -*- coding: utf-8 -*-
import json
import os
from io import BytesIO
from typing import Dict
from typing import List
from zipfile import ZipFile
from zipfile import ZipInfo

import pandas
from aws_xray_sdk.core import xray_recorder as xray

from slater.exceptions import SLaterImportEmptyWorkbookError
from slater.exceptions import SLaterImportGeneratedBadZipBundleError
from iobot.utils import init_config


@xray.capture()
def get_sheets_from_workbook(content: bytes,
                             filters: List[str] = None,
                             header: int = 5):  # -> pandas.DataFrame
    """ Reads an XLSX document into a DataFrame that can be operated against
    like a Dictionary.

    Note
    ----
    For the purposes of preserving left zero padding and formatting,
    cloud_account_number and master_payer are both handled as strings.

    Parameters
    ==========

    content: bytes
      The contents of the XLSX from an open(<file-like-obj>,'rb') operation

    filters: List[str]
      A list of sheets you want preserved when reading the workbook. All other
      sheets will be removed or 'ignored'. If filters is not passed, it defaults
      to:

          [
              'towers',
              'applications',
              'application contacts',
              'accounts',
              'account contacts'
          ]

    header: int
      Defaults to 5, meaning the rows 0-4 will be ignored and row 5 will be
      treated as the header row.

    Returns
    =======
    workbook : pandas.DataFrame
      Modified version the workbook bytes data you passed in.


    Raises
    ======
    slater.exceptions.SLaterImportEmptyWorkbookError :
      When at an empty workbook has been generated. Typically because none of
      the sheets matched the filters parameter.

    Validate we can parse excel xlsx content
    >>> file = open('tests/data/slater-load-sample.xlsx','rb')

    >>> content = file.read()

    >>> sheets = get_sheets_from_workbook(content)

    >>> sheets.keys()
        dict_keys(['towers', 'accounts', 'applications', 'application contacts', 'account contacts'])

    >>> sheets.get('towers')
                 tower_name  ... import
    0             Barad-dûr  ...    YES
    1              Isengard  ...    Yes
    2    Enterprise Systems  ...    yEs
    3   Customer Experience  ...    yes
    4               Content  ...    yes
    5             Inter Div  ...    yes
    6   Employee Experience  ...    yes
    7                   I&O  ...    yes
    8              Bitesize  ...    yes
    9                   PLA  ...    yes
    10               AppEng  ...    yes
    11            BayBridge  ...    yes
    12  Independent Managed  ...    yes
    13          Assessments  ...    yes
    14             Clinical  ...    yes
    15              Interco  ...    yes
    16         CISO-TechOps  ...    yes
    17             CISO-PSO  ...    yes
    18           CISO-ITSec  ...    yes
    19    Quality Assurance  ...    yes
    20                  VUE  ...    yes
    21                   ES  ...    yes
    22                  OBL  ...    yes
    23                  OPM  ...    yes
    24                 POLS  ...    yes
    25                  GBS  ...    yes
    26               ignore  ...     no
    <BLANKLINE>
    [27 rows x 4 columns]

    >>> sheets.get('accounts')
      cloud_account_number   t_appid  ... create_date import
    0         000000000000  svc00000  ...         NaN    yes
    1         111111111111  svc11111  ...         NaN    yes
    <BLANKLINE>
    [2 rows x 14 columns]


    Validate we can raise an exception if format processing fails.
    >>> file = open('tests/data/aws-slater-metadata-lite.xlsx','rb')

    >>> content = file.read()

    >>> empty = get_sheets_from_workbook(content)
    Traceback (most recent call last):
    ...
    slater.exceptions.SLaterImportEmptyWorkbookError: workbook should contain at minimum one of the following sheets: "towers, applications, application contacts, accounts, account contacts"
    """
    if not filters:
        filters = [
            'towers',
            'applications',
            'application contacts',
            'accounts',
            'account contacts'
        ]
    workbook = pandas.read_excel(content, sheet_name=None, header=header,
                                 dtype={'cloud_account_number': str, 'master_payer': str})

    ignore_sheets: list = []
    for sheet in workbook.keys():
        if sheet not in filters:
            ignore_sheets.append(sheet)

    for sheet in ignore_sheets:
        del workbook[sheet]

    if not workbook:
        valid_sheets: str = ", ".join(filters)
        raise SLaterImportEmptyWorkbookError(
            f'workbook should contain at minimum one of the following sheets: "{valid_sheets}"')

    return workbook


@xray.capture()
def convert_sheets_to_zip_bundle(sheets,  # : pandas.DataFrame
                                 slater_pad_rows: int = 5,
                                 slater_row_padding_value: str = ".,.,.,.,.\n"):
    """ Converts a SLater formatted pandas.DataFrame into a series of CSV
    files wrapped in a zip bundle.

    Note
    ----
    Sheets with spaces in the names are re-named to use underscores instead.

    Parameters
    ==========

    sheets: pandas.DataFrame
      Typically you generate this data from get_sheets_from_workbook() or
      convert_finops_aws_metadata_to_slater_import_sheets() operations.
      Any pandas.DataFrame should work with the right formatting.

    slater_pad_rows: int
      Defaults to 5. The number of rows that should be added to each sheet
      before the header is generated. SLater expects five.

    slater_row_padding_value: str
      Defaults to ".,.,.,.,.\n". Each sheet will be padded with:
      slater_row_padding_value * slater_pad_rows

    Returns
    =======

    buffer : io.BytesIO
      File-like object representation of the zip bundle generated

    Raises
    ======
    slater.exceptions.SLaterImportEmptyWorkbookError :
      When an empty workbook has been passed.

    SLaterImportGeneratedBadZipBundleError :
      When the generated zip bundle contains no files

    Validate we can render a DataFrame into multiple CSV files in a zip bundle
    >>> file = open('tests/data/slater-load-sample.xlsx','rb')

    >>> content = file.read()

    >>> sheets = get_sheets_from_workbook(content)

    >>> zip_data = convert_sheets_to_zip_bundle(sheets)

    >>> zip_data.getvalue()
      b"PK\\x03\\x04\\x14\\x00\\x00\\x00\\x00\\x00\\x00\\x00!\\x00\\xa9\\xb0\\xda\\x0e<\\x05\\x00\\x00<\\x05\\x00\\x00\\n\\x00\\x00\\x00towers.csv.,.,.,.,.\\n.,.,.,.,.\\n.,.,.,.,.\\n.,.,.,.,.\\n.,.,.,.,.\\ntower_name,tower_owner_name,tower_owner_email\\nBarad-d\\xc3\\xbbr,Sauron DarkLord,dark.lord@darktower.com\\nIsengard,Saruman WhiteHand,saruman.whitehand@lighttower.com\\nEnterprise Systems,Steve Santana,steve.santana@pearson.com\\nCustomer Experience,Corey Farrell,corey.farrell@pearson.com\\nContent,Eliot Pikoulis,eliot.pikoulis@pearson.com\\nInter Div,Iqbal Kahn,iqbal.kahn@pearson.com\\nEmployee Experience,Mark Barfield,mark.barfield@pearson.com\\nI&O,Ian Wright,ian.wright@pearson.com\\nBitesize,Ian Wright,ian.wright@pearson.com\\nPLA,Saleem Brahmanapalli,syedsaleemjavid.brahmanapalli@pearson.com\\nAppEng,Saleem Brahmanapalli,syedsaleemjavid.brahmanapalli@pearson.com\\nBayBridge,Saleem Brahmanapalli,syedsaleemjavid.brahmanapalli@pearson.com\\nIndependent Managed,Saleem Brahmanapalli,syedsaleemjavid.brahmanapalli@pearson.com\\nAssessments,,\\nClinical,,\\nInterco,,\\nCISO-TechOps,Vincent Blake,vincent.blake@pearson.com\\nCISO-PSO,Vincent Blake,vincent.blake@pearson.com\\nCISO-ITSec,Vincent Blake,vincent.blake@pearson.com\\nQuality Assurance,,\\nVUE,Guy Speier,guy.speier@pearson.com\\nES,Guy Speier,guy.speier@pearson.com\\nOBL,Eric Hilfer,eric.hilfer@pearson.com\\nOPM,Iwan Streichenberger,iwan.streichenberger@pearson.com\\nPOLS,Iwan Streichenberger,iwan.streichenberger@pearson.com\\nGBS,Eddie Monteiro,eddie.monteiro@pearson.com\\nPK\\x03\\x04\\x14\\x00\\x00\\x00\\x00\\x00\\x00\\x00!\\x00H\\xf0\\xe48\\x9f\\x02\\x00\\x00\\x9f\\x02\\x00\\x00\\x0c\\x00\\x00\\x00accounts.csv.,.,.,.,.\\n.,.,.,.,.\\n.,.,.,.,.\\n.,.,.,.,.\\n.,.,.,.,.\\ncloud_account_number, t_appid,cloud_account_name,cloud_account_root_email,master_payer_name,intake_number,account_type_name,governance_lob_name,budget_account_string,project_number,tower_name,snow_approval_group,create_date\\n000000000000,svc00000,Dark Tower Production,sauron@root.Barad-d\\xc3\\xbbr.com,222222222222,0,Production,Dark Lord,one-ring-to-rule-them-$all,000-00000-0000-0000,Barad-d\\xc3\\xbbr,e0a6bf6a-ba42-4fe9-bbe5-732bae268b94,\\n111111111111,svc11111,Light Tower Production,sarumon@root.Isengard.com,222222222222,1,Production,The Fellowship,and-bind-$them,111-11111-1111-1111,Isengard,08d065f4-58b4-4c05-b4d3-98dbdb6a688d,\\nPK\\x03\\x04\\x14\\x00\\x00\\x00\\x00\\x00\\x00\\x00!\\x00\\x10\\xf2\\xf1H\\xd5\\x01\\x00\\x00\\xd5\\x01\\x00\\x00\\x10\\x00\\x00\\x00applications.csv.,.,.,.,.\\n.,.,.,.,.\\n.,.,.,.,.\\n.,.,.,.,.\\n.,.,.,.,.\\napplication_name,t_appid,governance_lob_name,business_unit_name,application_budget_account_string,application_description,pagerduty_url,tower_name,create_date\\nDark Tower,svc00000,Dark Lord,Orcs,one-ring-to-rule-them-$all,Souron's Evil App,https://pagerduty.com/service/evilapp,Barad-d\\xc3\\xbbr,\\nLight Tower,svc11111,The Fellowship,Elves,and-bind-$them,Saruman's Orc Builder,https://pagerduty.com/service/orcbuilder,Isengard,\\nPK\\x03\\x04\\x14\\x00\\x00\\x00\\x00\\x00\\x00\\x00!\\x00\\xdb.\\x8c\\x05\\xf3\\x00\\x00\\x00\\xf3\\x00\\x00\\x00\\x18\\x00\\x00\\x00application_contacts.csv.,.,.,.,.\\n.,.,.,.,.\\n.,.,.,.,.\\n.,.,.,.,.\\n.,.,.,.,.\\nt_appid,contact_type,contact_name,contact_email\\nsvc00000,application_owner,Sauron DarkLord,dark.lord@darktower.com\\nsvc11111,application_owner,Saruman WhiteHand,saruman.whitehand@lighttower.com\\nPK\\x03\\x04\\x14\\x00\\x00\\x00\\x00\\x00\\x00\\x00!\\x00l\\xdcZ\\xde\\xf4\\x01\\x00\\x00\\xf4\\x01\\x00\\x00\\x14\\x00\\x00\\x00account_contacts.csv.,.,.,.,.\\n.,.,.,.,.\\n.,.,.,.,.\\n.,.,.,.,.\\n.,.,.,.,.\\ncloud_account_number,contact_type,contact_name,contact_email\\n000000000000,account_owner,Sauron DarkLord,dark.lord@darktower.com\\n111111111111,account_owner,Saruman WhiteHand,saruman.whitehand@lighttower.com\\n111111111111,tech_owner,Jeff Schmidt,jeffrey.schimdt@pearson.com\\n111111111111,tdm_owner,Jake Halvorsen,jake.halvorsen@pearson.com\\n000000000000,budget_vp,Ian Wright,ian.wright@pearson.com\\n111111111111,budget_vp,Ian Wright,ian.wright@pearson.com\\nPK\\x01\\x02\\x14\\x03\\x14\\x00\\x00\\x00\\x00\\x00\\x00\\x00!\\x00\\xa9\\xb0\\xda\\x0e<\\x05\\x00\\x00<\\x05\\x00\\x00\\n\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x01\\x00\\x00\\x00\\x00towers.csvPK\\x01\\x02\\x14\\x03\\x14\\x00\\x00\\x00\\x00\\x00\\x00\\x00!\\x00H\\xf0\\xe48\\x9f\\x02\\x00\\x00\\x9f\\x02\\x00\\x00\\x0c\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x01d\\x05\\x00\\x00accounts.csvPK\\x01\\x02\\x14\\x03\\x14\\x00\\x00\\x00\\x00\\x00\\x00\\x00!\\x00\\x10\\xf2\\xf1H\\xd5\\x01\\x00\\x00\\xd5\\x01\\x00\\x00\\x10\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x01-\\x08\\x00\\x00applications.csvPK\\x01\\x02\\x14\\x03\\x14\\x00\\x00\\x00\\x00\\x00\\x00\\x00!\\x00\\xdb.\\x8c\\x05\\xf3\\x00\\x00\\x00\\xf3\\x00\\x00\\x00\\x18\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x010\\n\\x00\\x00application_contacts.csvPK\\x01\\x02\\x14\\x03\\x14\\x00\\x00\\x00\\x00\\x00\\x00\\x00!\\x00l\\xdcZ\\xde\\xf4\\x01\\x00\\x00\\xf4\\x01\\x00\\x00\\x14\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x80\\x01Y\\x0b\\x00\\x00account_contacts.csvPK\\x05\\x06\\x00\\x00\\x00\\x00\\x05\\x00\\x05\\x008\\x01\\x00\\x00\\x7f\\r\\x00\\x00\\x00\\x00"

    >>> myzip = ZipFile(zip_data,'r')

    Validate that
    >>> myzip.filelist  # noqa E501
      [<ZipInfo filename='towers.csv' filemode='?rw-------' file_size=1340>, <ZipInfo filename='accounts.csv' filemode='?rw-------' file_size=671>, <ZipInfo filename='applications.csv' filemode='?rw-------' file_size=469>, <ZipInfo filename='application_contacts.csv' filemode='?rw-------' file_size=243>, <ZipInfo filename='account_contacts.csv' filemode='?rw-------' file_size=500>]

    Validate that we see zeros preserved
    >>> myzip.read('accounts.csv')  # noqa E501
      b'.,.,.,.,.\\n.,.,.,.,.\\n.,.,.,.,.\\n.,.,.,.,.\\n.,.,.,.,.\\ncloud_account_number, t_appid,cloud_account_name,cloud_account_root_email,master_payer_name,intake_number,account_type_name,governance_lob_name,budget_account_string,project_number,tower_name,snow_approval_group,create_date\\n000000000000,svc00000,Dark Tower Production,sauron@root.Barad-d\\xc3\\xbbr.com,222222222222,0,Production,Dark Lord,one-ring-to-rule-them-$all,000-00000-0000-0000,Barad-d\\xc3\\xbbr,e0a6bf6a-ba42-4fe9-bbe5-732bae268b94,\\n111111111111,svc11111,Light Tower Production,sarumon@root.Isengard.com,222222222222,1,Production,The Fellowship,and-bind-$them,111-11111-1111-1111,Isengard,08d065f4-58b4-4c05-b4d3-98dbdb6a688d,\\n'

    Validate unicode preserves
    >>> myzip.read('towers.csv')  # noqa E501
      b'.,.,.,.,.\\n.,.,.,.,.\\n.,.,.,.,.\\n.,.,.,.,.\\n.,.,.,.,.\\ntower_name,tower_owner_name,tower_owner_email\\nBarad-d\\xc3\\xbbr,Sauron DarkLord,dark.lord@darktower.com\\nIsengard,Saruman WhiteHand,saruman.whitehand@lighttower.com\\nEnterprise Systems,Steve Santana,steve.santana@pearson.com\\nCustomer Experience,Corey Farrell,corey.farrell@pearson.com\\nContent,Eliot Pikoulis,eliot.pikoulis@pearson.com\\nInter Div,Iqbal Kahn,iqbal.kahn@pearson.com\\nEmployee Experience,Mark Barfield,mark.barfield@pearson.com\\nI&O,Ian Wright,ian.wright@pearson.com\\nBitesize,Ian Wright,ian.wright@pearson.com\\nPLA,Saleem Brahmanapalli,syedsaleemjavid.brahmanapalli@pearson.com\\nAppEng,Saleem Brahmanapalli,syedsaleemjavid.brahmanapalli@pearson.com\\nBayBridge,Saleem Brahmanapalli,syedsaleemjavid.brahmanapalli@pearson.com\\nIndependent Managed,Saleem Brahmanapalli,syedsaleemjavid.brahmanapalli@pearson.com\\nAssessments,,\\nClinical,,\\nInterco,,\\nCISO-TechOps,Vincent Blake,vincent.blake@pearson.com\\nCISO-PSO,Vincent Blake,vincent.blake@pearson.com\\nCISO-ITSec,Vincent Blake,vincent.blake@pearson.com\\nQuality Assurance,,\\nVUE,Guy Speier,guy.speier@pearson.com\\nES,Guy Speier,guy.speier@pearson.com\\nOBL,Eric Hilfer,eric.hilfer@pearson.com\\nOPM,Iwan Streichenberger,iwan.streichenberger@pearson.com\\nPOLS,Iwan Streichenberger,iwan.streichenberger@pearson.com\\nGBS,Eddie Monteiro,eddie.monteiro@pearson.com\\n'

    >>> sheets.get('towers')
                 tower_name  ... import
    0             Barad-dûr  ...    YES
    1              Isengard  ...    Yes
    2    Enterprise Systems  ...    yEs
    3   Customer Experience  ...    yes
    4               Content  ...    yes
    5             Inter Div  ...    yes
    6   Employee Experience  ...    yes
    7                   I&O  ...    yes
    8              Bitesize  ...    yes
    9                   PLA  ...    yes
    10               AppEng  ...    yes
    11            BayBridge  ...    yes
    12  Independent Managed  ...    yes
    13          Assessments  ...    yes
    14             Clinical  ...    yes
    15              Interco  ...    yes
    16         CISO-TechOps  ...    yes
    17             CISO-PSO  ...    yes
    18           CISO-ITSec  ...    yes
    19    Quality Assurance  ...    yes
    20                  VUE  ...    yes
    21                   ES  ...    yes
    22                  OBL  ...    yes
    23                  OPM  ...    yes
    24                 POLS  ...    yes
    25                  GBS  ...    yes
    26               ignore  ...     no
    <BLANKLINE>
    [27 rows x 4 columns]

    Validate that emtpy sheets are omitted from the zip
    >>> sheets.get('towers').drop([0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26], inplace=True)

    >>> sheets.get('towers')
      Empty DataFrame
      Columns: [tower_name, tower_owner_name, tower_owner_email, import]
      Index: []

    >>> tower_omitted_data = convert_sheets_to_zip_bundle(sheets)

    >>> tower_omitted  = ZipFile(tower_omitted_data, 'r')

    >>> tower_omitted.filelist
      [<ZipInfo filename='accounts.csv' filemode='?rw-------' file_size=671>, <ZipInfo filename='applications.csv' filemode='?rw-------' file_size=469>, <ZipInfo filename='application_contacts.csv' filemode='?rw-------' file_size=243>, <ZipInfo filename='account_contacts.csv' filemode='?rw-------' file_size=500>]

    >>> file.close()

    Validate that rows with "import" <> "yes" are ignored
    >>> file = open('tests/data/slater-load-sample.xlsx','rb')

    >>> content = file.read()

    >>> zip_data = convert_sheets_to_zip_bundle(get_sheets_from_workbook(content))

    >>> file.close()

    >>> myzip = ZipFile(zip_data,'r')

    >>> myzip.read('towers.csv')
      b'.,.,.,.,.\\n.,.,.,.,.\\n.,.,.,.,.\\n.,.,.,.,.\\n.,.,.,.,.\\ntower_name,tower_owner_name,tower_owner_email\\nBarad-d\\xc3\\xbbr,Sauron DarkLord,dark.lord@darktower.com\\nIsengard,Saruman WhiteHand,saruman.whitehand@lighttower.com\\nEnterprise Systems,Steve Santana,steve.santana@pearson.com\\nCustomer Experience,Corey Farrell,corey.farrell@pearson.com\\nContent,Eliot Pikoulis,eliot.pikoulis@pearson.com\\nInter Div,Iqbal Kahn,iqbal.kahn@pearson.com\\nEmployee Experience,Mark Barfield,mark.barfield@pearson.com\\nI&O,Ian Wright,ian.wright@pearson.com\\nBitesize,Ian Wright,ian.wright@pearson.com\\nPLA,Saleem Brahmanapalli,syedsaleemjavid.brahmanapalli@pearson.com\\nAppEng,Saleem Brahmanapalli,syedsaleemjavid.brahmanapalli@pearson.com\\nBayBridge,Saleem Brahmanapalli,syedsaleemjavid.brahmanapalli@pearson.com\\nIndependent Managed,Saleem Brahmanapalli,syedsaleemjavid.brahmanapalli@pearson.com\\nAssessments,,\\nClinical,,\\nInterco,,\\nCISO-TechOps,Vincent Blake,vincent.blake@pearson.com\\nCISO-PSO,Vincent Blake,vincent.blake@pearson.com\\nCISO-ITSec,Vincent Blake,vincent.blake@pearson.com\\nQuality Assurance,,\\nVUE,Guy Speier,guy.speier@pearson.com\\nES,Guy Speier,guy.speier@pearson.com\\nOBL,Eric Hilfer,eric.hilfer@pearson.com\\nOPM,Iwan Streichenberger,iwan.streichenberger@pearson.com\\nPOLS,Iwan Streichenberger,iwan.streichenberger@pearson.com\\nGBS,Eddie Monteiro,eddie.monteiro@pearson.com\\n'

    >>> bad_zip = convert_sheets_to_zip_bundle(b'')
    Traceback (most recent call last):
    ...
    slater.exceptions.SLaterImportEmptyWorkbookError: cannot zip an empty workbook

    >>> from io import StringIO
    >>> one = pandas.read_csv(StringIO('foo\\n'))
    >>> two = pandas.read_csv(StringIO('bar\\n'))
    >>> empty = {'one': one, 'two': two}

    >>> bad_zip = convert_sheets_to_zip_bundle(empty)
    Traceback (most recent call last):
    ...
    slater.exceptions.SLaterImportGeneratedBadZipBundleError: generated zip bundle with no files, bad input?
    """
    if not sheets:
        raise SLaterImportEmptyWorkbookError('cannot zip an empty workbook')

    buffer = BytesIO()
    with ZipFile(buffer, "w") as zip_in_mem:
        for sheet in sheets.keys():
            if len(sheets[sheet].index) > 0:
                filtered_sheet = None
                if 'import' in sheets[sheet].keys():
                    filter_mask = sheets[sheet]['import'].str.lower() == 'yes'
                    import_rows_sheet = sheets[sheet][filter_mask]
                    filtered_sheet = import_rows_sheet.drop(['import'], axis=1)
                else:
                    filtered_sheet = sheets[sheet]
                info = ZipInfo(f"{sheet.replace(' ','_')}.csv")
                zip_in_mem.writestr(
                    info, f"{slater_row_padding_value * slater_pad_rows}{filtered_sheet.to_csv(index=False)}")

    with ZipFile(buffer, "r") as zip_in_mem:
        if len(zip_in_mem.namelist()) == 0:
            raise SLaterImportGeneratedBadZipBundleError(
                'generated zip bundle with no files, bad input?')

    return buffer


@xray.capture()
def convert_finops_aws_metadata_to_slater_import_sheets(content: bytes,
                                                        accounts_sheet: str = 'Business mapping&Accounts',
                                                        applications_sheet: str = 'Application Contacts',
                                                        filters: List[str] = [],
                                                        header: int = 1,
                                                        config: Dict = None,
                                                        config_file: str = "iobot.ini"):  # -> pandas.DataFrame
    """ Converts an Excel XLSX document from one workbook format to the SLater
    standard import version. It takes a flat row and breaks it into accounts
    and account_contacts sheets. Each contact type is broken into its own row.

    Parameters
    ==========

    content: bytes
      The contents of the XLSX from an open(<file-like-obj>,'rb') operation

    accounts_sheet: str
      The name of the sheet that contains accounts and account contacts.

    applications_sheet: str
      The name of the sheet that contains applications and application contacts.

    filters: List[str]
      The sheet within a workbook that contains the finops metadata to be
      converted into standard SLater format. If you set this, you must match
      accounts_sheet, and applications_sheet to the source workbook sheet
      names.

    header: int
      Defaults to 1, meaning row zero will be ignored and row 1 will be
      treated as the header row. Currently all sheets must use the same header.

    config : configparser.ConfigParser()
        Per https://docs.python.org/2.0/ref/function.html set config
        explicitly to None and evaluate how it should be set if no value
        is passed. See iobot.utils for more information.
    config_file: str
        Defaults to iobot.ini
        If config is not passed, we will attempt to load config from this file.


    Returns
    =======
    workbook : pandas.DataFrame
      Modified version the workbook bytes data you passed in.

    Raises
    ======
    slater.exceptions.SLaterImportEmptyWorkbookError :
      When at an empty workbook has been generated. Typically because none of
      the sheets matched the filters parameter.

    Validate we can parse excel xlsx content
    >>> file = open('tests/data/aws-slater-metadata-lite.xlsx','rb')

    >>> content = file.read()

    >>> sheets = convert_finops_aws_metadata_to_slater_import_sheets(content)

    >>> sheets
    {'accounts':    cloud_account_number  t_appid  ... snow_approval_group create_date
    0          138512462893      NaN  ...                None        None
    1          227490007503      NaN  ...                None        None
    2          838875724762      NaN  ...                None        None
    3          465750819367      NaN  ...                None        None
    4          540736820077      NaN  ...                None        None
    5          677411076568      NaN  ...                None        None
    6          837457484803      NaN  ...                None        None
    7          886735820831      NaN  ...                None        None
    8          502045333824      NaN  ...                None        None
    9          361247020131      NaN  ...                None        None
    10         598717697088      NaN  ...                None        None
    <BLANKLINE>
    [11 rows x 13 columns], 'account_contacts':    cloud_account_number  ...                  contact_email
    0          138512462893  ...        Adam.Krapfl@pearson.com
    1          138512462893  ...        Adam.Krapfl@pearson.com
    2          138512462893  ...                        distros
    3          138512462893  ...                            NaN
    4          227490007503  ...        Adam.Krapfl@pearson.com
    5          227490007503  ...        Adam.Krapfl@pearson.com
    6          227490007503  ...                            NaN
    7          838875724762  ...        Adam.Krapfl@pearson.com
    8          838875724762  ...        Adam.Krapfl@pearson.com
    9          838875724762  ...                            NaN
    10         465750819367  ...    Andrew.Lawrence@pearson.com
    11         465750819367  ...  Andrew.D.Lawrence@pearson.com
    12         465750819367  ...      jason.gormley@pearson.com
    13         540736820077  ...    Andrew.Lawrence@pearson.com
    14         540736820077  ...  Andrew.D.Lawrence@pearson.com
    15         540736820077  ...     jake.halvorsen@pearson.com
    16         677411076568  ...      Angela.Street@pearson.com
    17         677411076568  ...      Angela.Street@pearson.com
    18         837457484803  ...      Angela.Street@pearson.com
    19         837457484803  ...      Angela.Street@pearson.com
    20         886735820831  ...      Angela.Street@pearson.com
    21         886735820831  ...      Angela.Street@pearson.com
    22         502045333824  ...   Arya.Ordoubadian@pearson.com
    23         502045333824  ...   Arya.Ordoubadian@pearson.com
    24         361247020131  ...    Ashley.Hromatko@pearson.com
    25         361247020131  ...    Ashley.Hromatko@pearson.com
    26         598717697088  ...    Ashley.Hromatko@pearson.com
    27         598717697088  ...    Ashley.Hromatko@pearson.com
    <BLANKLINE>
    [28 rows x 4 columns], 'applications':                        application_name  ... create_date
    0                         Asset Library  ...         NaN
    1      Asset Management Data Hub (AMDH)  ...         NaN
    2                 BR CombosTreinamentos  ...         NaN
    3                             BR ConCOC  ...         NaN
    4                             BR Concur  ...         NaN
    5                                   CAS  ...         NaN
    6                             Cassandra  ...         NaN
    7                      Cassandra Ascend  ...         NaN
    8                                  CaST  ...         NaN
    9                           DPW Gateway  ...         NaN
    10  DRA (Diagnostic Reading Assessment)  ...         NaN
    11      Enterprise Key Management (EKM)  ...         NaN
    12                  FishEye (Atlassian)  ...         NaN
    13                     Flexible Scoring  ...         NaN
    14                   Gitlab-assessments  ...         NaN
    15                Gradebook Gateway API  ...         NaN
    16                      Grader Services  ...         NaN
    17                                  ICE  ...         NaN
    18                                LUCID  ...         NaN
    19                             NewRelic  ...         NaN
    20                           Oasis (UK)  ...         NaN
    21      OBL Azure DevOps (Build Server)  ...         NaN
    22                          PowerSchool  ...         NaN
    23                                  PQS  ...         NaN
    24         PSS (Pearson School Support)  ...         NaN
    25                     PsychCorp Center  ...         NaN
    26                             Redshift  ...         NaN
    27                            RMS - RMM  ...         NaN
    28  SAMS - Sales and marketing database  ...         NaN
    29                                  SAP  ...         NaN
    30                            SchoolNet  ...         NaN
    31                          TestNav 7.x  ...         NaN
    32                            TestNav 8  ...         NaN
    33                        TN8 Previewer  ...         NaN
    34                                  VUE  ...         NaN
    35                                  WPS  ...         NaN
    36                  XL Question Metrics  ...         NaN
    37                               Zabbix  ...         NaN
    38                             Zalenium  ...         NaN
    39                    Zen Desk - CA ERP  ...         NaN
    40                               Zenoss  ...         NaN
    <BLANKLINE>
    [41 rows x 9 columns], 'application_contacts':       t_appid  ...               contact_email
    0    SVC01946  ...  david.jolliffe@pearson.com
    1    SVC01946  ...     Nick.Gowler@pearson.com
    2    SVC01946  ...       paul.silk@pearson.com
    3    SVC03057  ...  david.jolliffe@pearson.com
    4    SVC03057  ...   Georgie.Stone@pearson.com
    ..        ...  ...                         ...
    118  SVC02730  ...  Lisa.Lessard@pearsoned.com
    119  SVC02730  ...   Ilyas.Mohamed@pearson.com
    120  SVC00175  ...  Dennis.Stetzel@pearson.com
    121  SVC00175  ...   Lahiru.Perera@Pearson.com
    122  SVC00175  ...       Ian.White@pearson.com
    <BLANKLINE>
    [123 rows x 4 columns]}

    >>> len(sheets['accounts'].get('cloud_account_number'))
      11

    >>> sheets['accounts'].get('master_payer_name')
    0     Assessments
    1     Assessments
    2     Assessments
    3     Assessments
    4     Assessments
    5     Assessments
    6     Assessments
    7     Assessments
    8
    9     Assessments
    10
    Name: master_payer_name, dtype: object

    >>> len(sheets['account_contacts'].get('cloud_account_number'))
    28

    >>> len(sheets['applications'].get('t_appid'))
      41

    >>> sheets['applications'].get('governance_lob_name')
    0               Global Product
    1               Global Product
    2       Information Technology
    3       Information Technology
    4       Information Technology
    5       Information Technology
    6                  Assessments
    7               Global Product
    8       Information Technology
    9       Information Technology
    10                 Assessments
    11              Tech Assurance
    12      Information Technology
    13                 Assessments
    14                 Assessments
    15                   Higher Ed
    16                   Higher Ed
    17      Information Technology
    18      Information Technology
    19      Information Technology
    20      Information Technology
    21                         OBL
    22      Information Technology
    23      Information Technology
    24                         VUE
    25                 Assessments
    26      Information Technology
    27      Information Technology
    28      Information Technology
    29      Information Technology
    30                 Assessments
    31                 Assessments
    32                 Assessments
    33                 Assessments
    34                         VUE
    35                   Higher Ed
    36              Global Product
    37      Information Technology
    38              Tech Assurance
    39      Information Technology
    40    Global Business Services
    Name: governance_lob_name, dtype: object
    >>> len(sheets['application_contacts'].get('contact_name'))
      123

    >>> current = sheets['account_contacts']

    >>> current = current.loc[current['contact_type'] == 'TDM Owner']

    >>> current.loc[current['contact_email'] == 'jake.halvorsen@pearson.com']
       cloud_account_number  ...               contact_email
    15         540736820077  ...  jake.halvorsen@pearson.com
    <BLANKLINE>
    [1 rows x 4 columns]

    Validate we can raise an exception if format processing fails.
    >>> file = open('tests/data/slater-load-sample.xlsx','rb')

    >>> content = file.read()

    >>> empty = convert_finops_aws_metadata_to_slater_import_sheets(content)
    Traceback (most recent call last):
    ...
    slater.exceptions.SLaterImportEmptyWorkbookError: workbook should contain at minimum one of the following sheets: "Business mapping&Accounts, Application Contacts"
    """
    if not config:
        config = init_config(config_file=config_file, env=dict(os.environ))

    if not filters:
        filters = [accounts_sheet, applications_sheet]

    sheets = get_sheets_from_workbook(content, filters=filters, header=header)

    if accounts_sheet in sheets:
        metadata = sheets[accounts_sheet]

        # build the 'accounts' sheet with a subset from the source sheet
        sheets['accounts'] = metadata.loc[:,
                                          (
                                              'cloud_account_number',
                                              'funding_account_application',  # rename to -> t_appid
                                              'cloud_account_name',
                                              # cloud_account_root_email -> None
                                              'master_payer',  # rename to -> master_payer_name & translate
                                              # intake_number -> None
                                              'account_type',  # rename to -> account_type_name
                                              'governance_lob',  # rename to -> governance_lob_name
                                              'budget_account_string',
                                              'project_number',
                                              'budget_account_tower'  # rename to -> tower_name
                                              # snow_approval_group -> None
                                              # create_date default to -> None
                                          )
                                          ]

        # replace the contents of master_payer with a translated value
        master_payer_lookup: Dict = dict(config["MasterPayerLookup"])
        for key, value in master_payer_lookup.items():
            sheets['accounts'].replace({'master_payer': key},
                                       {'master_payer': value},
                                       regex=True,
                                       inplace=True)

        # add some missing columns with defaults or translations
        sheets['accounts'].insert(3, 'cloud_account_root_email', None)
        sheets['accounts'].insert(5, 'intake_number', None)
        sheets['accounts'].insert(11, 'snow_approval_group', None)
        sheets['accounts'].insert(12, 'create_date', None)

        # rename some of the original columns to align with slater import format
        sheets['accounts'].rename(columns={
            'account_type': 'account_type_name',
            'budget_account_tower': 'tower_name',
            'funding_account_application': 't_appid',
            'master_payer': 'master_payer_name',
            'governance_lob': 'governance_lob_name'
        }, inplace=True)

        # build an empty 'account_contacts' sheet that we will append to afterwards
        sheets['account_contacts'] = pandas.DataFrame([], columns=[
            'cloud_account_number',
            'contact_type',
            'contact_name',
            'contact_email'
        ])

        # iterate the rows and generate contact records
        # one 'row' becomes six contact records
        type_map: Dict = {
            "productowner": "Product Owner",
            "acctowner": "Account Owner",
            "techowner": "Technical Owner",
            "techowner.1": "Technical Owner",
            "TDM": "TDM Owner",
            "budgetvp": "Budget VP"
        }

        for row in metadata.iterrows():
            for contact_type in type_map.keys():
                name: str = row[1].get(f'contact_name_{contact_type}', None)
                email: str = row[1].get(f'contact_email_{contact_type}', None)

                if (not name and not email):
                    continue
                elif (str(name) == "nan" and str(email) == "nan"):
                    continue
                else:
                    sheets['account_contacts'] = sheets['account_contacts'].append(
                        pandas.DataFrame(
                            {
                                'cloud_account_number': [
                                    row[1].get('cloud_account_number')],
                                'contact_type': [
                                    type_map[contact_type]],
                                'contact_name': [name],
                                'contact_email': [email]
                            },
                            columns=[
                                'cloud_account_number',
                                'contact_type',
                                'contact_name',
                                'contact_email'
                            ]
                        ), ignore_index=True
                    )

        del sheets[accounts_sheet]

    if applications_sheet in sheets:
        metadata = sheets[applications_sheet]

        # build the 'applications' sheet with a subset from the source sheet
        sheets['applications'] = metadata.loc[:,
                                              (
                                                  'application_name',
                                                  't_appid',
                                                  'governance_lob_name',
                                                  'business_unit_name',
                                                  'application_budget_account_string',
                                                  'application_description',
                                                  'pagerduty_url',
                                                  'tower_name',
                                                  'create_date'
                                              )
                                              ]

        sheets['application_contacts'] = pandas.DataFrame([], columns=[
            't_appid',
            'contact_type',
            'contact_name',
            'contact_email'
        ])

        type_map: Dict = {
            "Service VP": "Service VP",
            "Primary Business Contact": "Primary Business Contact",
            "Secondary Business Contact": "Secondary Business Contact"
        }

        for row in metadata.iterrows():
            for contact_type in type_map.keys():
                name: str = row[1].get(f'{contact_type} Name', None)
                email: str = row[1].get(f'{contact_type} Email', None)

                if (not name and not email):
                    continue
                elif (str(name) == "nan" and str(email) == "nan"):
                    continue
                else:
                    sheets['application_contacts'] = sheets['application_contacts'].append(
                        pandas.DataFrame(
                            {
                                't_appid': [
                                    row[1].get('t_appid')],
                                'contact_type': [
                                    type_map[contact_type]],
                                'contact_name': [name],
                                'contact_email': [email]
                            },
                            columns=[
                                't_appid',
                                'contact_type',
                                'contact_name',
                                'contact_email'
                            ]
                        ), ignore_index=True
                    )

        del sheets[applications_sheet]

    return sheets


@xray.capture()
def service_now_to_service_later_catalog(content: bytes) -> pandas.DataFrame:
    """Take ServiceNow Formatted JSON (via table api) and convert it into a
    format that SLater understands.

    Parameters
    ==========
    content: bytes
      Bytes encoded JSON data

    Returns
    =======
    cmdb: pandas.DataFrame
      SLater friendly import formatted DataFrame

    Validate we convert snow content to slater format
    >>> file = open('tests/data/u_cmdb_ci_business_app_list-test.json','r')

    >>> content = file.read()

    >>> cmdb = service_now_to_service_later_catalog(content)

    >>> cmdb = cmdb[0:10]

    >>> len(cmdb)
    10

    >>> cmdb
                           application_name   t_appid  ...          create_date active
    0                 .Net Validation Tools  SVC02026  ...  2018-10-08 16:55:12   true
    1               1042 Pro/1042 Reporting  SVC02840  ...  2020-03-30 09:02:06   true
    2                               1099PRO  SVC02908  ...  2020-03-30 09:02:09   true
    3                                18 App  SVC02069  ...  2018-10-26 19:23:21   true
    4                              1E Nomad  SVC00109  ...  2017-10-05 14:03:12   true
    5                               3D Tool  SVC02436  ...  2019-11-13 11:02:29   true
    6                                3PL-CG  SVC01451  ...  2018-02-22 15:26:24   true
    7                                3PL-NG  SVC01173  ...  2018-01-16 04:57:44   true
    8  451 (Web Content Management Project)  SVC00395  ...  2017-10-20 06:17:22   true
    9                                   A2C  SVC00240  ...  2017-10-16 05:04:15   true
    <BLANKLINE>
    [10 rows x 10 columns]
    """
    data: Dict = json.loads(content)
    cmdb: pandas.DataFrame = pandas.DataFrame.from_records(data['records'])

    valid_columns: Dict = {
        'name': 'application_name',
        'u_number': 't_appid',
        'short_description': 'application_description',
        'sys_created_on': 'create_date',
        'u_active': 'active'
    }

    missing_columns: list = [
        'governance_lob_name',
        'business_unit_name',
        'application_budget_account_string',
        'pagerduty_url',
        'tower_name'
    ]

    cmdb_copy = cmdb
    for column in cmdb_copy.keys():
        if column not in valid_columns:
            cmdb = cmdb.drop(column, axis=1)

    cmdb.rename(columns=valid_columns, inplace=True)

    for column in missing_columns:
        cmdb[column] = None

    return cmdb[
        [
            'application_name',
            't_appid',
            'governance_lob_name',
            'business_unit_name',
            'application_budget_account_string',
            'application_description',
            'pagerduty_url',
            'tower_name',
            'create_date',
            'active'
        ]
    ]
