### Navigation

- index
- modules |
- CCHR 1.0 documentation »
- Module code »
- src »

# Source code for src.log

    """
    This module provides log writing and report generation capabilities to
    the CCHR conversion software.

    Public Classes:

    Log -- Class for working with log files.
    Report -- Class for generating and updating report information.

    Exceptions:

    LogError -- Exception raised when problem occurs in log manipulation.
    ReportError -- Exception raised when error occurs in Report generation.

    """

    from os import path, listdir, remove, mkdir
    from datetime import datetime
    import cardsharp as cs
    from MySQLdb import connect
    from errors import *
    import re
    import Queue
    from util import RUNDATETIME, PHASE_MAP
    from collections import defaultdict

    __all__ = ['Log']

    #TODO: add modifying this list to user interface
    #list of log files, used in log.delete() to assert deletion of only log files
    _log_files = [
        'ncic.txt', 'ainc.txt', 'achgdetx.txt', 'acnt.txt', 'achgsvr.txt',
        'adisp.txt', 'astate.txt','arrdatex.txt', 'asource.txt',

        'cnic.txt', 'cchgdet.txt', 'ccnt.txt', 'cchgsvr.txt',
        'cdsp.txt', 'cdispdatex.txt', 'snttext.txt', 'cstate.txt',
        'csource.txt', 'sntdatex.txt',

        'dctz2.txt', 'dpob2.txt', 'dsource.txt', 'dstatex.txt', 'gender.txt',
        'race.txt',

        'rapstate.txt'
    ]

    #mapping of log.type to (name, output Variables)
    _log_types = {0 : ('default', [('segment', 'integer'), 'variable',
                                   ('phase', 'integer'), ('id', 'integer'), 'convert_state',
                                   'convert_key', 'error_message', 'convert_time']),
                  3 : ('cnt', [('segment', 'integer'), 'variable',
                                   ('phase', 'integer'), ('id', 'integer'), 'convert_state',
                                   'convert_key', 'extracted_cts', 'error_message', 'convert_time']),
                  2 : ('offense', [('segment', 'integer'), 'variable',
                                   ('phase', 'integer'), ('id', 'integer'), 'convert_state', 'convert_key_names',
                                   'convert_key_values', 'predictions', 'error_message', 'convert_time']),
                  1 : ('fed', [('segment', 'integer'), 'variable',
                               ('phase', 'integer'), ('id', 'integer'), 'rapstatex',
                               ('convert_state', 'integer'), 'ori', 'arrname', ('fed', 'integer'), 'convert_time']),
                  77 : ('report', [('id', 'integer'), 'region', 'rapstate', 'var_name', 'orig_value', 'new_value']),
                  78 : ('summary_report', [('casenum', 'integer'), 'var_name', 'orig_value', 'new_value']),
                  90 : ('ref_integ_error', ['segment', 'variable', 'phase', 'bad_value', 'table_name', 'allowed_values', 'rule_loc', 'error_message']),
                  91 : ('regex_init', ['message', 'filename', 'regex_pattern', 'regex_result', 'regex-priority', 'add_info_1', 'add_info_2', 'add_info_3']),
                  99 : ('fatal_error', ['segment', 'variable', 'phase', 'error_message', 'time'])
                }

    #TODO add setters for segment, phase, var to allow log creation outside of vc
    class Log(object):
        """A class to support log file manipulation for CCHR data conversion
        software.

        Public Methods:
        write -- Writes one line of log data to a log file.
        load -- Loads log data into an internal cardsharp dataset object.
        delete -- Deletes the log file.

        """

        def __init__(self, **kw):
            """Initialize a new log object.

            Keyword Arguments:
            log_dir -- Path to the log file directory. **optional**

            time -- A timestamp for the log, if not specified will
                    be automatically generated. **optional**

            segment -- The segment id (integer) corresponding to the segment that
                       the log object is mapped to. **optional**

            var -- The variable id (integer) corresponding to the variable that
                   the log object is mapped to. **optional**

            phase -- The phase in the data conversion process which the
                     the log object is mapped to. **optional**

            verbose -- Set to true for more detailed output. **optional**

            """
            self.time = kw.get('time', datetime.now().isoformat('_').replace(':', '-'))
            self.segment = kw.get('segment')
            self.phase = kw.get('phase')
            self.log_dir = kw.get('log_dir')
            self.var = kw.get('var')
            self.verbose = kw.get('verbose')
            self.stored = Queue.Queue()

        def load(self, fn):
            """Load a log file. The variables of the log file are determined by _log_types.
            The fn needs to have a log type indicator before the .txt in the filename (fn).
            Example) ancic_0.txt

            @param fn: The filename of the log to load.
            @return: (log_data, log_type) -- Returns tuple with the first value being a cardsharp dataset containing log data
                     and the second value is the log type.

            """
            try:
                type = re.search('(?<=_)\d+(?=.txt)', fn).group()
                log_data = cs.load(source=path.join(self.log_dir, fn), format='text', var_names=_log_types[int(type)][1])
                return (log_data, int(type))
            except:
                raise LogError('Can not load: %s' % fn)

        def write(self, fn, vals, type=0, no_add_info=False, store=False):
            """Writes a line of log data to the log file.

            @param fn: The name of log file to write (append) to.
            @param vals: The list of the values to write to the log file.
            @param type: The type of log to write. Default is 0 (see _log_types for list of avaialbe log types).
            @param store: Whether to store the file in memory (can be dumped to file later) or to write to file immediately

            """
            try:
                if store:
                    if not no_add_info:
                        self.stored.put((fn,type,'\t'.join([str(self.segment), str(self.var), str(self.phase),
                                            '\t'.join(str(val) for val in vals), '%s\n' % self.time])))
                    else:
                        self.stored.put((fn,type,'\t'.join(['\t'.join(str(val) for val in vals), '%s\n' % self.time])))
                else:
                    with open(path.join(self.log_dir, '%s_%s.txt' % (fn, type)), 'a') as out:
                        if not no_add_info:
                            out.write('\t'.join([str(self.segment), str(self.var), str(self.phase),
                                      '\t'.join(str(val) for val in vals), '%s\n' % self.time]))
                        else:
                            out.write('\t'.join(['\t'.join(str(val) for val in vals), '%s\n' % self.time]))
            except:
                raise
                raise LogError('Failed to write to log.')

        def dump(self):
            """Empties the log queue and writes to all of the log files for which there
            is stored log output
            """

            logs = defaultdict(list)
            while not self.stored.empty():
                temp = self.stored.get()
                logs[(temp[0],temp[1])].append(temp[2])
            for log in logs.keys():
                for row in logs[log]:
                    with open(path.join(self.log_dir, '%s_%s.txt' % (log[0], log[1])), 'a') as out:
                        out.write(row)

        def write_compare(self, fn, vals, type=0):
            """Writes a line of log data to the log file.

            @param fn: The name of log file to write (append) to.
            @param vals: The list of the values to write to the log file.
            @param type: The type of log to write. Default is 0 (see _log_types for list of avaialbe log types).

            """
            try:
                with open(path.join(self.log_dir, '%s_%s.txt' % (fn, type)), 'a') as out:
                    out.write('\t'.join([str(val) for val in vals]) + '\n')
            except:
                raise LogError('Failed to write compare log.')

        def delete(self, fn):
            """Deletes a log file.

            Arguments:
            fn -- name of the desired log file to delete

            """

            try:
                assert(fn in _log_files)
                remove(path.join(self.log_dir, fn))
            except AssertionError:
                raise LogError('Unable to delete %s. Not listed as log file.' % fn)
            except:
                raise LogError('Unable to delete %s.' % fn)

### Navigation

- index
- modules |
- CCHR 1.0 documentation »
- Module code »
- src »