### Navigation

- index
- modules |
- CCHR 1.0 documentation »
- Module code »
- src »

# Source code for src.report

    #python
    from os import listdir
    from os.path import join
    from collections import defaultdict
    from re import sub, search, findall
    #src
    from .util import *
    from .log import *
    from .errors import *
    from .configuration import config
    #third-party
    import cardsharp as cs

    __all__ = ['Report']

    #mapping to determine unique values in a log file
    #log type: list of variable names
    _log_unique_key = {
        0:  ['convert_state', 'convert_key'],
        1:  ['rapstatex', 'convert_state', 'ori', 'fed'],
        2:  ['convert_state', 'convert_key_values'],
        3:  ['convert_state', 'convert_key', 'extracted_cts'],
        90: ['bad_value', 'table_name', 'rule_loc'],
        99: ['variable', 'error_message'],
    }

    def _dedupe(log_data, log_type):
        unique_rows = []
        for row in log_data:
            if log_type in [77,78,91]:
                break

            #if the key based on type, if log_type not listed raise error
            try:
                key = [row[var] for var in  _log_unique_key[log_type]]
            except:
                raise ReportError('Unrecognized log type')

            if key not in unique_rows:
                unique_rows.append(key)
            else:
                row.delete()

    class Report(object):
        """Class to create/update CCHR reports based on data contained in the log files.

        The reports are stored in the web interface database. The tables
        contain one row of data for each unique log entry per data run,
        where a unique entry is determined by a key of:
        segment_id + variable_id + phase_id + region_id + key_1 + key_2 + key_3.
        The Report class is called when during pre_process, process, and transform
        phases to generate reports based on the log output from each of the
        respective phases.

        The reports can also be generated as excel/text files.

        Public Methods:
        ...

        """

        def __init__(self, report_path, **kw):
            self.log = Log(**kw)
            self.report_path = report_path
            self.verbose = kw.get('verbose')

        def create_reports(self):
            for log_file in listdir(self.log.log_dir):

                if is_file(r'%s/%s' % (self.log.log_dir, log_file)):
                    if self.verbose:
                        print 'creating %s report' % log_file.replace('.txt', '')

                    try:
                        log_ds, type = self.log.load(log_file)
                        log_ds.wait()
                        _dedupe(log_ds, type)
                        save_format = 'spss' if len(log_ds.rows) > 65000 else 'excel'
                        out_file = 'report_%s' % sub('_\d+', '', log_file).replace('.txt', FORMAT_INFO[save_format])
                        log_ds.save(source=join(self.report_path, out_file), format=save_format, overwrite=True)
                        cs.wait()
                        #create prediction report
                        if 'ncic' in out_file and 'wvfbi' not in out_file:
                            cs.wait()
                            '''convert_key_names    convert_key_values    predictions
                            cstatnumx|cchglitx    |cocaine possess-w/int to distribute (federal)    |||9|9|99|9'''
                            #create one prediction report for every unique set of key names
                            _keys = defaultdict(list)

                            for row in log_ds:
                                _keys[row['convert_key_names']].append((row['convert_key_values'].split('|'), row['predictions'].split('|')))

                            for c, key in enumerate(_keys.iterkeys()):
                                _vars = ['key:%s' % k for k in key.split('|')]
                                _vars.extend(['value:%s' % v for v in ['ncic', 'chg', 'inc', 'dom', 'weap', 'cdv', 'pg']])
                                ds = cs.Dataset(_vars)
                                for value in _keys[key]:
                                    _val = [v if v else None for v in value[0]]
                                    _val.extend([v if v else None for v in value[1]])
                                    ds.add_row(_val)

                                for var in ds.variables:
                                    if 'key' in var.name: continue
                                    ds.variables[var.name].convert('integer')

                                ds.save(source=join(self.report_path, out_file), format=save_format, overwrite=True, dataset='predict_%i' % c)
                                cs.wait()

                    except LogError as e:
                        print 'skipping report for %s. %s' % (log_file, e)
            cs.wait()

### Navigation

- index
- modules |
- CCHR 1.0 documentation »
- Module code »
- src »