### Navigation

- index
- modules |
- CCHR 1.0 documentation »
- Module code »
- src »

# Source code for src.parse_rules

    """
    This module is responsible for creating the business rules for the CCHR
    conversion system.

    Exported Classes:

    Exported Methods:

    parse_rules -- This method is called during the stage and process phase to
    create the business rules for that phase.

    Exceptions:

    RuleParsingError

    ConvertError -- This error is raised when a value we are trying to convert
    can not be handled by a Crosswalk object.

    DataError

    FatalDataError

    """

    import os, imp
    from itertools import count, izip
    from datetime import datetime, date
    from time import strptime
    from itertools import izip
    from sys import exc_info
    import traceback

    import cardsharp as cs
    import log
    from errors import *
    from rules import *
    from util import *
    from configuration import config
    from offense_rules import OffenseRules
    from sentence_rules import SentenceRules
    from admrec_rules import AdmrecRules
    from region import *
    from util import regions as util_regions
    from metadata import *
    import re
    from segment import segment_dict
    from contextlib import closing
    import MySQLdb
    from collections import defaultdict

    _count = count().next

    def mod_counter():
        return 'bjs_' + str(_count())

    #TODO rework log so it is created outside of VC
    class VariableConverter(object):
        def __init__(self, time, segment, var, phase, **kw):
            self.func = None
            self.name = var
            self.id_count = None
            self.regions = dict()
            self.region_order = []
            self.format_setters = set()
            self.halt_on_error = kw.get('halt_on_error', False)
            self.var = var
            self.log = log.Log(time=time, segment=segment, var=var, phase=phase, log_dir=kw.get('log_dir'))
            self.options = kw

        def add_region(self, region, mod, crosswalk):
            if mod:
                self.format_setters.add(mod)

            if crosswalk:
                for key in crosswalk.iterkeys():
                    self.format_setters.add(crosswalk[key])

            self.options.update({'region':region.get_state()[0]})
            kw = self.options

            self.regions[region] = _get_region_func(self.log, self.var, mod, crosswalk, self.halt_on_error, **kw)
            self.region_order.append(region)
            self.region_order.sort(key = lambda r: r.priority())

        def create_func(self, mod, crosswalk, add_info):
            if mod:
                self.format_setters.add(mod)

            if crosswalk:
                for key in crosswalk.iterkeys():
                    self.format_setters.add(crosswalk[key])

            self.func = _get_region_func(self.log, self.var, mod, crosswalk, self.halt_on_error, add_info, **self.options)

        def set_format(self, format):
            for rs in self.format_setters:
                rs.format = format

        def set_auto_id(self, start_value=1):
            #TODO put this rule in loader
            """This modules sets the translate function to pass an id value to the conversion function. The first id
            will start at **start_value**, which has a default value of 0, and will increment by one for each row."""
            self.id_count = count(start_value).next

        def translate(self, state_var, value, row):
            try:
                #if regions are specified then loop over them and return the region specfic function
                for region in self.region_order:
                    #TODO pass in map of rules to get 1 iteration over dataset
                    if row[state_var].lower() in region.get_state():
                         if self.id_count is not None:
                             return self.regions[region](self.id_count())
                         else:
                             return self.regions[region](value, row)

                #no regions are specified than return global function
                if self.id_count is not None:
                    return self.func(self.id_count())
                else:
                    return self.func(value, row)

            except:
                if config.debug_halt_on_error:
                    raise
                else:
                    self.log.write('fatal_error', ([str(exc_info()[:2])]), 99) #traceback.format_exc()

    def _get_region_func(log, var, mod, crosswalk, error, add_info=None, **kw):
        if mod and crosswalk:
            mod.crosswalk = crosswalk

        if mod:
            #pass objects to mod
            off_rules = OffenseRules(log)

            sentence_rules = SentenceRules(log)
            mod.log = log
            mod.ConvertError = ConvertError
            mod.RuleError = cs.rules.RuleError
            mod.DataError = DataError
            mod.FatalDataError = FatalDataError
            mod.halt_on_error = error
            mod.regions = regions
            mod.key_as_str = key_as_str
            mod.get_state_key = get_state_key
            mod.default_offense_codes = off_rules.missing_codes
            mod.o_vars = off_rules.vars
            mod.o_default_map = off_rules.missing_map
            mod.offense_not_coded = off_rules.not_coded_map
            mod.convert_inc = off_rules.convert_inc
            mod.convert_offenses = off_rules.convert_offenses
            mod.convert_sentences = sentence_rules.convert_sentences
            mod.sentence_missing  = sentence_rules.missing
            mod.sentence_unknown = sentence_rules.unknown
            mod.sentence_rules = sentence_rules
            mod.re = re
            mod.izip = izip
            mod.date = date
            mod.strptime = strptime
            mod.add_info=add_info
            mod.math_rule = RegexMathRule
            mod.datetime = datetime
            mod.ma_duplicate = defaultdict(int)
            mod.current_set = config.current_set
            #TODO use regions instead of get_state_key
            mod.regions = util_regions
            mod.opt = kw
            mod.get_cnxn = get_cnxn

            #=======================================================================
            # Define the function which the variable converter will return to the
            # Cardsharp task
            #=======================================================================

            if var in ['ancic', 'cncic'] and kw.get('region'):
                om = OffenseMetadata(var, 'arrest' if var[0] == 'a' else 'sentence', kw.get('region'))
                #add a prediction rules attribute for the offense rules
                mod.offense_rules = om.offense_map

            if hasattr(mod, 'convert'):
                def func(value, row):
                    try:
                        return mod.convert(value, row)
                    #handle type excpetion for easier debugging
                    except TypeError as err:
                        raise ConvertError("""Unable to call convert on var %s. for id:%s with value:%s. error was: %s""" % (var, row['id'], value, err))
                return func

            elif hasattr(mod, 'assign_vals'):
                def func(value, row):
                    #TODO:snt_processor is deprecated remove from all modules
                    #snt_processor = SNTParser(row, mod.log)
                    return mod.assign_vals(value, row, None)

                return func

            elif hasattr(mod, 'translate'):
                def func(value, row):
                    return mod.translate(value)

                return func

            elif hasattr(mod, 'assign_id'):
                def func(value):
                    return mod.assign_id(value)

                return func

            elif hasattr(mod, 'validate'):
                return mod.validate

            elif hasattr(mod, 'cache'):
                def func(value, row):
                    return mod.cache(row)

                return func

        elif crosswalk:
            if var in ['ancic', 'cncic']:
                def func(value, row):
                    vars = convert_offenses(row, crosswalk, offense_rules)

                    for k, v in vars.iteritems():
                        row[var[0]+k] = v

                    return vars['ncic']

            #nonarr rules
            elif var == 'nonarr':
                admrec_meta = AdmrecMetadata(var, 'arrest', kw.get('region'))
                admrec_rules = AdmrecRules(log, 'arrest', crosswalk, admrec_meta.admrec_map)

                def func(value, row):
                    return admrec_rules.convert_admrec(row)

            #noncrt rules
            elif var == 'noncrt':
                admrec_meta = AdmrecMetadata(var, 'sentence', kw.get('region'))
                admrec_rules = AdmrecRules(log, 'court', crosswalk, admrec_meta.admrec_map)
                def func(value, row):
                    return admrec_rules.convert_admrec(row)

            else:
                def func(key, value, row):
                    return crosswalk[key].convert(value)

            return func

        else:
            raise RuleParsingError('error creating rule %s: kw:%s' % (var,str(kw)))

    def _get_where_state_from_id(segment_name, **opt):
        if 'id' in opt['in_where']:
            where_ids = re.findall('id=(\d+)', opt['in_where'])

        where_states = set()
        try:
            with closing(get_cnxn(opt['db_info']['name'],opt)) as cnx:
                with closing(cnx.cursor()) as c:
                    for _id in where_ids:
                        c.execute("""SELECT rapstatex
                                     FROM %sx
                                     WHERE id=%s""" % (
                                        segment_name,
                                        _id)
                                 )

                        where_states.update([get_state_key(c.fetchone()[0])])

        except TypeError:
            #we did not find the id in the dataset so return an empty set
            return set()

        return where_states

    _crosswalk_region_re = re.compile('crosswalk_([a-zA-Z]{2})(\.txt|_\d+.txt)')

    def _skip_crosswalk(where_regions, filename, verbose, region=None):
        if where_regions and region == 'wvfbi':
            crosswalk_region = _crosswalk_region_re.search(filename)
            if crosswalk_region:
                skip = True
                for r in where_regions:
                    if crosswalk_region.groups()[0] == get_state_name(r):
                        skip = False
                    elif get_state_name(r) == 'ne' and crosswalk_region.groups()[0] == 'nb':
                        skip = False
                if skip:
                    if verbose:
                        print 'skipping crosswalk %s' % filename
                    return True

        return False

    def parse_rules(rule_dir, segment, **opt):
        db_info = opt['db_info']
        phase = opt['phase_id']
        segment_objs = opt['segments']
        halt_on_error = opt['halt_on_err']
        verbose = opt['verbose']
        meta_dir = opt['meta_dir']
        prefix = ''
        xwalk_kw = {}

        #opt['segment_name'] = segment['label']

        #TODO: change db_info['name'] to 'filename' to allow db_info to be passed as keyword
        time = datetime.now().isoformat('_').replace(':', '-')
        _var_map, vars = {}, {}

        variables = cs.load(source=os.path.join(meta_dir, segment['label'], 'var_info.xls'),
                            dataset='vars', format='excel')
        cs.wait()

        for row in variables:
            _var_map[row['name'].lower()] = (row['name'], row['format'])

        additional_data = {}
        if re.search('pre_process_rules_3', rule_dir) and segment['label'] == 'sentence' and opt['phase_id'] != '3':
            print 'loading cfed data...'
            additional_data = {}
            #TODO change this into mssql query to avoid slowness of cardsharp
            options = {'source': db_info['name'], 'dataset': 'arrestx', 'format':'mysql',
                    'user': db_info['user'], 'pwd': db_info['pass'], 'load_as_null':[''],
                    'select': ['casenum', 'cycnum', 'afed', 'astate']}
            if opt.get('in_where'):
                options['where'] = opt.get('in_where')
            ds = cs.load(**options)
            cs.wait()
            for row in ds:
                additional_data['%s|%s' % (row['casenum'], row['cycnum'])] = (row['afed'], row['astate'])

            cs.wait()
            print 'complete.'

        #for each segment
        #loop over segment directories (arrest, demographic, sentence, supervision)

        #set where_regions
        where_regions = set()

        if opt.get('in_where'):
            where_regions = set(re.findall('state[ ]*=[ ]*(\d+)', opt['in_where']))

            if re.search('id[ ]*=[ ]*',opt['in_where']):
                _r = _get_where_state_from_id(segment['label'], **opt)
                if _r and 'wvfbi' not in _r: where_regions.update(_r) #do not remove wvfbi crosswalks

        for segment_dir in os.listdir(rule_dir):

            segment_name = segment_dir.lower()
            segment_dir = os.path.join(rule_dir, segment_dir)
            if os.path.isfile(segment_dir):
                continue #if file than not a segment directory, check next
            elif segment_name != segment['label']:
                continue #only load the segment we are runing rules for

            #for each variable
            #loop over all variable folders within a segment
            for var_dir in os.listdir(segment_dir):
                #if include rules are specified skip out of variables not in the include rules
                if opt.get('include_rules') and var_dir.lower() not in opt['include_rules']:
                    continue

                #if exclude rules are specified skip out of variables in the exclude rules
                if opt.get('exclude_rules') and var_dir.lower() in opt['exclude_rules']:
                    continue

                var_name = var_dir.lower()
                var_dir = os.path.join(segment_dir, var_dir)

                if os.path.isfile(var_dir):
                    continue #if file than not a variable directory, check next

                #create a variable converter
                vc = VariableConverter(time, segment_dict[segment_name]['id'], _var_map[var_name][0], phase, **opt)
                vc_mod = None
                vc_crosswalk = dict()

                #for each rule or region
                #loop over all files in the variable directory
                for filename in os.listdir(var_dir):
                    full_path = os.path.join(var_dir, filename)

                    #load crosswalk rule
                    if filename.startswith('crosswalk'):
                        if _skip_crosswalk(where_regions, verbose, filename):
                            continue

                        if verbose:
                            print 'Loading crosswalk for %s' % filename
                        if filename[len(filename) - 4:len(filename)].lower() != '.txt':
                            raise RuleParsingError('file type must be .txt not %s' % filename[len(filename) - 4:len(filename)])

                        vc_crosswalk[filename[10:len(filename) - 4].lower()] = Crosswalk(full_path, db_info, segment_objs[segment_name], vc.log)

                    #load func rule
                    elif filename == 'func.py':
                        if verbose:
                            print 'Loading functions for %s' % var_name
                        vc_mod = imp.load_source(mod_counter(), full_path)
                    elif filename.endswith('.pyc'):
                        pass
                    #load region rules
                    elif os.path.isdir(full_path):
                        region = get_region(filename)

                        crosswalk = dict()
                        mod = None

                        #TODO make a state_where object that contains region names
                        #check the where to see if we should only load state specific rules
                        if where_regions:
                            skip_region = True
                            for r in where_regions:
                                if get_state_key(r) in region.state:
                                    skip_region = False

                            if get_state_key('wvfbi') not in region.state and skip_region:
                                if verbose:
                                    print 'skipping %s' % region
                                continue

                        #for each rule
                        for filename in os.listdir(full_path):
                            file_path = os.path.join(full_path, filename)
                            filename = filename.lower()
                            #if we are in a wvfbi state
                            #load region crosswalk
                            if filename.startswith('crosswalk'):
                                if _skip_crosswalk(where_regions, filename, verbose, region.state[0]):
                                    continue

                                if verbose:
                                    print 'Loading crosswalk for %s' % var_name
                                if filename[len(filename) - 4:len(filename)].lower() != '.txt':
                                    raise RuleParsingError('file type must be .txt not %s' %
                                                           filename[len(filename) - 4:len(filename)])

                                crosswalk[filename[10:len(filename) - 4].lower()] = Crosswalk(file_path, db_info, segment_objs[segment_name], vc.log)
                            #load func rule
                            elif filename == 'func.py':
                                if verbose:
                                    print 'Loading functions for %s:%s' % (var_name, region)
                                mod = imp.load_source(mod_counter(), file_path)
                            elif filename.endswith('.pyc'):
                                pass
                            else:
                                raise RuleParsingError('Region directory %s for var %s can only contain "crosswalk.txt", or "func.py" files.  Found: %s' % (region, var_name, filename))

                        #add region rules to variable converter
                        #pass segment_name for log output
                        #region used for log and to store region func / crosswalk
                        #func is the function to be associated with the  converter
                        #crosswalk is the crosswalk to be used by the  converter (in addition to func or standalone)
                        vc.add_region(region, mod, crosswalk)
                    else:
                        raise RuleParsingError(' directory %s can only contain region directories, "func.py" files, or "crosswalk.txt" files.  Found: %s' % (var_name, filename))

                #create variable rule
                vc.create_func(vc_mod, vc_crosswalk, additional_data)

                vars[(segment_name, var_name)] = vc

        return vars

### Navigation

- index
- modules |
- CCHR 1.0 documentation »
- Module code »
- src »