### Navigation

- index
- modules |
- CCHR 1.0 documentation »
- Module code »
- src »

# Source code for src.rules

    from cardsharp.transform import metadata
    from collections import defaultdict
    from datetime import datetime
    from errors import *
    from itertools import izip
    from itertools import count as icount
    from util import *
    import cardsharp as cs
    import codecs
    import log
    import re
    import sre_constants
    from configuration import config

    __all__ = ['Rule', 'AllRule', 'RegexRule', 'Crosswalk', 'RegexMathRule']

    time = datetime.now().isoformat('_').replace(':', '-')

    class Rule(object):
        '''Default Crosswalk Rule defining a crosswalk conversion.

        When a crosswalk is loaded each line in the crosswalk file is turned
        into a rule. For this default rule the initial value is the key in the
        crosswalk and the result is the value in the crosswalk. See crosswalk
        for more details.

        Public Methods:
        handle -- Returns true if initial matches the value
        convert -- Returns the result

        '''
        def __init__(self, initial, result):
            self.initial = initial
            self.result = result
            self.return_type = 'result'
            self.type = 'match'

        def convert(self, **kw):
            if kw.get('split'):
                return [x for x in self.result.split(kw['split'])]
            else:
                return self.result

    class AllRule(object):
        '''Crosswalk rule to handle all values.
        If result is :copy: will copy the initial value to the result value,
        otherwise will output the result value.

        example)
        :all:    :copy:
        This will output exactly what is in the original dataset without
        doing any conversion.

        :all:    :none:
        This will output a None value.

        :all:    1111.11
        This will output all values to 1111.11
        '''
        def __init__(self, result):
            self.result = result
            self.return_type = 'result'
            self.copy = True if re.search(':copy:', result, re.I) else False
            self.none = True if re.search(':none:', result, re.I) else False
            self.type = 'all'
            self.out = []
            self._rules = {':copy:': lambda key: self.out.append(key[0]),
                           ':none:': lambda key: self.out.append('')}

        def convert(self, **kw):
            #if multiple out value are defined
            if kw.get('split'):
                if self.copy or self.none:
                    #add the result value to out list based on its rule (:copy:, :none: or the result value)
                    for x in self.result.split(kw['split']):
                        self._rules.get(x, lambda r: self.out.append(r[1]))((kw.get('value'), x))
                    return self.out
                #otherwise return result values as a list
                else:
                    return [x for x in self.result.split(kw['split'])]
            else:
                if self.copy or self.none:
                    self._rules.get(self.result)((kw.get('value'), self.result))
                    return self.out[0]

                return self.result

    class RegexRule(object):
        def __init__(self, regex, result, type, priority, escape=True, **kw):
            """
            @param regex: A regular expression pattern
            @param result: The value to be returned if the regular expression matches
            @param type: The type of regular expression rule
            #TODO better doc
                         math: return match object groups applied to math function
                         search: return re.search
                         match: return re.match

            """
            #escape regex as needed
            self.escape = escape
            if escape:
                '''# captures '#' or '#\d+' or '\d+' '''
                self.regex = re_escape(regex).replace('#', '(\d+|#\d+|#)')
            else:
                self.regex = regex

            #do an exact match
            if type == 'search':
                self.re_func = re.search
                self.delete_transform_regex = self.regex

            elif type == 'match':
                self.re_func = re.match
                self.delete_transform_regex = ''.join(['^', self.regex])

            elif type == 'exact':
                self.delete_transform_regex = ''.join(['^', self.regex, '$'])
                self.regex = ''.join([self.regex, '$'])
                self.re_func = re.match

            elif type == 'end':
                self.delete_transform_regex = ''.join([self.regex, '$'])
                self.regex = ''.join([self.regex, '$'])
                self.re_func = re.search

            elif type == 'findall':
                #TODO this is currently broken, this is not an exact match this is a match to the end of the string
                self.delete_transform_regex = self.regex
                self.regex = self.regex
                self.re_func = re.findall

            elif type == 'tilde':
                #TODO this is currently broken, this is not an exact match this is a match to the end of the string
                self.delete_transform_regex = ''.join(['(?:^|~)', self.regex, '(?:~|$)'])
                self.regex = ''.join(['(?:^|~)', self.regex, '[ ]*(?:~|$)'])
                self.re_func = re.search

            elif type == 'tilde-match':
                self.delete_transform_regex = ''.join([self.regex, '[ ]*(?:~|$)'])
                self.regex = ''.join([self.regex, '[ ]*(?:~|$)'])
                self.re_func = re.match

            elif type == 'tilde-begin':
                self.delete_transform_regex = ''.join(['(?:~|^)', self.regex])
                self.regex = ''.join(['(?:~|^)', self.regex])
                self.re_func = re.search

            self.regex = re.compile(self.regex, re.I) #actual regular expression string
            self.delete_transform_regex = re.compile(self.delete_transform_regex, re.I) #actual regular expression string

            self.return_type = 'result' #this rule will return a value
            self.result = self._validate_result(result)
            self.reg = regex
            self.type = type
            self.match = None #object to hold match object
            self.priority = priority

        #TODO change this to match RegexMathRule.handle()
        def handle(self, value):
            """Returns True if rule can handle the value. Convert will only be called
            if handle returns True."""
            if value in [None, '']:
                raise ConvertError
            #TODO this is not using compiled regular expression
            #should be self.regex.match
            self.match = self.re_func(self.regex, value)

            if self.match:
                self.groups = self.match.groups()

            #will be the match object or None
            return self.match

        def delete_transform(self, value):
            """Transform the value by removing the regex pattern from value. If a named group called **keep** is present
            than remove other but the pattern in the **keep** group."""
            return self.delete_transform_regex.sub(lambda match: match.groupdict().get('keep'), value)

        def convert(self, **kw):
            """Return result."""
            if kw.get('split'):
                return [x for x in self.result.split(kw['split'])]
            else:
                return self.result

        def _validate_result(self, result):
            """Helper function to validate the result string. This method can be overridden to do
            clean, formating, validation of the result in classes that inherit from RegexRule."""
            return result

    _formula_func_map = {
        'max': (re.compile('max\(|\)$'), max),
        'min': (re.compile('min\(|\)$'), min)
    }

    def _get_result(result, groups, type=None):
        '''result = the result we want to calculate for a output variable.
           groups = the groups object from the match object on the regex see handle below
           type = The type of function we want to calculate
        '''
        if type in _formula_func_map:
            _ff = _formula_func_map[type]
            result = _ff[0].sub('',result)
            _values = result.split(',')
            return _ff[1]([calc(re.sub('(#)(\d+)', lambda c: groups[int(c.group(2)) - 1].replace('#', ''), x)) for x in _values])

        else:
            return calc(re.sub('(#)(\d+)', lambda c: groups[int(c.group(2)) - 1].replace('#', ''), result))

    class RegexMathRule(RegexRule):
        def __init__(self, regex, result, type, priority, escape=True, **kw):
            RegexRule.__init__(self, regex, result, type, priority, escape, **kw)
            self.result = self.result.lower()
            self.has_result_formula_func = True if 'max' in self.result or 'min' in self.result else False

        def handle(self, value):
            RegexRule.handle(self, value)
            self.match = self.re_func(self.regex, value)

            if self.match:
                self.groups = self.match.groups()
                return True
            else:
                return False

        def convert(self, **kw):
            """
            For math regex apply groups extracted from value to arthimetic function within self.result
            Math:
            >>> r = RegexMathRule('Sent to #Day #Y', '#1 + #2(365.25)', 'math')
            >>> r.handle('Sent to 10Day 2Y')
            True
            >>> r.convert()
            740.5
            """
            #groups are base 0, research tema has been entering there # objects as base one
            #subtract 1 from every #\d+ match and return the matched group
            #example)
            #   regex = Sent to #Day #Y
            #   result = #1 + #2(365.25)
            #    value = 'Sent to 10Day 2Y'
            #return calc('10 + 2(365.25)')

            #if a transform result function is present then substitute the variable name in the results
            #with the current value in the row
            result = self.result
            if kw.get('transform_results'):
                for var in kw.get('transform_results'):
                    if var in result:
                        result = result.replace(var, str(kw['row'][var]))

            try:
                if kw.get('split'):
                    '''@todo: Make result a class so we do not need to check for max, min on every result value'''
                    result = [r if not is_none(r) else 'None' for r in result.split(kw['split'])]

                    if self.has_result_formula_func:
                        _out_values = []
                        for r in result:
                            if r.startswith('max'):
                                _out_values.append(_get_result(r, self.groups, 'max'))
                            elif r.startswith('min'):
                                _out_values.append(_get_result(r, self.groups, 'min'))
                            else:
                                _out_values.append(_get_result(r, self.groups))

                        return _out_values

                    else:
                        #same as above but condensed without evaluation
                        return [calc(re.sub('(#)(\d+)', lambda c: self.groups[int(c.group(2)) - 1].replace('#', ''), x)) for x in result]
                else:
                    if self.has_result_formula_func:
                        if result.startswith('max'):
                            return _get_result(result, self.groups, 'max')
                        elif result.startswith('min'):
                            return _get_result(result, self.groups, 'min')
                        else:
                            return _get_result(result, self.groups)
                    else:
                        return calc(re.sub('(#)(\d+)', lambda c: self.groups[int(c.group(2)) - 1].replace('#', ''), result))

            except CalcError:
                raise RuleError("CalcError: Invalid Result Formula: %s . Priority = %s" % (self.result, self.priority))
            except IndexError:
                raise RuleError("Invalid output value: key:%s | value:%s, number index out of range" % (self.regex.pattern, self.result))
            except SyntaxError:
                raise RuleError("Syntax Error: check rule syntax: key:%s | value:%s" % (self.regex.pattern,self.result))
            except AttributeError:
                raise RuleError("Attribute Error: check rule syntax: key:%s | value:%s" % (self.regex.pattern,self.result))
            except NameError:
                raise RuleError("NameError: make sure the variable you listed is in the ruleset.variables metadata: key:%s | result:%s | %s" % (self.regex.pattern,self.result, kw.get(['row'], {'id':''})['id']))

        def _validate_result(self, result):
            """Clean the result formulas"""
            #replace [ ] with ( )
            result = result.replace('[', '(')
            result = result.replace(']', ')')
            result = result.replace('{', '(')
            result = result.replace('}', ')')
            result = result.replace(' ', '')

            return result

    class RegexMathFindallRule(RegexMathRule):
        def handle(self, value):
            if value in [None, '']:
                raise ConvertError

            self.groups = self.re_func(self.regex, value)
            self.match = self.regex.search(value) #need match object for delete transform rule

            return self.match #return None or match object

        def convert(self, **kw):
            result = self.result
            if kw.get('transform_results'):
                for var in kw.get('transform_results'):
                    if var in result:
                        result = result.replace(var, str(kw['row'][var]))

            try:
                out = []

                if kw.get('split'):
                    '''@todo: Make result a class so we do not need to check for max, min on every result value'''
                    result = [r if not is_none(r) else 'None' for r in result.split(kw['split'])]

                    #findall has one or many groups we need to select the highest
                    for groups in self.groups:
                        if not isinstance(groups, tuple): groups = tuple([groups])
                        out.append([calc(re.sub('(#)(\d+)', lambda c: groups[int(c.group(2)) - 1].replace('#', ''), x)) for x in result])

                    #determine which result formula we want to evaluate max on
                    c = 0
                    for r in result:
                        if '#' in r: #if multiple formulas use first
                            eval_index = c
                            break
                        else:
                            c+=1
                    #return the largest item
                    return sorted(out, key = lambda k: k[eval_index])[-1]

                else:
                    for groups in self.groups:
                        out.append(calc(re.sub('(#)(\d+)', lambda c: groups[int(c.group(2)) - 1].replace('#', ''), result)))

                    return sorted(out)[-1]

            except CalcError:
                raise RuleError("CalcError: Invalid Result Formula: %s . Priority = %s" % (self.result, self.priority))
            except IndexError:
                raise RuleError("Invalid output value: key:%s | value:%s, number index out of range. Priority = %s" % (self.regex.pattern, self.result, self.priority))
            except SyntaxError:
                raise RuleError("Syntax Error: check rule syntax: key:%s | value:%s. Priority = %s" % (self.regex.pattern,self.result, self.priority))
            except AttributeError:
                raise RuleError("Attribute Error: check rule syntax: key:%s | value:%s. Priority = %s" % (self.regex.pattern,self.result, self.priority))
            except NameError:
                raise RuleError("NameError: make sure the variable you listed is in the ruleset.variables metadata: key:%s | result:%s | %s. Priority = %s" % (self.regex.pattern,self.result, kw.get(['row'], {'id':''})['id'], self.priority))

    class RegexReturnRule(RegexRule):
        """Return the Match Object of a match regular expression."""
        def __init__(self, regex, result, type, priority, escape=True):
            RegexRule.__init__(self, regex, result, type, priority, escape)
            self.return_type = 'match'

        def handle(self, value):
            RegexRule.handle(self, value)
            self.result = self.re_func(self.regex, value)

            return True if self.result else False

    class Lookup(object):
        def __init__(self, db, tables, log):
            self.values = []
            self.log = log
            self.lookup_tables = tables.split('|')
    #        for table in tables.split('|'):
    #            ds = cs.load(source=db['name'], format='mysql', dataset=table, user=db['user'], pwd=db['pass'])
    #            self.values.append(([row['id'] for row in ds], table))

        def validate(self, initial, value, filename):
            #iterate over all values and lookups checking to see if each value
            #in corresponding lookup list
            for value, table in izip(value.split('|'), self.lookup_tables):
                try:
                    if value not in ('', 'None', ':none:') and int(value) not in LOOKUPS[table]:
                        self.log.write('ref_integ_error', [value, table, LOOKUPS[table], filename, 'Value not present in lookup'], 90)
                        return False
                except:
                    self.log.write('ref_integ_error', [value, table, LOOKUPS[table], filename, 'Value not present in lookup'], 90)
                    raise RuleError("Rule validation failed for. Check log.")

            return True

    #        temp_value = value.split('|')
    #        for v, l in izip(temp_value, self.values):
    #            try:
    #                if v not in ('', 'None', ':none:') and int(v) not in l[0]:
    #                    #TODO add validate_rules
    #                    #if segment.validate_rules:
    #                    self.log.write('ref_integ_error', [v, l[1], l[0], filename, 'Value not present in lookup'], 90)
    #                    return False
    #            except:
    #                self.log.write('ref_integ_error', [v, l[1], l[0], filename, 'Value not present in lookup'], 90)
    #                raise RuleError("Rule validation failed for. Check log.")
    #
    #        return True

    class Crosswalk(object):
        """A Crosswalk object is used to do variable conversions either directly
        or by accessing the object in the func.py file for a CCHR variable.

        The Crosswalk object first loads the crosswalk.txt file specified by the
        **filename** parameter. Each line of this file is turned into a rule object,
        see _rules for more information. When crosswalk.convert(value) is called
        we iterate over the rules list checking to see if a rule can handle the
        value. If a rule can handle the value the rules result is returned,
        otherwise we raise a ConvertError. An additional **lookup** parameter can
        be defined to provide a list of valid return values.

        The column delimiter is a tab and the row delimiter is a newline.
        So each line in the crosswalk looks like:

        **key**\\t**value**\\n

        example)

        100\tmurder

        parameters:

        filename - Path to file containing crosswalk information.
        lookup - A list of valid values that can be returned by the convert function.

        """

        def __init__(self, filename, db_info, segment, log, **kw):
            self._re_pattern = re.compile(':re:.+?:')
            self.db_info = db_info
            #rules
            self.rules = defaultdict(dict)
            self.reg_rules = []
            self.lookup = None
            self.log = log
            #metadata
            self.format = 'string'
            self.missing = None
            self.unknown = None
            self._has_all = False
            self._key_rule_map = {}
            self.remove_key = False
            self.transform_results = False
            self.filename = filename
            self.vars = []
            self.key = {}
            self.skip_missing = []
            self.skip_unknown = []
            self.null_predicate = []
            self.key_predicate = None
            self.overwrite = []
            self.force_unknown = False

            self.rule_generator_map = {
                ':re:search:': lambda args: self._generate_re_rule(RegexRule, self.re_pattern, self.line_value, 'search', self.line_priority),
                ':re:match:': lambda args: self._generate_re_rule(RegexRule, self.re_pattern,self.line_value, 'match', self.line_priority),
                ':re:exact:': lambda args: self._generate_re_rule(RegexRule, self.re_pattern,self.line_value, 'exact', self.line_priority),
                ':re:end:': lambda args: self._generate_re_rule(RegexRule, self.re_pattern,self.line_value, 'end', self.line_priority),
                ':re:tilde:': lambda args: self._generate_re_rule(RegexRule, self.re_pattern,self.line_value, 'tilde', self.line_priority),
                ':re:tilde-match:': lambda args: self._generate_re_rule(RegexRule, self.re_pattern,self.line_value, 'tilde-match', self.line_priority),
                ':re:tilde-begin:': lambda args: self._generate_re_rule(RegexRule, self.re_pattern,self.line_value, 'tilde-begin', self.line_priority),
                ':re:math-s:': lambda args: self._generate_re_rule(RegexMathRule, self.re_pattern,self.line_value, 'search', self.line_priority),
                ':re:math-m:': lambda args: self._generate_re_rule(RegexMathRule, self.re_pattern, self.line_value, 'match', self.line_priority),
                ':re:math-e:': lambda args: self._generate_re_rule(RegexMathRule, self.re_pattern,self.line_value, 'exact', self.line_priority),
                ':re:math-end:': lambda args: self._generate_re_rule(RegexMathRule, self.re_pattern,self.line_value, 'end', self.line_priority),
                ':re:math-f:': lambda args: self._generate_re_rule(RegexMathFindallRule, self.re_pattern,self.line_value, 'findall', self.line_priority),
                ':re:math-t:': lambda args: self._generate_re_rule(RegexMathRule, self.re_pattern,self.line_value, 'tilde', self.line_priority),
                ':re:math-t-m:': lambda args: self._generate_re_rule(RegexMathRule, self.re_pattern,self.line_value, 'tilde-match', self.line_priority),
                ':re:math-t-b:': lambda args: self._generate_re_rule(RegexMathRule, self.re_pattern,self.line_value, 'tilde-begin', self.line_priority),
                ':re:regex-s:': lambda args: self._generate_re_rule(RegexRule, self.re_pattern,self.line_value, 'search', self.line_priority, False),
                ':re:regex-m:': lambda args: self._generate_re_rule(RegexRule, self.re_pattern,self.line_value, 'match', self.line_priority, False),
                ':re:regex-e:': lambda args: self._generate_re_rule(RegexRule, self.re_pattern,self.line_value, 'exact', self.line_priority, False),
                ':re:regex-end:': lambda args: self._generate_re_rule(RegexRule, self.re_pattern,self.line_value, 'end', self.line_priority, False),
                ':re:regex-t:': lambda args: self._generate_re_rule(RegexRule, self.re_pattern,self.line_value, 'tilde', self.line_priority, False),
                ':re:regex-t-m:': lambda args: self._generate_re_rule(RegexRule, self.re_pattern,self.line_value, 'tilde-match', self.line_priority, False),
                ':re:regex-t-b:': lambda args: self._generate_re_rule(RegexRule, self.re_pattern,self.line_value, 'tilde-begin', self.line_priority, False),
                ':re:regex-math-s:': lambda args: self._generate_re_rule(RegexMathRule, self.re_pattern,self.line_value, 'search', self.line_priority, False),
                ':re:regex-math-m:': lambda args: self._generate_re_rule(RegexMathRule, self.re_pattern,self.line_value, 'match', self.line_priority, False),
                ':re:regex-math-e:': lambda args: self._generate_re_rule(RegexMathRule, self.re_pattern,self.line_value, 'exact', self.line_priority, False),
                ':re:regex-math-end:': lambda args: self._generate_re_rule(RegexMathRule, self.re_pattern,self.line_value, 'end', self.line_priority, False),
                ':re:regex-math-f:': lambda args: self._generate_re_rule(RegexMathFindallRule, self.re_pattern,self.line_value, 'findall', self.line_priority, False),
                ':re:regex-math-t:': lambda args: self._generate_re_rule(RegexMathRule, self.re_pattern,self.line_value, 'tilde', self.line_priority, False),
                ':re:regex-math-t-m:': lambda args: self._generate_re_rule(RegexMathRule, self.re_pattern,self.line_value, 'tilde-match', self.line_priority, False),
                ':re:regex-math-t-b:': lambda args: self._generate_re_rule(RegexMathRule, self.re_pattern,self.line_value, 'tilde-begin', self.line_priority, False),
                ':re:return-m:': lambda args: self._generate_re_rule(RegexReturnRule, self.re_pattern, None, 'match', None, False),
                ':re:return-s:': lambda args: self._generate_re_rule(RegexReturnRule, self.re_pattern, None, 'search', None, False),
                ':re:return-e:': lambda args: self._generate_re_rule(RegexReturnRule, self.re_pattern, None, 'exact', None,False),
                ':all:': lambda args: self._create_all_rule(),
                ':key_predicate:': lambda args: self._set_key_predicate(),
                ':value_predicate:': lambda args: self._set_value_predicate(args[0]),
                ':overwrite:': lambda args: self._set_overwrite(self.line_value),
                ':null_predicate:': lambda args: self._set_null_predicate(args[0], args[1]),
                ':missing:': lambda args: self._set_missing(self.line_value),
                ':unknown:': lambda args: self._set_unknown(self.line_value),
                ':lookup:': lambda args: self._set_lookup(args[0], args[1]),
                ':vars:': lambda args: self._set_vars(args[0], args[1]),
                ':keys:': lambda args: self._set_keys(args[0], args[1]),
                ':skip_missing:': lambda args: self._set_skip_missing(args[0]),
                ':skip_unknown:': lambda args: self._set_skip_unknown(args[0]),
                ':remove_key:': lambda args: self._set_remove_key(self.line_value),
                ':transform_results:': lambda args: self._set_transform_results(self.line_value),
                ':key:': lambda args: self._set_key_rule(args[0]),
                ':force_unknown:': lambda args: self._set_force_unknown(self.line_value),
            }

            #load the rules
            self.load(filename, db_info, segment, kw.get('prediction', False))

            #create a VariableSet of the output variables for the crosswalk
            out_vars = cs.variables.VariableSet([(var[0], var[1]) for var in self.vars])

            #create a sorted list of KeyMetadata objects
            key_meta = [k_m for k_m in sorted([k for k in self.key.itervalues()], key= lambda k:k.order)]

            opts = dict(missing=self.missing, unknown=self.unknown, force_unknown=self.force_unknown)

            #create the RuleSetMetadata
            self.metadata = cs.transform.RuleSetMetadata(key_meta, out_vars, **opts)

            self.validate()

        def _create_all_rule(self):
            self.rules[self.line_priority][self.line_initial] = AllRule(self.line_value)
            self._has_all = True

        def _generate_re_rule(self, rule_class, pattern, return_value, search_method, priority, escape=True):

            if priority is not None:
                self.reg_rules.append(rule_class(pattern, return_value, search_method, priority, escape=escape))
            else:
                self.reg_rules.append(rule_class(pattern, return_value, search_method, escape=escape))

        #metadata setters
        def _set_lookup(self, line, segment):
            self.lookup = Lookup(self.db_info, line.split('\t')[1], self.log)

        def _set_skip_missing(self, line):
            self.skip_missing = ([k for k in line.split('\t')[1].split('|')])

        def _set_force_unknown(self, line_value):
            self.force_unknown = str_to_boolean(line_value)

        def _set_skip_unknown(self, line):
            self.skip_unknown = ([k for k in line.split('\t')[1].split('|')])

        def _set_key_predicate(self):
            #self.key_predicate = self.line_value
            self.key_predicate = re.compile(self.line_value)

        def _set_value_predicate(self, line):
            if '|' in value:
                _vars = [v for v in line[1].split('|')]
                values = [v for v in line[2].split('|')]

                self.value_predicate = izip(vars, values)
            else:
                self.value_predicate = (line[1], line[2] if line[2] != ':none:' else None)

        def _set_overwrite(self, overwrite):
            if '|' in overwrite:
                self.overwrite = [o for o in overwrite.split('|')]
            else:
                self.overwrite.append(overwrite)

        def _set_missing(self, missing):
            self.missing = [m for m in missing.split('|')] if '|' in missing else missing

        def _set_unknown(self, unknown):
            self.unknown = [u for u in unknown.split('|')] if '|' in unknown else unknown

        def _set_null_predicate(self, line, segment):
            #all values must be None
            #TODO this should be combined with value_predicate
            self.null_predicate = line.split('\t')[1].split('|')
            self._validate_vars(self.null_predicate, segment)

        def _set_vars(self, line, segment):
            vars = line.split('\t')[1].split('|')

            try:
                for v in vars:
                    v_info = segment.var_dict[v]
                    self.vars.append((v, v_info[0], v_info[2], v_info[3]))

            except KeyError as e:
                raise RuleError('Invalid variable for crosswalk: %s. %s' % (vars, e))

        def _set_keys(self, line, segment):
            #TODO make this python-etl variables object by looking at metadata
            keys = line.split('\t')[1].split('|')

            try:
                for i, k in enumerate(keys):
                    v_info = segment.var_dict[k]
                    key_var = cs.transform.metadata.KeyMetadata(cs.variables.Variable(k, v_info[0]), i, as_str=True, inner_trim=True)
                    self.key[k] = key_var
                    #self.key[k] = dict(format=v_info[0], length=v_info[2], precision=v_info[3], order=_c)

            except KeyError as e:
                raise RuleError('Invalid variable for crosswalk: %s. %s' % (keys, e))

        def _set_remove_key(self, r):
            self.remove_key = str_to_boolean(r)

        def _set_transform_results(self, transform_results):
            self.transform_results = [k for k in transform_results.split('|')]

        def _set_key_rule(self, line):
            rule_info = line.split('\t')
            #need to store in map so we can keep track of multiple rules within one crosswalk
            self._key_rule_map[rule_info[1]] = rule_info

            #rule to replace the key
            #:key:    cchglitx    :replace:    0    o
            if re.search('replace', rule_info[2]):
                def _replace(key, var_name):
                    _rule_info = self._key_rule_map[var_name]
                    return key.replace(_rule_info[3], _rule_info[4] if _rule_info[4] != ':none:' else '')

                try:
                    self.key[rule_info[1]].transform = _replace
                except KeyError as e:
                    raise RuleError('KeyError %s: invalid variable in %s' % (e, self.filename))

            elif re.search('sub', rule_info[2]):
                #compile the regex
                self._key_rule_map[rule_info[1]][3] = re.compile(rule_info[3])

                def _sub(key, var_name):
                    _rule_info = self._key_rule_map[var_name]
                    return _rule_info[3].sub(_rule_info[4] if _rule_info[4] != ':none:' else '', key)

                try:
                    self.key[rule_info[1]].transform = _sub
                except KeyError as e:
                    raise RuleError('KeyError %s: invalid variable in %s' % (e, self.filename))

            elif re.search('predicate_re_replace', rule_info[2]):
                #search for the predicate regular expression on key
                #if a match is found do a replace on the match.group() of the second regex with the third input string
                self._key_rule_map[rule_info[1]][3] = re.compile(rule_info[3])
                self._key_rule_map[rule_info[1]][4] = re.compile(rule_info[4])
                self._key_rule_map[rule_info[1]][5] = rule_info[5] if rule_info[5] != ':none:' else ''

                def _predicate_re_transform(key, var_name):
                    _rule_info = self._key_rule_map[var_name]
                    if _rule_info[3].search(key):
                        replace = _rule_info[4].search(key)
                        return key.replace(replace.group(), _rule_info[5])

                self.key[rule_info[1]].transform = _predicate_re_transform

            else:
                raise RuleError('Invalid task for crosswalk: %s' % rule_info[2])

        def load(self, filename, db_info, segment, no_space=False):
            #for line in codecs.open(filename, 'r', 'utf-8'):
            for line in codecs.open(filename, 'r'):
                line = line.strip()
                split = line.split('\t')
                self.line_initial, self.line_value, self.line_priority = None, None, None

                if line:
                    try:
                        self.line_initial, self.line_value = split[:2]
                        if no_space:
                            self.line_initial = self.line_initial.replace(' ', '')

                        if self.line_initial.startswith(':re:'):
                            self.re_pattern = self._re_pattern.sub('', self.line_initial)
                            self.line_initial = self._re_pattern.match(self.line_initial).group()

                    except ValueError:
                        print line
                        print self.filename
                        raise

                    if len(split) == 3:
                        self.line_priority = int(split[2])
                    else:
                        self.line_priority = 1

                    try:
                        if self.lookup is not None:
                            if not self.lookup.validate(self.line_initial, self.line_value, filename):
                                continue

                        if self.line_initial in self.rule_generator_map:
                            self.rule_generator_map.get(self.line_initial)((line, segment))
                        else:
                            self.rules[self.line_priority][self.line_initial] = (Rule(self.line_initial.lower(), self.line_value))

                    except sre_constants.error as e:
                        print "Skipping regular expression rule. Check regex_init log for more details."
                        self.log.write('regex_init', [e, filename, self.line_initial, self.line_value, self.line_priority], 91, no_add_info=True)

        def _validate_vars(self, vars, segment):
            for v in vars:
                try:
                    assert(segment.var_dict.get(v))
                except AssertionError:
                    raise RuleError('Invalid variable for crosswalk: %s' % v)

        def _validate_regex_rules(self):
            patterns = dict()

            #TODO MongoDB for log reports
            for regex_rule in sorted(self.reg_rules, key=lambda rule: rule.priority, reverse=True):
                #validate result matches variables

                if len(self.metadata.out_vars) != len(regex_rule.result.split('|')) and 'remove' not in self.filename:
                    self.log.write('regex_init', ['error:result does not match out_variables', self.filename, regex_rule.regex.pattern, regex_rule.result, regex_rule.priority, self.metadata.out_vars], 91, no_add_info=True)

                #math rules validate the same number of parans
                if isinstance(regex_rule, RegexMathRule):
                    if len(re.findall('\(', regex_rule.result)) != len(re.findall('\)', regex_rule.result)):
                        self.log.write('regex_init', ['error:Syntax Error, unbalanced parentheses', self.filename, regex_rule.regex.pattern, regex_rule.result, regex_rule.priority], 91, no_add_info=True)

                    if re.search('#[ ]*(?!\d+)', regex_rule.result):
                        self.log.write('regex_init', ['error:Syntax Error, repeating digit group (#) not followed by number reference', self.filename, regex_rule.regex.pattern, regex_rule.result, regex_rule.priority], 91, no_add_info=True)

                    _digit_group = [int(x) for x in re.findall('#[ ]*(\d+)', regex_rule.result)] or [0]
                    if int(max(_digit_group)) > len(re.findall('%s|%s' % (re.escape('(\d+|#\d+|#)'), re.escape('(\d+)')), regex_rule.regex.pattern)):
                        self.log.write('regex_init', ['error:Syntax Error, formula number index out of range', self.filename, regex_rule.regex.pattern, regex_rule.result, regex_rule.priority], 91, no_add_info=True)

                    #validate that the result does not have 4# instead of #4
                    if re.search('\d#', regex_rule.result):
                        self.log.write('regex_init', ['potential error:Syntax Error, formula number index before number', self.filename, regex_rule.regex.pattern, regex_rule.result, regex_rule.priority], 91, no_add_info=True)

                #validate rule logic
                if regex_rule.type in ('exact', 'tilde') and regex_rule.escape:
                    continue

                for pattern, regex in patterns.iteritems():
                    if regex_rule.regex.pattern in pattern:
                        if regex[0] != regex_rule.result.replace(' ', ''):
                            err = '%serror:inconsistent result' % ('potential ' if regex_rule.type in ('exact', 'tilde', 'match') else '')
                            self.log.write('regex_init', [err, self.filename, regex_rule.regex.pattern, regex_rule.result.replace(' ', ''), regex_rule.priority, pattern, regex[0], regex[1]], 91, no_add_info=True)
                        else:
                            self.log.write('regex_init', ['warning:redundent rule', self.filename, regex_rule.regex.pattern, regex_rule.result.replace(' ', ''), regex_rule.priority, pattern, regex[0], regex[1]], 91, no_add_info=True)

                patterns[regex_rule.regex.pattern] = (regex_rule.result.replace(' ', ''), regex_rule.priority)

        def validate(self):
            '''Validate the ruleset.'''
            if config.debug_validate_regex_rules:
               self._validate_regex_rules()

        def _convert(self, rule, value, **kw):
            """Helper function for convert"""
            if rule.type == 'all':
                kw['value'] = value

            out_value = rule.convert(**kw)
            out = []
            if 'split' in kw:
                for val, var in izip(out_value, self.vars):
                    if var[1] in FORMAT_DICT:
                        if is_none(val):
                            o = None
                        else:
                            if var[1] == 'float':
                                try:
                                    #hacky fix to deal with precision without transforming to a decimal (I think this is faster)
                                    o = round(round(FORMAT_DICT[var[1]](val), var[3]+1), var[3])
                                except ValueError:
                                    raise ConvertError("Unable to convert. Check rule regex: %s" % rule.regex.pattern)
                            else:
                                o = FORMAT_DICT[var[1]](val)

                        out.append(o)

                    else:
                        raise ConvertError('Unable to convert to %s' % self.format)

                return out

            if rule.return_type == 'match':
                return out_value

            if self.vars[0][1] in FORMAT_DICT:
                if is_none(out_value):
                    return None
                else:
                    #convert to correct format
                    if self.vars[0][1] == 'string':
                        return out_value
                    elif self.vars[0][1] == 'float':
                        #hacky fix to deal with precision without transforming to a decimal (I think this is faster)
                        return round(round(FORMAT_DICT[self.vars[0][1]](out_value), self.vars[0][3]+1), self.vars[0][3])
                    else:
                        return FORMAT_DICT[self.vars[0][1]](out_value)

            else:
                raise ConvertTypeError('Unable to convert to %s' % self.format)

        def convert(self, key, **kw):
            """
            Returns a converted value based on key or raises a ConvertError
            if key can not be converted.

            Looks to see if there is a rule that can handle a given key.
            Split keyword returns the converted result as a list of values,
            using the value of the split keyword as the delimiter for the result
            list.
            """

            #check to see if :all: rule is present
            if not self._has_all:
                str_key = None if key is None else str(key)
            else:
                str_key = ':all:'

            for rule_group in self.rules.itervalues():
                if str_key in rule_group:
                    return self._convert(rule_group[str_key], str_key, **kw)

            if self.reg_rules: #if key not found in rules dict then see if regex rule can handle the key
                for rule in sorted(self.reg_rules, key=lambda rule: rule.priority):
                    if rule.handle(str_key):
                        if kw.get('remove'):
                            return (self._convert(rule, str_key, **kw), rule.match.group())
                        else:
                            return self._convert(rule, str_key, **kw)

            raise ConvertError('Unable to convert to %s' % key)

        def delete_transform(self, key, count=0, **kw):
            """Transform the given key by applying the rules in the crosswalk.
            If a rule can handle the key it will remove the matched text from the key.
            After the rules are applied the transformed key will be returned.

            The optional argument of **count** is the maximum number of times a transform
            will be applied to the key. By default (count=0) all rules will be checked to see
            if they can handle the key. If count is specified than the crosswalk will
            incrememnt a counter for each time a rule handles the key. Once the counter = count
            the crosswalk will return the transformed key.
            """
            if key is None or key == '':
                return key

            #initialize a counter
            _count = icount().next

            _key = key

            #check the exact match non-regular expression rules and remove the key
            for rule_group in self.rules.itervalues():
                if _key in rule_group:
                    #we ignore count here because if a match is found the entire string will be replaced
                    return ''

            #if key not found in rules dict then see if regex rule can handle the key
            if self.reg_rules:
                for rule in sorted(self.reg_rules, key=lambda rule: rule.priority):
                    if count:
                        if _count() == count:
                            return _key

                    _key = rule.delete_transform(_key)

                return _key

            return key

### Navigation

- index
- modules |
- CCHR 1.0 documentation »
- Module code »
- src »