from cardsharp.transform import metadata
from collections import defaultdict
from datetime import datetime
from errors import *
from itertools import izip
from itertools import count as icount
from util import *
import cardsharp as cs
import codecs
import log
import re
import sre_constants
from configuration import config
import traceback
import sys

__all__ = ['Rule', 'AllRule', 'RegexRule', 'Crosswalk', 'RegexMathRule']

time = datetime.now().isoformat('_').replace(':', '-')

class Rule(object):
    '''Default Crosswalk Rule defining a crosswalk conversion. 
    
    When a crosswalk is loaded each line in the crosswalk file is turned 
    into a rule. For this default rule the initial value is the key in the 
    crosswalk and the result is the value in the crosswalk. See crosswalk 
    for more details. 
    
    Public Methods:
    handle -- Returns true if initial matches the value
    convert -- Returns the result
    
    '''
    def __init__(self, initial, result, rule_id):
        self.initial = initial
        self.result = result
        self.id = rule_id
        self.return_type = 'result' 
        self.type = 'match'

    def convert(self, **kw):
        if kw.get('split'):
            return [x for x in self.result.split(kw['split'])]  
        else:
            return self.result
      
class AllRule(object):
    '''Crosswalk rule to handle all values. 
    If result is :copy: will copy the initial value to the result value,
    otherwise will output the result value.
    
    example)
    :all:    :copy:
    This will output exactly what is in the original dataset without 
    doing any conversion.
    
    :all:    :none:
    This will output a None value.
    
    :all:    1111.11
    This will output all values to 1111.11
    '''
    def __init__(self, result, rule_id):
        self.result = result
        self.id = rule_id
        self.return_type = 'result'
        self.copy = True if re.search(':copy:', result, re.I) else False
        self.none = True if re.search(':none:', result, re.I) else False
        self.type = 'all'
        self.out = []
        self._rules = {':copy:': lambda key: self.out.append(key[0]),
                       ':none:': lambda key: self.out.append('')}
        
    def convert(self, **kw):
        #if multiple out value are defined
        if kw.get('split'):
            if self.copy or self.none:
                #add the result value to out list based on its rule (:copy:, :none: or the result value)
                for x in self.result.split(kw['split']):
                    self._rules.get(x, lambda r: self.out.append(r[1]))((kw.get('value'), x))
                return self.out
            #otherwise return result values as a list
            else:
                return [x for x in self.result.split(kw['split'])]   
        else:
            if self.copy or self.none:
                self._rules.get(self.result)((kw.get('value'), self.result))
                return self.out[0]
            
            return self.result


class RegexRule(object):
    def __init__(self, regex, result, type, priority, rule_id, escape=True, lookbehind=None, lookahead=None, region=None, **kwargs):
        """
        @param regex: A regular expression pattern
        @param result: The value to be returned if the regular expression matches
        @param type: The type of regular expression rule
        #TODO better doc
                     math: return match object groups applied to math function
                     search: return re.search
                     match: return re.match
        
        """
        #escape regex as needed
        self.escape = escape
        if escape:
            '''# captures '#' or '#\d+' or '\d+' '''
            if region not in {'sd', 'or', 'ia', 'sc', 'fl', 'ga', 'mi', 'nc'}:
                self.regex = re_escape(regex).replace('#', '(\d+|#\d+|#)')
            else:
                self.regex = re_escape(regex).replace('#', '(\d+\.\d+|\.\d+|\d+|#\d+|#)')
        else:
            self.regex = regex

        if lookbehind:
            if lookbehind.startswith('!'):
                self.regex = '(?<!%s)' % lookbehind + self.regex
            else:
                self.regex = '(?<=%s)' % lookbehind + self.regex

        if lookahead:
            if lookahead.startswith('!'):
                self.regex = self.regex + '(?%s)' % lookahead
            else:
                self.regex = self.regex + '(?=%s)' % lookahead

        #do an exact match
        if type == 'search':
            self.re_func = re.search
            self.delete_transform_regex = self.regex
            
        elif type == 'match':
            self.re_func = re.match
            self.delete_transform_regex = ''.join(['^', self.regex])
            
        elif type == 'exact': 
            self.delete_transform_regex = ''.join(['^', self.regex, '$'])
            self.regex = ''.join([self.regex, '$'])
            self.re_func = re.match
        
        elif type == 'end': 
            self.delete_transform_regex = ''.join([self.regex, '$'])
            self.regex = ''.join([self.regex, '$'])
            self.re_func = re.search
            
        elif type == 'findall':
            #TODO this is currently broken, this is not an exact match this is a match to the end of the string 
            self.delete_transform_regex = self.regex
            self.regex = self.regex
            self.re_func = re.findall
            
        elif type == 'tilde':
            #TODO this is currently broken, this is not an exact match this is a match to the end of the string 
            self.delete_transform_regex = ''.join(['(?:^|~)', self.regex, '(?:~|$)'])
            self.regex = ''.join(['(?:^|~)', self.regex, '[ ]*(?:~|$)'])
            self.re_func = re.search
        
        elif type == 'tilde-match':
            self.delete_transform_regex = ''.join([self.regex, '[ ]*(?:~|$)'])
            self.regex = ''.join([self.regex, '[ ]*(?:~|$)'])
            self.re_func = re.match
        
        elif type == 'tilde-begin':
            self.delete_transform_regex = ''.join(['(?:~|^)', self.regex])
            self.regex = ''.join(['(?:~|^)', self.regex])
            self.re_func = re.search
        
        self.regex = re.compile(self.regex, re.I) #actual regular expression string
        self.delete_transform_regex = re.compile(self.delete_transform_regex, re.I) #actual regular expression string

        self.id = rule_id
            
        self.return_type = 'result' #this rule will return a value
        self.result = self._validate_result(result)
        self.reg = regex
        self.type = type
        self.match = None #object to hold match object
        self.priority = priority
    
    #TODO change this to match RegexMathRule.handle()
    def handle(self, value):
        """Returns True if rule can handle the value. Convert will only be called 
        if handle returns True."""
        if value in [None, '']:
            raise ConvertError
        #TODO this is not using compiled regular expression
        #should be self.regex.match

        self.match = self.re_func(self.regex, value)

        if self.match:
            self.groups = self.match.groups()
          
        #will be the match object or None  
        return self.match
        
    def delete_transform(self, value):
        """Transform the value by removing the regex pattern from value. If a named group called **keep** is present 
        than remove other but the pattern in the **keep** group.""" 
        return self.delete_transform_regex.sub(lambda match: match.groupdict().get('keep'), value)
        
    def convert(self, **kw):
        """Return result."""
        if kw.get('split'):
            return [x for x in self.result.split(kw['split'])]  
        else:
            return self.result
    
    def _validate_result(self, result):
        """Helper function to validate the result string. This method can be overridden to do 
        clean, formating, validation of the result in classes that inherit from RegexRule."""
        return result


_formula_func_map = {
    'max': (re.compile('max\(|\)$'), max),
    'min': (re.compile('min\(|\)$'), min)
}

def _get_result(result, groups, type=None):
    '''result = the result we want to calculate for a output variable.
       groups = the groups object from the match object on the regex see handle below
       type = The type of function we want to calculate 
    '''
    if type in _formula_func_map:
        _ff = _formula_func_map[type]
        result = _ff[0].sub('',result)
        _values = result.split(',')
        return _ff[1]([calc(re.sub('(#)(\d+)', lambda c: groups[int(c.group(2)) - 1].replace('#', ''), x)) for x in _values])
    
    else:
        return calc(re.sub('(#)(\d+)', lambda c: groups[int(c.group(2)) - 1].replace('#', ''), result))

class RegexMathRule(RegexRule):
    def __init__(self, regex, result, type, priority, rule_id, escape=True, lookbehind=None, lookahead=None, region=None, **kw):
        RegexRule.__init__(self, regex, result, type, priority, rule_id, escape, lookbehind, lookahead, region, **kw)
        self.result = self.result.lower()
        self.has_result_formula_func = True if 'max' in self.result or 'min' in self.result else False
        
    def handle(self, value):
        RegexRule.handle(self, value)
        self.match = self.re_func(self.regex, value)
        
        if self.match:
            self.groups = self.match.groups()
            return True
        else:
            return False

    def convert(self, **kw):
        """
        For math regex apply groups extracted from value to arthimetic function within self.result
        Math:
        >>> r = RegexMathRule('Sent to #Day #Y', '#1 + #2(365.25)', 'math')
        >>> r.handle('Sent to 10Day 2Y')
        True
        >>> r.convert()
        740.5
        """
        #groups are base 0, research tema has been entering there # objects as base one
        #subtract 1 from every #\d+ match and return the matched group
        #example) 
        #   regex = Sent to #Day #Y
        #   result = #1 + #2(365.25)
        #    value = 'Sent to 10Day 2Y'
        #return calc('10 + 2(365.25)')
        
        #if a transform result function is present then substitute the variable name in the results
        #with the current value in the row
        result = self.result
        if kw.get('transform_results'):
            for var in kw.get('transform_results'):
                if var in result:
                    result = result.replace(var, str(kw['row'][var]))
        
        try:
            if kw.get('split'):
                '''@todo: Make result a class so we do not need to check for max, min on every result value'''
                result = [r if not is_none(r) else 'None' for r in result.split(kw['split'])]

                if self.has_result_formula_func:
                    _out_values = []
                    for r in result:
                        if r.startswith('max'):
                            _out_values.append(_get_result(r, self.groups, 'max'))
                        elif r.startswith('min'):
                            _out_values.append(_get_result(r, self.groups, 'min'))
                        else:
                            _out_values.append(_get_result(r, self.groups))
                    
                    return _out_values
                
                else:
                    #same as above but condensed without evaluation
                    return [calc(re.sub('(#)(\d+)', lambda c: self.groups[int(c.group(2)) - 1].replace('#', ''), x)) for x in result]
            else:
                if self.has_result_formula_func:
                    if result.startswith('max'):
                        return _get_result(result, self.groups, 'max')
                    elif result.startswith('min'):
                        return _get_result(result, self.groups, 'min')
                    else:
                        return _get_result(result, self.groups)
                else:
                    return calc(re.sub('(#)(\d+)', lambda c: self.groups[int(c.group(2)) - 1].replace('#', ''), result))
            
        except CalcError:
            raise RuleError("CalcError: Invalid Result Formula: %s . Priority = %s" % (self.result, self.priority))
        except IndexError:
            raise RuleError("Invalid output value: key:%s | value:%s, number index out of range" % (self.regex.pattern, self.result))
        except SyntaxError:
            raise RuleError("Syntax Error: check rule syntax: key:%s | value:%s" % (self.regex.pattern,self.result))
        except AttributeError:
            raise RuleError("Attribute Error: check rule syntax: key:%s | value:%s" % (self.regex.pattern,self.result))
        except NameError:
            raise RuleError("NameError: make sure the variable you listed is in the ruleset.variables metadata: key:%s | result:%s | %s" % (self.regex.pattern,self.result, kw.get(['row'], {'id':''})['id']))
    
    def _validate_result(self, result):
        """Clean the result formulas"""
        #replace [ ] with ( )
        result = result.replace('[', '(')
        result = result.replace(']', ')')
        result = result.replace('{', '(')
        result = result.replace('}', ')')
        result = result.replace(' ', '')
        
        return result

class RegexMathFindallRule(RegexMathRule):
    def handle(self, value):
        if value in [None, '']:
            raise ConvertError
        
        self.groups = self.re_func(self.regex, value)
        self.match = self.regex.search(value) #need match object for delete transform rule
        
        return self.match #return None or match object
    
    def convert(self, **kw):
        result = self.result
        if kw.get('transform_results'):
            for var in kw.get('transform_results'):
                if var in result:
                    result = result.replace(var, str(kw['row'][var]))
        
        try:
            out = []
            
            if kw.get('split'):
                '''@todo: Make result a class so we do not need to check for max, min on every result value'''
                result = [r if not is_none(r) else 'None' for r in result.split(kw['split'])]

                #findall has one or many groups we need to select the highest                
                for groups in self.groups:
                    if not isinstance(groups, tuple): groups = tuple([groups])
                    out.append([calc(re.sub('(#)(\d+)', lambda c: groups[int(c.group(2)) - 1].replace('#', ''), x)) for x in result])

                #determine which result formula we want to evaluate max on
                c = 0
                for r in result:
                    if '#' in r: #if multiple formulas use first
                        eval_index = c
                        break
                    else:
                        c+=1
                #return the largest item
                return sorted(out, key = lambda k: k[eval_index])[-1]        
                
            else:
                for groups in self.groups:
                    out.append(calc(re.sub('(#)(\d+)', lambda c: groups[int(c.group(2)) - 1].replace('#', ''), result)))
                
                return sorted(out)[-1]
                
        except CalcError:
            raise RuleError("CalcError: Invalid Result Formula: %s . Priority = %s" % (self.result, self.priority))    
        except IndexError:
            raise RuleError("Invalid output value: key:%s | value:%s, number index out of range. Priority = %s" % (self.regex.pattern, self.result, self.priority))
        except SyntaxError:
            raise RuleError("Syntax Error: check rule syntax: key:%s | value:%s. Priority = %s" % (self.regex.pattern,self.result, self.priority))
        except AttributeError:
            raise RuleError("Attribute Error: check rule syntax: key:%s | value:%s. Priority = %s" % (self.regex.pattern,self.result, self.priority))
        except NameError:
            raise RuleError("NameError: make sure the variable you listed is in the ruleset.variables metadata: key:%s | result:%s | %s. Priority = %s" % (self.regex.pattern,self.result, kw.get(['row'], {'id':''})['id'], self.priority))


class RegexReturnRule(RegexRule):
    """Return the Match Object of a match regular expression."""
    def __init__(self, regex, result, type, priority, rule_id, escape=True, lookbehind=None, lookahead=None, region=None):
        RegexRule.__init__(self, regex, result, type, priority, rule_id, escape, lookbehind, lookahead, region)
        self.return_type = 'match'
            
    def handle(self, value):
        RegexRule.handle(self, value)  
        self.result = self.re_func(self.regex, value) 
        
        return True if self.result else False


class Lookup(object):
    def __init__(self, db, tables, log):
        self.values = []
        self.log = log
        self.lookup_tables = tables.split('|')
#        for table in tables.split('|'):
#            ds = cs.load(source=db['name'], format='mysql', dataset=table, user=db['user'], pwd=db['pass'])
#            self.values.append(([row['id'] for row in ds], table))

    def validate(self, initial, value, filename):
        #iterate over all values and lookups checking to see if each value
        #in corresponding lookup list
        for value, table in izip(value.split('|'), self.lookup_tables):
            try:
                if value not in ('', 'None', ':none:', None) and int(value.replace('.0', '')) not in LOOKUPS[table]:
                    if (table.startswith("a") or table.startswith("c")) and table != 'cnfreltype' and table != 'agencytype':
                        if int(value) not in LOOKUPS[table[1:]]:
                            self.log.write('ref_integ_error', [value, table[1:], LOOKUPS[table[1:]], filename, 'Value not present in lookup'], 90)
                            return False
                    else:
                        self.log.write('ref_integ_error', [value, table, LOOKUPS[table], filename, 'Value not present in lookup'], 90)
                        return False
            except Exception as e:
                print 'Invalid Value: %s' % str(value)
                print 'Error: %s' % e
                self.log.write('ref_integ_error', [value, table, LOOKUPS[table], filename, 'Value not present in lookup'], 90)
                raise RuleError("Rule validation failed for. Check log.")
        
        return True
        
#        temp_value = value.split('|')
#        for v, l in izip(temp_value, self.values):
#            try:
#                if v not in ('', 'None', ':none:') and int(v) not in l[0]:
#                    #TODO add validate_rules
#                    #if segment.validate_rules:
#                    self.log.write('ref_integ_error', [v, l[1], l[0], filename, 'Value not present in lookup'], 90)
#                    return False
#            except:
#                self.log.write('ref_integ_error', [v, l[1], l[0], filename, 'Value not present in lookup'], 90)
#                raise RuleError("Rule validation failed for. Check log.")
#    
#        return True


class Crosswalk(object):
    """A Crosswalk object is used to do variable conversions either directly
    or by accessing the object in the func.py file for a CCHR variable. 
    
    The Crosswalk object first loads the crosswalk.txt file specified by the 
    **filename** parameter. Each line of this file is turned into a rule object, 
    see _rules for more information. When crosswalk.convert(value) is called
    we iterate over the rules list checking to see if a rule can handle the 
    value. If a rule can handle the value the rules result is returned,
    otherwise we raise a ConvertError. An additional **lookup** parameter can
    be defined to provide a list of valid return values. 
    
    The column delimiter is a tab and the row delimiter is a newline. 
    So each line in the crosswalk looks like: 
    
    **key**\\t**value**\\n 
    
    example)
    
    100\tmurder
    
    parameters:
    
    filename - Path to file containing crosswalk information.
    lookup - A list of valid values that can be returned by the convert function.
    
    """
    
    def __init__(self, filename, db_info, segment, log, crosswalk_id, **kw):
        self._re_pattern = re.compile(':re:.+?:')
        self.db_info = db_info
        #rules
        self.rules = defaultdict(dict)
        self.reg_rules = []
        self.lookup = None
        self.log = log
        self.crosswalk_id = crosswalk_id
        #metadata
        self.format = 'string'
        self.missing = None
        self.unknown = None
        self._has_all = False
        self._key_rule_map = {}
        self.remove_key = False
        self.transform_results = False
        self.filename = filename
        self.vars = []
        self.key = {}
        self.skip_missing = []
        self.skip_unknown = []
        self.null_predicate = []
        self.key_predicate = None
        self.overwrite = []
        self.force_unknown = False
        self.lookbehind = None
        self.lookahead = None
        
        self.rule_generator_map = {
            ':re:search:': lambda args: self._generate_re_rule(RegexRule, self.re_pattern, self.line_value, 'search', self.line_priority, self.rule_id, region=args[2]),
            ':re:match:': lambda args: self._generate_re_rule(RegexRule, self.re_pattern, self.line_value, 'match', self.line_priority, self.rule_id, region=args[2]),
            ':re:exact:': lambda args: self._generate_re_rule(RegexRule, self.re_pattern, self.line_value, 'exact', self.line_priority, self.rule_id, region=args[2]),
            ':re:end:': lambda args: self._generate_re_rule(RegexRule, self.re_pattern, self.line_value, 'end', self.line_priority, self.rule_id, region=args[2]),
            ':re:tilde:': lambda args: self._generate_re_rule(RegexRule, self.re_pattern, self.line_value, 'tilde', self.line_priority, self.rule_id, region=args[2]),
            ':re:tilde-match:': lambda args: self._generate_re_rule(RegexRule, self.re_pattern, self.line_value, 'tilde-match', self.line_priority, self.rule_id, region=args[2]),
            ':re:tilde-begin:': lambda args: self._generate_re_rule(RegexRule, self.re_pattern, self.line_value, 'tilde-begin', self.line_priority, self.rule_id, region=args[2]),
            ':re:math-s:': lambda args: self._generate_re_rule(RegexMathRule, self.re_pattern, self.line_value, 'search', self.line_priority, self.rule_id, region=args[2]),
            ':re:math-m:': lambda args: self._generate_re_rule(RegexMathRule, self.re_pattern, self.line_value, 'match', self.line_priority, self.rule_id, region=args[2]),
            ':re:math-e:': lambda args: self._generate_re_rule(RegexMathRule, self.re_pattern,self.line_value, 'exact', self.line_priority, self.rule_id, region=args[2]),
            ':re:math-end:': lambda args: self._generate_re_rule(RegexMathRule, self.re_pattern,self.line_value, 'end', self.line_priority, self.rule_id, region=args[2]),
            ':re:math-f:': lambda args: self._generate_re_rule(RegexMathFindallRule, self.re_pattern, self.line_value, 'findall', self.line_priority, self.rule_id, region=args[2]),
            ':re:math-t:': lambda args: self._generate_re_rule(RegexMathRule, self.re_pattern, self.line_value, 'tilde', self.line_priority, self.rule_id, region=args[2]),
            ':re:math-t-m:': lambda args: self._generate_re_rule(RegexMathRule, self.re_pattern, self.line_value, 'tilde-match', self.line_priority, self.rule_id, region=args[2]),
            ':re:math-t-b:': lambda args: self._generate_re_rule(RegexMathRule, self.re_pattern, self.line_value, 'tilde-begin', self.line_priority, self.rule_id, region=args[2]),
            ':re:regex-s:': lambda args: self._generate_re_rule(RegexRule, self.re_pattern, self.line_value, 'search', self.line_priority, self.rule_id, False),
            ':re:regex-m:': lambda args: self._generate_re_rule(RegexRule, self.re_pattern,self.line_value, 'match', self.line_priority, self.rule_id, False),
            ':re:regex-e:': lambda args: self._generate_re_rule(RegexRule, self.re_pattern,self.line_value, 'exact', self.line_priority, self.rule_id, False),
            ':re:regex-end:': lambda args: self._generate_re_rule(RegexRule, self.re_pattern,self.line_value, 'end', self.line_priority, self.rule_id, False),
            ':re:regex-t:': lambda args: self._generate_re_rule(RegexRule, self.re_pattern,self.line_value, 'tilde', self.line_priority, self.rule_id, False),
            ':re:regex-t-m:': lambda args: self._generate_re_rule(RegexRule, self.re_pattern,self.line_value, 'tilde-match', self.line_priority, self.rule_id, False),
            ':re:regex-t-b:': lambda args: self._generate_re_rule(RegexRule, self.re_pattern,self.line_value, 'tilde-begin', self.line_priority, self.rule_id, False),
            ':re:regex-math-s:': lambda args: self._generate_re_rule(RegexMathRule, self.re_pattern,self.line_value, 'search', self.line_priority, self.rule_id, False),
            ':re:regex-math-m:': lambda args: self._generate_re_rule(RegexMathRule, self.re_pattern,self.line_value, 'match', self.line_priority, self.rule_id, False),
            ':re:regex-math-e:': lambda args: self._generate_re_rule(RegexMathRule, self.re_pattern,self.line_value, 'exact', self.line_priority, self.rule_id, False),
            ':re:regex-math-end:': lambda args: self._generate_re_rule(RegexMathRule, self.re_pattern,self.line_value, 'end', self.line_priority, self.rule_id, False),
            ':re:regex-math-f:': lambda args: self._generate_re_rule(RegexMathFindallRule, self.re_pattern,self.line_value, 'findall', self.line_priority, self.rule_id, False),
            ':re:regex-math-t:': lambda args: self._generate_re_rule(RegexMathRule, self.re_pattern,self.line_value, 'tilde', self.line_priority, self.rule_id, False),
            ':re:regex-math-t-m:': lambda args: self._generate_re_rule(RegexMathRule, self.re_pattern,self.line_value, 'tilde-match', self.line_priority, self.rule_id, False),
            ':re:regex-math-t-b:': lambda args: self._generate_re_rule(RegexMathRule, self.re_pattern,self.line_value, 'tilde-begin', self.line_priority, self.rule_id, False),
            ':re:return-m:': lambda args: self._generate_re_rule(RegexReturnRule, self.re_pattern, None, 'match', None, self.rule_id, False),
            ':re:return-s:': lambda args: self._generate_re_rule(RegexReturnRule, self.re_pattern, None, 'search', None, self.rule_id, False),
            ':re:return-e:': lambda args: self._generate_re_rule(RegexReturnRule, self.re_pattern, None, 'exact', None, self.rule_id, False),
            ':all:': lambda args: self._create_all_rule(),
            ':key_predicate:': lambda args: self._set_key_predicate(),
            ':value_predicate:': lambda args: self._set_value_predicate(args[0]),
            ':overwrite:': lambda args: self._set_overwrite(self.line_value),
            ':null_predicate:': lambda args: self._set_null_predicate(args[0], args[1]),
            ':lookbehind:': lambda args: self._set_lookbehind(args[0].split('\t')[1]),
            ':lookahead:': lambda args: self._set_lookahead(args[0].split('\t')[1]),
            ':missing:': lambda args: self._set_missing(self.line_value),
            ':unknown:': lambda args: self._set_unknown(self.line_value),
            ':lookup:': lambda args: self._set_lookup(args[0], args[1]),
            ':vars:': lambda args: self._set_vars(args[0], args[1]),
            ':keys:': lambda args: self._set_keys(args[0], args[1]),
            ':skip_missing:': lambda args: self._set_skip_missing(args[0]),
            ':skip_unknown:': lambda args: self._set_skip_unknown(args[0]),
            ':remove_key:': lambda args: self._set_remove_key(self.line_value),
            ':transform_results:': lambda args: self._set_transform_results(self.line_value),
            ':key:': lambda args: self._set_key_rule(args[0]),
            ':force_unknown:': lambda args: self._set_force_unknown(self.line_value),
        } 
    
        #load the rules
        try:
            self.load(filename, db_info, segment, kw.get('prediction', False))
        except Exception as e:
            print filename
            print traceback.print_exc(file=sys.stdout)
            raise e
        
        #create a VariableSet of the output variables for the crosswalk
        out_vars = cs.variables.VariableSet([(var[0], var[1]) for var in self.vars])
        
        #create a sorted list of KeyMetadata objects
        key_meta = [k_m for k_m in sorted([k for k in self.key.itervalues()], key= lambda k:k.order)]
        
        opts = dict(missing=self.missing, unknown=self.unknown, force_unknown=self.force_unknown)
        
        #create the RuleSetMetadata
        self.metadata = cs.transform.RuleSetMetadata(key_meta, out_vars, **opts)
        
        self.validate()
        
    def _create_all_rule(self):
        self.rules[self.line_priority][self.line_initial] = AllRule(self.line_value, self.rule_id)
        self._has_all = True
    
    def _generate_re_rule(self, rule_class, pattern, return_value, search_method, priority, rule_id, escape=True, region=None, **kwargs):
        
        if priority is not None:
            self.reg_rules.append(rule_class(pattern, return_value, search_method, priority, rule_id, escape=escape, lookbehind=self.lookbehind, lookahead=self.lookahead, region=region, **kwargs))
        else:
            self.reg_rules.append(rule_class(pattern, return_value, search_method, escape=escape, lookbehind=self.lookbehind, lookahead=self.lookahead, region=region, **kwargs))
    
    #metadata setters
    def _set_lookup(self, line, segment):
        self.lookup = Lookup(self.db_info, line.split('\t')[1], self.log)

    def _set_lookbehind(self, pattern):
        print '##%s##' % pattern
        self.lookbehind = pattern

    def _set_lookahead(self, pattern):
        self.lookahead = pattern
    
    def _set_skip_missing(self, line):
        self.skip_missing = ([k for k in line.split('\t')[1].split('|')])
    
    def _set_force_unknown(self, line_value):
        self.force_unknown = str_to_boolean(line_value)
        
    def _set_skip_unknown(self, line):
        self.skip_unknown = ([k for k in line.split('\t')[1].split('|')])
        
    def _set_key_predicate(self):
        #self.key_predicate = self.line_value 
        self.key_predicate = re.compile(self.line_value)
        
    def _set_value_predicate(self, line):
        if '|' in value:
            _vars = [v for v in line[1].split('|')]
            values = [v for v in line[2].split('|')]
            
            self.value_predicate = izip(vars, values) 
        else:
            self.value_predicate = (line[1], line[2] if line[2] != ':none:' else None)
    
    def _set_overwrite(self, overwrite):
        if '|' in overwrite:
            self.overwrite = [o for o in overwrite.split('|')]
        else:
            self.overwrite.append(overwrite)

    def _set_missing(self, missing):
        self.missing = [m for m in missing.split('|')] if '|' in missing else missing
    
    def _set_unknown(self, unknown):
        self.unknown = [u for u in unknown.split('|')] if '|' in unknown else unknown
        
    def _set_null_predicate(self, line, segment):
        #all values must be None
        #TODO this should be combined with value_predicate
        self.null_predicate = line.split('\t')[1].split('|')
        self._validate_vars(self.null_predicate, segment)
    
    def _set_vars(self, line, segment): 
        vars = line.split('\t')[1].split('|')
        
        try:
            for v in vars:
                v_info = segment.var_dict[v]
                self.vars.append((v, v_info[0], v_info[2], v_info[3]))
            
        except KeyError as e:
            raise RuleError('Invalid variable for crosswalk: %s, %s. %s' % (line, vars, e))
    
    def _set_keys(self, line, segment):
        #TODO make this python-etl variables object by looking at metadata
        keys = line.split('\t')[1].split('|')
        
        try:
            for i, k in enumerate(keys):
                v_info = segment.var_dict[k]
                key_var = cs.transform.metadata.KeyMetadata(cs.variables.Variable(k, v_info[0]), i, as_str=True, inner_trim=True) 
                self.key[k] = key_var 
                #self.key[k] = dict(format=v_info[0], length=v_info[2], precision=v_info[3], order=_c)
        
        except KeyError as e:
            raise RuleError('Invalid variable for crosswalk: %s. %s' % (keys, e))
    
    def _set_remove_key(self, r):
        self.remove_key = str_to_boolean(r)
    
    def _set_transform_results(self, transform_results):
        self.transform_results = [k for k in transform_results.split('|')]
    
    def _set_key_rule(self, line):
        rule_info = line.split('\t')
        #need to store in map so we can keep track of multiple rules within one crosswalk
        self._key_rule_map[rule_info[1]] = rule_info
        
        #rule to replace the key
        #:key:    cchglitx    :replace:    0    o
        if re.search('replace', rule_info[2]):
            def _replace(key, var_name):
                _rule_info = self._key_rule_map[var_name]
                return key.replace(_rule_info[3], _rule_info[4] if _rule_info[4] != ':none:' else '')
        
            try:  
                self.key[rule_info[1]].transform = _replace
            except KeyError as e:
                raise RuleError('KeyError %s: invalid variable in %s' % (e, self.filename))
            
        elif re.search('last_char', rule_info[2]):
            def _last(key, var_name):
                return key[-1:]

            try:
                self.key[rule_info[1]].transform = _last
            except KeyError as e:
                raise RuleError('KeyError %s: invalid variable in %s' % (e, self.filename))

        elif re.search('sub', rule_info[2]):
            #compile the regex
            self._key_rule_map[rule_info[1]][3] = re.compile(rule_info[3])
            
            def _sub(key, var_name):
                _rule_info = self._key_rule_map[var_name]
                return _rule_info[3].sub(_rule_info[4] if _rule_info[4] != ':none:' else '', key)
            
            try:
                self.key[rule_info[1]].transform = _sub
            except KeyError as e:
                raise RuleError('KeyError %s: invalid variable in %s' % (e, self.filename))
            
        elif re.search('predicate_re_replace', rule_info[2]):
            #search for the predicate regular expression on key
            #if a match is found do a replace on the match.group() of the second regex with the third input string
            self._key_rule_map[rule_info[1]][3] = re.compile(rule_info[3])
            self._key_rule_map[rule_info[1]][4] = re.compile(rule_info[4])
            self._key_rule_map[rule_info[1]][5] = rule_info[5] if rule_info[5] != ':none:' else ''
            
            def _predicate_re_transform(key, var_name):
                _rule_info = self._key_rule_map[var_name]
                if _rule_info[3].search(key):
                    replace = _rule_info[4].search(key)
                    return key.replace(replace.group(), _rule_info[5])
            
            self.key[rule_info[1]].transform = _predicate_re_transform
        
        else:
            raise RuleError('Invalid task for crosswalk: %s' % rule_info[2])
                 
    def load(self, filename, db_info, segment, no_space=False):
        #for line in codecs.open(filename, 'r', 'utf-8'):
        header_end = 0
        for i, line in enumerate(codecs.open(filename, 'r')):
            line = line.strip()
            split = line.split('\t')
            self.line_initial, self.line_value, self.line_priority = None, None, None
            
            if line:
                try:
                    self.line_initial, self.line_value = split[:2]
                    if no_space:
                        self.line_initial = self.line_initial.replace(' ', '') 
                        
                    if self.line_initial.startswith(':re:'):
                        self.re_pattern = self._re_pattern.sub('', self.line_initial)
                        self.line_initial = self._re_pattern.match(self.line_initial).group()

                    if self.line_initial.startswith(':') and not (self.line_initial.startswith(':re:') or self.line_initial.startswith(':all:')):
                        header_end += 1
                    else:
                        self.rule_id = i - header_end + 1

                        
                except ValueError:
                    print line
                    print self.filename
                    raise
                
                if len(split) == 3:
                    self.line_priority = int(split[2])
                else:
                    self.line_priority = 1

                try:
                    if self.lookup is not None:
                        if not self.lookup.validate(self.line_initial, self.line_value, filename):
                            continue
                    
                    if self.line_initial in self.rule_generator_map:
                        result = re.search(r'\W+snttextx\W+([a-z]{2})\W+', filename)
                        self.rule_generator_map.get(self.line_initial)((line, segment, result.group(1) if result else None))
                    else:
                        self.rules[self.line_priority][self.line_initial] = (Rule(self.line_initial.lower(), self.line_value, self.rule_id))
                
                except sre_constants.error as e:
                    print "Skipping regular expression rule. Check regex_init log for more details."
                    self.log.write('regex_init', [e, filename, self.line_initial, self.line_value, self.line_priority], 91, no_add_info=True)
         
    def _validate_vars(self, vars, segment):
        for v in vars:
            try:
                assert(segment.var_dict.get(v))
            except AssertionError:
                raise RuleError('Invalid variable for crosswalk: %s' % v)
    
    def _validate_regex_rules(self):
        patterns = dict()

        for regex_rule in sorted(self.reg_rules, key=lambda rule: rule.priority, reverse=True):
            #validate result matches variables
            
            if len(self.metadata.out_vars) != len(regex_rule.result.split('|')) and 'remove' not in self.filename:
                self.log.write('regex_init', ['error:result does not match out_variables', self.filename, regex_rule.regex.pattern, regex_rule.result, regex_rule.priority, self.metadata.out_vars], 91, no_add_info=True)
            
            #math rules validate the same number of parans
            if isinstance(regex_rule, RegexMathRule):
                if len(re.findall('\(', regex_rule.result)) != len(re.findall('\)', regex_rule.result)):
                    self.log.write('regex_init', ['error:Syntax Error, unbalanced parentheses', self.filename, regex_rule.regex.pattern, regex_rule.result, regex_rule.priority], 91, no_add_info=True)
                
                if re.search('#[ ]*(?!\d+)', regex_rule.result):
                    self.log.write('regex_init', ['error:Syntax Error, repeating digit group (#) not followed by number reference', self.filename, regex_rule.regex.pattern, regex_rule.result, regex_rule.priority], 91, no_add_info=True)
                
                _digit_group = [int(x) for x in re.findall('#[ ]*(\d+)', regex_rule.result)] or [0]
                if int(max(_digit_group)) > len(re.findall('%s|%s' % (re.escape('(\d+\.\d+|\.\d+|\d+|#\d+|#)'), re.escape('(\d+)')), regex_rule.regex.pattern)):
                    self.log.write('regex_init', ['error:Syntax Error, formula number index out of range', self.filename, regex_rule.regex.pattern, regex_rule.result, regex_rule.priority], 91, no_add_info=True)
                
                #validate that the result does not have 4# instead of #4
                if re.search('\d#', regex_rule.result):
                    self.log.write('regex_init', ['potential error:Syntax Error, formula number index before number', self.filename, regex_rule.regex.pattern, regex_rule.result, regex_rule.priority], 91, no_add_info=True)
                    
            #validate rule logic
            if regex_rule.type in ('exact', 'tilde') and regex_rule.escape:
                continue
            
            for pattern, regex in patterns.iteritems():
                if regex_rule.regex.pattern in pattern:
                    if regex[0] != regex_rule.result.replace(' ', ''):
                        err = '%serror:inconsistent result' % ('potential ' if regex_rule.type in ('exact', 'tilde', 'match') else '')
                        self.log.write('regex_init', [err, self.filename, regex_rule.regex.pattern, regex_rule.result.replace(' ', ''), regex_rule.priority, pattern, regex[0], regex[1]], 91, no_add_info=True)
                    else:
                        self.log.write('regex_init', ['warning:redundent rule', self.filename, regex_rule.regex.pattern, regex_rule.result.replace(' ', ''), regex_rule.priority, pattern, regex[0], regex[1]], 91, no_add_info=True)
                
            patterns[regex_rule.regex.pattern] = (regex_rule.result.replace(' ', ''), regex_rule.priority)
    
    def validate(self):
        '''Validate the ruleset.'''
        if config.debug_validate_regex_rules:
           self._validate_regex_rules() 

    def _convert(self, rule, value, **kw):
        """Helper function for convert"""
        if rule.type == 'all':
            kw['value'] = value
            
        out_value = rule.convert(**kw)
        out = []
        if 'split' in kw:
            for val, var in izip(out_value, self.vars):
                if var[1] in FORMAT_DICT:
                    if is_none(val):
                        o = None
                    elif isinstance(val, str) and val.startswith('${') and val.endswith('}'):
                        o = val
                    else:
                        if var[1] == 'float':
                            try:
                                #hacky fix to deal with precision without transforming to a decimal (I think this is faster)
                                o = round(round(FORMAT_DICT[var[1]](val), var[3]+1), var[3])
                            except ValueError:
                                raise ConvertError("Unable to convert. Check rule regex: %s" % rule.regex.pattern)
                        else:
                            if var[1] == 'integer' and isinstance(val, str) and val.endswith('.0'):
                                val = val.split('.')[0]
                            o = FORMAT_DICT[var[1]](val)

                    out.append(o)
                    
                else:
                    raise ConvertError('1.  Unable to convert to %s' % self.format)

            if kw.get('audit', False):
                self.log.write('audit', [kw.get('case_id', 0), value.replace('\\', ''), '|'.join([var[0] for var in self.vars]) if len(self.vars) > 1 else self.vars[0][0], self.crosswalk_id, rule.id, rule.regex.pattern if hasattr(rule, 'regex') else rule.initial.replace('\\', '')], 79)
            return out
        
        if rule.return_type == 'match':
            if kw.get('audit', False):
                self.log.write('audit', [kw.get('case_id', 0), value.replace('\\', ''), '|'.join([var[0] for var in self.vars]) if len(self.vars) > 1 else self.vars[0][0], self.crosswalk_id, rule.id, rule.regex.pattern if hasattr(rule, 'regex') else rule.initial.replace('\\', '')], 79)
            return out_value

        if self.vars[0][1] in FORMAT_DICT:
            if is_none(out_value):
                return None
            else:
                #convert to correct format
                if self.vars[0][1] == 'string':
                    if kw.get('audit', False):
                        self.log.write('audit', [kw.get('case_id', 0), value.replace('\\', ''), '|'.join([var[0] for var in self.vars]) if len(self.vars) > 1 else self.vars[0][0], self.crosswalk_id, rule.id,
                                                 rule.regex.pattern if hasattr(rule, 'regex') else rule.initial.replace('\\', '')], 79)
                    return out_value
                elif self.vars[0][1] == 'float':
                    #hacky fix to deal with precision without transforming to a decimal (I think this is faster)
                    if kw.get('audit', False):
                        self.log.write('audit', [kw.get('case_id', 0), value.replace('\\', ''), '|'.join([var[0] for var in self.vars]) if len(self.vars) > 1 else self.vars[0][0], self.crosswalk_id, rule.id,
                                                 rule.regex.pattern if hasattr(rule, 'regex') else rule.initial.replace('\\', '')], 79)
                    return round(round(FORMAT_DICT[self.vars[0][1]](out_value), self.vars[0][3]+1), self.vars[0][3]) 
                else:
                    if kw.get('audit', False):
                        self.log.write('audit', [kw.get('case_id', 0), value.replace('\\', ''), '|'.join([var[0] for var in self.vars]) if len(self.vars) > 1 else self.vars[0][0], self.crosswalk_id, rule.id,
                                                 rule.regex.pattern if hasattr(rule, 'regex') else rule.initial.replace('\\', '')], 79)
                    return FORMAT_DICT[self.vars[0][1]](out_value)
                
        else:
            raise ConvertTypeError('Unable to convert to %s' % self.format)
      
    def convert(self, key, **kw):
        """
        Returns a converted value based on key or raises a ConvertError
        if key can not be converted.
        
        Looks to see if there is a rule that can handle a given key.
        Split keyword returns the converted result as a list of values,
        using the value of the split keyword as the delimiter for the result
        list.
        """
        
        #check to see if :all: rule is present
        if not self._has_all:
            str_key = None if key is None else str(key)
        else:
            str_key = ':all:'
            
        for rule_group in self.rules.itervalues():
            if str_key in rule_group:
                return self._convert(rule_group[str_key], str_key, **kw)
        
        if self.reg_rules: #if key not found in rules dict then see if regex rule can handle the key
            for rule in sorted(self.reg_rules, key=lambda rule: rule.priority):
                if rule.handle(str_key):
                    if kw.get('remove'):
                        return (self._convert(rule, str_key, **kw), rule.match.group())
                    else:
                        return self._convert(rule, str_key, **kw)
        
        raise ConvertError('2.  Unable to convert to %s' % key)
    
    def delete_transform(self, key, count=0, **kw):
        """Transform the given key by applying the rules in the crosswalk.
        If a rule can handle the key it will remove the matched text from the key. 
        After the rules are applied the transformed key will be returned. 
        
        The optional argument of **count** is the maximum number of times a transform
        will be applied to the key. By default (count=0) all rules will be checked to see
        if they can handle the key. If count is specified than the crosswalk will 
        incrememnt a counter for each time a rule handles the key. Once the counter = count 
        the crosswalk will return the transformed key.
        """
        if key is None or key == '':
            return key
        
        #initialize a counter
        _count = icount().next

        _key = key
        
        #check the exact match non-regular expression rules and remove the key
        for rule_group in self.rules.itervalues():
            if _key in rule_group:
                #we ignore count here because if a match is found the entire string will be replaced
                return ''
        
        #if key not found in rules dict then see if regex rule can handle the key
        if self.reg_rules: 
            for rule in sorted(self.reg_rules, key=lambda rule: rule.priority):
                if count:
                    if _count() == count:
                        return _key
                  
                _key = rule.delete_transform(_key)
                
            return _key
        
        return key
        