"""
This module is responsible for creating the business rules for the CCHR 
conversion system. 

Exported Classes:

Exported Methods:

parse_rules -- This method is called during the stage and process phase to 
create the business rules for that phase. 

Exceptions:

RuleParsingError

ConvertError -- This error is raised when a value we are trying to convert 
can not be handled by a Crosswalk object.

DataError 

FatalDataError
 
"""

import os, imp, sys
from itertools import count, izip
from datetime import datetime, date
from time import strptime
from itertools import izip
from sys import exc_info
import traceback

import cardsharp as cs
import log
from errors import *
from rules import *
from util import *
from configuration import config
from offense_rules import OffenseRules
from sentence_rules import SentenceRules
from supervision_rules import SupervisionRules
from admrec_rules import AdmrecRules
from region import *
from util import regions as util_regions
from util import fill_vars
from metadata import *
import re
from segment import segment_dict
from contextlib import closing
import MySQLdb
from collections import defaultdict

_count = count().next

def mod_counter():
    return 'bjs_' + str(_count())

#TODO rework log so it is created outside of VC
class VariableConverter(object):
    def __init__(self, time, segment, var, phase, **kw):
        self.func = None
        self.name = var
        self.id_count = None
        self.regions = dict()
        self.region_order = []
        self.format_setters = set()
        self.halt_on_error = kw.get('halt_on_error', False)
        self.var = var
        self.log = log.Log(time=time, segment=segment, var=var, phase=phase, log_dir=kw.get('log_dir'))
        self.options = kw
        
    def add_region(self, region, mod, crosswalk):
        if mod:
            self.format_setters.add(mod)
            
        if crosswalk:
            for key in crosswalk.iterkeys():
                self.format_setters.add(crosswalk[key])

        self.options.update({'region': region.get_state()[0]})

        kw = self.options
        
        self.regions[region] = _get_region_func(self.log, self.var, mod, crosswalk, self.halt_on_error, **kw)
        self.region_order.append(region)
        self.region_order.sort(key=lambda r: r.priority())
        
    def create_func(self, mod, crosswalk, add_info):
        if mod:
            self.format_setters.add(mod)
            
        if crosswalk:
            for key in crosswalk.iterkeys():
                self.format_setters.add(crosswalk[key])

        #this is a hack to make admrecs not load everything as wvfbi rule
        self.options['base_admrec'] = True
        self.func = _get_region_func(self.log, self.var, mod, crosswalk, self.halt_on_error, add_info, **self.options)
    
    def set_format(self, format):
        for rs in self.format_setters:
            rs.format = format
    
    def set_auto_id(self, start_value=1):
        #TODO put this rule in loader
        """This modules sets the translate function to pass an id value to the conversion function. The first id
        will start at **start_value**, which has a default value of 0, and will increment by one for each row.""" 
        self.id_count = count(start_value).next
    
    def translate(self, state_var, value, row):
        try:
            #if regions are specified then loop over them and return the region specfic function
            for region in self.region_order:
                #TODO pass in map of rules to get 1 iteration over dataset
                if row[state_var].lower() in region.get_state():
                     if self.id_count is not None:
                         return self.regions[region](self.id_count())  
                     else:
                         return self.regions[region](value, row)
                    
            
            #no regions are specified than return global function
            if self.id_count is not None:
                return self.func(self.id_count())     
            else:
                return self.func(value, row)
            
        except:
            if config.debug_halt_on_error:
                raise
            else:
                self.log.write('fatal_error', ([str(exc_info()[:2]), row]), 99) #traceback.format_exc()
                 
def _get_region_func(log, var, mod, crosswalk, error, add_info=None, **kw):
    if mod and crosswalk:
        mod.crosswalk = crosswalk
            
    if mod:
        #pass objects to mod
        off_rules = OffenseRules(log, kw.get('offense_code_pkl', ''))
        
        sentence_rules = SentenceRules(log)
        supervision_rules = SupervisionRules(log)
        mod.log = log
        mod.ConvertError = ConvertError
        mod.RuleError = cs.rules.RuleError
        mod.DataError = DataError
        mod.FatalDataError = FatalDataError
        mod.halt_on_error = error
        mod.regions = regions
        mod.key_as_str = key_as_str
        mod.get_state_key = get_state_key
        mod.default_offense_codes = off_rules.missing_codes
        mod.o_vars = off_rules.vars
        mod.o_default_map = off_rules.missing_map
        mod.offense_not_coded = off_rules.not_coded_map
        mod.offense_pkl_dump = off_rules.pickle_dump
        mod.convert_inc = off_rules.convert_inc
        mod.convert_offenses = off_rules.convert_offenses
        mod.convert_sentences = sentence_rules.convert_sentences
        mod.sentence_missing = sentence_rules.missing
        mod.sentence_unknown = sentence_rules.unknown
        mod.sentence_rules = sentence_rules
        mod.supervision_rules = supervision_rules
        mod.re = re
        mod.izip = izip
        mod.date = date
        mod.strptime = strptime
        mod.add_info=add_info
        mod.math_rule = RegexMathRule
        mod.datetime = datetime
        mod.ma_duplicate = defaultdict(int)
        mod.current_set = config.current_set
        #TODO use regions instead of get_state_key
        mod.regions = util_regions
        mod.opt = kw
        mod.get_cnxn = get_cnxn
        
        
        #=======================================================================
        # Define the function which the variable converter will return to the
        # Cardsharp task
        #=======================================================================
        
        if var in ['ancic', 'cncic', 'pncic'] and kw.get('region'):
            if var[0] == 'a':
                _seg = 'arrest'
            elif var[0] == 'c':
                _seg = 'sentence'
            else:
                _seg = 'prosecution'
            om = OffenseMetadata(var, _seg, kw.get('region'))
            #add a prediction rules attribute for the offense rules
            mod.offense_rules = om.offense_map
            
        
        if hasattr(mod, 'convert'):
            def func(value, row):
                try:
                    return mod.convert(value, row)
                #handle type excpetion for easier debugging
                except TypeError as err:
                    print traceback.print_exc(file=sys.stdout)
                    raise ConvertError("""Unable to call convert on var %s. for id:%s with value:%s. error was: %s""" % (var, row['id'], value, err))
                except Exception as e:
                    print traceback.print_exc(file=sys.stdout)
                    raise e
            return func
        
        elif hasattr(mod, 'assign_vals'):
            def func(value, row):
                #TODO:snt_processor is deprecated remove from all modules
                #snt_processor = SNTParser(row, mod.log)
                return mod.assign_vals(value, row, None)
            
            return func
        
        elif hasattr(mod, 'translate'):
            def func(value, row):
                return mod.translate(value)
            
            return func
        
        elif hasattr(mod, 'assign_id'):
            def func(value):
                return mod.assign_id(value)
            
            return func
        
        elif hasattr(mod, 'validate'):
            return mod.validate
        
        elif hasattr(mod, 'cache'):
            def func(value, row):
                return mod.cache(row)
            
            return func
        
    elif crosswalk:
        if var in ['ancic', 'cncic']:
            def func(value, row):
                vars = convert_offenses(row, crosswalk, offense_rules)
                
                for k, v in vars.iteritems():
                    row[var[0]+k] = v
                    
                return vars['ncic']
            
        #nonarr rules
        elif var == 'nonarr':
            admrec_meta = AdmrecMetadata(var, 'arrest', kw.get('region') if not kw.get('base_admrec') else None)
            admrec_rules = AdmrecRules(log, 'arrest', crosswalk, admrec_meta.admrec_map)
            
            def func(value, row):
                try:
                    return admrec_rules.convert_admrec(row)
                except Exception as e:
                    print traceback.print_exc(file=sys.stdout)
                    raise e
            
        #noncrt rules
        elif var == 'noncrt':
            admrec_meta = AdmrecMetadata(var, 'sentence', kw.get('region') if not kw.get('base_admrec') else None)
            admrec_rules = AdmrecRules(log, 'court', crosswalk, admrec_meta.admrec_map)
            def func(value, row):
                return admrec_rules.convert_admrec(row)

        elif var == 'nonprs':
            admrec_meta = AdmrecMetadata(var, 'prosecution', kw.get('region') if not kw.get('base_admrec') else None)
            admrec_rules = AdmrecRules(log, 'prosecution', crosswalk, admrec_meta.admrec_map)
            def func(value, row):
                return admrec_rules.convert_admrec(row)
        
        else:
            def func(key, value, row):
                return crosswalk[key].convert(value)
             
        return func
        
    else:
        raise RuleParsingError('error creating rule %s: kw:%s' % (var,str(kw)))

def _get_where_state_from_id(segment_name, **opt):
    if 'id' in opt['in_where']:
        where_ids = re.findall('id=(\d+)', opt['in_where'])
    
    where_states = set()
    try:
        with closing(get_cnxn(opt['db_info']['name'],opt)) as cnx:
            with closing(cnx.cursor()) as c:
                for _id in where_ids:
                    c.execute("""SELECT rapstatex 
                                 FROM %sx 
                                 WHERE id=%s""" % (
                                    segment_name, 
                                    _id)
                             )
                    
                    where_states.update([get_state_key(c.fetchone()[0])])
    
    except TypeError:
        #we did not find the id in the dataset so return an empty set
        return set()

    return where_states
  
_crosswalk_region_re = re.compile('crosswalk_([a-zA-Z]{2})(\.txt|_\d+.txt)')

def _skip_crosswalk(where_regions, filename, verbose, region=None):
    if where_regions and region == 'wvfbi':
        crosswalk_region = _crosswalk_region_re.search(filename)

        if crosswalk_region:
            skip = True
            for r in where_regions:
                if crosswalk_region.groups()[0] == get_state_name(r):
                    skip = False
                elif get_state_name(r) == 'ne' and crosswalk_region.groups()[0] == 'nb':
                    skip = False
            if skip:
                if verbose:
                    print 'skipping crosswalk %s' % filename
                return True
            
    return False
         
def parse_rules(rule_dir, segment, **opt):
    db_info = opt['db_info']
    phase = opt['phase_id']
    segment_objs = opt['segments']
    halt_on_error = opt['halt_on_err']
    verbose = opt['verbose']
    meta_dir = opt['meta_dir']
    prefix = ''
    xwalk_kw = {}
    
    #opt['segment_name'] = segment['label']
    
    #TODO: change db_info['name'] to 'filename' to allow db_info to be passed as keyword
    time = datetime.now().isoformat('_').replace(':', '-')
    _var_map, vars = {}, {}

    variables = cs.load(source=os.path.join(meta_dir, segment['label'], 'var_info.xls'), 
                        dataset='vars', format='excel')
    cs.wait()
   
    for row in variables:
        _var_map[row['name'].lower()] = (row['name'], row['format'])
    
    additional_data = {}
    if re.search('pre_process_rules_3', rule_dir) and segment['label'] in ('sentence', 'prosecution', 'supervision') and opt['phase_id'] != '3':
        print 'loading cfed data...'
        additional_data = {}
        #TODO change this into mssql query to avoid slowness of cardsharp
        options = {'source': db_info['name'], 'dataset': 'arrestx', 'format':'mysql',
                'user': db_info['user'], 'pwd': db_info['pass'], 'load_as_null':[''],
                'select': ['casenum', 'cycnum', 'afed', 'astate']}
        if opt.get('in_where'):
            options['where'] = opt.get('in_where')
        
        ds = cs.load(**options) 
        cs.wait()
        for row in ds:
            additional_data['%s|%s' % (row['casenum'], row['cycnum'])] = (row['afed'], row['astate'])
            
        cs.wait()
        print 'complete.'
    

    #for each segment
    #loop over segment directories (arrest, demographic, sentence, supervision)
    
    #set where_regions
    where_regions = set()
    
    if opt.get('in_where'):
        where_regions = set(re.findall('state[ ]*=[ ]*(\d+)', opt['in_where']))
        
        if re.search('id[ ]*=[ ]*',opt['in_where']):
            _r = _get_where_state_from_id(segment['label'], **opt)
            if _r and 'wvfbi' not in _r: where_regions.update(_r) #do not remove wvfbi crosswalks
              
    for segment_dir in os.listdir(rule_dir):
        if verbose:
            print 'SEGMENT_DIR: %s' % segment_dir

        segment_name = segment_dir.lower()
        segment_dir = os.path.join(rule_dir, segment_dir)

        if os.path.isfile(segment_dir):
            continue # if file than not a segment directory, check next

        elif segment_name != segment['label']:
            continue # only load the segment we are runing rules for
        
        # for each variable
        # loop over all variable folders within a segment
        for var_dir in os.listdir(segment_dir):
            if verbose:
                print 'VAR_DIR: %s' % var_dir
            # if include rules are specified skip out of variables not in the include rules
            if opt.get('include_rules') and var_dir.lower() not in opt['include_rules']:
                continue
            
            # if exclude rules are specified skip out of variables in the exclude rules
            if opt.get('exclude_rules') and var_dir.lower() in opt['exclude_rules']:
                continue
            
            var_name = var_dir.lower()
            var_dir = os.path.join(segment_dir, var_dir)
            
            if os.path.isfile(var_dir):
                continue # if file than not a variable directory, check next
            
            # create a variable converter
            vc = VariableConverter(time, segment_dict[segment_name]['id'], _var_map[var_name][0], phase, **opt)
            vc_mod = None
            vc_crosswalk = dict()
            
            #for each rule or region
            #loop over all files in the variable directory
            for filename in os.listdir(var_dir):
                if verbose:
                    print 'FILE: %s' % filename

                full_path = os.path.join(var_dir, filename)
                
                #load crosswalk rule
                if filename.startswith('crosswalk'):
                    if _skip_crosswalk(where_regions, verbose, filename):
                        continue
                    
                    if verbose:
                        print 'Loading crosswalk for %s' % filename
                    if filename[len(filename) - 4:len(filename)].lower() != '.txt':
                        raise RuleParsingError('file type must be .txt not %s' % filename[len(filename) - 4:len(filename)])

                    crosswalk_id = filename[10:len(filename) - 4].lower()
                    vc_crosswalk[crosswalk_id] = Crosswalk(full_path, db_info, segment_objs[segment_name], vc.log, crosswalk_id)
                
                #load func rule
                elif filename == 'func.py':
                    if verbose:
                        print 'Loading functions for %s' % var_name
                    vc_mod = imp.load_source(mod_counter(), full_path)
                elif filename.endswith('.pyc'):
                    pass                               
                #load region rules
                elif os.path.isdir(full_path):
                    region = get_region(filename)
                    
                    crosswalk = dict()
                    mod = None
                    
                    #TODO make a state_where object that contains region names
                    #check the where to see if we should only load state specific rules

                    if where_regions:
                        skip_region = True
                        for r in where_regions:
                            _state = get_state_key(r)
                            if _state in region.state:
                                skip_region = False
                            elif _state == 'ne' and 'nb' in region.state:
                                skip_region = False
                        
                        if get_state_key('wvfbi') not in region.state and skip_region:
                            if verbose:
                                print 'skipping %s' % region
                            continue 
                        
                    #for each rule
                    for filename in os.listdir(full_path):
                        file_path = os.path.join(full_path, filename)
                        filename = filename.lower()
                        #if we are in a wvfbi state
                        #load region crosswalk
                        if filename.startswith('crosswalk'):
                            if _skip_crosswalk(where_regions, filename, verbose, region.state[0]):
                                continue
                                    
                            if verbose:
                                print 'Loading crosswalk for %s' % var_name
                            if filename[len(filename) - 4:len(filename)].lower() != '.txt':
                                raise RuleParsingError('file type must be .txt not %s' % 
                                                       filename[len(filename) - 4:len(filename)])
                            
                            #TODO get this working to handle predictions for no space crosswalks
#                            if 'ncic' in var_dir and not re.search('pred|inc'): #create prediction crosswalk using current crosswalk w/o spaces
#                                prefix = 'pred_'
#                                xwalk_kw['prediction'] = True
#                                name = ''.join(['pred_', filename[10:len(filename) - 4].lower()])
#                                crosswalk[name] = Crosswalk(file_path, db_info, segment_objs[segment_name], vc.log, **{'prediction':True})
                            crosswalk_id = filename[10:len(filename) - 4].lower()
                            crosswalk[crosswalk_id] = Crosswalk(file_path, db_info, segment_objs[segment_name], vc.log, crosswalk_id)
                        #load func rule
                        elif filename == 'func.py':
                            if verbose:
                                print 'Loading functions for %s:%s' % (var_name, region)
                            mod = imp.load_source(mod_counter(), file_path)
                        elif filename.endswith('.pyc'):
                            pass
                        else:
                            raise RuleParsingError('Region directory %s for var %s can only contain "crosswalk.txt", or "func.py" files.  Found: %s' % (region, var_name, filename))
                    
                    #add region rules to variable converter
                    #pass segment_name for log output
                    #region used for log and to store region func / crosswalk
                    #func is the function to be associated with the  converter
                    #crosswalk is the crosswalk to be used by the  converter (in addition to func or standalone)
                    vc.add_region(region, mod, crosswalk)
                else:
                    raise RuleParsingError(' directory %s can only contain region directories, "func.py" files, or "crosswalk.txt" files.  Found: %s' % (var_name, filename))
           
            #create variable rule
            vc.create_func(vc_mod, vc_crosswalk, additional_data)
            
            vars[(segment_name, var_name)] = vc
    
    return vars