#python
import os, re
from collections import OrderedDict, defaultdict
from contextlib import closing
#third-party
import cardsharp as cs
#src
from ..configuration import config
from ..segment import Segment
from ..errors import LoadError
from ..util import copy_to_review
from pandas import read_csv

__all__ = ['set_varcodes', 'get_variables', 'get_demo_variables', 'set_variables', 'load_text', 'load_spss', 'get_variable_metadata']

def set_varcodes(name, varcodes=defaultdict(dict)):
    seg = Segment(name, 'load_recid_flat', **{'meta_dir':config.meta_dir, 'verbose':False})
    seg.get_metadata('flat' if name == 'demographic' else 'var_info')
    
    for var in seg.metadata.keys():
        if not seg.metadata[var]['value_labels']: continue
        for label in seg.metadata[var]['value_labels']:
            lbl_txt = label[1].lower() 
            if lbl_txt == "not applicable": 
                varcodes[var]['na'] = label[0]
            elif lbl_txt == "missing" or 'unknown/no' in lbl_txt:
                varcodes[var]['missing'] = label[0]
            
        if not varcodes[var].get('missing'):                    
            varcodes[var]['missing'] = int('9' * int(seg.metadata[var]['length']))
        if not varcodes[var].get('na'):
            varcodes[var]['na'] = 8 if seg.metadata[var]['length'] == 1 else int('9' * int(seg.metadata[var]['length'] - 1) + '8')
    
    return varcodes


def get_variables(max_cycles):
    vars = ['casenum']
    for i in range(1, max_cycles + 1): 
        cycstr = (3-len(str(i)))*"0" + str(i)
        vars.extend(['A%sYR' % cycstr, 'A%sMO' % cycstr, 'A%sDA' % cycstr, 'A%sST' % cycstr, 'A%sTOT' % cycstr])
        for j in range(1, 4):
            vars.extend(['A%sOFF%s' % (cycstr,str(j)),'A%sOFD%s' % (cycstr,str(j)),'A%sCNT%s' % (cycstr,str(j))])
            vars.extend(['A%sFED%s' % (cycstr,str(j)),'A%sSVR%s' % (cycstr,str(j)),'A%sDSP%s' % (cycstr,str(j))])
            vars.extend(['A%sDOM%s' % (cycstr,str(j)),'A%sWPN%s' % (cycstr,str(j)),'A%sCDV%s' % (cycstr,str(j))])
            
    for i in range(1, max_cycles + 1): 
        cycstr = (3-len(str(i)))*"0" + str(i)
        vars.extend(['J%sYR' % cycstr, 'J%sMO' % cycstr, 'J%sDA' % cycstr, 'J%sST' % cycstr, 'J%sTOT' % cycstr])
        for j in range(1, 4):
            vars.extend(['J%sOFF%s' % (cycstr,str(j)),'J%sOFD%s' % (cycstr,str(j)),'J%sCNT%s' % (cycstr,str(j))])
            vars.extend(['J%sFED%s' % (cycstr,str(j)),'J%sSVR%s' % (cycstr,str(j)),'J%sDSP%s' % (cycstr,str(j))])
            vars.extend(['J%sDOM%s' % (cycstr,str(j)),'J%sWPN%s' % (cycstr,str(j)),'J%sCDV%s' % (cycstr,str(j))])
            vars.extend(['J%sPJP%s' % (cycstr,str(j)),'J%sCNF%s' % (cycstr,str(j)),'J%sPMX%s' % (cycstr,str(j)),'J%sPRB%s' % (cycstr,str(j))])
            
    return vars

def get_demo_variables(max_profiles):
    vars = ['casenum']
    
    for p in xrange(1, max_profiles+1):
        _vars = []
        for x in ['dstate', 'sex', 'race', 'hispanic', 'plcbth', 'ctz', 'dead']:
            _vars.append('%s%i' % (x,p))
        
        vars.extend(_vars)
    
    return vars
def get_variable_metadata(segment):
    path = os.path.join(config.meta_dir,segment, 'var_info.xls')
    _var_meta_ds = cs.load(source=path, format='excel', dataset='vars')
    _vm = OrderedDict()
    
    #create variable metadata mapping
    for row in _var_meta_ds:
        _vm[row['name']] = (row['format'], row['length'], row['precision'])
        
    return _vm
        
def set_variables(dataset, var_names, var_meta):
    for var_name in var_names:
        #set the format, length, precision
        _var_name = ''.join([var_name[0], var_name[4:]]) if var_name != 'casenum' else var_name
        
        print dataset.variables[var_name]
        print var_meta[_var_name]
        print _var_name
        print var_name
        
        dataset.variables[var_name].convert(var_meta[_var_name][0])
        dataset.variables[var_name].rules.length = int(var_meta[_var_name][1].replace('.0', ''))
        
        if var_meta[_var_name][2]:
            dataset.variables[var_name].rules.precision = int(var_meta[_var_name][2].replace('.0', ''))
    
     
def load_text(chflat, chflat_var_names, position, chsum, chsum_var_names):
    from threads import WriteThread
    #start a write thread
    segment = 'flat'
    var_meta = get_variable_metadata(segment)
    
    t = WriteThread('flat', chflat, 
                    'crim_hist_%s_%s.sav' % (segment, position), 
                    chflat_var_names)
    t.setDaemon(True)
    t.start()
    
    segment = 'summary'
    var_meta = get_variable_metadata(segment)
    
    t = WriteThread('summary', chsum, 
                    'crim_hist_%s_%s.sav' % (segment, position), 
                    chsum_var_names)
    t.setDaemon(True)
    t.start()
    #let the threads do their work
    #chflat.join()
    #chsum.join()

def _save_sps(labels, value_labels, position, path, filename):
    print 'Saving spss file #%i...' % position
    labels = '%s.' % labels 
    value_labels = '%s.' % value_labels
    syntax = '%s \n %s \n EXECUTE.' % (labels, value_labels)
    save_path = os.path.join(path, 'criminal_history_%s_%s.sps' % (filename, position))
    
    syntax_file = open(save_path, 'wb')
    syntax_file.write(syntax.encode('ascii', 'xmlcharrefreplace'))
    syntax_file.close()

def _get_var_name(filename, var_name, cycle):
    #get the variable name
    if filename != 'demographic':
        return ''.join([var_name[0], (3-len(cycle))*"0" + cycle, var_name[1:]])
    else:
        return ''.join([var_name, cycle])

class LoadSyntax(object):
    def __init__(self, name, in_path, out_path):
        self.name = 'criminal_history_flat' if name != 'demographic' else 'demographic_flat'
        self.out = open(os.path.join(out_path, 'load_%s.sps' % self.name), 'wb')
        self.out.write('\r\n '.join(['GET DATA', 
                                     '/TYPE=TXT', 
                                     '/FILE="%s"' % in_path,
                                     '/DELCASE=LINE',
                                     '/DELIMITERS="|"',
                                     '/ARRANGEMENT=DELIMITED',
                                     '/FIRSTCASE=2',
                                     '/IMPORTCASE=ALL',
                                     '/VARIABLES=']))


    def add_var(self, name, meta):
        format = meta['format'].lower().strip()
        precision = str(meta.get('precision', ''))
        length = str(meta['length'])
        
        if format == 'string':
            _f = ''
        elif format == 'float':
            _f = '.%s' % precision
        else:
            _f = '.0'
        
        self.out.write('\r\n%s %s%s%s' % (name, 
                                          'F' if format != 'string' else 'A', 
                                          length, 
                                          _f))
        
    def close(self):
        self.out.write('\r\n'.join([
                            '.', 
                            'CACHE.', 
                            'EXECUTE.', 
                            'DATASET NAME %s WINDOW=FRONT.' % self.name]))
        self.out.close()
        

# def _combine_raw(path, combined_path, sort=True):
#     print '...combining files...'
#     if os.path.isfile(combined_path):
#         os.remove(combined_path)
#
#     file_c = 0
#     out_vars = None
#     combined_data = []
#     out_file = open(combined_path, 'a')
#
#     for file in os.listdir(path):
#         #skip directories, files not ending in .txt, and combined file
#         if not file.endswith('.txt') or os.path.isdir(os.path.join(path, file)) or 'combined' in file:
#             continue
#
#         with closing(open(os.path.join(path, file), 'rb')) as in_file:
#             first_line = True
#             for line in in_file:
#                 line = line.replace('\n', '')
#
#                 #get header information
#                 if first_line:
#                     #if first file than define output variables
#                     if not out_vars:
#                         if sort:
#                             out_vars = line
#                         else:
#                             out_file.write(line)
#
#                     #validate that previous file and current file have same header information
#                     elif out_vars != line:
#                         raise LoadError("Error combining thread output files. Headers do not match. %s vs %s" % (out_vars, line))
#
#                     first_line = False
#
#                 else:
#                     if sort:
#                         combined_data.append(line.split('|'))
#                     else:
#                         out_file.write(line)
#
#             file_c += 1
#
#     #sort the combined data by 1st variable (casenum for summary and id for crim hist flat) and output results to txt file
#     if sort:
#         out_file.write(out_vars)
#         for line in sorted(combined_data, key=lambda k: int(k[0])):
#             out_file.write('|'.join(line))
#
#     out_file.close()

def _combine_raw(path, combined_path, sort=True):
    print '...combining files...'
    if os.path.isfile(combined_path):
        os.remove(combined_path)

    combined_data = None

    for file in os.listdir(path):
        # skip directories, files not ending in .txt, and combined file
        total_path = os.path.join(path, file)
        print '...reading file %s...' % total_path
        if not file.endswith('.txt') or os.path.isdir(total_path) or 'combined' in file:
            continue

        na_filter = True if 'demographic' not in total_path else False
        if combined_data is None:
            combined_data = read_csv(total_path, sep='|', low_memory=False, na_filter=na_filter)
        else:
            combined_data = combined_data.append(read_csv(total_path, sep='|', low_memory=False, na_filter=na_filter), ignore_index=True)

    # sort the combined data by 1st variable (casenum for summary and id for crim hist flat) and output results to txt file
    if sort:
        print "...sorting data..."
        combined_data.sort_values(list(combined_data.columns)[0], inplace=True)

    print '...writing combined output...'
    combined_data.to_csv(combined_path, sep='|', index=False)
    
class LoadSPSS(object):
    def __init__(self, name, var_metadata, max_cycle):
        self.name = name
        self.max_cycle = max_cycle
        self.meta = var_metadata
    
    def load_data(self, path):
        self.ds = cs.load(source=path, format='text', delimiter='|', line_delimiter='\r')
    
    
    def add_demographic_metadata(self):
        for i in range(1, self.max_cycle+1):
            for var_name, meta in self.meta.iteritems():
                if var_name == 'casenum':
                    if i == 1:
                        if meta['value_labels']:
                            self.ds.variables[var_name].value_labels = [(k.strip() if isinstance(k, str) else k, v.strip() if isinstance(v, str) else v) for k,v in meta['value_labels']] 
                        self.ds.variables[var_name].label = meta['label']
                        self.ds.variables[var_name].convert(meta['format'])
                    continue
                
                _v = '%s%s' % (var_name, i) 
                if meta['value_labels']:
                    self.ds.variables[_v].value_labels = [(k.strip() if isinstance(k, str) else k, v.strip() if isinstance(v, str) else v) for k,v in meta['value_labels']]
                self.ds.variables[_v].label = 'Profile %i: %s' % (i, meta['label'])
                self.ds.variables[_v].convert(meta['format'])
                

    def add_summary_metadata(self):
        for var_name, meta in self.meta.iteritems():
            var_name=var_name.lower().strip()
            if meta['value_labels']:
                self.ds.variables[var_name].value_labels = [(k.strip() if isinstance(k, str) else k, v.strip() if isinstance(v, str) else v) for k,v in meta['value_labels']] 
            self.ds.variables[var_name].label = meta['label']
            self.ds.variables[var_name].convert(meta['format'])
            
    def save(self, path):
        _name = 'criminal_history_summary.sav' if self.name == 'summary' else '%s_flat.sav' % self.name 
        save_path = os.path.join(path, _name)
        
        self.ds.save(source=save_path, format='spss', overwrite=True)
        cs.wait()
        #copy the saved file to review after save
        copy_to_review(self.name, '', save_path)
        print '%s save complete.' % _name
        
def load_spss(filename, path, sort=True, max_cycles=None):
    seg = Segment(filename, 'load_recid_flat', 
                  **{'meta_dir':config.meta_dir, 'verbose':False})
    
    seg.get_metadata('flat' if filename == 'demographic' else 'var_info')
    
    
    combined_path = os.path.join(path, '%s_combined.txt' % filename)
    
    #combine the raw input files
    _combine_raw(path, combined_path, sort)
    
    print '...converting variables and setting value labels...'
    #format the summary dataset
    if filename in ['summary', 'demographic']:
        #initalize load class and load data
        ls = LoadSPSS(filename, seg.metadata, max_cycles)
        ls.load_data(combined_path)
        
        #add variable metadata
        if filename[0] == 's': ls.add_summary_metadata()
        else: ls.add_demographic_metadata()
        #save the dataset
        ls.save(path)
        
    #create the criminal history flat file syntax files
    else:
        _meta = {}
        labels = ''
        value_labels = ''
        _file_num = 1
        
        ls = LoadSyntax(filename, combined_path, path)
        
        for type in ['meta', 'arrest', 'court']: 
            for i in xrange(1, max_cycles+1):
                if type == 'meta' and i > 1: 
                    break
                
                for var_name, meta in sorted(seg.metadata.iteritems(), key=lambda k: k[1]['index']):
                    var_name=var_name.lower().strip()
                    _value_label = ''
                    _var_name = _get_var_name(filename, var_name, str(i))
                    
                    
                    #handle casenum
                    if var_name.lower() == 'casenum':
                        #if not meta then skip to next variable 
                        if type != 'meta': continue 
                        
                        if not value_labels:
                            labels = "VARIABLE LABELS\ncasenum '%s'\n" % meta['label']
                        if i == 1:
                            ls.add_var(var_name, meta)
                        
                        #if meta then add casenum and break
                        if type == 'meta': 
                            break
                        
                    #group the arrest and sentence labels together
                    try:
                        if type == 'arrest' and meta['type'].lower().strip() != 'arrest':
                            continue
                        elif type == 'court' and meta['type'].lower().strip() != 'court':
                            continue
                    except KeyError as e:
                        print var_name
                        raise e
                    
                    #add the variable label
                    if filename == 'demographic':
                        labels = '%s%s %s %s / \n' %(labels, _var_name, meta['label'], i)
                    else:
                        labels = "%s%s 'Cycle %i: %s' / \n" % (labels, _var_name, i, meta['label'])
                    
                    ls.add_var(_var_name, meta)
                    
                    #add the variable value labels
                    if meta['value_labels']:
                        if filename == 'demographic' and re.search('plcbth|ctz',_var_name, re.I):
                            _value_label = ''.join(["'%s' '%s'\n" % (k, v) for k,v in meta['value_labels']])
                        else:
                            _value_label = ''.join(["%s '%s'\n" % (k, v) for k,v in meta['value_labels']])
                        value_labels = "%s%s%s %s" %(value_labels if value_labels else 'VALUE LABELS \n%s' % value_labels,
                                                 '/ \n' if value_labels else '',  
                                                 _var_name, 
                                                 _value_label)
                        
                if i % 15 == 0 and filename != 'demographic': 
                    _save_sps(labels, value_labels, _file_num, path, filename)
                    #begin new file
                    labels = 'VARIABLE LABELS\n'
                    value_labels = ''
                    _file_num += 1
            
        _save_sps(labels, value_labels, _file_num, path, filename)
        ls.close()
        
    
    