import os, Queue, time
from threading import Thread
from contextlib import closing
from datetime import datetime
from itertools import izip_longest
from random import random
from traceback import format_exc, extract_stack, format_list

from ..recid_rules_db.extract import get_select_meta
from ..recid_rules_db.transform.rulesets.loader import get_rulesets
from ..segment import Segment
from ..log import Log
from ..util import get_cnxn
from ..threads import TransformThread
    
def run(**opt):
    """Transform the standardized arrest and sentence into format suitable for recidivism 
    database load."""
    try:
        print "Starting transform at %s:" % (datetime.now().isoformat(' '))
        
        #TODO delete all existing data in the out_dir
        
        #initialize objects
        queue = Queue.Queue()
        arrest_del_queue = Queue.Queue()
        sentence_del_queue = Queue.Queue()
        
        active_threads = []
        arrest = Segment('arrest', 'transform', **opt)
        sentence = Segment('sentence', 'transform', **opt)
        
        #load the metadata
        arrest.load_metadata_variables()
        sentence.load_metadata_variables()
        
        opt['log'] = Log(**opt)
        
        #load the crosswalks
        opt['arrest_rulesets'] = get_rulesets(arrest, **opt)
        opt['sentence_rulesets'] = get_rulesets(sentence, **opt)
        
        #get the subject data to gather data to feed to the threads
        with closing(get_cnxn(opt['db_info']['name'],opt)) as cnx:
            with closing(cnx.cursor()) as c:
                c.execute('SET GLOBAL MAX_CONNECTIONS = 10000')
                c.execute('SELECT * FROM Subject limit %s, %s' % (opt['start'], opt['step']))
                subjects = c.fetchall()
                c.execute('SHOW COLUMNS FROM SUBJECT')
                columns = c.fetchall()
        
        #set the select_str and column_names to be used in loading records
        opt['select_str'], opt['column_names'] = get_select_meta({'arrest': arrest.orig_var_names, 'sentence': sentence.orig_var_names})
        opt['variables'] = {'arrest': arrest.variables, 'sentence':sentence.variables}
        
        #start the threads
        c = 0
        
        for i in xrange(int(opt.get('threads'))):
            _db_info = opt['db_info']
            c+= 1 
            active_threads.append(TransformThread(queue, _db_info, '%s_%i.txt' % (opt['start'], c), arrest_del_queue, sentence_del_queue))
            active_threads[-1].setDaemon(True)
            active_threads[-1].start()
        
        #populate the queue
        columns = [col[0] for col in columns]
        
        for s in subjects:
            _opt = opt.copy()
            for column, value in izip_longest(columns, s):
                _opt[column] = value
                
            queue.put(_opt)
            
        #let the threads do their work
        queue.join()
        
        #shutdown the threads
        for thread in active_threads:
            thread.arrest_file.close()
            thread.sentence_file.close()
            thread.db.close()
            #thread.kill_received = True
        
        
        for seg in ['arrest', 'sentence']:
            for flag in ['delflag', 'wvfbi_overlap_flag', 'adateflag',
                         'follow_up_flag', 'juv_flag', 'off_del_flag', 'arrest_match_flag']:
                if flag not in opt['variables'][seg]:
                    if flag == 'arrest_match_flag' and seg == 'arrest':
                        continue
                    opt['variables'][seg].extend([flag])    
        
        #set paths, make the directories, create file headers for deletion records output
        _a_del_path = os.path.join(opt['tmp_out_dir'], r'master\del_records\arrest\%s.txt' % opt['position'])
        _c_del_path = os.path.join(opt['tmp_out_dir'], r'master\del_records\sentence\%s.txt' % opt['position'])
        
        for seg in ['arrest', 'sentence']:
            time.sleep(random())
            if not os.path.isdir(os.path.join(opt['tmp_out_dir'], r'master\del_records\%s' % seg)):
                os.makedirs(os.path.join(opt['tmp_out_dir'], r'master\del_records\%s' % seg))
            
        if os.path.isfile(_a_del_path):
            os.unlink(_a_del_path)
        
        if os.path.isfile(_c_del_path):
            os.unlink(_c_del_path)
        
        with closing(open(_a_del_path, 'a')) as file:
            file.write('%s\n' % '|'.join([var.name for var in opt['variables']['arrest']]))
        with closing(open(_c_del_path, 'a')) as file:
            file.write('%s\n' % '|'.join([var.name for var in opt['variables']['sentence']]))
        
        #create the thread workers    
        def arrest_worker():
            while True:
                with closing(open(_a_del_path, 'a')) as file:
                    items = arrest_del_queue.get()
                    for i in items:
                        out = []
                        for var in opt['variables']['arrest']:
                             out.append(str(i.get(var.name)).replace('\\', '\\\\') if i.get(var.name) is not None else '')
                        file.write('%s\n' % '|'.join(out))
                    arrest_del_queue.task_done()
        
        def sentence_worker():
            while True:
                with closing(open(_c_del_path, 'a')) as file:
                    items = sentence_del_queue.get()
                    for i in items:
                        out = []
                        for var in opt['variables']['sentence']:
                             out.append(str(i.get(var.name)).replace('\\', '\\\\') if i.get(var.name) is not None else '')
                        file.write('%s\n' % '|'.join(out))
                    sentence_del_queue.task_done()
        
        
        #start the threads
        for i in range(1):
             t = Thread(target=arrest_worker)
             t.daemon = True
             t.start()
        
        for i in range(1):
             t = Thread(target=sentence_worker)
             t.daemon = True
             t.start()
             
        arrest_del_queue.join()
        sentence_del_queue.join()
    

        opt['log'].dump()
        print '...done.'
        return 1
    except:
        if opt['verbose']:
            raise
        return (format_exc(), format_list(extract_stack())) 