#system
import os, json, Queue
from contextlib import closing
from itertools import izip_longest
#local
from ..util import *
from ..configuration import config
from load import *
import extract
from threads import *
#third-party
import MySQLdb

def run(**opt):
    try:
        queue = Queue.Queue()
        demo_queue = Queue.Queue()
        out_queue = Queue.Queue()
        chflat_write_queue = Queue.Queue()
        chsummary_write_queue = Queue.Queue()
        active_threads = []
        
        #get the subject data01
        with closing(get_cnxn(opt['db_info']['name'],opt)) as cnx:
            with closing(cnx.cursor()) as c:
                c.execute('SET GLOBAL MAX_CONNECTIONS = 10000')
                c.execute('SELECT * FROM Subject limit %s, %s' % (opt['start'], opt['step']))
                subjects = c.fetchall()
                c.execute('SHOW COLUMNS FROM SUBJECT')
                columns = [col[0] for col in c.fetchall()]
       
        if opt['reload_flat']:
            #get highest cycle count from recidivism database
            max_cycles = extract.get_max_cycles(opt['db_info'])
            #create the output variables
            variables = get_variables(max_cycles)
                    
            try:
                print '...loading flat file metadata...'
                varcodes = set_varcodes('flat')
                varcodes = set_varcodes('summary', varcodes)
                print '...metadata load complete...'
            except:
                print 'METADATA LOAD FAILED'
                raise
    
            #start the threads
            for i in xrange(int(opt.get('threads'))*8):
                #create copies of the info we need to give each thread their own objects to avoid blocking
                _db_info = opt['db_info']
                _variables = []
                _variables.extend(variables)
                _varcodes = varcodes.copy()
                active_threads.append(TransformThread(queue, chflat_write_queue, chsummary_write_queue, _db_info, _variables, max_cycles, _varcodes))
                active_threads[-1].setDaemon(True)
                active_threads[-1].start()
            
            #populate the queue
            for s in subjects:
                _opt = opt.copy()
                for column, value in izip_longest(columns, s):
                    _opt[column] = value

                 #for debugging
                #if _opt['id'] in _range:#[1, 2, 15, 17, 128, 177227]:
                queue.put(_opt)
                
            
            #create the flat files
            _variables = []
            _variables.extend(variables)
            load_text(chflat_write_queue, _variables, opt['position'], chsummary_write_queue, [k for k in get_variable_metadata('summary').iterkeys()])
            
            #let the threads do their work
            
            queue.join()
            chsummary_write_queue.join()
            chflat_write_queue.join()
        
        if opt['reload_demo']:
            print 'Creating Demographic Flat file'
            
            #shutdown the threads
            for thread in active_threads:
                thread.db.close()
            
            
            _db_info = opt['db_info']
            max_profiles = extract.get_max_profiles(_db_info)
            demo_variables = get_demo_variables(max_profiles)
            varcodes = set_varcodes('demographic')
            #TODO make gender match sex in metadata
            varcodes['gender'] = varcodes['sex'] 
            varcodes['ctzshp'] = varcodes['ctz']
            t = DemoTransformThread(demo_queue, out_queue, _db_info, demo_variables, max_profiles, varcodes)
            t.setDaemon(True)
            t.start()
            
            for s in subjects:
                _opt = opt.copy()
                for column, value in izip_longest(columns, s):
                    _opt[column] = value
         
                demo_queue.put(_opt)
            
            segment = 'demographic'
            t = WriteThread(segment, out_queue, '%s_%s.sav' % (segment, opt['position']), demo_variables)
            t.setDaemon(True)
            t.start()
        
            demo_queue.join()
            out_queue.join()
        return 1
    except:
        from traceback import format_exc, extract_stack, format_list
        
        if opt['verbose']:
            raise
        return (format_exc(), format_list(extract_stack())) 