### Navigation

- index
- modules |
- CCHR 1.0 documentation »
- Module code »
- src »

# Source code for src.cmd.pre_process

    from util import segment_run_map
    from ..util import check_file
    from ..configuration import config, dir_map, db_info

    def pre_process(cmd):
        print "Start PreProcessing..."

        args = cmd.args
        _segments = [segment_run_map[seg] for seg in args.segments.split('|')]

        #validate user data
        opt = {'out_dir': check_file(config.out_dir, as_dir = True),
               'meta_dir': check_file(config.meta_dir, as_dir = True),
               'rules': dir_map['pre_process_rules_dir'],
               'out_format': 'mysql',
               'out_delimiter': '|',
               'verbose': args.verbose,
               'db_info': db_info['stand'],
               'limit': args.limit,
               'halt_on_err': config.debug_halt_on_error,
               'phase_id': '0',
        }

        for segment in sorted(_segments):
            if segment in ('sentence', 'supervision'): #removed supervision from pre_processing
                continue
            _opt = opt.copy()
            _opt['segment'] = segment
            cmd.task_queue.put(_opt)

        cmd.start() #start the arrest, demographic, and supervision workers
        cmd.task_queue.join() #wait for the workers to finish

        opt['segment'] = 'sentence'
        cmd.task_queue.put(opt)
        cmd.start() #start the sentence worker
        cmd.task_queue.join() #wait for the workers to finish
        cmd.flush() #flush the cmd logs

### Navigation

- index
- modules |
- CCHR 1.0 documentation »
- Module code »
- src »