import logging
import os.path
import sys
from optparse import OptionParser
from gensim.corpora import WikiCorpus
 
 
def parse_corpus(infile, outfile):
    '''parse the corpus of the infile into the outfile'''
    space = ' '
    i = 0
    with open(outfile, 'w', encoding='utf-8') as fout:
        wiki = WikiCorpus(infile, lemmatize=False, dictionary={})  # gensim中的维基百科处理类WikiCorpus
        for text in wiki.get_texts():
            fout.write(space.join(text) + 'n')
            i += 1
            if i % 10000 == 0:
                logger.info('Saved ' + str(i) + ' articles')
 
 
if __name__ == '__main__':
    program = os.path.basename(sys.argv[0])
    logging.basicConfig(level = logging.INFO, format = '%(asctime)s - %(levelname)s - %(message)s')
    logger = logging.getLogger(program)  # logging.getLogger(logger_name)
    logger.info('running ' + program + ': parse the chinese corpus')
 
    # parse the parameters
    parser = OptionParser()
    parser.add_option('-i','--input',dest='infile',default='zhwiki-latest-pages-articles.xml.bz2',help='input: Wiki corpus')
    parser.add_option('-o','--output',dest='outfile',default='corpus.zhwiki.txt',help='output: Wiki corpus')
 
    (options,args) = parser.parse_args()
 
    infile = options.infile
    outfile = options.outfile
 
    try:
        parse_corpus(infile, outfile)
        logger.info('Finished Saved ' + str(i) + 'articles')
    except Exception as err:
        logger.info(err)
 
 
# python parse_zhwiki_corpus.py -i zhwiki-latest-pages-articles.xml.bz2 -o corpus.zhwiki.txt
 

import os, sys
 
# 解析完的维基百科数据路径 
wiki_path = './wiki.txt/'
 
# 获取路径下面的所有文件
wiki_list = os.listdir(wiki_path)
# 或者文件下面的所有txt文件
for per_file in wiki_list:
    if per_file == '.DS_Store':
        continue
    # 文件路径
    file_path = os.path.join( wiki_path, per_file )
    txt_list = os.listdir(file_path)
    # 或者每一个txt 
    for per_txt in txt_list:
        if per_txt == '.DS_Store':
            continue
        # 每一个txt文件的路径
        txt_path = os.path.join( wiki_path, per_file, per_txt ) 
        # cat file0.txt >> file.txt   将file0.txt追加到file.txt的末尾  
        cms = 'cat {} >> corpus.zhwiki.txt'.format( txt_path )
        print (cms)
        os.system( cms )