import matplotlib.pyplot as plt
from tqdm import tqdm
import time
from numpy import *
import pandas as pd

print('count: 200000') #詞典sen中，每個(gè)標(biāo)簽對(duì)應(yīng)其所有句子的二維列表
print('average: '+str(sum([[sum(sen[i][j]) for j in range(len(sen[i]))] for i in sen])/200000))
x = []
y = []
for key,value in tag.items(): #詞典tag中，每個(gè)標(biāo)簽對(duì)應(yīng)該標(biāo)簽下的句子數(shù)目
    x.append(key)
    y.append(value)
plt.bar(x,y)
plt.show()

train_df = open('./data/train_set.csv').readlines()[1:]
train = train_df[0:190000]
test = train_df[190000:200000]
true_test = open('./data/test_a.csv').readlines()[1:]
tag = {str(i):0 for i in range(0,14)}
sen = {str(i):{} for i in range(0,14)}
words={}
stop_words = {'4149': 1, '1519': 1, '2465': 1, '7539': 1, ...... } 

for line in tqdm(train_df):
    cur_line = line.split('\t')
    cur_tag = cur_line[0]
    tag[cur_tag] += 1
    cur_line = cur_line[1][:-1].split(' ')
    for i in cur_line:
        if i not in words:
            words[i] = 1
        else:    
            words[i] += 1
        if i not in sen[cur_tag]:
            sen[cur_tag][i] = 1
        else:
            sen[cur_tag][i] += 1

def mul(l):
    res = 1
    for i in l:
        res *= i
    return res
def prob_clas(clas):
    return tag[clas]/(sum([tag[i] for i in tag]))
def probability(char,clas):  #P(特征|類別)
    if char not in sen[clas]:
        num_char = 0
    else:
        num_char = sen[clas][char]
    return (1+num_char)/(len(sen[clas])+len(words))

PRED = []
for line in tqdm(true_test):
    result = {str(i):0 for i in range(0,14)}
    cur_line = line[:-1].split(' ')
    clas = cur_tag
    for i in result:
        prob = []
        for j in cur_line:
            if j in stop_words:
                continue
            prob.append(log(probability(j,i)))
        result[i] = log(prob_clas(i))+sum(prob)
    for key,value in result.items():
        if(value == max(result.values())):
            pred = int(key)
    PRED.append(pred)

res=pd.DataFrame()
res['label']=PRED
res.to_csv('test_TL.csv')

from tqdm import tqdm
import pandas as pd
import time
import matplotlib.pyplot as plt
import seaborn as sns
from numpy import *
from sklearn import metrics
from sklearn.preprocessing import LabelEncoder,OneHotEncoder
from keras.models import Model
from keras.layers import LSTM, Activation, Dense, Dropout, Input, Embedding
from keras.optimizers import rmsprop_v2
from keras.preprocessing import sequence
from keras.callbacks import EarlyStopping
from keras.models import load_model
import os.path

max_words = 7549 #字典最大編號(hào)
# 可通過(guò)調(diào)節(jié)max_len調(diào)整模型效果和學(xué)習(xí)速度
max_len = 600 #句子的最大長(zhǎng)度
stop_words = {}

def to_seq(dataframe):
    x = []
    y = array([[0]*int(i)+[1]+[0]*(13-int(i)) for i in dataframe['label']])
    for i in tqdm(dataframe['text']):
        cur_sentense = []
        for word in i.split(' '):
            if word not in stop_words: #最終并未采用停用詞列表
                cur_sentense.append(word)
        x.append(cur_sentense)
    return sequence.pad_sequences(x,maxlen=max_len),y

def test_file(text,value,train,val):
    ## 定義LSTM模型
    inputs = Input(name='inputs',shape=[max_len])
    ## Embedding(詞匯表大小,batch大小,每個(gè)新聞的詞長(zhǎng))
    layer = Embedding(max_words+1,128,input_length=max_len)(inputs)
    layer = LSTM(128)(layer)
    layer = Dense(128,activation="relu",name="FC1")(layer)
    layer = Dropout(0.5)(layer)
    layer = Dense(14,activation="softmax",name="FC2")(layer)
    model = Model(inputs=inputs,outputs=layer)
    model.summary()
    model.compile(loss="categorical_crossentropy",optimizer=rmsprop_v2.RMSprop(),metrics=["accuracy"])
    
    ## 模型建立好之后開始訓(xùn)練，如果已經(jīng)保存訓(xùn)練文件（.h5格式），則直接調(diào)取即可
    if os.path.exists('my_model.h5') == True:
        model = load_model('my_model.h5')
    else:
        train_seq_mat,train_y = to_seq(train)
        val_seq_mat,val_y = to_seq(val)
        model.fit(train_seq_mat,train_y,batch_size=128,epochs=10,  #可通過(guò)epochs數(shù)來(lái)調(diào)整準(zhǔn)確率和運(yùn)算速度
                      validation_data=(val_seq_mat,val_y))
        model.save('my_model.h5') 
    ## 開始預(yù)測(cè)
    test_pre = model.predict(text)
    ##計(jì)算混淆函數(shù)
    confm = metrics.confusion_matrix(argmax(test_pre,axis=1),argmax(value,axis=1))
    print(metrics.classification_report(argmax(test_pre,axis=1),argmax(value,axis=1)))
    return confm

def plot_fig(matrix):
    Labname = [str(i) for i in range(14)]
    plt.figure(figsize=(8,8))
    sns.heatmap(matrix.T, square=True, annot=True,
                fmt='d', cbar=False,linewidths=.8,
                cmap="YlGnBu")
    plt.xlabel('True label',size = 14)
    plt.ylabel('Predicted label',size = 14)
    plt.xticks(arange(14)+0.5,Labname,size = 12)
    plt.yticks(arange(14)+0.3,Labname,size = 12)
    plt.show()
    return

def test_main():
    train_df = pd.read_csv("./data/train_set.csv",sep='\t',nrows=200000)
    train = train_df.iloc[0:150000,:]
    test = train_df.iloc[150000:180000,:]
    val = train_df.iloc[180000:,:]
    test_seq_mat,test_y = to_seq(test)
    Confm = test_file(test_seq_mat,test_y,train,val)
    plot_fig(Confm)

def train():
    train_df = pd.read_csv("./data/train_set.csv",sep='\t',nrows=200000)
    train = train_df.iloc[0:170000,:]
    test = train_df.iloc[0:10000,:]
    val = train_df.iloc[170000:,:]
    test_seq_mat,test_y = to_seq(test)
    Confm = test_file(test_seq_mat,test_y,train,val)
    plot_fig(Confm)

def pred_file():
    test_df = pd.read_csv('./data/test_a.csv')
    test_seq_mat = sequence.pad_sequences([i.split(' ') for i in tqdm(test_df['text'])],maxlen=max_len)
    
    inputs = Input(name='inputs',shape=[max_len])
    ## Embedding(詞匯表大小,batch大小,每個(gè)新聞的詞長(zhǎng))
    layer = Embedding(max_words+1,128,input_length=max_len)(inputs)
    layer = LSTM(128)(layer)
    layer = Dense(128,activation="relu",name="FC1")(layer)
    layer = Dropout(0.5)(layer)
    layer = Dense(14,activation="softmax",name="FC2")(layer)
    model = Model(inputs=inputs,outputs=layer)
    model.summary()
    model.compile(loss="categorical_crossentropy",optimizer=rmsprop_v2.RMSprop(),metrics=["accuracy"])
    model = load_model('my_model.h5')
    
    test_pre = model.predict(test_seq_mat)

    pred_result = [i.tolist().index(max(i.tolist())) for i in test_pre]
    res=pd.DataFrame()
    res['label']=pred_result
    res.to_csv('test_DL.csv')

#如果想要訓(xùn)練，取消下行注釋，訓(xùn)練之前需先刪除原訓(xùn)練文件（.h5）
#train()

#如果想要查看模型效果，取消下行注釋（訓(xùn)練集：檢驗(yàn)集：測(cè)試集=15：2：3）
# test_main()

#如果想預(yù)測(cè)并生成csv文件，取消下行注釋
# pred_file()