1주차 세션 정리노트(LEVEL.2 예시)

$ kaggle competitions download -c quora-question-pairs

print("파일 크기: ") for file in os.listdir(DATA_IN_PATH): if 'csv' in file and 'zip' not in file: print(file.ljust(30) + str(round(os.path.getsize(DATA_IN_PATH + file) / 1000000,2)) + 'MB')

파일 크기: test.csv 314.02MB train.csv 63.4MB sample_submission.csv 22.35MB

print('전체 학습 데이터의 개수: {}'.format(len(train_data)))

전체 학습 데이터의 개수: 404290

train_set = pd.Series(train_data['question1'].tolist() + train_data['question2'].tolist()).astype(str) train_set.head()

print('train 데이터의 총 질문 수: {}'.format(len(np.unique(train_set)))) print('반복해서 나타나는 질문의 수: {}'.format(np.sum(train_set.value_counts() > 1)))

plt.figure(figsize=(12,5)) plt.hist(train_set.value_counts(), bins=50, alpha=0.5, color='r', label='word') plt.yscale('log', nonposy='clip') plt.title('Log-Histogram of question appearance counts') plt.xlabel('Number of occurremces of question') plt.ylabel('Number of questions')

print('중복 최대 개수: {}'.format(np.max(train_set.value_counts()))) print('중복 최소 개수: {}'.format(np.min(train_set.value_counts()))) print('중복 평균 개수: {:.2f}'.format(np.mean(train_set.value_counts()))) print('중복 표준편차: {:.2f}'.format(np.std(train_set.value_counts()))) print('중복 중간길이: {}'.format(np.median(train_set.value_counts()))) # 사분위의 대한 경우는 0~100 스케일로 되어있음 print('제 1 사분위 중복: {}'.format(np.percentile(train_set.value_counts(), 25))) print('제 3 사분위 중복: {}'.format(np.percentile(train_set.value_counts(), 75)))

plt.figure(figsize=(12, 5)) plt.boxplot([train_set.value_counts()], labels=['counts'], showmeans=True)

from wordcloud import WordCloud cloud = WordCloud(width=800, height=600).generate(" ".join(train_set.astype(str))) plt.figure(figsize=(15, 10)) plt.imshow(cloud) plt.axis('off')

fig, axe = plt.subplots(ncols=1) fig.set_size_inches(6, 3) sns.countplot(train_data['is_duplicate'])

train_length = train_set.apply(len) # 문자 단위로 분석하기 위한 데이터 길이를 담은 변수 생성 plt.figure(figsize=(12,5)) plt.hist(train_length, bins=200, range = [0, 200], facecolor='r', density=True, label='train') plt.title('Normalised histogram of character count in questions', fontsize=15) plt.legend() plt.xlabel('Number of characters', fontsize=15) plt.ylabel('Probaability', fontsize=15)

print('질문 길이 최대 값: {}'.format(np.max(train_length))) print('질문 길이 평균 값: {:.2f}'.format(np.mean(train_length))) print('질문 길이 표준편차: {:.2f}'.format(np.std(train_length))) print('질문 길이 중간 값: {}'.format(np.median(train_length))) print('질문 길이 제 1 사분위: {}'.format(np.percentile(train_length, 25))) print('질문 길이 제 3 사분위: {}'.format(np.percentile(train_length, 75)))

plt.figure(figsize=(12, 5)) plt.boxplot(train_length, labels=['char counts'], showmeans=True)

train_word_counts = train_set.apply(lambda x: len(x.split(' '))) plt.figure(figsize=(12,5)) plt.hist(train_word_counts, bins=50, range = [0, 50], facecolor='r', density=True, label='train') plt.title('Normalised histogram of word count in questions', fontsize=15) plt.legend() plt.xlabel('Number of words', fontsize=15) plt.ylabel('Probaability', fontsize=15)

print('질문 단어 개수 최대 값: {}'.format(np.max(train_word_counts))) print('질문 단어 개수 평균 값: {:.2f}'.format(np.mean(train_word_counts))) print('질문 단어 개수 표준편차: {:.2f}'.format(np.std(train_word_counts))) print('질문 단어 개수 중간 값: {}'.format(np.median(train_word_counts))) print('질문 단어 개수 제 1 사분위: {}'.format(np.percentile(train_word_counts, 25))) print('질문 단어 개수 제 3 사분위: {}'.format(np.percentile(train_word_counts, 75))) print('질문 단어 개수 99 퍼센트: {}'.format(np.percentile(train_word_counts, 99)))

plt.figure(figsize=(12, 5)) plt.boxplot(train_word_counts, labels=['counts'], showmeans=True)

qmarks = np.mean(train_set.apply(lambda x: '?' in x)) # 물음표가 구두점으로 쓰임 math = np.mean(train_set.apply(lambda x: '[math]' in x)) # [] fullstop = np.mean(train_set.apply(lambda x: '.' in x)) # 마침표 capital_first = np.mean(train_set.apply(lambda x: x[0].isupper())) # 첫번째 대문자 capitals = np.mean(train_set.apply(lambda x: max([y.isupper() for y in x]))) # 대문자가 몇개 numbers = np.mean(train_set.apply(lambda x: max([y.isdigit() for y in x]))) # 숫자가 몇개 print('물음표가있는 질문: {:.2f}%'.format(qmarks * 100)) print('수학 태그가있는 질문: {:.2f}%'.format(math * 100)) print('마침표를 포함한 질문: {:.2f}%'.format(fullstop * 100)) print('첫 글자가 대문자 인 질문: {:.2f}%'.format(capital_first * 100)) print('대문자가있는 질문: {:.2f}%'.format(capitals * 100)) print('숫자가있는 질문: {:.2f}%'.format(numbers * 100))

import pandas as pd import numpy as np import re import json from tensorflow.python.keras.preprocessing.text import Tokenizer from tensorflow.python.keras.preprocessing.sequence import pad_sequences

train_data = pd.read_csv(DATA_IN_PATH + 'train.csv', encoding='utf-8') #데이터 불러오기 train_pos_data = train_data.loc[train_data['is_duplicate'] == 1] train_neg_data = train_data.loc[train_data['is_duplicate'] == 0] class_difference = len(train_neg_data) - len(train_pos_data) sample_frac = 1 - (class_difference / len(train_neg_data)) train_neg_data = train_neg_data.sample(frac = sample_frac) print("중복 질문 개수: {}".format(len(train_pos_data))) print("중복이 아닌 질문 개수: {}".format(len(train_neg_data)))

train_data = pd.concat([train_neg_data, train_pos_data])

FILTERS = "([~.,!?\"':;)(])" change_filter = re.compile(FILTERS) question1 = [str(s) for s in train_data['question1']] question2 = [str(s) for s in train_data['question2']] filtered_question1 = list() filtered_question2 = list() for q in question1: filtered_question1.append(re.sub(change_filter, "", q).lower()) for q in question2: filtered_question2.append(re.sub(change_filter, "", q).lower())

tokenizer = Tokenizer() tokenizer.fit_on_texts(filtered_question1 + filtered_question2) question1_sequence = tokenizer.texts_to_sequences(filtered_question1) question2_sequence = tokenizer.texts_to_sequences(filtered_question2)

MAX_SEQUENCE_LENGTH = 31 q1_data = pad_sequences(question1_sequence, maxlen=MAX_SEQUENCE_LENGTH, padding='post') q2_data = pad_sequences(question2_sequence, maxlen=MAX_SEQUENCE_LENGTH, padding='post')

word_vocab = {} word_vocab = tokenizer.word_index word_vocab["<PAD>"] = 0 labels = np.array(train_data['is_duplicate'], dtype=int) print('Shape of question1 data: {}'.format(q1_data.shape)) print('Shape of question2 data: {}'.format(q2_data.shape)) print('Shape of label: {}'.format(labels.shape)) print('Words in index: {}'.format(len(word_vocab)))

data_configs = {} data_configs['vocab'] = word_vocab data_configs['vocab_size'] = len(word_vocab) TRAIN_Q1_DATA = 'q1_train.npy' TRAIN_Q2_DATA = 'q2_train.npy' TRAIN_LABEL_DATA = 'label_train.npy' DATA_CONFIGS = 'data_configs.npy' np.save(open(DATA_IN_PATH + TRAIN_Q1_DATA, 'wb'), q1_data) np.save(open(DATA_IN_PATH + TRAIN_Q2_DATA, 'wb'), q2_data) np.save(open(DATA_IN_PATH + TRAIN_LABEL_DATA, 'wb'), labels) json.dump(data_configs, open(DATA_IN_PATH + DATA_CONFIGS,'w'))

test_data = pd.read_csv(DATA_IN_PATH + 'test.csv', encoding='utf-8') valid_ids = [type(x) ==int for x in test_data.test_id] test_data = test_data[valid_ids].drop_duplicates() test_questions1 = [str(s) for s in test_data['question1']] test_questions2 = [str(s) for s in test_data['question2']] filtered_test_questions1 = list() filtered_test_questions2 = list() for q in test_questions1: filtered_test_questions1.append(re.sub(change_filter, "", q).lower()) for q in test_questions2: filtered_test_questions2.append(re.sub(change_filter, "", q).lower()) test_questions1_sequence = tokenizer.texts_to_sequences(filtered_test_questions1) test_questions2_sequence = tokenizer.texts_to_sequences(filtered_test_questions2) test_q1_data = pad_sequences(test_questions1_sequence, maxlen=MAX_SEQUENCE_LENGTH, padding='post') test_q2_data = pad_sequences(test_questions2_sequence, maxlen=MAX_SEQUENCE_LENGTH, padding='post') test_id = np.array(test_data['test_id']) print('Shape of question1 data: {}'.format(test_q1_data.shape)) print('Shape of question2 data:{}'.format(test_q2_data.shape)) print('Shape of ids: {}'.format(test_id.shape))

TEST_Q1_DATA = 'test_q1.npy' TEST_Q2_DATA = 'test_q2.npy' TEST_ID_DATA = 'test_id.npy' np.save(open(DATA_IN_PATH + TEST_Q1_DATA, 'wb'), test_q1_data) np.save(open(DATA_IN_PATH + TEST_Q2_DATA , 'wb'), test_q2_data) np.save(open(DATA_IN_PATH + TEST_ID_DATA , 'wb'), test_id)

TRAIN_Q1_DATA_FILE = 'q1_train.npy' TRAIN_Q2_DATA_FILE = 'q2_train.npy' TRAIN_LABEL_DATA_FILE = 'label_train.npy' # 훈련 데이터를 가져옵니다. train_q1_data = np.load(open(DATA_IN_PATH + TRAIN_Q1_DATA_FILE, 'rb')) train_q2_data = np.load(open(DATA_IN_PATH + TRAIN_Q2_DATA_FILE, 'rb')) train_labels = np.load(open(DATA_IN_PATH + TRAIN_LABEL_DATA_FILE, 'rb'))

train_input = np.stack((train_q1_data, train_q2_data), axis=1) print(train_input.shape)

from sklearn.model_selection import train_test_split train_input, eval_input, train_label, eval_label = train_test_split(train_input, train_labels, test_size=0.2, random_state=42)

import xgboost as xgb train_data = xgb.DMatrix(train_input.sum(axis=1), label=train_label) eval_data = xgb.DMatrix(eval_input.sum(axis=1), label=eval_label) data_list = [(train_data, 'train'), (eval_data, 'valid')]

params = {} params['objective'] = 'binary:logistic' params['eval_metric'] = 'rmse' bst = xgb.train(params, train_data, num_boost_round=1000, evals = data_list, early_stopping_rounds=10)

# test set 불러오기 TEST_Q1_DATA_FILE = 'test_q1.npy' TEST_Q2_DATA_FILE = 'test_q2.npy' TEST_ID_DATA_FILE = 'test_id.npy' test_q1_data = np.load(open(DATA_IN_PATH + TEST_Q1_DATA_FILE, 'rb')) test_q2_data = np.load(open(DATA_IN_PATH + TEST_Q2_DATA_FILE, 'rb')) test_id_data = np.load(open(DATA_IN_PATH + TEST_ID_DATA_FILE, 'rb')) # test set predict test_input = np.stack((test_q1_data, test_q2_data), axis=1) test_data = xgb.DMatrix(test_input.sum(axis=1)) test_predict = bst.predict(test_data) # 예측값 저장 DATA_OUT_PATH = './data_out/quora/' if not os.path.exists(DATA_OUT_PATH): os.makedirs(DATA_OUT_PATH) output = pd.DataFrame({'test_id': test_id_data, 'is_duplicate': test_predict}) output.to_csv(DATA_OUT_PATH + 'simple_xgb.csv', index=False)

from tensorflow.keras import layers from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint DATA_IN_PATH = './data_in/quora/' DATA_OUT_PATH = './data_out/quora/' TRAIN_Q1_DATA_FILE = 'q1_train.npy' TRAIN_Q2_DATA_FILE = 'q2_train.npy' TRAIN_LABEL_DATA_FILE = 'label_train.npy' DATA_CONFIGS = 'data_configs.npy' q1_data = np.load(open(DATA_IN_PATH + TRAIN_Q1_DATA_FILE, 'rb')) q2_data = np.load(open(DATA_IN_PATH + TRAIN_Q2_DATA_FILE, 'rb')) labels = np.load(open(DATA_IN_PATH + TRAIN_LABEL_DATA_FILE, 'rb')) prepro_configs = json.load(open(DATA_IN_PATH + DATA_CONFIGS, 'r'))

import tensorflow as tf class SentenceEmbedding(layers.Layer): def __init__(self, args): super(SentenceEmbedding, self).__init__() self.conv = layers.Conv1D(args.conv_num_filters, args.conv_window_size, activation = tf.keras.activations.relu, padding='same') self.max_pool = layers.MaxPool1D(args.max_pool_seq_len, 1) self.dense = layers.Dense(args.sent_embedding_dimension, activation=tf.keras.activations.relu) def call(self, x): x = self.conv(x) x = self.max_pool(x) x = self.dense(x) return tf.squeeze(x, 1)

class SentenceSimilarityModel(tf.keras.Model): def __init__(self, args): super(SentenceSimilarityModel, self).__init__(name = args.model_name) self.word_embedding = layers.Embedding(args.vocab_size, args.word_embedding_dimension) self.base_encoder = SentenceEmbedding(args) self.hypo_encoder = SentenceEmbedding(args) self.dense = layers.Dense(args.hidden_dimension, activation = tf.keras.activations.relu) self.logit = layers.Dense(1, activation = tf.keras.activations.sigmoid) self.dropout = layers.Dropout(args.dropout_rate) def call(self, x): x1, x2 = x b_x = self.word_embedding(x1) h_x = self.word_embedding(x2) b_x = self.dropout(b_x) h_x = self.dropout(h_x) b_x = self.base_encoder(b_x) h_x = self.hypo_encoder(h_x) e_x = tf.concat([b_x, h_x], -1) e_x = self.dense(e_x) e_x = self.dropout(e_x) return self.logit(e_x)

from argparse import Namespace model_name = 'cnn_similarity' BATCH_SIZE = 1024 NUM_EPOCHS = 100 VALID_SPLIT = 0.1 MAX_LEN = 31 args = Namespace( model_name= model_name, vocab_size = prepro_configs['vocab_size'], word_embedding_dimension = 100, conv_num_filters = 300, conv_window_size = 3, max_pool_seq_len = MAX_LEN, sent_embedding_dimension = 128, dropout_rate = 0.2, hidden_dimension = 200, output_dimension =1 )

model = SentenceSimilarityModel(args) model.compile(optimizer=tf.keras.optimizers.Adam(1e-3), loss=tf.keras.losses.BinaryCrossentropy(), metrics=[tf.keras.metrics.BinaryAccuracy(name='accuracy')])

# 오버피팅을 막기 위한 ealrystop earlystop_callback = EarlyStopping(monitor='val_accuracy', min_delta=0.0001, patience=1) checkpoint_path = DATA_OUT_PATH + model_name +'/weights.h5' checkpoint_dir = os.path.dirname(checkpoint_path) if os.path.exists(checkpoint_dir): print("{} -- Folder already exists \n".format(checkpoint_dir)) else: os.makedirs(checkpoint_dir, exist_ok=True) print("{} -- Folder create complete \n".format(checkpoint_dir)) cp_callback = ModelCheckpoint(checkpoint_path, monitor='val_accuracy', verbose=1, save_best_only=True, save_weights_only=True) #모델 학습 history = model.fit((q1_data, q2_data), labels, batch_size=BATCH_SIZE, epochs=NUM_EPOCHS, validation_split=VALID_SPLIT, callbacks=[earlystop_callback, cp_callback])

def plot_graphs(history, string): plt.plot(history.history[string]) plt.plot(history.history['val_'+string], '') plt.xlabel("Epochs") plt.ylabel(string) plt.legend([string, 'val_'+string]) plt.show() plot_graphs(history, 'loss')

plot_graphs(history, 'accuracy')

class MaLSTM(tf.keras.Model): def __init__(self, **kargs): super(MaLSTM, self).__init__(name=model_name) self.embedding = layers.Embedding(input_dim=kargs['vocab_size'], output_dim=kargs['embedding_dimension']) self.lstm = layers.LSTM(units=kargs['lstm_dimension']) def call(self, x): x1, x2 = x x1 = self.embedding(x1) x2 = self.embedding(x2) x1 = self.lstm(x1) x2 = self.lstm(x2) x = tf.exp(-tf.reduce_sum(tf.abs(x1 - x2), axis=1)) return x

model_name = 'malstm_similarity' BATCH_SIZE = 128 NUM_EPOCHS = 5 VALID_SPLIT = 0.1 kargs = { 'vocab_size': prepro_configs['vocab_size'], 'embedding_dimension': 100, 'lstm_dimension': 150, }

model = MaLSTM(**kargs) model.compile(optimizer=tf.keras.optimizers.Adam(1e-3), loss=tf.keras.losses.BinaryCrossentropy(), metrics=[tf.keras.metrics.BinaryAccuracy(name='accuracy')])

# overfitting을 막기 위한 ealrystop 추가 earlystop_callback = EarlyStopping(monitor='val_accuracy', min_delta=0.0001, patience=1) # min_delta: the threshold that triggers the termination (acc should at least improve 0.0001) # patience: no improvment epochs (patience = 1, 1번 이상 상승이 없으면 종료)\ checkpoint_path = DATA_OUT_PATH + model_name + '/weights.h5' checkpoint_dir = os.path.dirname(checkpoint_path) # Create path if exists if os.path.exists(checkpoint_dir): print("{} -- Folder already exists \n".format(checkpoint_dir)) else: os.makedirs(checkpoint_dir, exist_ok=True) print("{} -- Folder create complete \n".format(checkpoint_dir)) cp_callback = ModelCheckpoint( checkpoint_path, monitor='val_accuracy', verbose=1, save_best_only=True, save_weights_only=True) #모델 학습 history = model.fit((q1_data, q2_data), labels, batch_size=BATCH_SIZE, epochs=NUM_EPOCHS, validation_split=VALID_SPLIT, callbacks=[earlystop_callback, cp_callback])

plot_graphs(history, 'loss')

plot_graphs(history, 'accuracy')

predictions = model.predict((test_q1_data, test_q2_data), batch_size=BATCH_SIZE) output = pd.DataFrame( data={"test_id":test_id_data, "is_duplicate": list(predictions)} ) output.to_csv(DATA_OUT_PATH+"rnn_predict.csv", index=False, quoting=3)