[ML]주택가격예측(EDA+keras)
Author : tmlab / Date : 2017. 12. 29. 19:14 / Category : Analytics
In [3]:
import os
import pandas as pd
os.chdir("D:\STUDY\kaggle" )
train = pd.read_csv('train.csv')
test = pd.read_csv('test.csv')
Out[3]:
In [7]:
train.head()
Out[7]:
In [8]:
test.head()
Out[8]:
In [23]:
train.describe()
Out[23]:
In [229]:
def cnt_NA(df):
colname = df.columns.tolist()
for i in colname:
if sum(pd.isnull(df[i])) != 0:
na = sum(pd.isnull(df[i]))
print(i + ":" + str(na)+ ", NA_ratio:" + str(na/len(df)))
print("NA test end")
In [230]:
cnt_NA(train)
In [42]:
train["SalePrice"].describe()
Out[42]:
In [52]:
%matplotlib inline
import seaborn as sns
sns.distplot(train['SalePrice'])
Out[52]:
In [118]:
cols = train.columns #전체컬럼
num_cols = train._get_numeric_data().columns # 연속형변수
num_cols = list(num_cols)
In [119]:
cate_cols = list(set(cols) - set(num_cols)) #이산형변수
In [84]:
numeric_data = train._get_numeric_data()
numeric_data.hist(bins=50, figsize=(20,15))
Out[84]:
In [129]:
cate_data = train[cate_cols]
cate_data.head()
Out[129]:
In [130]:
cate_data.describe()
Out[130]:
In [181]:
#43개 컬럼
cate_data[cate_cols[0]].value_counts().plot(kind = "bar")
Out[181]:
In [153]:
numeric_data.plot.scatter(x=num_cols[4], y='SalePrice', ylim=(0,800000));
In [182]:
from pandas.tools.plotting import scatter_matrix
scatter_matrix(numeric_data, figsize=(12,8))
Out[182]:
In [183]:
corr_matrix = numeric_data.corr()
corr_matrix["SalePrice"].sort_values(ascending=False)
Out[183]:
In [297]:
import seaborn as sns
plt.subplots(figsize=(12,9))
sns.heatmap(corr_matrix, vmax=0.9, square=True)
Out[297]:
Out[297]:
In [440]:
num_data = train._get_numeric_data()
cnt_NA(num_data)
In [441]:
# 결측값 얼마안돼는 경우 그냥 drop시킴
num_data = num_data.dropna(subset=["MasVnrArea"])
num_data = num_data.dropna(subset=["GarageYrBlt"])
len(num_data)
Out[441]:
In [442]:
# LotFrontage은 중앙값으로 채움
median = num_data["LotFrontage"].median()
num_data["LotFrontage"] = num_data["LotFrontage"].fillna(median)
In [443]:
cnt_NA(num_data)
In [444]:
num_data.info()
In [461]:
from sklearn.model_selection import train_test_split
X = num_data.iloc[:, :-1].values
y = num_data.iloc[:, -1].values
X_train, X_test, Y_train, Y_test = train_test_split(X, y, test_size=0.3, random_state=1)
print(len(X_train), len(X_test), len(Y_train), len(Y_test))
In [462]:
from sklearn.preprocessing import StandardScaler
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
X_test = sc.transform(X_test)
In [463]:
from keras.models import Sequential
from keras.layers import Dense
model = Sequential([
Dense(80, input_dim=37, kernel_initializer='normal', activation='selu'),
Dense(40, kernel_initializer='normal', activation='selu'),
Dense(1, kernel_initializer='normal'),
])
In [464]:
model.summary()
In [465]:
model.compile(loss='mse', optimizer='adam')
In [466]:
history = model.fit(X_train, Y_train, batch_size=3, epochs=100)
In [470]:
plt.plot(history.history["loss"])
plt.title("Loss")
plt.show()
Out[470]:
Out[470]:
In [471]:
score = model.evaluate(X_test, Y_test, verbose=0)
print(model.metrics_names)
print(score)
In [472]:
predictions = model.predict(X_test)
from sklearn.metrics import mean_absolute_error
print("Mean Absolute Error : " + str(mean_absolute_error(predictions, Y_test)))