In [196]:
import numpy as np
import pandas as pd
%matplotlib inline
import matplotlib.pyplot as plt
import seaborn as sns
In [197]:
train = pd.read_csv('train.csv')
test = pd.read_csv('test.csv')
データの基本統計量を見てみる¶
In [198]:
train.describe(include='all')
Out[198]:
データの型を見てみる¶
In [199]:
print (train.info())
欠測値の確認¶
全データ数が891に対しての欠測値の数が、以下のようになっている。
Ageが177でおよそ20%の欠測
Cabinが687でおよそ77%の欠測
Embarkedが2の欠測であった。
Cabinは欠測率が高い ＝＞削除した方がいいかも
Ageは欠測率が小さい=>補完する
Embarkedは欠測値が少ないので、補完する
In [200]:
print("cout=", 891)
pd.isnull(train).sum()
Out[200]:
In [201]:
test.isnull().sum()
Out[201]:
データの種類¶
- Survived:カテゴリデータ
- Pclass:カテゴリデータ
- Name:カテゴリデータ
- Age:数値データ
- SibSp:カテゴリデータ
- Parch:カテゴリデータ
- Ticket:カテゴリデータ
- Fare:数値データ
- Cabin:カテゴリデータ
- Embarked:カテゴリデータ
In [202]:
p_id = train["PassengerId"]
train = train.drop(["PassengerId"], axis=1)
Survived¶
生存フラグ
0:死亡
1:生存
In [203]:
sns.countplot(x="Survived", data=train)
Out[203]:
Pclass¶
Ticket Class
1:1st
2:2nd
3:3rd
In [204]:
print('unique_val:', train['Pclass'].unique())
print('unique_val_counts')
print('Number of Name unique:', train['Pclass'].nunique())
print(train['Pclass'].value_counts())
sns.countplot(x="Survived", hue='Pclass', data=train)
Out[204]:
In [205]:
print (train[['Pclass', 'Survived']].groupby(['Pclass'], as_index=False).mean())
Name¶
固有名称が891と分類が難しい
In [206]:
print('Number of Name unique:', train['Name'].nunique())
print(train['Name'].value_counts())
Sex¶
female or male
In [207]:
print('unique_val:', train['Sex'].unique())
print('Number of Name unique:', train['Sex'].nunique())
print('unique_val_counts')
print(train['Sex'].value_counts())
#0 is dead, 1is survival
sns.countplot(x="Survived", hue="Sex",data=train)
Out[207]:
In [208]:
print (train[["Sex", "Survived"]].groupby(['Sex'], as_index=False).mean())
数値化する¶
In [209]:
train['Sex'] = train['Sex'].map({'male':0, 'female':1})
Age¶
In [210]:
train['Age'].hist()
Out[210]:
In [211]:
plt.title('Age Distribution by Dead')
train[train['Survived'] == 0]['Age'].hist()
Out[211]:
In [212]:
plt.title('Age Distribution by Survived')
train[train['Survived'] == 1]['Age'].hist()
Out[212]:
SibSp¶
兄弟・配偶者の人数
In [213]:
train["SibSp"].plot("hist")
Out[213]:
In [214]:
print('unique_val:', train['SibSp'].unique())
print('unique_val_counts')
print('Number of Name unique:', train['SibSp'].nunique())
print(train['SibSp'].value_counts())
sns.countplot(x="Survived", hue='SibSp', data=train)
Out[214]:
Parch¶
親・子供の人数
In [215]:
train["Parch"].plot('hist')
Out[215]:
In [216]:
print('unique_val:', train['Parch'].unique())
print('unique_val_counts')
print('Number of Name unique:', train['Parch'].nunique())
print(train['Parch'].value_counts())
sns.countplot(x="Survived", hue='Parch', data=train)
Out[216]:
In [217]:
#print('unique_val:', train['Ticket'].unique())
print('unique_val_counts')
print('Number of Name unique:', train['Ticket'].nunique())
print(train['Ticket'].value_counts());
#sns.countplot(x="Survived", hue='Ticket', data=train)
In [218]:
print(train[train['Ticket'] == 'CA. 2343']['Survived'].value_counts())
print(train[train['Ticket'] == '1601']['Survived'].value_counts())
print(train[train['Ticket'] == '347082']['Survived'].value_counts())
print(train[train['Ticket'] == 'CA 2144']['Survived'].value_counts())
Fare¶
料金
In [219]:
#print('unique_val:', train['Fare'].unique())
print('unique_val_counts')
print('Number of Name unique:', train['Fare'].nunique())
print(train['Fare'].value_counts().head())
train['Fare'].plot('hist')
plt.show()
#sns.countplot(x='Survived', hue='Fare', data=train)
Cabin¶
Cabin number
In [220]:
#print('unique_val:', train['Cabin'].unique())
print('unique_val_counts')
print('Number of Name unique:', train['Cabin'].nunique())
print(train['Cabin'].value_counts().head(3))
#sns.countplot(x="Survived", hue='Cabin', data=train)
Embarked¶
出発した港
C = Cherbourg
Q = Queenstown
S = Southampton
In [221]:
print('unique_val:', train['Embarked'].unique())
print('unique_val_counts')
print('Number of Name unique:', train['Embarked'].nunique())
print(train['Embarked'].value_counts().head(3))
sns.countplot(x="Survived", hue='Embarked', data=train)
Out[221]:
Pclass and Sex¶
In [222]:
pclass_1 = train[train['Pclass'] == 1]
p1_f = pclass_1[pclass_1['Sex'] == 1]
p1_m = pclass_1[pclass_1['Sex'] == 0]
fig, ax = plt.subplots(figsize=(16, 5))
plt.subplot(1, 2, 1)
g = sns.countplot(x='Survived', data=p1_f)
plt.title('Pclass=1 and Sex = female')
plt.subplot(1, 2, 2)
g = sns.countplot(x='Survived', data=p1_m)
plt.title('Pclass=1 and Sex = female')
Out[222]:
In [223]:
pclass_2 = train[train['Pclass'] == 2]
p2_f = pclass_2[pclass_2['Sex'] == 1]
p2_m = pclass_2[pclass_2['Sex'] == 0]
fig, ax = plt.subplots(figsize=(16, 5))
plt.subplot(1, 2, 1)
g = sns.countplot(x='Survived', data=p2_f)
plt.title('Pclass=2 and Sex = female')
plt.subplot(1, 2, 2)
g = sns.countplot(x='Survived', data=p2_m)
plt.title('Pclass=2 and Sex = female')
Out[223]:
In [224]:
pclass_3 = train[train['Pclass'] == 3]
p3_f = pclass_3[pclass_3['Sex'] == 1]
p3_m = pclass_3[pclass_3['Sex'] == 0]
fig, ax = plt.subplots(figsize=(16, 5))
plt.subplot(1, 2, 1)
g = sns.countplot(x='Survived', data=p3_f)
plt.title('Pclass=3 and Sex = female')
plt.subplot(1, 2, 2)
g = sns.countplot(x='Survived', data=p3_m)
plt.title('Pclass=3 and Sex = male')
Out[224]:
In [225]:
fig, axes = plt.subplots(figsize=(9, 6))
sns.heatmap(train.corr(),annot=True)
Out[225]:
方針¶
- カテゴリ変数はエンコーディング(数値化)しよう
- 欠測値の多い'Age'と'Cabin'は削除しよう
- Embarkedは欠測が２個なので、補完しよう
- 'Name'と'Ticket'は、文字データなので扱いづらいので、削除しよう
欠測値 Embarkedの補完¶
a = train['Embarked'].fillna('S')
b = train.drop(['Embarked'], axis=1)
train = pd.concat([b,a], axis=1)
train.isnull().sum()
In [226]:
train['Embarked'] = train['Embarked'].fillna('S')
In [227]:
train.isnull().sum()
Out[227]:
Embarkedのエンコーディング¶
ダミー変換をする
In [228]:
train['Embarked'] = train['Embarked'].map({'S':0, 'C':1, 'Q':2})
データの削除¶
In [229]:
drop_col = ['Survived', 'Name', 'Ticket', 'Cabin', 'Age']
y = train['Survived']
X = train.drop(drop_col, axis=1)
テストデータも整形する¶
In [230]:
p_id = test['PassengerId']
drop_col_test = ['PassengerId', 'Name', 'Ticket', 'Cabin', 'Age']
test['Embarked'] = test['Embarked'].fillna('S')
test['Embarked'] = test['Embarked'].map({'S':0, 'C':1, 'Q':2})
In [232]:
test['Fare'] = test['Fare'].fillna(test['Fare'].median())
test['Sex'] = test['Sex'].map({'male':0, 'female':1})
test = test.drop(drop_col_test, axis=1)
In [233]:
test.isnull().sum()
Out[233]:
In [234]:
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.2, random_state=1)
ロジスティック回帰¶
In [235]:
# Logistic Regression
from sklearn.linear_model import LogisticRegression
logreg = LogisticRegression()
logreg.fit(X_train, y_train)
y_pred = logreg.predict(X_val)
acc_logreg = round(accuracy_score(y_pred, y_val) * 100, 2)
print(acc_logreg)
ランダムフォレスト¶
In [236]:
# Random Forest
from sklearn.ensemble import RandomForestClassifier
randomforest = RandomForestClassifier()
randomforest.fit(X_train, y_train)
y_pred = randomforest.predict(X_val)
acc_randomforest = round(accuracy_score(y_pred, y_val) * 100, 2)
print(acc_randomforest)
予測して、提出ファイルを作成¶
In [237]:
pred = randomforest.predict(test)
In [243]:
submit = pd.DataFrame({'PassengerId':p_id, 'Survived':pred})
In [245]:
submit.to_csv('submit_randomforest.csv', index=None)